test/python/tools/test_migration.py

   1 # SPDX-License-Identifier: GPL-2.0-only
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2022 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Tests for migration functions
   9 """
  10 import pytest
  11 import psycopg2.extras
  12
  13 from nominatim.tools import migration
  14 from nominatim.errors import UsageError
  15 import nominatim.version
  16
  17 class DummyTokenizer:
  18
  19     def update_sql_functions(self, config):
  20         pass
  21
  22
  23 @pytest.fixture
  24 def postprocess_mock(monkeypatch):
  25     monkeypatch.setattr(migration.refresh, 'create_functions', lambda *args: args)
  26     monkeypatch.setattr(migration.tokenizer_factory, 'get_tokenizer_for_db',
  27                         lambda *args: DummyTokenizer())
  28
  29
  30 def test_no_migration_old_versions(temp_db_with_extensions, table_factory, def_config):
  31     table_factory('country_name', 'name HSTORE, country_code TEXT')
  32
  33     with pytest.raises(UsageError, match='Migration not possible'):
  34         migration.migrate(def_config, {})
  35
  36
  37 def test_set_up_migration_for_36(temp_db_with_extensions, temp_db_cursor,
  38                                  table_factory, def_config, monkeypatch,
  39                                  postprocess_mock):
  40     psycopg2.extras.register_hstore(temp_db_cursor)
  41     # don't actually run any migration, except the property table creation
  42     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  43                         [((3, 5, 0, 99), migration.add_nominatim_property_table)])
  44     # Use a r/o user name that always exists
  45     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
  46
  47     table_factory('country_name', 'name HSTORE, country_code TEXT',
  48                   (({str(x): 'a' for x in range(200)}, 'gb'),))
  49
  50     assert not temp_db_cursor.table_exists('nominatim_properties')
  51
  52     assert migration.migrate(def_config, {}) == 0
  53
  54     assert temp_db_cursor.table_exists('nominatim_properties')
  55
  56     assert 1 == temp_db_cursor.scalar(""" SELECT count(*) FROM nominatim_properties
  57                                           WHERE property = 'database_version'""")
  58
  59
  60 def test_already_at_version(def_config, property_table):
  61
  62     property_table.set('database_version',
  63                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION))
  64
  65     assert migration.migrate(def_config, {}) == 0
  66
  67
  68 def test_no_migrations_necessary(def_config, temp_db_cursor, property_table,
  69                                  monkeypatch):
  70     oldversion = [x for x in nominatim.version.NOMINATIM_VERSION]
  71     oldversion[0] -= 1
  72     property_table.set('database_version',
  73                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(oldversion))
  74
  75     oldversion[0] = 0
  76     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  77                         [(tuple(oldversion), lambda **attr: True)])
  78
  79     assert migration.migrate(def_config, {}) == 0
  80
  81
  82 def test_run_single_migration(def_config, temp_db_cursor, property_table,
  83                               monkeypatch, postprocess_mock):
  84     oldversion = [x for x in nominatim.version.NOMINATIM_VERSION]
  85     oldversion[0] -= 1
  86     property_table.set('database_version',
  87                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(oldversion))
  88
  89     done = {'old': False, 'new': False}
  90     def _migration(**_):
  91         """ Dummy migration"""
  92         done['new'] = True
  93
  94     def _old_migration(**_):
  95         """ Dummy migration"""
  96         done['old'] = True
  97
  98     oldversion[0] = 0
  99     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
 100                         [(tuple(oldversion), _old_migration),
 101                          (nominatim.version.NOMINATIM_VERSION, _migration)])
 102
 103     assert migration.migrate(def_config, {}) == 0
 104
 105     assert done['new']
 106     assert not done['old']
 107     assert property_table.get('database_version') == \
 108            '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION)
 109
 110
 111 ###### Tests for specific migrations
 112 #
 113 # Each migration should come with two tests:
 114 #  1. Test that migration from old to new state works as expected.
 115 #  2. Test that the migration can be rerun on the new state without side effects.
 116
 117
 118 @pytest.mark.parametrize('in_attr', ('', 'with time zone'))
 119 def test_import_status_timestamp_change(temp_db_conn, temp_db_cursor,
 120                                         table_factory, in_attr):
 121     table_factory('import_status',
 122                   f"""lastimportdate timestamp {in_attr},
 123                      sequence_id integer,
 124                      indexed boolean""")
 125
 126     migration.import_status_timestamp_change(temp_db_conn)
 127     temp_db_conn.commit()
 128
 129     assert temp_db_cursor.scalar("""SELECT data_type FROM information_schema.columns
 130                                     WHERE table_name = 'import_status'
 131                                       and column_name = 'lastimportdate'""")\
 132             == 'timestamp with time zone'
 133
 134
 135 def test_add_nominatim_property_table(temp_db_conn, temp_db_cursor,
 136                                       def_config, monkeypatch):
 137     # Use a r/o user name that always exists
 138     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
 139
 140     assert not temp_db_cursor.table_exists('nominatim_properties')
 141
 142     migration.add_nominatim_property_table(temp_db_conn, def_config)
 143     temp_db_conn.commit()
 144
 145     assert temp_db_cursor.table_exists('nominatim_properties')
 146
 147
 148 def test_add_nominatim_property_table_repeat(temp_db_conn, temp_db_cursor,
 149                                              def_config, property_table):
 150     assert temp_db_cursor.table_exists('nominatim_properties')
 151
 152     migration.add_nominatim_property_table(temp_db_conn, def_config)
 153     temp_db_conn.commit()
 154
 155     assert temp_db_cursor.table_exists('nominatim_properties')
 156
 157
 158 def test_change_housenumber_transliteration(temp_db_conn, temp_db_cursor,
 159                                             word_table, placex_table):
 160     placex_table.add(housenumber='3A')
 161
 162     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION make_standard_name(name TEXT)
 163                               RETURNS TEXT AS $$ SELECT lower(name) $$ LANGUAGE SQL """)
 164     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
 165                               RETURNS INTEGER AS $$ SELECT 4325 $$ LANGUAGE SQL """)
 166
 167     migration.change_housenumber_transliteration(temp_db_conn)
 168     temp_db_conn.commit()
 169
 170     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 171
 172     migration.change_housenumber_transliteration(temp_db_conn)
 173     temp_db_conn.commit()
 174
 175     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 176
 177
 178 def test_switch_placenode_geometry_index(temp_db_conn, temp_db_cursor, placex_table):
 179     temp_db_cursor.execute("""CREATE INDEX idx_placex_adminname
 180                               ON placex (place_id)""")
 181
 182     migration.switch_placenode_geometry_index(temp_db_conn)
 183     temp_db_conn.commit()
 184
 185     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 186     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 187
 188
 189 def test_switch_placenode_geometry_index_repeat(temp_db_conn, temp_db_cursor, placex_table):
 190     temp_db_cursor.execute("""CREATE INDEX idx_placex_geometry_placenode
 191                               ON placex (place_id)""")
 192
 193     migration.switch_placenode_geometry_index(temp_db_conn)
 194     temp_db_conn.commit()
 195
 196     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 197     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 198     assert temp_db_cursor.scalar("""SELECT indexdef from pg_indexes
 199                                     WHERE tablename = 'placex'
 200                                       and indexname = 'idx_placex_geometry_placenode'
 201                                  """).endswith('(place_id)')
 202
 203
 204 def test_install_legacy_tokenizer(temp_db_conn, temp_db_cursor, project_env,
 205                                   property_table, table_factory, monkeypatch,
 206                                   tmp_path):
 207     table_factory('placex', 'place_id BIGINT')
 208     table_factory('location_property_osmline', 'place_id BIGINT')
 209
 210     # Setting up the tokenizer is problematic
 211     class MiniTokenizer:
 212         def migrate_database(self, config):
 213             pass
 214
 215     monkeypatch.setattr(migration.tokenizer_factory, 'create_tokenizer',
 216                         lambda cfg, **kwargs: MiniTokenizer())
 217
 218     migration.install_legacy_tokenizer(temp_db_conn, project_env)
 219     temp_db_conn.commit()
 220
 221
 222
 223 def test_install_legacy_tokenizer_repeat(temp_db_conn, temp_db_cursor,
 224                                          def_config, property_table):
 225
 226     property_table.set('tokenizer', 'dummy')
 227     migration.install_legacy_tokenizer(temp_db_conn, def_config)
 228     temp_db_conn.commit()
 229
 230
 231 def test_create_tiger_housenumber_index(temp_db_conn, temp_db_cursor, table_factory):
 232     table_factory('location_property_tiger',
 233                   'parent_place_id BIGINT, startnumber INT, endnumber INT')
 234
 235     migration.create_tiger_housenumber_index(temp_db_conn)
 236     temp_db_conn.commit()
 237
 238     if temp_db_conn.server_version_tuple() >= (11, 0, 0):
 239         assert temp_db_cursor.index_exists('location_property_tiger',
 240                                            'idx_location_property_tiger_housenumber_migrated')
 241
 242     migration.create_tiger_housenumber_index(temp_db_conn)
 243     temp_db_conn.commit()