test/python/tools/test_migration.py

   1 """
   2 Tests for migration functions
   3 """
   4 import pytest
   5 import psycopg2.extras
   6
   7 from nominatim.tools import migration
   8 from nominatim.errors import UsageError
   9 import nominatim.version
  10
  11 class DummyTokenizer:
  12
  13     def update_sql_functions(self, config):
  14         pass
  15
  16
  17 @pytest.fixture
  18 def postprocess_mock(monkeypatch):
  19     monkeypatch.setattr(migration.refresh, 'create_functions', lambda *args: args)
  20     monkeypatch.setattr(migration.tokenizer_factory, 'get_tokenizer_for_db',
  21                         lambda *args: DummyTokenizer())
  22
  23
  24 def test_no_migration_old_versions(temp_db_with_extensions, table_factory, def_config):
  25     table_factory('country_name', 'name HSTORE, country_code TEXT')
  26
  27     with pytest.raises(UsageError, match='Migration not possible'):
  28         migration.migrate(def_config, {})
  29
  30
  31 def test_set_up_migration_for_36(temp_db_with_extensions, temp_db_cursor,
  32                                  table_factory, def_config, monkeypatch,
  33                                  postprocess_mock):
  34     psycopg2.extras.register_hstore(temp_db_cursor)
  35     # don't actually run any migration, except the property table creation
  36     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  37                         [((3, 5, 0, 99), migration.add_nominatim_property_table)])
  38     # Use a r/o user name that always exists
  39     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
  40
  41     table_factory('country_name', 'name HSTORE, country_code TEXT',
  42                   (({str(x): 'a' for x in range(200)}, 'gb'),))
  43
  44     assert not temp_db_cursor.table_exists('nominatim_properties')
  45
  46     assert migration.migrate(def_config, {}) == 0
  47
  48     assert temp_db_cursor.table_exists('nominatim_properties')
  49
  50     assert 1 == temp_db_cursor.scalar(""" SELECT count(*) FROM nominatim_properties
  51                                           WHERE property = 'database_version'""")
  52
  53
  54 def test_already_at_version(def_config, property_table):
  55
  56     property_table.set('database_version',
  57                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION))
  58
  59     assert migration.migrate(def_config, {}) == 0
  60
  61
  62 def test_no_migrations_necessary(def_config, temp_db_cursor, property_table,
  63                                  monkeypatch):
  64     oldversion = [x for x in nominatim.version.NOMINATIM_VERSION]
  65     oldversion[0] -= 1
  66     property_table.set('database_version',
  67                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(oldversion))
  68
  69     oldversion[0] = 0
  70     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  71                         [(tuple(oldversion), lambda **attr: True)])
  72
  73     assert migration.migrate(def_config, {}) == 0
  74
  75
  76 def test_run_single_migration(def_config, temp_db_cursor, property_table,
  77                               monkeypatch, postprocess_mock):
  78     oldversion = [x for x in nominatim.version.NOMINATIM_VERSION]
  79     oldversion[0] -= 1
  80     property_table.set('database_version',
  81                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(oldversion))
  82
  83     done = {'old': False, 'new': False}
  84     def _migration(**_):
  85         """ Dummy migration"""
  86         done['new'] = True
  87
  88     def _old_migration(**_):
  89         """ Dummy migration"""
  90         done['old'] = True
  91
  92     oldversion[0] = 0
  93     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  94                         [(tuple(oldversion), _old_migration),
  95                          (nominatim.version.NOMINATIM_VERSION, _migration)])
  96
  97     assert migration.migrate(def_config, {}) == 0
  98
  99     assert done['new']
 100     assert not done['old']
 101     assert property_table.get('database_version') == \
 102            '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION)
 103
 104
 105 ###### Tests for specific migrations
 106 #
 107 # Each migration should come with two tests:
 108 #  1. Test that migration from old to new state works as expected.
 109 #  2. Test that the migration can be rerun on the new state without side effects.
 110
 111
 112 @pytest.mark.parametrize('in_attr', ('', 'with time zone'))
 113 def test_import_status_timestamp_change(temp_db_conn, temp_db_cursor,
 114                                         table_factory, in_attr):
 115     table_factory('import_status',
 116                   f"""lastimportdate timestamp {in_attr},
 117                      sequence_id integer,
 118                      indexed boolean""")
 119
 120     migration.import_status_timestamp_change(temp_db_conn)
 121     temp_db_conn.commit()
 122
 123     assert temp_db_cursor.scalar("""SELECT data_type FROM information_schema.columns
 124                                     WHERE table_name = 'import_status'
 125                                       and column_name = 'lastimportdate'""")\
 126             == 'timestamp with time zone'
 127
 128
 129 def test_add_nominatim_property_table(temp_db_conn, temp_db_cursor,
 130                                       def_config, monkeypatch):
 131     # Use a r/o user name that always exists
 132     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
 133
 134     assert not temp_db_cursor.table_exists('nominatim_properties')
 135
 136     migration.add_nominatim_property_table(temp_db_conn, def_config)
 137     temp_db_conn.commit()
 138
 139     assert temp_db_cursor.table_exists('nominatim_properties')
 140
 141
 142 def test_add_nominatim_property_table_repeat(temp_db_conn, temp_db_cursor,
 143                                              def_config, property_table):
 144     assert temp_db_cursor.table_exists('nominatim_properties')
 145
 146     migration.add_nominatim_property_table(temp_db_conn, def_config)
 147     temp_db_conn.commit()
 148
 149     assert temp_db_cursor.table_exists('nominatim_properties')
 150
 151
 152 def test_change_housenumber_transliteration(temp_db_conn, temp_db_cursor,
 153                                             word_table, placex_table):
 154     placex_table.add(housenumber='3A')
 155
 156     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION make_standard_name(name TEXT)
 157                               RETURNS TEXT AS $$ SELECT lower(name) $$ LANGUAGE SQL """)
 158     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
 159                               RETURNS INTEGER AS $$ SELECT 4325 $$ LANGUAGE SQL """)
 160
 161     migration.change_housenumber_transliteration(temp_db_conn)
 162     temp_db_conn.commit()
 163
 164     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 165
 166     migration.change_housenumber_transliteration(temp_db_conn)
 167     temp_db_conn.commit()
 168
 169     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 170
 171
 172 def test_switch_placenode_geometry_index(temp_db_conn, temp_db_cursor, placex_table):
 173     temp_db_cursor.execute("""CREATE INDEX idx_placex_adminname
 174                               ON placex (place_id)""")
 175
 176     migration.switch_placenode_geometry_index(temp_db_conn)
 177     temp_db_conn.commit()
 178
 179     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 180     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 181
 182
 183 def test_switch_placenode_geometry_index_repeat(temp_db_conn, temp_db_cursor, placex_table):
 184     temp_db_cursor.execute("""CREATE INDEX idx_placex_geometry_placenode
 185                               ON placex (place_id)""")
 186
 187     migration.switch_placenode_geometry_index(temp_db_conn)
 188     temp_db_conn.commit()
 189
 190     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 191     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 192     assert temp_db_cursor.scalar("""SELECT indexdef from pg_indexes
 193                                     WHERE tablename = 'placex'
 194                                       and indexname = 'idx_placex_geometry_placenode'
 195                                  """).endswith('(place_id)')
 196
 197
 198 def test_install_legacy_tokenizer(temp_db_conn, temp_db_cursor, project_env,
 199                                   property_table, table_factory, monkeypatch,
 200                                   tmp_path):
 201     table_factory('placex', 'place_id BIGINT')
 202     table_factory('location_property_osmline', 'place_id BIGINT')
 203
 204     # Setting up the tokenizer is problematic
 205     class MiniTokenizer:
 206         def migrate_database(self, config):
 207             pass
 208
 209     monkeypatch.setattr(migration.tokenizer_factory, 'create_tokenizer',
 210                         lambda cfg, **kwargs: MiniTokenizer())
 211
 212     migration.install_legacy_tokenizer(temp_db_conn, project_env)
 213     temp_db_conn.commit()
 214
 215
 216
 217 def test_install_legacy_tokenizer_repeat(temp_db_conn, temp_db_cursor,
 218                                          def_config, property_table):
 219
 220     property_table.set('tokenizer', 'dummy')
 221     migration.install_legacy_tokenizer(temp_db_conn, def_config)
 222     temp_db_conn.commit()
 223
 224
 225 def test_create_tiger_housenumber_index(temp_db_conn, temp_db_cursor, table_factory):
 226     table_factory('location_property_tiger',
 227                   'parent_place_id BIGINT, startnumber INT, endnumber INT')
 228
 229     migration.create_tiger_housenumber_index(temp_db_conn)
 230     temp_db_conn.commit()
 231
 232     if temp_db_conn.server_version_tuple() >= (11, 0, 0):
 233         assert temp_db_cursor.index_exists('location_property_tiger',
 234                                            'idx_location_property_tiger_housenumber_migrated')
 235
 236     migration.create_tiger_housenumber_index(temp_db_conn)
 237     temp_db_conn.commit()