test/python/tools/test_migration.py

   1 # SPDX-License-Identifier: GPL-2.0-only
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2022 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Tests for migration functions
   9 """
  10 import pytest
  11 import psycopg2.extras
  12
  13 from nominatim.tools import migration
  14 from nominatim.errors import UsageError
  15 import nominatim.version
  16
  17 from mock_legacy_word_table import MockLegacyWordTable
  18
  19 class DummyTokenizer:
  20
  21     def update_sql_functions(self, config):
  22         pass
  23
  24
  25 @pytest.fixture
  26 def postprocess_mock(monkeypatch):
  27     monkeypatch.setattr(migration.refresh, 'create_functions', lambda *args: args)
  28     monkeypatch.setattr(migration.tokenizer_factory, 'get_tokenizer_for_db',
  29                         lambda *args: DummyTokenizer())
  30
  31 @pytest.fixture
  32 def legacy_word_table(temp_db_conn):
  33     return MockLegacyWordTable(temp_db_conn)
  34
  35
  36 def test_no_migration_old_versions(temp_db_with_extensions, table_factory, def_config):
  37     table_factory('country_name', 'name HSTORE, country_code TEXT')
  38
  39     with pytest.raises(UsageError, match='Migration not possible'):
  40         migration.migrate(def_config, {})
  41
  42
  43 def test_set_up_migration_for_36(temp_db_with_extensions, temp_db_cursor,
  44                                  table_factory, def_config, monkeypatch,
  45                                  postprocess_mock):
  46     psycopg2.extras.register_hstore(temp_db_cursor)
  47     # don't actually run any migration, except the property table creation
  48     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  49                         [((3, 5, 0, 99), migration.add_nominatim_property_table)])
  50     # Use a r/o user name that always exists
  51     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
  52
  53     table_factory('country_name', 'name HSTORE, country_code TEXT',
  54                   (({str(x): 'a' for x in range(200)}, 'gb'),))
  55
  56     assert not temp_db_cursor.table_exists('nominatim_properties')
  57
  58     assert migration.migrate(def_config, {}) == 0
  59
  60     assert temp_db_cursor.table_exists('nominatim_properties')
  61
  62     assert 1 == temp_db_cursor.scalar(""" SELECT count(*) FROM nominatim_properties
  63                                           WHERE property = 'database_version'""")
  64
  65
  66 def test_already_at_version(def_config, property_table):
  67
  68     property_table.set('database_version',
  69                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION))
  70
  71     assert migration.migrate(def_config, {}) == 0
  72
  73
  74 def test_no_migrations_necessary(def_config, temp_db_cursor, property_table,
  75                                  monkeypatch):
  76     oldversion = [x for x in nominatim.version.NOMINATIM_VERSION]
  77     oldversion[0] -= 1
  78     property_table.set('database_version',
  79                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(oldversion))
  80
  81     oldversion[0] = 0
  82     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  83                         [(tuple(oldversion), lambda **attr: True)])
  84
  85     assert migration.migrate(def_config, {}) == 0
  86
  87
  88 def test_run_single_migration(def_config, temp_db_cursor, property_table,
  89                               monkeypatch, postprocess_mock):
  90     oldversion = [x for x in nominatim.version.NOMINATIM_VERSION]
  91     oldversion[0] -= 1
  92     property_table.set('database_version',
  93                        '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(oldversion))
  94
  95     done = {'old': False, 'new': False}
  96     def _migration(**_):
  97         """ Dummy migration"""
  98         done['new'] = True
  99
 100     def _old_migration(**_):
 101         """ Dummy migration"""
 102         done['old'] = True
 103
 104     oldversion[0] = 0
 105     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
 106                         [(tuple(oldversion), _old_migration),
 107                          (nominatim.version.NOMINATIM_VERSION, _migration)])
 108
 109     assert migration.migrate(def_config, {}) == 0
 110
 111     assert done['new']
 112     assert not done['old']
 113     assert property_table.get('database_version') == \
 114            '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(nominatim.version.NOMINATIM_VERSION)
 115
 116
 117 ###### Tests for specific migrations
 118 #
 119 # Each migration should come with two tests:
 120 #  1. Test that migration from old to new state works as expected.
 121 #  2. Test that the migration can be rerun on the new state without side effects.
 122
 123
 124 @pytest.mark.parametrize('in_attr', ('', 'with time zone'))
 125 def test_import_status_timestamp_change(temp_db_conn, temp_db_cursor,
 126                                         table_factory, in_attr):
 127     table_factory('import_status',
 128                   f"""lastimportdate timestamp {in_attr},
 129                      sequence_id integer,
 130                      indexed boolean""")
 131
 132     migration.import_status_timestamp_change(temp_db_conn)
 133     temp_db_conn.commit()
 134
 135     assert temp_db_cursor.scalar("""SELECT data_type FROM information_schema.columns
 136                                     WHERE table_name = 'import_status'
 137                                       and column_name = 'lastimportdate'""")\
 138             == 'timestamp with time zone'
 139
 140
 141 def test_add_nominatim_property_table(temp_db_conn, temp_db_cursor,
 142                                       def_config, monkeypatch):
 143     # Use a r/o user name that always exists
 144     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
 145
 146     assert not temp_db_cursor.table_exists('nominatim_properties')
 147
 148     migration.add_nominatim_property_table(temp_db_conn, def_config)
 149     temp_db_conn.commit()
 150
 151     assert temp_db_cursor.table_exists('nominatim_properties')
 152
 153
 154 def test_add_nominatim_property_table_repeat(temp_db_conn, temp_db_cursor,
 155                                              def_config, property_table):
 156     assert temp_db_cursor.table_exists('nominatim_properties')
 157
 158     migration.add_nominatim_property_table(temp_db_conn, def_config)
 159     temp_db_conn.commit()
 160
 161     assert temp_db_cursor.table_exists('nominatim_properties')
 162
 163
 164 def test_change_housenumber_transliteration(temp_db_conn, temp_db_cursor,
 165                                             legacy_word_table, placex_table):
 166     placex_table.add(housenumber='3A')
 167
 168     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION make_standard_name(name TEXT)
 169                               RETURNS TEXT AS $$ SELECT lower(name) $$ LANGUAGE SQL """)
 170     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
 171                               RETURNS INTEGER AS $$ SELECT 4325 $$ LANGUAGE SQL """)
 172
 173     migration.change_housenumber_transliteration(temp_db_conn)
 174     temp_db_conn.commit()
 175
 176     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 177
 178     migration.change_housenumber_transliteration(temp_db_conn)
 179     temp_db_conn.commit()
 180
 181     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 182
 183
 184 def test_switch_placenode_geometry_index(temp_db_conn, temp_db_cursor, placex_table):
 185     temp_db_cursor.execute("""CREATE INDEX idx_placex_adminname
 186                               ON placex (place_id)""")
 187
 188     migration.switch_placenode_geometry_index(temp_db_conn)
 189     temp_db_conn.commit()
 190
 191     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 192     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 193
 194
 195 def test_switch_placenode_geometry_index_repeat(temp_db_conn, temp_db_cursor, placex_table):
 196     temp_db_cursor.execute("""CREATE INDEX idx_placex_geometry_placenode
 197                               ON placex (place_id)""")
 198
 199     migration.switch_placenode_geometry_index(temp_db_conn)
 200     temp_db_conn.commit()
 201
 202     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 203     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 204     assert temp_db_cursor.scalar("""SELECT indexdef from pg_indexes
 205                                     WHERE tablename = 'placex'
 206                                       and indexname = 'idx_placex_geometry_placenode'
 207                                  """).endswith('(place_id)')
 208
 209
 210 def test_install_legacy_tokenizer(temp_db_conn, temp_db_cursor, project_env,
 211                                   property_table, table_factory, monkeypatch,
 212                                   tmp_path):
 213     table_factory('placex', 'place_id BIGINT')
 214     table_factory('location_property_osmline', 'place_id BIGINT')
 215
 216     # Setting up the tokenizer is problematic
 217     class MiniTokenizer:
 218         def migrate_database(self, config):
 219             pass
 220
 221     monkeypatch.setattr(migration.tokenizer_factory, 'create_tokenizer',
 222                         lambda cfg, **kwargs: MiniTokenizer())
 223
 224     migration.install_legacy_tokenizer(temp_db_conn, project_env)
 225     temp_db_conn.commit()
 226
 227
 228
 229 def test_install_legacy_tokenizer_repeat(temp_db_conn, temp_db_cursor,
 230                                          def_config, property_table):
 231
 232     property_table.set('tokenizer', 'dummy')
 233     migration.install_legacy_tokenizer(temp_db_conn, def_config)
 234     temp_db_conn.commit()
 235
 236
 237 def test_create_tiger_housenumber_index(temp_db_conn, temp_db_cursor, table_factory):
 238     table_factory('location_property_tiger',
 239                   'parent_place_id BIGINT, startnumber INT, endnumber INT')
 240
 241     migration.create_tiger_housenumber_index(temp_db_conn)
 242     temp_db_conn.commit()
 243
 244     if temp_db_conn.server_version_tuple() >= (11, 0, 0):
 245         assert temp_db_cursor.index_exists('location_property_tiger',
 246                                            'idx_location_property_tiger_housenumber_migrated')
 247
 248     migration.create_tiger_housenumber_index(temp_db_conn)
 249     temp_db_conn.commit()