test/python/tools/test_migration.py

   1 # SPDX-License-Identifier: GPL-3.0-or-later
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2024 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Tests for migration functions
   9 """
  10 import pytest
  11 import psycopg2.extras
  12
  13 from nominatim_db.tools import migration
  14 from nominatim_db.errors import UsageError
  15 from nominatim_db.db.connection import server_version_tuple
  16 import nominatim_db.version
  17
  18 from mock_legacy_word_table import MockLegacyWordTable
  19
  20 class DummyTokenizer:
  21
  22     def update_sql_functions(self, config):
  23         pass
  24
  25
  26 @pytest.fixture
  27 def postprocess_mock(monkeypatch):
  28     monkeypatch.setattr(migration.refresh, 'create_functions', lambda *args: args)
  29     monkeypatch.setattr(migration.tokenizer_factory, 'get_tokenizer_for_db',
  30                         lambda *args: DummyTokenizer())
  31
  32 @pytest.fixture
  33 def legacy_word_table(temp_db_conn):
  34     return MockLegacyWordTable(temp_db_conn)
  35
  36
  37 def test_no_migration_old_versions(temp_db_with_extensions, table_factory, def_config):
  38     table_factory('country_name', 'name HSTORE, country_code TEXT')
  39
  40     with pytest.raises(UsageError, match='Migration not possible'):
  41         migration.migrate(def_config, {})
  42
  43
  44 def test_set_up_migration_for_36(temp_db_with_extensions, temp_db_cursor,
  45                                  table_factory, def_config, monkeypatch,
  46                                  postprocess_mock):
  47     psycopg2.extras.register_hstore(temp_db_cursor)
  48     # don't actually run any migration, except the property table creation
  49     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  50                         [((3, 5, 0, 99), migration.add_nominatim_property_table)])
  51     # Use a r/o user name that always exists
  52     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
  53
  54     table_factory('country_name', 'name HSTORE, country_code TEXT',
  55                   (({str(x): 'a' for x in range(200)}, 'gb'),))
  56
  57     assert not temp_db_cursor.table_exists('nominatim_properties')
  58
  59     assert migration.migrate(def_config, {}) == 0
  60
  61     assert temp_db_cursor.table_exists('nominatim_properties')
  62
  63     assert 1 == temp_db_cursor.scalar(""" SELECT count(*) FROM nominatim_properties
  64                                           WHERE property = 'database_version'""")
  65
  66
  67 def test_already_at_version(temp_db_with_extensions, def_config, property_table):
  68
  69     property_table.set('database_version',
  70                        str(nominatim_db.version.NOMINATIM_VERSION))
  71
  72     assert migration.migrate(def_config, {}) == 0
  73
  74
  75 def test_run_single_migration(temp_db_with_extensions, def_config, temp_db_cursor,
  76                               property_table, monkeypatch, postprocess_mock):
  77     oldversion = [x for x in nominatim_db.version.NOMINATIM_VERSION]
  78     oldversion[0] -= 1
  79     property_table.set('database_version',
  80                        str(nominatim_db.version.NominatimVersion(*oldversion)))
  81
  82     done = {'old': False, 'new': False}
  83     def _migration(**_):
  84         """ Dummy migration"""
  85         done['new'] = True
  86
  87     def _old_migration(**_):
  88         """ Dummy migration"""
  89         done['old'] = True
  90
  91     oldversion[0] = 0
  92     monkeypatch.setattr(migration, '_MIGRATION_FUNCTIONS',
  93                         [(tuple(oldversion), _old_migration),
  94                          (nominatim_db.version.NOMINATIM_VERSION, _migration)])
  95
  96     assert migration.migrate(def_config, {}) == 0
  97
  98     assert done['new']
  99     assert not done['old']
 100     assert property_table.get('database_version') == str(nominatim_db.version.NOMINATIM_VERSION)
 101
 102
 103 ###### Tests for specific migrations
 104 #
 105 # Each migration should come with two tests:
 106 #  1. Test that migration from old to new state works as expected.
 107 #  2. Test that the migration can be rerun on the new state without side effects.
 108
 109
 110 @pytest.mark.parametrize('in_attr', ('', 'with time zone'))
 111 def test_import_status_timestamp_change(temp_db_conn, temp_db_cursor,
 112                                         table_factory, in_attr):
 113     table_factory('import_status',
 114                   f"""lastimportdate timestamp {in_attr},
 115                      sequence_id integer,
 116                      indexed boolean""")
 117
 118     migration.import_status_timestamp_change(temp_db_conn)
 119     temp_db_conn.commit()
 120
 121     assert temp_db_cursor.scalar("""SELECT data_type FROM information_schema.columns
 122                                     WHERE table_name = 'import_status'
 123                                       and column_name = 'lastimportdate'""")\
 124             == 'timestamp with time zone'
 125
 126
 127 def test_add_nominatim_property_table(temp_db_conn, temp_db_cursor,
 128                                       def_config, monkeypatch):
 129     # Use a r/o user name that always exists
 130     monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'postgres')
 131
 132     assert not temp_db_cursor.table_exists('nominatim_properties')
 133
 134     migration.add_nominatim_property_table(temp_db_conn, def_config)
 135     temp_db_conn.commit()
 136
 137     assert temp_db_cursor.table_exists('nominatim_properties')
 138
 139
 140 def test_add_nominatim_property_table_repeat(temp_db_conn, temp_db_cursor,
 141                                              def_config, property_table):
 142     assert temp_db_cursor.table_exists('nominatim_properties')
 143
 144     migration.add_nominatim_property_table(temp_db_conn, def_config)
 145     temp_db_conn.commit()
 146
 147     assert temp_db_cursor.table_exists('nominatim_properties')
 148
 149
 150 def test_change_housenumber_transliteration(temp_db_conn, temp_db_cursor,
 151                                             legacy_word_table, placex_table):
 152     placex_table.add(housenumber='3A')
 153
 154     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION make_standard_name(name TEXT)
 155                               RETURNS TEXT AS $$ SELECT lower(name) $$ LANGUAGE SQL """)
 156     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
 157                               RETURNS INTEGER AS $$ SELECT 4325 $$ LANGUAGE SQL """)
 158
 159     migration.change_housenumber_transliteration(temp_db_conn)
 160     temp_db_conn.commit()
 161
 162     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 163
 164     migration.change_housenumber_transliteration(temp_db_conn)
 165     temp_db_conn.commit()
 166
 167     assert temp_db_cursor.scalar('SELECT housenumber from placex') == '3a'
 168
 169
 170 def test_switch_placenode_geometry_index(temp_db_conn, temp_db_cursor, placex_table):
 171     temp_db_cursor.execute("""CREATE INDEX idx_placex_adminname
 172                               ON placex (place_id)""")
 173
 174     migration.switch_placenode_geometry_index(temp_db_conn)
 175     temp_db_conn.commit()
 176
 177     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 178     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 179
 180
 181 def test_switch_placenode_geometry_index_repeat(temp_db_conn, temp_db_cursor, placex_table):
 182     temp_db_cursor.execute("""CREATE INDEX idx_placex_geometry_placenode
 183                               ON placex (place_id)""")
 184
 185     migration.switch_placenode_geometry_index(temp_db_conn)
 186     temp_db_conn.commit()
 187
 188     assert temp_db_cursor.index_exists('placex', 'idx_placex_geometry_placenode')
 189     assert not temp_db_cursor.index_exists('placex', 'idx_placex_adminname')
 190     assert temp_db_cursor.scalar("""SELECT indexdef from pg_indexes
 191                                     WHERE tablename = 'placex'
 192                                       and indexname = 'idx_placex_geometry_placenode'
 193                                  """).endswith('(place_id)')
 194
 195
 196 def test_install_legacy_tokenizer(temp_db_conn, temp_db_cursor, project_env,
 197                                   property_table, table_factory, monkeypatch,
 198                                   tmp_path):
 199     table_factory('placex', 'place_id BIGINT')
 200     table_factory('location_property_osmline', 'place_id BIGINT')
 201
 202     # Setting up the tokenizer is problematic
 203     class MiniTokenizer:
 204         def migrate_database(self, config):
 205             pass
 206
 207     monkeypatch.setattr(migration.tokenizer_factory, 'create_tokenizer',
 208                         lambda cfg, **kwargs: MiniTokenizer())
 209
 210     migration.install_legacy_tokenizer(temp_db_conn, project_env)
 211     temp_db_conn.commit()
 212
 213
 214
 215 def test_install_legacy_tokenizer_repeat(temp_db_conn, temp_db_cursor,
 216                                          def_config, property_table):
 217
 218     property_table.set('tokenizer', 'dummy')
 219     migration.install_legacy_tokenizer(temp_db_conn, def_config)
 220     temp_db_conn.commit()
 221
 222
 223 def test_create_tiger_housenumber_index(temp_db_conn, temp_db_cursor, table_factory):
 224     table_factory('location_property_tiger',
 225                   'parent_place_id BIGINT, startnumber INT, endnumber INT')
 226
 227     migration.create_tiger_housenumber_index(temp_db_conn)
 228     temp_db_conn.commit()
 229
 230     if server_version_tuple(temp_db_conn) >= (11, 0, 0):
 231         assert temp_db_cursor.index_exists('location_property_tiger',
 232                                            'idx_location_property_tiger_housenumber_migrated')
 233
 234     migration.create_tiger_housenumber_index(temp_db_conn)
 235     temp_db_conn.commit()