test/python/test_indexing.py

   1 """
   2 Tests for running the indexing.
   3 """
   4 import itertools
   5 import pytest
   6
   7 from nominatim.indexer import indexer
   8 from nominatim.tokenizer import factory
   9
  10 class IndexerTestDB:
  11
  12     def __init__(self, conn):
  13         self.placex_id = itertools.count(100000)
  14         self.osmline_id = itertools.count(500000)
  15         self.postcode_id = itertools.count(700000)
  16
  17         self.conn = conn
  18         self.conn.set_isolation_level(0)
  19         with self.conn.cursor() as cur:
  20             cur.execute('CREATE EXTENSION hstore')
  21             cur.execute("""CREATE TABLE placex (place_id BIGINT,
  22                                                 name HSTORE,
  23                                                 class TEXT,
  24                                                 type TEXT,
  25                                                 linked_place_id BIGINT,
  26                                                 rank_address SMALLINT,
  27                                                 rank_search SMALLINT,
  28                                                 indexed_status SMALLINT,
  29                                                 indexed_date TIMESTAMP,
  30                                                 partition SMALLINT,
  31                                                 admin_level SMALLINT,
  32                                                 address HSTORE,
  33                                                 token_info JSONB,
  34                                                 geometry_sector INTEGER)""")
  35             cur.execute("""CREATE TABLE location_property_osmline (
  36                                place_id BIGINT,
  37                                osm_id BIGINT,
  38                                address HSTORE,
  39                                token_info JSONB,
  40                                indexed_status SMALLINT,
  41                                indexed_date TIMESTAMP,
  42                                geometry_sector INTEGER)""")
  43             cur.execute("""CREATE TABLE location_postcode (
  44                                place_id BIGINT,
  45                                indexed_status SMALLINT,
  46                                indexed_date TIMESTAMP,
  47                                country_code varchar(2),
  48                                postcode TEXT)""")
  49             cur.execute("""CREATE OR REPLACE FUNCTION date_update() RETURNS TRIGGER
  50                            AS $$
  51                            BEGIN
  52                              IF NEW.indexed_status = 0 and OLD.indexed_status != 0 THEN
  53                                NEW.indexed_date = now();
  54                              END IF;
  55                              RETURN NEW;
  56                            END; $$ LANGUAGE plpgsql;""")
  57             cur.execute("""CREATE OR REPLACE FUNCTION placex_prepare_update(p placex,
  58                                                       OUT name HSTORE,
  59                                                       OUT address HSTORE,
  60                                                       OUT country_feature VARCHAR,
  61                                                       OUT linked_place_id BIGINT)
  62                            AS $$
  63                            BEGIN
  64                             address := p.address;
  65                             name := p.name;
  66                            END;
  67                            $$ LANGUAGE plpgsql STABLE;
  68                         """)
  69             cur.execute("""CREATE OR REPLACE FUNCTION
  70                              get_interpolation_address(in_address HSTORE, wayid BIGINT)
  71                            RETURNS HSTORE AS $$
  72                            BEGIN
  73                              RETURN in_address;
  74                            END;
  75                            $$ LANGUAGE plpgsql STABLE;
  76                         """)
  77
  78             for table in ('placex', 'location_property_osmline', 'location_postcode'):
  79                 cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
  80                                FOR EACH ROW EXECUTE PROCEDURE date_update()
  81                             """.format(table))
  82
  83     def scalar(self, query):
  84         with self.conn.cursor() as cur:
  85             cur.execute(query)
  86             return cur.fetchone()[0]
  87
  88     def add_place(self, cls='place', typ='locality',
  89                   rank_search=30, rank_address=30, sector=20):
  90         next_id = next(self.placex_id)
  91         with self.conn.cursor() as cur:
  92             cur.execute("""INSERT INTO placex
  93                               (place_id, class, type, rank_search, rank_address,
  94                                indexed_status, geometry_sector)
  95                               VALUES (%s, %s, %s, %s, %s, 1, %s)""",
  96                         (next_id, cls, typ, rank_search, rank_address, sector))
  97         return next_id
  98
  99     def add_admin(self, **kwargs):
 100         kwargs['cls'] = 'boundary'
 101         kwargs['typ'] = 'administrative'
 102         return self.add_place(**kwargs)
 103
 104     def add_osmline(self, sector=20):
 105         next_id = next(self.osmline_id)
 106         with self.conn.cursor() as cur:
 107             cur.execute("""INSERT INTO location_property_osmline
 108                               (place_id, osm_id, indexed_status, geometry_sector)
 109                               VALUES (%s, %s, 1, %s)""",
 110                         (next_id, next_id, sector))
 111         return next_id
 112
 113     def add_postcode(self, country, postcode):
 114         next_id = next(self.postcode_id)
 115         with self.conn.cursor() as cur:
 116             cur.execute("""INSERT INTO location_postcode
 117                             (place_id, indexed_status, country_code, postcode)
 118                             VALUES (%s, 1, %s, %s)""",
 119                         (next_id, country, postcode))
 120         return next_id
 121
 122     def placex_unindexed(self):
 123         return self.scalar('SELECT count(*) from placex where indexed_status > 0')
 124
 125     def osmline_unindexed(self):
 126         return self.scalar("""SELECT count(*) from location_property_osmline
 127                               WHERE indexed_status > 0""")
 128
 129
 130 @pytest.fixture
 131 def test_db(temp_db_conn):
 132     yield IndexerTestDB(temp_db_conn)
 133
 134
 135 @pytest.fixture
 136 def test_tokenizer(tokenizer_mock, def_config, tmp_path):
 137     def_config.project_dir = tmp_path
 138     return factory.create_tokenizer(def_config)
 139
 140
 141 @pytest.mark.parametrize("threads", [1, 15])
 142 def test_index_all_by_rank(test_db, threads, test_tokenizer):
 143     for rank in range(31):
 144         test_db.add_place(rank_address=rank, rank_search=rank)
 145     test_db.add_osmline()
 146
 147     assert test_db.placex_unindexed() == 31
 148     assert test_db.osmline_unindexed() == 1
 149
 150     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 151     idx.index_by_rank(0, 30)
 152
 153     assert test_db.placex_unindexed() == 0
 154     assert test_db.osmline_unindexed() == 0
 155
 156     assert test_db.scalar("""SELECT count(*) from placex
 157                              WHERE indexed_status = 0 and indexed_date is null""") == 0
 158     # ranks come in order of rank address
 159     assert test_db.scalar("""
 160         SELECT count(*) FROM placex p WHERE rank_address > 0
 161           AND indexed_date >= (SELECT min(indexed_date) FROM placex o
 162                                WHERE p.rank_address < o.rank_address)""") == 0
 163     # placex rank < 30 objects come before interpolations
 164     assert test_db.scalar(
 165         """SELECT count(*) FROM placex WHERE rank_address < 30
 166              AND indexed_date >
 167                    (SELECT min(indexed_date) FROM location_property_osmline)""") == 0
 168     # placex rank = 30 objects come after interpolations
 169     assert test_db.scalar(
 170         """SELECT count(*) FROM placex WHERE rank_address = 30
 171              AND indexed_date <
 172                    (SELECT max(indexed_date) FROM location_property_osmline)""") == 0
 173     # rank 0 comes after rank 29 and before rank 30
 174     assert test_db.scalar(
 175         """SELECT count(*) FROM placex WHERE rank_address < 30
 176              AND indexed_date >
 177                    (SELECT min(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
 178     assert test_db.scalar(
 179         """SELECT count(*) FROM placex WHERE rank_address = 30
 180              AND indexed_date <
 181                    (SELECT max(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
 182
 183
 184 @pytest.mark.parametrize("threads", [1, 15])
 185 def test_index_partial_without_30(test_db, threads, test_tokenizer):
 186     for rank in range(31):
 187         test_db.add_place(rank_address=rank, rank_search=rank)
 188     test_db.add_osmline()
 189
 190     assert test_db.placex_unindexed() == 31
 191     assert test_db.osmline_unindexed() == 1
 192
 193     idx = indexer.Indexer('dbname=test_nominatim_python_unittest',
 194                           test_tokenizer, threads)
 195     idx.index_by_rank(4, 15)
 196
 197     assert test_db.placex_unindexed() == 19
 198     assert test_db.osmline_unindexed() == 1
 199
 200     assert test_db.scalar("""
 201                     SELECT count(*) FROM placex
 202                       WHERE indexed_status = 0 AND not rank_address between 4 and 15""") == 0
 203
 204
 205 @pytest.mark.parametrize("threads", [1, 15])
 206 def test_index_partial_with_30(test_db, threads, test_tokenizer):
 207     for rank in range(31):
 208         test_db.add_place(rank_address=rank, rank_search=rank)
 209     test_db.add_osmline()
 210
 211     assert test_db.placex_unindexed() == 31
 212     assert test_db.osmline_unindexed() == 1
 213
 214     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 215     idx.index_by_rank(28, 30)
 216
 217     assert test_db.placex_unindexed() == 27
 218     assert test_db.osmline_unindexed() == 0
 219
 220     assert test_db.scalar("""
 221                     SELECT count(*) FROM placex
 222                       WHERE indexed_status = 0 AND rank_address between 1 and 27""") == 0
 223
 224 @pytest.mark.parametrize("threads", [1, 15])
 225 def test_index_boundaries(test_db, threads, test_tokenizer):
 226     for rank in range(4, 10):
 227         test_db.add_admin(rank_address=rank, rank_search=rank)
 228     for rank in range(31):
 229         test_db.add_place(rank_address=rank, rank_search=rank)
 230     test_db.add_osmline()
 231
 232     assert test_db.placex_unindexed() == 37
 233     assert test_db.osmline_unindexed() == 1
 234
 235     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 236     idx.index_boundaries(0, 30)
 237
 238     assert test_db.placex_unindexed() == 31
 239     assert test_db.osmline_unindexed() == 1
 240
 241     assert test_db.scalar("""
 242                     SELECT count(*) FROM placex
 243                       WHERE indexed_status = 0 AND class != 'boundary'""") == 0
 244
 245
 246 @pytest.mark.parametrize("threads", [1, 15])
 247 def test_index_postcodes(test_db, threads, test_tokenizer):
 248     for postcode in range(1000):
 249         test_db.add_postcode('de', postcode)
 250     for postcode in range(32000, 33000):
 251         test_db.add_postcode('us', postcode)
 252
 253     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 254     idx.index_postcodes()
 255
 256     assert test_db.scalar("""SELECT count(*) FROM location_postcode
 257                                   WHERE indexed_status != 0""") == 0
 258
 259
 260 @pytest.mark.parametrize("analyse", [True, False])
 261 def test_index_full(test_db, analyse, test_tokenizer):
 262     for rank in range(4, 10):
 263         test_db.add_admin(rank_address=rank, rank_search=rank)
 264     for rank in range(31):
 265         test_db.add_place(rank_address=rank, rank_search=rank)
 266     test_db.add_osmline()
 267     for postcode in range(1000):
 268         test_db.add_postcode('de', postcode)
 269
 270     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4)
 271     idx.index_full(analyse=analyse)
 272
 273     assert test_db.placex_unindexed() == 0
 274     assert test_db.osmline_unindexed() == 0
 275     assert test_db.scalar("""SELECT count(*) FROM location_postcode
 276                              WHERE indexed_status != 0""") == 0
 277
 278
 279 @pytest.mark.parametrize("threads", [1, 15])
 280 def test_index_reopen_connection(test_db, threads, monkeypatch, test_tokenizer):
 281     monkeypatch.setattr(indexer.WorkerPool, "REOPEN_CONNECTIONS_AFTER", 15)
 282
 283     for _ in range(1000):
 284         test_db.add_place(rank_address=30, rank_search=30)
 285
 286     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 287     idx.index_by_rank(28, 30)
 288
 289     assert test_db.placex_unindexed() == 0