test/python/test_indexing.py

   1 """
   2 Tests for running the indexing.
   3 """
   4 import itertools
   5 import pytest
   6
   7 from nominatim.indexer import indexer
   8 from nominatim.tokenizer import factory
   9
  10 class IndexerTestDB:
  11
  12     def __init__(self, conn):
  13         self.placex_id = itertools.count(100000)
  14         self.osmline_id = itertools.count(500000)
  15         self.postcode_id = itertools.count(700000)
  16
  17         self.conn = conn
  18         self.conn.set_isolation_level(0)
  19         with self.conn.cursor() as cur:
  20             cur.execute('CREATE EXTENSION hstore')
  21             cur.execute("""CREATE TABLE placex (place_id BIGINT,
  22                                                 name HSTORE,
  23                                                 class TEXT,
  24                                                 type TEXT,
  25                                                 linked_place_id BIGINT,
  26                                                 rank_address SMALLINT,
  27                                                 rank_search SMALLINT,
  28                                                 indexed_status SMALLINT,
  29                                                 indexed_date TIMESTAMP,
  30                                                 partition SMALLINT,
  31                                                 admin_level SMALLINT,
  32                                                 country_code TEXT,
  33                                                 address HSTORE,
  34                                                 token_info JSONB,
  35                                                 geometry_sector INTEGER)""")
  36             cur.execute("""CREATE TABLE location_property_osmline (
  37                                place_id BIGINT,
  38                                osm_id BIGINT,
  39                                address HSTORE,
  40                                token_info JSONB,
  41                                indexed_status SMALLINT,
  42                                indexed_date TIMESTAMP,
  43                                geometry_sector INTEGER)""")
  44             cur.execute("""CREATE TABLE location_postcode (
  45                                place_id BIGINT,
  46                                indexed_status SMALLINT,
  47                                indexed_date TIMESTAMP,
  48                                country_code varchar(2),
  49                                postcode TEXT)""")
  50             cur.execute("""CREATE OR REPLACE FUNCTION date_update() RETURNS TRIGGER
  51                            AS $$
  52                            BEGIN
  53                              IF NEW.indexed_status = 0 and OLD.indexed_status != 0 THEN
  54                                NEW.indexed_date = now();
  55                              END IF;
  56                              RETURN NEW;
  57                            END; $$ LANGUAGE plpgsql;""")
  58             cur.execute("DROP TYPE IF EXISTS prepare_update_info CASCADE")
  59             cur.execute("""CREATE TYPE prepare_update_info AS (
  60                              name HSTORE,
  61                              address HSTORE,
  62                              rank_address SMALLINT,
  63                              country_code TEXT,
  64                              class TEXT,
  65                              type TEXT,
  66                              linked_place_id BIGINT
  67                            )""")
  68             cur.execute("""CREATE OR REPLACE FUNCTION placex_indexing_prepare(p placex,
  69                                                      OUT result prepare_update_info)
  70                            AS $$
  71                            BEGIN
  72                              result.address := p.address;
  73                              result.name := p.name;
  74                              result.class := p.class;
  75                              result.type := p.type;
  76                              result.country_code := p.country_code;
  77                              result.rank_address := p.rank_address;
  78                            END;
  79                            $$ LANGUAGE plpgsql STABLE;
  80                         """)
  81             cur.execute("""CREATE OR REPLACE FUNCTION
  82                              get_interpolation_address(in_address HSTORE, wayid BIGINT)
  83                            RETURNS HSTORE AS $$
  84                            BEGIN
  85                              RETURN in_address;
  86                            END;
  87                            $$ LANGUAGE plpgsql STABLE;
  88                         """)
  89
  90             for table in ('placex', 'location_property_osmline', 'location_postcode'):
  91                 cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
  92                                FOR EACH ROW EXECUTE PROCEDURE date_update()
  93                             """.format(table))
  94
  95     def scalar(self, query):
  96         with self.conn.cursor() as cur:
  97             cur.execute(query)
  98             return cur.fetchone()[0]
  99
 100     def add_place(self, cls='place', typ='locality',
 101                   rank_search=30, rank_address=30, sector=20):
 102         next_id = next(self.placex_id)
 103         with self.conn.cursor() as cur:
 104             cur.execute("""INSERT INTO placex
 105                               (place_id, class, type, rank_search, rank_address,
 106                                indexed_status, geometry_sector)
 107                               VALUES (%s, %s, %s, %s, %s, 1, %s)""",
 108                         (next_id, cls, typ, rank_search, rank_address, sector))
 109         return next_id
 110
 111     def add_admin(self, **kwargs):
 112         kwargs['cls'] = 'boundary'
 113         kwargs['typ'] = 'administrative'
 114         return self.add_place(**kwargs)
 115
 116     def add_osmline(self, sector=20):
 117         next_id = next(self.osmline_id)
 118         with self.conn.cursor() as cur:
 119             cur.execute("""INSERT INTO location_property_osmline
 120                               (place_id, osm_id, indexed_status, geometry_sector)
 121                               VALUES (%s, %s, 1, %s)""",
 122                         (next_id, next_id, sector))
 123         return next_id
 124
 125     def add_postcode(self, country, postcode):
 126         next_id = next(self.postcode_id)
 127         with self.conn.cursor() as cur:
 128             cur.execute("""INSERT INTO location_postcode
 129                             (place_id, indexed_status, country_code, postcode)
 130                             VALUES (%s, 1, %s, %s)""",
 131                         (next_id, country, postcode))
 132         return next_id
 133
 134     def placex_unindexed(self):
 135         return self.scalar('SELECT count(*) from placex where indexed_status > 0')
 136
 137     def osmline_unindexed(self):
 138         return self.scalar("""SELECT count(*) from location_property_osmline
 139                               WHERE indexed_status > 0""")
 140
 141
 142 @pytest.fixture
 143 def test_db(temp_db_conn):
 144     yield IndexerTestDB(temp_db_conn)
 145
 146
 147 @pytest.fixture
 148 def test_tokenizer(tokenizer_mock, def_config, tmp_path):
 149     def_config.project_dir = tmp_path
 150     return factory.create_tokenizer(def_config)
 151
 152
 153 @pytest.mark.parametrize("threads", [1, 15])
 154 def test_index_all_by_rank(test_db, threads, test_tokenizer):
 155     for rank in range(31):
 156         test_db.add_place(rank_address=rank, rank_search=rank)
 157     test_db.add_osmline()
 158
 159     assert test_db.placex_unindexed() == 31
 160     assert test_db.osmline_unindexed() == 1
 161
 162     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 163     idx.index_by_rank(0, 30)
 164
 165     assert test_db.placex_unindexed() == 0
 166     assert test_db.osmline_unindexed() == 0
 167
 168     assert test_db.scalar("""SELECT count(*) from placex
 169                              WHERE indexed_status = 0 and indexed_date is null""") == 0
 170     # ranks come in order of rank address
 171     assert test_db.scalar("""
 172         SELECT count(*) FROM placex p WHERE rank_address > 0
 173           AND indexed_date >= (SELECT min(indexed_date) FROM placex o
 174                                WHERE p.rank_address < o.rank_address)""") == 0
 175     # placex rank < 30 objects come before interpolations
 176     assert test_db.scalar(
 177         """SELECT count(*) FROM placex WHERE rank_address < 30
 178              AND indexed_date >
 179                    (SELECT min(indexed_date) FROM location_property_osmline)""") == 0
 180     # placex rank = 30 objects come after interpolations
 181     assert test_db.scalar(
 182         """SELECT count(*) FROM placex WHERE rank_address = 30
 183              AND indexed_date <
 184                    (SELECT max(indexed_date) FROM location_property_osmline)""") == 0
 185     # rank 0 comes after rank 29 and before rank 30
 186     assert test_db.scalar(
 187         """SELECT count(*) FROM placex WHERE rank_address < 30
 188              AND indexed_date >
 189                    (SELECT min(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
 190     assert test_db.scalar(
 191         """SELECT count(*) FROM placex WHERE rank_address = 30
 192              AND indexed_date <
 193                    (SELECT max(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
 194
 195
 196 @pytest.mark.parametrize("threads", [1, 15])
 197 def test_index_partial_without_30(test_db, threads, test_tokenizer):
 198     for rank in range(31):
 199         test_db.add_place(rank_address=rank, rank_search=rank)
 200     test_db.add_osmline()
 201
 202     assert test_db.placex_unindexed() == 31
 203     assert test_db.osmline_unindexed() == 1
 204
 205     idx = indexer.Indexer('dbname=test_nominatim_python_unittest',
 206                           test_tokenizer, threads)
 207     idx.index_by_rank(4, 15)
 208
 209     assert test_db.placex_unindexed() == 19
 210     assert test_db.osmline_unindexed() == 1
 211
 212     assert test_db.scalar("""
 213                     SELECT count(*) FROM placex
 214                       WHERE indexed_status = 0 AND not rank_address between 4 and 15""") == 0
 215
 216
 217 @pytest.mark.parametrize("threads", [1, 15])
 218 def test_index_partial_with_30(test_db, threads, test_tokenizer):
 219     for rank in range(31):
 220         test_db.add_place(rank_address=rank, rank_search=rank)
 221     test_db.add_osmline()
 222
 223     assert test_db.placex_unindexed() == 31
 224     assert test_db.osmline_unindexed() == 1
 225
 226     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 227     idx.index_by_rank(28, 30)
 228
 229     assert test_db.placex_unindexed() == 27
 230     assert test_db.osmline_unindexed() == 0
 231
 232     assert test_db.scalar("""
 233                     SELECT count(*) FROM placex
 234                       WHERE indexed_status = 0 AND rank_address between 1 and 27""") == 0
 235
 236 @pytest.mark.parametrize("threads", [1, 15])
 237 def test_index_boundaries(test_db, threads, test_tokenizer):
 238     for rank in range(4, 10):
 239         test_db.add_admin(rank_address=rank, rank_search=rank)
 240     for rank in range(31):
 241         test_db.add_place(rank_address=rank, rank_search=rank)
 242     test_db.add_osmline()
 243
 244     assert test_db.placex_unindexed() == 37
 245     assert test_db.osmline_unindexed() == 1
 246
 247     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 248     idx.index_boundaries(0, 30)
 249
 250     assert test_db.placex_unindexed() == 31
 251     assert test_db.osmline_unindexed() == 1
 252
 253     assert test_db.scalar("""
 254                     SELECT count(*) FROM placex
 255                       WHERE indexed_status = 0 AND class != 'boundary'""") == 0
 256
 257
 258 @pytest.mark.parametrize("threads", [1, 15])
 259 def test_index_postcodes(test_db, threads, test_tokenizer):
 260     for postcode in range(1000):
 261         test_db.add_postcode('de', postcode)
 262     for postcode in range(32000, 33000):
 263         test_db.add_postcode('us', postcode)
 264
 265     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 266     idx.index_postcodes()
 267
 268     assert test_db.scalar("""SELECT count(*) FROM location_postcode
 269                                   WHERE indexed_status != 0""") == 0
 270
 271
 272 @pytest.mark.parametrize("analyse", [True, False])
 273 def test_index_full(test_db, analyse, test_tokenizer):
 274     for rank in range(4, 10):
 275         test_db.add_admin(rank_address=rank, rank_search=rank)
 276     for rank in range(31):
 277         test_db.add_place(rank_address=rank, rank_search=rank)
 278     test_db.add_osmline()
 279     for postcode in range(1000):
 280         test_db.add_postcode('de', postcode)
 281
 282     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4)
 283     idx.index_full(analyse=analyse)
 284
 285     assert test_db.placex_unindexed() == 0
 286     assert test_db.osmline_unindexed() == 0
 287     assert test_db.scalar("""SELECT count(*) FROM location_postcode
 288                              WHERE indexed_status != 0""") == 0
 289
 290
 291 @pytest.mark.parametrize("threads", [1, 15])
 292 def test_index_reopen_connection(test_db, threads, monkeypatch, test_tokenizer):
 293     monkeypatch.setattr(indexer.WorkerPool, "REOPEN_CONNECTIONS_AFTER", 15)
 294
 295     for _ in range(1000):
 296         test_db.add_place(rank_address=30, rank_search=30)
 297
 298     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 299     idx.index_by_rank(28, 30)
 300
 301     assert test_db.placex_unindexed() == 0