test/python/test_indexing.py

   1 """
   2 Tests for running the indexing.
   3 """
   4 import itertools
   5 import psycopg2
   6 import pytest
   7
   8 from nominatim.indexer import indexer
   9 from nominatim.tokenizer import factory
  10
  11 class IndexerTestDB:
  12
  13     def __init__(self, conn):
  14         self.placex_id = itertools.count(100000)
  15         self.osmline_id = itertools.count(500000)
  16         self.postcode_id = itertools.count(700000)
  17
  18         self.conn = conn
  19         self.conn.set_isolation_level(0)
  20         with self.conn.cursor() as cur:
  21             cur.execute('CREATE EXTENSION hstore')
  22             cur.execute("""CREATE TABLE placex (place_id BIGINT,
  23                                                 class TEXT,
  24                                                 type TEXT,
  25                                                 rank_address SMALLINT,
  26                                                 rank_search SMALLINT,
  27                                                 indexed_status SMALLINT,
  28                                                 indexed_date TIMESTAMP,
  29                                                 partition SMALLINT,
  30                                                 admin_level SMALLINT,
  31                                                 address HSTORE,
  32                                                 token_info JSONB,
  33                                                 geometry_sector INTEGER)""")
  34             cur.execute("""CREATE TABLE location_property_osmline (
  35                                place_id BIGINT,
  36                                indexed_status SMALLINT,
  37                                indexed_date TIMESTAMP,
  38                                geometry_sector INTEGER)""")
  39             cur.execute("""CREATE TABLE location_postcode (
  40                                place_id BIGINT,
  41                                indexed_status SMALLINT,
  42                                indexed_date TIMESTAMP,
  43                                country_code varchar(2),
  44                                postcode TEXT)""")
  45             cur.execute("""CREATE OR REPLACE FUNCTION date_update() RETURNS TRIGGER
  46                            AS $$
  47                            BEGIN
  48                              IF NEW.indexed_status = 0 and OLD.indexed_status != 0 THEN
  49                                NEW.indexed_date = now();
  50                              END IF;
  51                              RETURN NEW;
  52                            END; $$ LANGUAGE plpgsql;""")
  53             cur.execute("""CREATE OR REPLACE FUNCTION placex_prepare_update(p placex,
  54                                                       OUT name HSTORE,
  55                                                       OUT address HSTORE,
  56                                                       OUT country_feature VARCHAR)
  57                            AS $$
  58                            BEGIN
  59                             address := p.address;
  60                             name := p.address;
  61                            END;
  62                            $$ LANGUAGE plpgsql STABLE;
  63                         """)
  64             for table in ('placex', 'location_property_osmline', 'location_postcode'):
  65                 cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
  66                                FOR EACH ROW EXECUTE PROCEDURE date_update()
  67                             """.format(table))
  68
  69     def scalar(self, query):
  70         with self.conn.cursor() as cur:
  71             cur.execute(query)
  72             return cur.fetchone()[0]
  73
  74     def add_place(self, cls='place', typ='locality',
  75                   rank_search=30, rank_address=30, sector=20):
  76         next_id = next(self.placex_id)
  77         with self.conn.cursor() as cur:
  78             cur.execute("""INSERT INTO placex
  79                               (place_id, class, type, rank_search, rank_address,
  80                                indexed_status, geometry_sector)
  81                               VALUES (%s, %s, %s, %s, %s, 1, %s)""",
  82                         (next_id, cls, typ, rank_search, rank_address, sector))
  83         return next_id
  84
  85     def add_admin(self, **kwargs):
  86         kwargs['cls'] = 'boundary'
  87         kwargs['typ'] = 'administrative'
  88         return self.add_place(**kwargs)
  89
  90     def add_osmline(self, sector=20):
  91         next_id = next(self.osmline_id)
  92         with self.conn.cursor() as cur:
  93             cur.execute("""INSERT INTO location_property_osmline
  94                               (place_id, indexed_status, geometry_sector)
  95                               VALUES (%s, 1, %s)""",
  96                         (next_id, sector))
  97         return next_id
  98
  99     def add_postcode(self, country, postcode):
 100         next_id = next(self.postcode_id)
 101         with self.conn.cursor() as cur:
 102             cur.execute("""INSERT INTO location_postcode
 103                             (place_id, indexed_status, country_code, postcode)
 104                             VALUES (%s, 1, %s, %s)""",
 105                         (next_id, country, postcode))
 106         return next_id
 107
 108     def placex_unindexed(self):
 109         return self.scalar('SELECT count(*) from placex where indexed_status > 0')
 110
 111     def osmline_unindexed(self):
 112         return self.scalar('SELECT count(*) from location_property_osmline where indexed_status > 0')
 113
 114
 115 @pytest.fixture
 116 def test_db(temp_db_conn):
 117     yield IndexerTestDB(temp_db_conn)
 118
 119
 120 @pytest.fixture
 121 def test_tokenizer(tokenizer_mock, def_config, tmp_path):
 122     def_config.project_dir = tmp_path
 123     return factory.create_tokenizer(def_config)
 124
 125
 126 @pytest.mark.parametrize("threads", [1, 15])
 127 def test_index_all_by_rank(test_db, threads, test_tokenizer):
 128     for rank in range(31):
 129         test_db.add_place(rank_address=rank, rank_search=rank)
 130     test_db.add_osmline()
 131
 132     assert 31 == test_db.placex_unindexed()
 133     assert 1 == test_db.osmline_unindexed()
 134
 135     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 136     idx.index_by_rank(0, 30)
 137
 138     assert 0 == test_db.placex_unindexed()
 139     assert 0 == test_db.osmline_unindexed()
 140
 141     assert 0 == test_db.scalar("""SELECT count(*) from placex
 142                                WHERE indexed_status = 0 and indexed_date is null""")
 143     # ranks come in order of rank address
 144     assert 0 == test_db.scalar("""
 145         SELECT count(*) FROM placex p WHERE rank_address > 0
 146           AND indexed_date >= (SELECT min(indexed_date) FROM placex o
 147                                WHERE p.rank_address < o.rank_address)""")
 148     # placex rank < 30 objects come before interpolations
 149     assert 0 == test_db.scalar(
 150         """SELECT count(*) FROM placex WHERE rank_address < 30
 151              AND indexed_date > (SELECT min(indexed_date) FROM location_property_osmline)""")
 152     # placex rank = 30 objects come after interpolations
 153     assert 0 == test_db.scalar(
 154         """SELECT count(*) FROM placex WHERE rank_address = 30
 155              AND indexed_date < (SELECT max(indexed_date) FROM location_property_osmline)""")
 156     # rank 0 comes after rank 29 and before rank 30
 157     assert 0 == test_db.scalar(
 158         """SELECT count(*) FROM placex WHERE rank_address < 30
 159              AND indexed_date > (SELECT min(indexed_date) FROM placex WHERE rank_address = 0)""")
 160     assert 0 == test_db.scalar(
 161         """SELECT count(*) FROM placex WHERE rank_address = 30
 162              AND indexed_date < (SELECT max(indexed_date) FROM placex WHERE rank_address = 0)""")
 163
 164
 165 @pytest.mark.parametrize("threads", [1, 15])
 166 def test_index_partial_without_30(test_db, threads, test_tokenizer):
 167     for rank in range(31):
 168         test_db.add_place(rank_address=rank, rank_search=rank)
 169     test_db.add_osmline()
 170
 171     assert 31 == test_db.placex_unindexed()
 172     assert 1 == test_db.osmline_unindexed()
 173
 174     idx = indexer.Indexer('dbname=test_nominatim_python_unittest',
 175                           test_tokenizer, threads)
 176     idx.index_by_rank(4, 15)
 177
 178     assert 19 == test_db.placex_unindexed()
 179     assert 1 == test_db.osmline_unindexed()
 180
 181     assert 0 == test_db.scalar("""
 182                     SELECT count(*) FROM placex
 183                       WHERE indexed_status = 0 AND not rank_address between 4 and 15""")
 184
 185
 186 @pytest.mark.parametrize("threads", [1, 15])
 187 def test_index_partial_with_30(test_db, threads, test_tokenizer):
 188     for rank in range(31):
 189         test_db.add_place(rank_address=rank, rank_search=rank)
 190     test_db.add_osmline()
 191
 192     assert 31 == test_db.placex_unindexed()
 193     assert 1 == test_db.osmline_unindexed()
 194
 195     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 196     idx.index_by_rank(28, 30)
 197
 198     assert 27 == test_db.placex_unindexed()
 199     assert 0 == test_db.osmline_unindexed()
 200
 201     assert 0 == test_db.scalar("""
 202                     SELECT count(*) FROM placex
 203                       WHERE indexed_status = 0 AND rank_address between 1 and 27""")
 204
 205 @pytest.mark.parametrize("threads", [1, 15])
 206 def test_index_boundaries(test_db, threads, test_tokenizer):
 207     for rank in range(4, 10):
 208         test_db.add_admin(rank_address=rank, rank_search=rank)
 209     for rank in range(31):
 210         test_db.add_place(rank_address=rank, rank_search=rank)
 211     test_db.add_osmline()
 212
 213     assert 37 == test_db.placex_unindexed()
 214     assert 1 == test_db.osmline_unindexed()
 215
 216     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 217     idx.index_boundaries(0, 30)
 218
 219     assert 31 == test_db.placex_unindexed()
 220     assert 1 == test_db.osmline_unindexed()
 221
 222     assert 0 == test_db.scalar("""
 223                     SELECT count(*) FROM placex
 224                       WHERE indexed_status = 0 AND class != 'boundary'""")
 225
 226
 227 @pytest.mark.parametrize("threads", [1, 15])
 228 def test_index_postcodes(test_db, threads, test_tokenizer):
 229     for postcode in range(1000):
 230         test_db.add_postcode('de', postcode)
 231     for postcode in range(32000, 33000):
 232         test_db.add_postcode('us', postcode)
 233
 234     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 235     idx.index_postcodes()
 236
 237     assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode
 238                                   WHERE indexed_status != 0""")
 239
 240
 241 @pytest.mark.parametrize("analyse", [True, False])
 242 def test_index_full(test_db, analyse, test_tokenizer):
 243     for rank in range(4, 10):
 244         test_db.add_admin(rank_address=rank, rank_search=rank)
 245     for rank in range(31):
 246         test_db.add_place(rank_address=rank, rank_search=rank)
 247     test_db.add_osmline()
 248     for postcode in range(1000):
 249         test_db.add_postcode('de', postcode)
 250
 251     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4)
 252     idx.index_full(analyse=analyse)
 253
 254     assert 0 == test_db.placex_unindexed()
 255     assert 0 == test_db.osmline_unindexed()
 256     assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode
 257                                   WHERE indexed_status != 0""")
 258
 259
 260 @pytest.mark.parametrize("threads", [1, 15])
 261 def test_index_reopen_connection(test_db, threads, monkeypatch, test_tokenizer):
 262     monkeypatch.setattr(indexer.WorkerPool, "REOPEN_CONNECTIONS_AFTER", 15)
 263
 264     for _ in range(1000):
 265         test_db.add_place(rank_address=30, rank_search=30)
 266
 267     idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
 268     idx.index_by_rank(28, 30)
 269
 270     assert 0 == test_db.placex_unindexed()