X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/9397bf54b89c1ae0b41115673d2f38582cd03ed9..a33f2c0f5ba43df533b6e137f9151fe67feb20a1:/test/python/test_indexing.py?ds=sidebyside diff --git a/test/python/test_indexing.py b/test/python/test_indexing.py index 223a599e..ff84e379 100644 --- a/test/python/test_indexing.py +++ b/test/python/test_indexing.py @@ -6,6 +6,7 @@ import psycopg2 import pytest from nominatim.indexer import indexer +from nominatim.tokenizer import factory class IndexerTestDB: @@ -28,9 +29,13 @@ class IndexerTestDB: partition SMALLINT, admin_level SMALLINT, address HSTORE, + token_info JSONB, geometry_sector INTEGER)""") cur.execute("""CREATE TABLE location_property_osmline ( place_id BIGINT, + osm_id BIGINT, + address HSTORE, + token_info JSONB, indexed_status SMALLINT, indexed_date TIMESTAMP, geometry_sector INTEGER)""") @@ -59,6 +64,14 @@ class IndexerTestDB: END; $$ LANGUAGE plpgsql STABLE; """) + cur.execute("""CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT) + RETURNS HSTORE AS $$ + BEGIN + RETURN in_address; + END; + $$ LANGUAGE plpgsql STABLE; + """) + for table in ('placex', 'location_property_osmline', 'location_postcode'): cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0} FOR EACH ROW EXECUTE PROCEDURE date_update() @@ -89,9 +102,9 @@ class IndexerTestDB: next_id = next(self.osmline_id) with self.conn.cursor() as cur: cur.execute("""INSERT INTO location_property_osmline - (place_id, indexed_status, geometry_sector) - VALUES (%s, 1, %s)""", - (next_id, sector)) + (place_id, osm_id, indexed_status, geometry_sector) + VALUES (%s, %s, 1, %s)""", + (next_id, next_id, sector)) return next_id def add_postcode(self, country, postcode): @@ -115,8 +128,14 @@ def test_db(temp_db_conn): yield IndexerTestDB(temp_db_conn) +@pytest.fixture +def test_tokenizer(tokenizer_mock, def_config, tmp_path): + def_config.project_dir = tmp_path + return factory.create_tokenizer(def_config) + + @pytest.mark.parametrize("threads", [1, 15]) -def test_index_all_by_rank(test_db, threads): +def test_index_all_by_rank(test_db, threads, test_tokenizer): for rank in range(31): test_db.add_place(rank_address=rank, rank_search=rank) test_db.add_osmline() @@ -124,7 +143,7 @@ def test_index_all_by_rank(test_db, threads): assert 31 == test_db.placex_unindexed() assert 1 == test_db.osmline_unindexed() - idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads) + idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx.index_by_rank(0, 30) assert 0 == test_db.placex_unindexed() @@ -155,7 +174,7 @@ def test_index_all_by_rank(test_db, threads): @pytest.mark.parametrize("threads", [1, 15]) -def test_index_partial_without_30(test_db, threads): +def test_index_partial_without_30(test_db, threads, test_tokenizer): for rank in range(31): test_db.add_place(rank_address=rank, rank_search=rank) test_db.add_osmline() @@ -163,7 +182,8 @@ def test_index_partial_without_30(test_db, threads): assert 31 == test_db.placex_unindexed() assert 1 == test_db.osmline_unindexed() - idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads) + idx = indexer.Indexer('dbname=test_nominatim_python_unittest', + test_tokenizer, threads) idx.index_by_rank(4, 15) assert 19 == test_db.placex_unindexed() @@ -175,7 +195,7 @@ def test_index_partial_without_30(test_db, threads): @pytest.mark.parametrize("threads", [1, 15]) -def test_index_partial_with_30(test_db, threads): +def test_index_partial_with_30(test_db, threads, test_tokenizer): for rank in range(31): test_db.add_place(rank_address=rank, rank_search=rank) test_db.add_osmline() @@ -183,7 +203,7 @@ def test_index_partial_with_30(test_db, threads): assert 31 == test_db.placex_unindexed() assert 1 == test_db.osmline_unindexed() - idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads) + idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx.index_by_rank(28, 30) assert 27 == test_db.placex_unindexed() @@ -194,7 +214,7 @@ def test_index_partial_with_30(test_db, threads): WHERE indexed_status = 0 AND rank_address between 1 and 27""") @pytest.mark.parametrize("threads", [1, 15]) -def test_index_boundaries(test_db, threads): +def test_index_boundaries(test_db, threads, test_tokenizer): for rank in range(4, 10): test_db.add_admin(rank_address=rank, rank_search=rank) for rank in range(31): @@ -204,7 +224,7 @@ def test_index_boundaries(test_db, threads): assert 37 == test_db.placex_unindexed() assert 1 == test_db.osmline_unindexed() - idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads) + idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx.index_boundaries(0, 30) assert 31 == test_db.placex_unindexed() @@ -216,13 +236,13 @@ def test_index_boundaries(test_db, threads): @pytest.mark.parametrize("threads", [1, 15]) -def test_index_postcodes(test_db, threads): +def test_index_postcodes(test_db, threads, test_tokenizer): for postcode in range(1000): test_db.add_postcode('de', postcode) for postcode in range(32000, 33000): test_db.add_postcode('us', postcode) - idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads) + idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx.index_postcodes() assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode @@ -230,7 +250,7 @@ def test_index_postcodes(test_db, threads): @pytest.mark.parametrize("analyse", [True, False]) -def test_index_full(test_db, analyse): +def test_index_full(test_db, analyse, test_tokenizer): for rank in range(4, 10): test_db.add_admin(rank_address=rank, rank_search=rank) for rank in range(31): @@ -239,7 +259,7 @@ def test_index_full(test_db, analyse): for postcode in range(1000): test_db.add_postcode('de', postcode) - idx = indexer.Indexer('dbname=test_nominatim_python_unittest', 4) + idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4) idx.index_full(analyse=analyse) assert 0 == test_db.placex_unindexed() @@ -249,13 +269,13 @@ def test_index_full(test_db, analyse): @pytest.mark.parametrize("threads", [1, 15]) -def test_index_reopen_connection(test_db, threads, monkeypatch): +def test_index_reopen_connection(test_db, threads, monkeypatch, test_tokenizer): monkeypatch.setattr(indexer.WorkerPool, "REOPEN_CONNECTIONS_AFTER", 15) for _ in range(1000): test_db.add_place(rank_address=30, rank_search=30) - idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads) + idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx.index_by_rank(28, 30) assert 0 == test_db.placex_unindexed()