Merge pull request #2326 from lonvia/wokerpool-for-tiger-data

[nominatim.git] / test / python / test_indexing.py

diff --git a/test/python/test_indexing.py b/test/python/test_indexing.py
index 6692eba61d349b8908415d793078fba17b0db3af..ff84e37964b660a33be2faa0908ba2ad0c1519a7 100644 (file)
--- a/test/python/test_indexing.py
+++ b/test/python/test_indexing.py
@@ -6,6 +6,7 @@ import psycopg2
 import pytest
 
 from nominatim.indexer import indexer
 import pytest
 
 from nominatim.indexer import indexer

+from nominatim.tokenizer import factory

 
 class IndexerTestDB:
 
 
 class IndexerTestDB:
 


@@ -17,6 +18,7 @@ class IndexerTestDB:
         self.conn = conn
         self.conn.set_isolation_level(0)
         with self.conn.cursor() as cur:
         self.conn = conn
         self.conn.set_isolation_level(0)
         with self.conn.cursor() as cur:

+            cur.execute('CREATE EXTENSION hstore')

             cur.execute("""CREATE TABLE placex (place_id BIGINT,
                                                 class TEXT,
                                                 type TEXT,
             cur.execute("""CREATE TABLE placex (place_id BIGINT,
                                                 class TEXT,
                                                 type TEXT,


@@ -26,9 +28,14 @@ class IndexerTestDB:
                                                 indexed_date TIMESTAMP,
                                                 partition SMALLINT,
                                                 admin_level SMALLINT,
                                                 indexed_date TIMESTAMP,
                                                 partition SMALLINT,
                                                 admin_level SMALLINT,

+                                                address HSTORE,
+                                                token_info JSONB,

                                                 geometry_sector INTEGER)""")
             cur.execute("""CREATE TABLE location_property_osmline (
                                place_id BIGINT,
                                                 geometry_sector INTEGER)""")
             cur.execute("""CREATE TABLE location_property_osmline (
                                place_id BIGINT,

+                               osm_id BIGINT,
+                               address HSTORE,
+                               token_info JSONB,

                                indexed_status SMALLINT,
                                indexed_date TIMESTAMP,
                                geometry_sector INTEGER)""")
                                indexed_status SMALLINT,
                                indexed_date TIMESTAMP,
                                geometry_sector INTEGER)""")


@@ -46,6 +53,25 @@ class IndexerTestDB:
                              END IF;
                              RETURN NEW;
                            END; $$ LANGUAGE plpgsql;""")
                              END IF;
                              RETURN NEW;
                            END; $$ LANGUAGE plpgsql;""")

+            cur.execute("""CREATE OR REPLACE FUNCTION placex_prepare_update(p placex,
+                                                      OUT name HSTORE,
+                                                      OUT address HSTORE,
+                                                      OUT country_feature VARCHAR)
+                           AS $$
+                           BEGIN
+                            address := p.address;
+                            name := p.address;
+                           END;
+                           $$ LANGUAGE plpgsql STABLE;
+                        """)
+            cur.execute("""CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT)
+                           RETURNS HSTORE AS $$
+                           BEGIN
+                             RETURN in_address;
+                           END;
+                           $$ LANGUAGE plpgsql STABLE;
+                        """)
+

             for table in ('placex', 'location_property_osmline', 'location_postcode'):
                 cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
                                FOR EACH ROW EXECUTE PROCEDURE date_update()
             for table in ('placex', 'location_property_osmline', 'location_postcode'):
                 cur.execute("""CREATE TRIGGER {0}_update BEFORE UPDATE ON {0}
                                FOR EACH ROW EXECUTE PROCEDURE date_update()


@@ -76,9 +102,9 @@ class IndexerTestDB:
         next_id = next(self.osmline_id)
         with self.conn.cursor() as cur:
             cur.execute("""INSERT INTO location_property_osmline
         next_id = next(self.osmline_id)
         with self.conn.cursor() as cur:
             cur.execute("""INSERT INTO location_property_osmline

-                              (place_id, indexed_status, geometry_sector)
-                              VALUES (%s, 1, %s)""",
-                        (next_id, sector))
+                              (place_id, osm_id, indexed_status, geometry_sector)
+                              VALUES (%s, %s, 1, %s)""",
+                        (next_id, next_id, sector))

         return next_id
 
     def add_postcode(self, country, postcode):
         return next_id
 
     def add_postcode(self, country, postcode):


@@ -102,8 +128,14 @@ def test_db(temp_db_conn):
     yield IndexerTestDB(temp_db_conn)
 
 
     yield IndexerTestDB(temp_db_conn)
 
 

+@pytest.fixture
+def test_tokenizer(tokenizer_mock, def_config, tmp_path):
+    def_config.project_dir = tmp_path
+    return factory.create_tokenizer(def_config)
+
+

 @pytest.mark.parametrize("threads", [1, 15])
 @pytest.mark.parametrize("threads", [1, 15])

-def test_index_all_by_rank(test_db, threads):
+def test_index_all_by_rank(test_db, threads, test_tokenizer):

     for rank in range(31):
         test_db.add_place(rank_address=rank, rank_search=rank)
     test_db.add_osmline()
     for rank in range(31):
         test_db.add_place(rank_address=rank, rank_search=rank)
     test_db.add_osmline()


@@ -111,7 +143,7 @@ def test_index_all_by_rank(test_db, threads):
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 

-    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)

     idx.index_by_rank(0, 30)
 
     assert 0 == test_db.placex_unindexed()
     idx.index_by_rank(0, 30)
 
     assert 0 == test_db.placex_unindexed()


@@ -142,7 +174,7 @@ def test_index_all_by_rank(test_db, threads):
 
 
 @pytest.mark.parametrize("threads", [1, 15])
 
 
 @pytest.mark.parametrize("threads", [1, 15])

-def test_index_partial_without_30(test_db, threads):
+def test_index_partial_without_30(test_db, threads, test_tokenizer):

     for rank in range(31):
         test_db.add_place(rank_address=rank, rank_search=rank)
     test_db.add_osmline()
     for rank in range(31):
         test_db.add_place(rank_address=rank, rank_search=rank)
     test_db.add_osmline()


@@ -150,7 +182,8 @@ def test_index_partial_without_30(test_db, threads):
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 

-    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest',
+                          test_tokenizer, threads)

     idx.index_by_rank(4, 15)
 
     assert 19 == test_db.placex_unindexed()
     idx.index_by_rank(4, 15)
 
     assert 19 == test_db.placex_unindexed()


@@ -162,7 +195,7 @@ def test_index_partial_without_30(test_db, threads):
 
 
 @pytest.mark.parametrize("threads", [1, 15])
 
 
 @pytest.mark.parametrize("threads", [1, 15])

-def test_index_partial_with_30(test_db, threads):
+def test_index_partial_with_30(test_db, threads, test_tokenizer):

     for rank in range(31):
         test_db.add_place(rank_address=rank, rank_search=rank)
     test_db.add_osmline()
     for rank in range(31):
         test_db.add_place(rank_address=rank, rank_search=rank)
     test_db.add_osmline()


@@ -170,7 +203,7 @@ def test_index_partial_with_30(test_db, threads):
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
     assert 31 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 

-    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)

     idx.index_by_rank(28, 30)
 
     assert 27 == test_db.placex_unindexed()
     idx.index_by_rank(28, 30)
 
     assert 27 == test_db.placex_unindexed()


@@ -181,7 +214,7 @@ def test_index_partial_with_30(test_db, threads):
                       WHERE indexed_status = 0 AND rank_address between 1 and 27""")
 
 @pytest.mark.parametrize("threads", [1, 15])
                       WHERE indexed_status = 0 AND rank_address between 1 and 27""")
 
 @pytest.mark.parametrize("threads", [1, 15])

-def test_index_boundaries(test_db, threads):
+def test_index_boundaries(test_db, threads, test_tokenizer):

     for rank in range(4, 10):
         test_db.add_admin(rank_address=rank, rank_search=rank)
     for rank in range(31):
     for rank in range(4, 10):
         test_db.add_admin(rank_address=rank, rank_search=rank)
     for rank in range(31):


@@ -191,7 +224,7 @@ def test_index_boundaries(test_db, threads):
     assert 37 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 
     assert 37 == test_db.placex_unindexed()
     assert 1 == test_db.osmline_unindexed()
 

-    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)

     idx.index_boundaries(0, 30)
 
     assert 31 == test_db.placex_unindexed()
     idx.index_boundaries(0, 30)
 
     assert 31 == test_db.placex_unindexed()


@@ -203,13 +236,13 @@ def test_index_boundaries(test_db, threads):
 
 
 @pytest.mark.parametrize("threads", [1, 15])
 
 
 @pytest.mark.parametrize("threads", [1, 15])

-def test_index_postcodes(test_db, threads):
+def test_index_postcodes(test_db, threads, test_tokenizer):

     for postcode in range(1000):
         test_db.add_postcode('de', postcode)
     for postcode in range(32000, 33000):
         test_db.add_postcode('us', postcode)
 
     for postcode in range(1000):
         test_db.add_postcode('de', postcode)
     for postcode in range(32000, 33000):
         test_db.add_postcode('us', postcode)
 

-    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)

     idx.index_postcodes()
 
     assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode
     idx.index_postcodes()
 
     assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode


@@ -217,7 +250,7 @@ def test_index_postcodes(test_db, threads):
 
 
 @pytest.mark.parametrize("analyse", [True, False])
 
 
 @pytest.mark.parametrize("analyse", [True, False])

-def test_index_full(test_db, analyse):
+def test_index_full(test_db, analyse, test_tokenizer):

     for rank in range(4, 10):
         test_db.add_admin(rank_address=rank, rank_search=rank)
     for rank in range(31):
     for rank in range(4, 10):
         test_db.add_admin(rank_address=rank, rank_search=rank)
     for rank in range(31):


@@ -226,7 +259,7 @@ def test_index_full(test_db, analyse):
     for postcode in range(1000):
         test_db.add_postcode('de', postcode)
 
     for postcode in range(1000):
         test_db.add_postcode('de', postcode)
 

-    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', 4)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4)

     idx.index_full(analyse=analyse)
 
     assert 0 == test_db.placex_unindexed()
     idx.index_full(analyse=analyse)
 
     assert 0 == test_db.placex_unindexed()


@@ -236,13 +269,13 @@ def test_index_full(test_db, analyse):
 
 
 @pytest.mark.parametrize("threads", [1, 15])
 
 
 @pytest.mark.parametrize("threads", [1, 15])

-def test_index_reopen_connection(test_db, threads, monkeypatch):
+def test_index_reopen_connection(test_db, threads, monkeypatch, test_tokenizer):

     monkeypatch.setattr(indexer.WorkerPool, "REOPEN_CONNECTIONS_AFTER", 15)
 
     for _ in range(1000):
         test_db.add_place(rank_address=30, rank_search=30)
 
     monkeypatch.setattr(indexer.WorkerPool, "REOPEN_CONNECTIONS_AFTER", 15)
 
     for _ in range(1000):
         test_db.add_place(rank_address=30, rank_search=30)
 

-    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', threads)
+    idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)

     idx.index_by_rank(28, 30)
 
     assert 0 == test_db.placex_unindexed()
     idx.index_by_rank(28, 30)
 
     assert 0 == test_db.placex_unindexed()