Merge pull request #3467 from mtmail/docs-datrie

[nominatim.git] / test / bdd / steps / steps_db_ops.py

diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py
index 2f598f3d6c8f3479cbaf3b3b5168bd36b6cbaba8..441198fdd4dfe46391d8fdcffad199d39f74efb5 100644 (file)
--- a/test/bdd/steps/steps_db_ops.py
+++ b/test/bdd/steps/steps_db_ops.py
@@ -1,8 +1,8 @@
-# SPDX-License-Identifier: GPL-2.0-only
+# SPDX-License-Identifier: GPL-3.0-or-later

 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #

-# Copyright (C) 2022 by the Nominatim developer community.
+# Copyright (C) 2024 by the Nominatim developer community.

 # For a full list of authors see the git log.
 import logging
 from itertools import chain
 # For a full list of authors see the git log.
 import logging
 from itertools import chain


@@ -12,8 +12,8 @@ import psycopg2.extras
 from place_inserter import PlaceColumn
 from table_compare import NominatimID, DBRow
 
 from place_inserter import PlaceColumn
 from table_compare import NominatimID, DBRow
 

-from nominatim.indexer import indexer
-from nominatim.tokenizer import factory as tokenizer_factory
+from nominatim_db.indexer import indexer
+from nominatim_db.tokenizer import factory as tokenizer_factory

 
 def check_database_integrity(context):
     """ Check some generic constraints on the tables.
 
 def check_database_integrity(context):
     """ Check some generic constraints on the tables.


@@ -52,33 +52,52 @@ def add_data_to_planet_relations(context):
         for tests on data that looks up members.
     """
     with context.db.cursor() as cur:
         for tests on data that looks up members.
     """
     with context.db.cursor() as cur:

-        for r in context.table:
-            last_node = 0
-            last_way = 0
-            parts = []
-            if r['members']:
-                members = []
-                for m in r['members'].split(','):
-                    mid = NominatimID(m)
-                    if mid.typ == 'N':
-                        parts.insert(last_node, int(mid.oid))
-                        last_node += 1
-                        last_way += 1
-                    elif mid.typ == 'W':
-                        parts.insert(last_way, int(mid.oid))
-                        last_way += 1
-                    else:
-                        parts.append(int(mid.oid))
-
-                    members.extend((mid.typ.lower() + mid.oid, mid.cls or ''))
-            else:
-                members = None
-
-            tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")])
-
-            cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, parts, members, tags)
-                           VALUES (%s, %s, %s, %s, %s, %s)""",
-                        (r['id'], last_node, last_way, parts, members, list(tags)))
+        cur.execute("SELECT value FROM osm2pgsql_properties WHERE property = 'db_format'")
+        row = cur.fetchone()
+        if row is None or row[0] == '1':
+            for r in context.table:
+                last_node = 0
+                last_way = 0
+                parts = []
+                if r['members']:
+                    members = []
+                    for m in r['members'].split(','):
+                        mid = NominatimID(m)
+                        if mid.typ == 'N':
+                            parts.insert(last_node, int(mid.oid))
+                            last_node += 1
+                            last_way += 1
+                        elif mid.typ == 'W':
+                            parts.insert(last_way, int(mid.oid))
+                            last_way += 1
+                        else:
+                            parts.append(int(mid.oid))
+
+                        members.extend((mid.typ.lower() + mid.oid, mid.cls or ''))
+                else:
+                    members = None
+
+                tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")])
+
+                cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, parts, members, tags)
+                               VALUES (%s, %s, %s, %s, %s, %s)""",
+                            (r['id'], last_node, last_way, parts, members, list(tags)))
+        else:
+            for r in context.table:
+                if r['members']:
+                    members = []
+                    for m in r['members'].split(','):
+                        mid = NominatimID(m)
+                        members.append({'ref': mid.oid, 'role': mid.cls or '', 'type': mid.typ})
+                else:
+                    members = []
+
+                tags = {h[5:]: r[h] for h in r.headings if h.startswith("tags+")}
+
+                cur.execute("""INSERT INTO planet_osm_rels (id, tags, members)
+                               VALUES (%s, %s, %s)""",
+                            (r['id'], psycopg2.extras.Json(tags),
+                             psycopg2.extras.Json(members)))

 
 @given("the ways")
 def add_data_to_planet_ways(context):
 
 @given("the ways")
 def add_data_to_planet_ways(context):


@@ -86,12 +105,19 @@ def add_data_to_planet_ways(context):
         tests on that that looks up node ids in this table.
     """
     with context.db.cursor() as cur:
         tests on that that looks up node ids in this table.
     """
     with context.db.cursor() as cur:

+        cur.execute("SELECT value FROM osm2pgsql_properties WHERE property = 'db_format'")
+        row = cur.fetchone()
+        json_tags = row is not None and row[0] != '1'

         for r in context.table:
         for r in context.table:

-            tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")])
+            if json_tags:
+                tags = psycopg2.extras.Json({h[5:]: r[h] for h in r.headings if h.startswith("tags+")})
+            else:
+                tags = list(chain.from_iterable([(h[5:], r[h])
+                                                 for h in r.headings if h.startswith("tags+")]))

             nodes = [ int(x.strip()) for x in r['nodes'].split(',') ]
 
             cur.execute("INSERT INTO planet_osm_ways (id, nodes, tags) VALUES (%s, %s, %s)",
             nodes = [ int(x.strip()) for x in r['nodes'].split(',') ]
 
             cur.execute("INSERT INTO planet_osm_ways (id, nodes, tags) VALUES (%s, %s, %s)",

-                        (r['id'], nodes, list(tags)))
+                        (r['id'], nodes, tags))

 
 ################################ WHEN ##################################
 
 
 ################################ WHEN ##################################
 


@@ -118,7 +144,10 @@ def update_place_table(context):
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
         for row in context.table:
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
         for row in context.table:

-            PlaceColumn(context).add_row(row, False).db_insert(cur)
+            col = PlaceColumn(context).add_row(row, False)
+            col.db_delete(cur)
+            col.db_insert(cur)
+        cur.execute('SELECT flush_deleted_places()')

 
     context.nominatim.reindex_placex(context.db)
     check_database_integrity(context)
 
     context.nominatim.reindex_placex(context.db)
     check_database_integrity(context)


@@ -143,8 +172,10 @@ def delete_places(context, oids):
     """
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
     """
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:

+        cur.execute('TRUNCATE place_to_be_deleted')

         for oid in oids.split(','):
             NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')
         for oid in oids.split(','):
             NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')

+        cur.execute('SELECT flush_deleted_places()')

 
     context.nominatim.reindex_placex(context.db)
 
 
     context.nominatim.reindex_placex(context.db)
 


@@ -159,7 +190,7 @@ def delete_places(context, oids):
 def check_place_contents(context, table, exact):
     """ Check contents of place/placex tables. Each row represents a table row
         and all data must match. Data not present in the expected table, may
 def check_place_contents(context, table, exact):
     """ Check contents of place/placex tables. Each row represents a table row
         and all data must match. Data not present in the expected table, may

-        be arbitry. The rows are identified via the 'object' column which must
+        be arbitrary. The rows are identified via the 'object' column which must

         have an identifier of the form '<NRW><osm id>[:<class>]'. When multiple
         rows match (for example because 'class' was left out and there are
         multiple entries for the given OSM object) then all must match. All
         have an identifier of the form '<NRW><osm id>[:<class>]'. When multiple
         rows match (for example because 'class' was left out and there are
         multiple entries for the given OSM object) then all must match. All


@@ -185,7 +216,10 @@ def check_place_contents(context, table, exact):
 
         if exact:
             cur.execute('SELECT osm_type, osm_id, class from {}'.format(table))
 
         if exact:
             cur.execute('SELECT osm_type, osm_id, class from {}'.format(table))

-            assert expected_content == set([(r[0], r[1], r[2]) for r in cur])
+            actual = set([(r[0], r[1], r[2]) for r in cur])
+            assert expected_content == actual, \
+                   f"Missing entries: {expected_content - actual}\n" \
+                   f"Not expected in table: {actual - expected_content}"

 
 
 @then("(?P<table>placex|place) has no entry for (?P<oid>.*)")
 
 
 @then("(?P<table>placex|place) has no entry for (?P<oid>.*)")


@@ -203,7 +237,7 @@ def check_place_has_entry(context, table, oid):
 def check_search_name_contents(context, exclude):
     """ Check contents of place/placex tables. Each row represents a table row
         and all data must match. Data not present in the expected table, may
 def check_search_name_contents(context, exclude):
     """ Check contents of place/placex tables. Each row represents a table row
         and all data must match. Data not present in the expected table, may

-        be arbitry. The rows are identified via the 'object' column which must
+        be arbitrary. The rows are identified via the 'object' column which must

         have an identifier of the form '<NRW><osm id>[:<class>]'. All
         expected rows are expected to be present with at least one database row.
     """
         have an identifier of the form '<NRW><osm id>[:<class>]'. All
         expected rows are expected to be present with at least one database row.
     """


@@ -252,7 +286,7 @@ def check_search_name_has_entry(context, oid):
 def check_location_postcode(context):
     """ Check full contents for location_postcode table. Each row represents a table row
         and all data must match. Data not present in the expected table, may
 def check_location_postcode(context):
     """ Check full contents for location_postcode table. Each row represents a table row
         and all data must match. Data not present in the expected table, may

-        be arbitry. The rows are identified via 'country' and 'postcode' columns.
+        be arbitrary. The rows are identified via 'country' and 'postcode' columns.

         All rows must be present as excepted and there must not be additional
         rows.
     """
         All rows must be present as excepted and there must not be additional
         rows.
     """


@@ -309,7 +343,7 @@ def check_word_table_for_postcodes(context, exclude, postcodes):
 def check_place_addressline(context):
     """ Check the contents of the place_addressline table. Each row represents
         a table row and all data must match. Data not present in the expected
 def check_place_addressline(context):
     """ Check the contents of the place_addressline table. Each row represents
         a table row and all data must match. Data not present in the expected

-        table, may be arbitry. The rows are identified via the 'object' column,
+        table, may be arbitrary. The rows are identified via the 'object' column,

         representing the addressee and the 'address' column, representing the
         address item.
     """
         representing the addressee and the 'address' column, representing the
         address item.
     """


@@ -372,4 +406,49 @@ def check_location_property_osmline(context, oid, neg):
 
         assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
 
 
         assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
 

+@then("location_property_osmline contains(?P<exact> exactly)?")
+def check_place_contents(context, exact):
+    """ Check contents of the interpolation table. Each row represents a table row
+        and all data must match. Data not present in the expected table, may
+        be arbitrary. The rows are identified via the 'object' column which must
+        have an identifier of the form '<osm id>[:<startnumber>]'. When multiple
+        rows match (for example because 'startnumber' was left out and there are
+        multiple entries for the given OSM object) then all must match. All
+        expected rows are expected to be present with at least one database row.
+        When 'exactly' is given, there must not be additional rows in the database.
+    """
+    with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
+        expected_content = set()
+        for row in context.table:
+            if ':' in row['object']:
+                nid, start = row['object'].split(':', 2)
+                start = int(start)
+            else:
+                nid, start = row['object'], None
+
+            query = """SELECT *, ST_AsText(linegeo) as geomtxt,
+                              ST_GeometryType(linegeo) as geometrytype
+                       FROM location_property_osmline WHERE osm_id=%s"""
+
+            if ':' in row['object']:
+                query += ' and startnumber = %s'
+                params = [int(val) for val in row['object'].split(':', 2)]
+            else:
+                params = (int(row['object']), )
+
+            cur.execute(query, params)
+            assert cur.rowcount > 0, "No rows found for " + row['object']
+
+            for res in cur:
+                if exact:
+                    expected_content.add((res['osm_id'], res['startnumber']))
+
+                DBRow(nid, res, context).assert_row(row, ['object'])
+
+        if exact:
+            cur.execute('SELECT osm_id, startnumber from location_property_osmline')
+            actual = set([(r[0], r[1]) for r in cur])
+            assert expected_content == actual, \
+                   f"Missing entries: {expected_content - actual}\n" \
+                   f"Not expected in table: {actual - expected_content}"