X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/90b29aa8080ffae08084cf556b45e05f9f2d3198..8e5a19b39209f20f41b7bd10ff7dbcdb8a3dcb9c:/test/bdd/steps/steps_db_ops.py?ds=sidebyside diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py index 14ae5d52..441198fd 100644 --- a/test/bdd/steps/steps_db_ops.py +++ b/test/bdd/steps/steps_db_ops.py @@ -1,8 +1,8 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: GPL-3.0-or-later # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2022 by the Nominatim developer community. +# Copyright (C) 2024 by the Nominatim developer community. # For a full list of authors see the git log. import logging from itertools import chain @@ -12,8 +12,8 @@ import psycopg2.extras from place_inserter import PlaceColumn from table_compare import NominatimID, DBRow -from nominatim.indexer import indexer -from nominatim.tokenizer import factory as tokenizer_factory +from nominatim_db.indexer import indexer +from nominatim_db.tokenizer import factory as tokenizer_factory def check_database_integrity(context): """ Check some generic constraints on the tables. @@ -52,33 +52,52 @@ def add_data_to_planet_relations(context): for tests on data that looks up members. """ with context.db.cursor() as cur: - for r in context.table: - last_node = 0 - last_way = 0 - parts = [] - if r['members']: - members = [] - for m in r['members'].split(','): - mid = NominatimID(m) - if mid.typ == 'N': - parts.insert(last_node, int(mid.oid)) - last_node += 1 - last_way += 1 - elif mid.typ == 'W': - parts.insert(last_way, int(mid.oid)) - last_way += 1 - else: - parts.append(int(mid.oid)) - - members.extend((mid.typ.lower() + mid.oid, mid.cls or '')) - else: - members = None - - tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")]) - - cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, parts, members, tags) - VALUES (%s, %s, %s, %s, %s, %s)""", - (r['id'], last_node, last_way, parts, members, list(tags))) + cur.execute("SELECT value FROM osm2pgsql_properties WHERE property = 'db_format'") + row = cur.fetchone() + if row is None or row[0] == '1': + for r in context.table: + last_node = 0 + last_way = 0 + parts = [] + if r['members']: + members = [] + for m in r['members'].split(','): + mid = NominatimID(m) + if mid.typ == 'N': + parts.insert(last_node, int(mid.oid)) + last_node += 1 + last_way += 1 + elif mid.typ == 'W': + parts.insert(last_way, int(mid.oid)) + last_way += 1 + else: + parts.append(int(mid.oid)) + + members.extend((mid.typ.lower() + mid.oid, mid.cls or '')) + else: + members = None + + tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")]) + + cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, parts, members, tags) + VALUES (%s, %s, %s, %s, %s, %s)""", + (r['id'], last_node, last_way, parts, members, list(tags))) + else: + for r in context.table: + if r['members']: + members = [] + for m in r['members'].split(','): + mid = NominatimID(m) + members.append({'ref': mid.oid, 'role': mid.cls or '', 'type': mid.typ}) + else: + members = [] + + tags = {h[5:]: r[h] for h in r.headings if h.startswith("tags+")} + + cur.execute("""INSERT INTO planet_osm_rels (id, tags, members) + VALUES (%s, %s, %s)""", + (r['id'], psycopg2.extras.Json(tags), + psycopg2.extras.Json(members))) @given("the ways") def add_data_to_planet_ways(context): @@ -86,12 +105,19 @@ def add_data_to_planet_ways(context): tests on that that looks up node ids in this table. """ with context.db.cursor() as cur: + cur.execute("SELECT value FROM osm2pgsql_properties WHERE property = 'db_format'") + row = cur.fetchone() + json_tags = row is not None and row[0] != '1' for r in context.table: - tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")]) + if json_tags: + tags = psycopg2.extras.Json({h[5:]: r[h] for h in r.headings if h.startswith("tags+")}) + else: + tags = list(chain.from_iterable([(h[5:], r[h]) + for h in r.headings if h.startswith("tags+")])) nodes = [ int(x.strip()) for x in r['nodes'].split(',') ] cur.execute("INSERT INTO planet_osm_ways (id, nodes, tags) VALUES (%s, %s, %s)", - (r['id'], nodes, list(tags))) + (r['id'], nodes, tags)) ################################ WHEN ################################## @@ -164,7 +190,7 @@ def delete_places(context, oids): def check_place_contents(context, table, exact): """ Check contents of place/placex tables. Each row represents a table row and all data must match. Data not present in the expected table, may - be arbitry. The rows are identified via the 'object' column which must + be arbitrary. The rows are identified via the 'object' column which must have an identifier of the form '[:]'. When multiple rows match (for example because 'class' was left out and there are multiple entries for the given OSM object) then all must match. All @@ -211,7 +237,7 @@ def check_place_has_entry(context, table, oid): def check_search_name_contents(context, exclude): """ Check contents of place/placex tables. Each row represents a table row and all data must match. Data not present in the expected table, may - be arbitry. The rows are identified via the 'object' column which must + be arbitrary. The rows are identified via the 'object' column which must have an identifier of the form '[:]'. All expected rows are expected to be present with at least one database row. """ @@ -260,7 +286,7 @@ def check_search_name_has_entry(context, oid): def check_location_postcode(context): """ Check full contents for location_postcode table. Each row represents a table row and all data must match. Data not present in the expected table, may - be arbitry. The rows are identified via 'country' and 'postcode' columns. + be arbitrary. The rows are identified via 'country' and 'postcode' columns. All rows must be present as excepted and there must not be additional rows. """ @@ -317,7 +343,7 @@ def check_word_table_for_postcodes(context, exclude, postcodes): def check_place_addressline(context): """ Check the contents of the place_addressline table. Each row represents a table row and all data must match. Data not present in the expected - table, may be arbitry. The rows are identified via the 'object' column, + table, may be arbitrary. The rows are identified via the 'object' column, representing the addressee and the 'address' column, representing the address item. """ @@ -384,7 +410,7 @@ def check_location_property_osmline(context, oid, neg): def check_place_contents(context, exact): """ Check contents of the interpolation table. Each row represents a table row and all data must match. Data not present in the expected table, may - be arbitry. The rows are identified via the 'object' column which must + be arbitrary. The rows are identified via the 'object' column which must have an identifier of the form '[:]'. When multiple rows match (for example because 'startnumber' was left out and there are multiple entries for the given OSM object) then all must match. All