From 1ccd4360b4c4c9a861aea022cec084a9eddad283 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 13 May 2021 09:59:34 +0200 Subject: [PATCH] correctly handle removing all postcodes for country --- nominatim/tools/postcodes.py | 12 ++++++++++++ test/bdd/steps/steps_db_ops.py | 2 +- test/python/dummy_tokenizer.py | 2 +- test/python/test_tools_postcodes.py | 11 +++++++++-- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/nominatim/tools/postcodes.py b/nominatim/tools/postcodes.py index e172a77c..418b814f 100644 --- a/nominatim/tools/postcodes.py +++ b/nominatim/tools/postcodes.py @@ -142,6 +142,13 @@ def update_postcodes(dsn, project_dir, tokenizer): """ with tokenizer.name_analyzer() as analyzer: with connect(dsn) as conn: + # First get the list of countries that currently have postcodes. + # (Doing this before starting to insert, so it is fast on import.) + with conn.cursor() as cur: + cur.execute("SELECT DISTINCT country_code FROM location_postcode") + todo_countries = set((row[0] for row in cur)) + + # Recompute the list of valid postcodes from placex. with conn.cursor(name="placex_postcodes") as cur: cur.execute("""SELECT country_code, pc, ST_X(centroid), ST_Y(centroid) FROM ( @@ -161,11 +168,16 @@ def update_postcodes(dsn, project_dir, tokenizer): if collector is not None: collector.commit(conn, analyzer, project_dir) collector = _CountryPostcodesCollector(country) + todo_countries.discard(country) collector.add(postcode, x, y) if collector is not None: collector.commit(conn, analyzer, project_dir) + # Now handle any countries that are only in the postcode table. + for country in todo_countries: + _CountryPostcodesCollector(country).commit(conn, analyzer, project_dir) + conn.commit() analyzer.update_postcodes_from_db() diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py index 6d7bc188..8285f938 100644 --- a/test/bdd/steps/steps_db_ops.py +++ b/test/bdd/steps/steps_db_ops.py @@ -251,7 +251,7 @@ def check_location_postcode(context): with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: cur.execute("SELECT *, ST_AsText(geometry) as geomtxt FROM location_postcode") assert cur.rowcount == len(list(context.table)), \ - "Postcode table has {} rows, expected {}.".foramt(cur.rowcount, len(list(context.table))) + "Postcode table has {} rows, expected {}.".format(cur.rowcount, len(list(context.table))) results = {} for row in cur: diff --git a/test/python/dummy_tokenizer.py b/test/python/dummy_tokenizer.py index 8402e160..0a86ba8d 100644 --- a/test/python/dummy_tokenizer.py +++ b/test/python/dummy_tokenizer.py @@ -54,7 +54,7 @@ class DummyNameAnalyzer: def normalize_postcode(self, postcode): return postcode - def add_postcodes_from_db(self): + def update_postcodes_from_db(self): pass def update_special_phrases(self, phrases): diff --git a/test/python/test_tools_postcodes.py b/test/python/test_tools_postcodes.py index f0838b9e..4348b019 100644 --- a/test/python/test_tools_postcodes.py +++ b/test/python/test_tools_postcodes.py @@ -71,8 +71,7 @@ def test_import_postcodes_add_new(dsn, placex_table, postcode_table, tmp_path, t postcodes.update_postcodes(dsn, tmp_path, tokenizer) - assert postcode_table.row_set == {('xx', '9486', 10, 12), - ('yy', '9486', 99, 34)} + assert postcode_table.row_set == {('xx', '9486', 10, 12), } def test_import_postcodes_replace_coordinates(dsn, placex_table, postcode_table, tmp_path, tokenizer): @@ -105,6 +104,14 @@ def test_import_postcodes_remove(dsn, placex_table, postcode_table, tmp_path, to assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)} +def test_import_postcodes_remove_all(dsn, placex_table, postcode_table, tmp_path, tokenizer): + postcode_table.add('ch', '5613', 10, 12) + + postcodes.update_postcodes(dsn, tmp_path, tokenizer) + + assert not postcode_table.row_set + + def test_import_postcodes_multi_country(dsn, placex_table, postcode_table, tmp_path, tokenizer): placex_table.add(country='de', geom='POINT(10 12)', address=dict(postcode='54451')) -- 2.39.5