call osm2pgsql postprocessing flush_deleted_places() when adding data

diff --git a/nominatim/clicmd/add_data.py b/nominatim/clicmd/add_data.py
index 8905bc2183fc1c75a4c500530c3134c14a0efa6f..3f31d216b45a9dce1299b47c2767d226aca07e06 100644 (file)
--- a/nominatim/clicmd/add_data.py
+++ b/nominatim/clicmd/add_data.py
@@ -76,21 +76,25 @@ class UpdateAddData:
 
         osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
         if args.file or args.diff:
 
         osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
         if args.file or args.diff:

-            return add_osm_data.add_data_from_file(cast(str, args.file or args.diff),
+            return add_osm_data.add_data_from_file(args.config.get_libpq_dsn(),
+                                                   cast(str, args.file or args.diff),

                                                    osm2pgsql_params)
 
         if args.node:
                                                    osm2pgsql_params)
 
         if args.node:

-            return add_osm_data.add_osm_object('node', args.node,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'node', args.node,

                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.way:
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.way:

-            return add_osm_data.add_osm_object('way', args.way,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'way', args.way,

                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.relation:
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.relation:

-            return add_osm_data.add_osm_object('relation', args.relation,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'relation', args.relation,

                                                args.use_main_api,
                                                osm2pgsql_params)
 
                                                args.use_main_api,
                                                osm2pgsql_params)
 

diff --git a/nominatim/tools/add_osm_data.py b/nominatim/tools/add_osm_data.py
index fc016fec80d6538268bb6b441b16660680b6bb38..1814b921e79dcaae29abddca708ba632872756f1 100644 (file)
--- a/nominatim/tools/add_osm_data.py
+++ b/nominatim/tools/add_osm_data.py
@@ -12,23 +12,34 @@ from pathlib import Path
 import logging
 import urllib
 
 import logging
 import urllib
 

+from nominatim.db.connection import connect

 from nominatim.tools.exec_utils import run_osm2pgsql, get_url
 
 LOG = logging.getLogger()
 
 from nominatim.tools.exec_utils import run_osm2pgsql, get_url
 
 LOG = logging.getLogger()
 

-def add_data_from_file(fname: str, options: MutableMapping[str, Any]) -> int:
+def _run_osm2pgsql(dsn: str, options: MutableMapping[str, Any]) -> None:
+    run_osm2pgsql(options)
+
+    # Handle deletions
+    with connect(dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute('SELECT flush_deleted_places()')
+        conn.commit()
+
+
+def add_data_from_file(dsn: str, fname: str, options: MutableMapping[str, Any]) -> int:

     """ Adds data from a OSM file to the database. The file may be a normal
         OSM file or a diff file in all formats supported by libosmium.
     """
     options['import_file'] = Path(fname)
     options['append'] = True
     """ Adds data from a OSM file to the database. The file may be a normal
         OSM file or a diff file in all formats supported by libosmium.
     """
     options['import_file'] = Path(fname)
     options['append'] = True

-    run_osm2pgsql(options)
+    _run_osm2pgsql(dsn, options)

 
     # No status update. We don't know where the file came from.
     return 0
 
 
 
     # No status update. We don't know where the file came from.
     return 0
 
 

-def add_osm_object(osm_type: str, osm_id: int, use_main_api: bool,
+def add_osm_object(dsn: str, osm_type: str, osm_id: int, use_main_api: bool,

                    options: MutableMapping[str, Any]) -> int:
     """ Add or update a single OSM object from the latest version of the
         API.
                    options: MutableMapping[str, Any]) -> int:
     """ Add or update a single OSM object from the latest version of the
         API.


@@ -51,6 +62,6 @@ def add_osm_object(osm_type: str, osm_id: int, use_main_api: bool,
     options['append'] = True
     options['import_data'] = get_url(base_url).encode('utf-8')
 
     options['append'] = True
     options['import_data'] = get_url(base_url).encode('utf-8')
 

-    run_osm2pgsql(options)
+    _run_osm2pgsql(dsn, options)

 
     return 0
 
     return 0

diff --git a/test/python/tools/test_add_osm_data.py b/test/python/tools/test_add_osm_data.py
index 5e808c6aed5fce51d2c405ad6a70a0a5ed4ca1ac..a9bf9b2abbc5dfc7d7b4846709bb2aed527c92ee 100644 (file)
--- a/test/python/tools/test_add_osm_data.py
+++ b/test/python/tools/test_add_osm_data.py
@@ -24,10 +24,14 @@ class CaptureGetUrl:
         return '<xml></xml>'
 
 
         return '<xml></xml>'
 
 

-def test_import_osm_file_simple(table_factory, osm2pgsql_options, capfd):
-    table_factory('place', content=((1, ), ))
+@pytest.fixture(autouse=True)
+def setup_delete_postprocessing(temp_db_cursor):
+    temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION flush_deleted_places()
+                              RETURNS INTEGER AS $$ SELECT 1 $$ LANGUAGE SQL""")

 
 

-    assert add_osm_data.add_data_from_file(Path('change.osm'), osm2pgsql_options) == 0
+def test_import_osm_file_simple(dsn, table_factory, osm2pgsql_options, capfd):
+
+    assert add_osm_data.add_data_from_file(dsn, Path('change.osm'), osm2pgsql_options) == 0

     captured = capfd.readouterr()
 
     assert '--append' in captured.out
     captured = capfd.readouterr()
 
     assert '--append' in captured.out


@@ -41,11 +45,11 @@ def test_import_osm_file_simple(table_factory, osm2pgsql_options, capfd):
 @pytest.mark.parametrize("osm_type", ['node', 'way', 'relation'])
 @pytest.mark.parametrize("main_api,url", [(True, 'https://www.openstreetmap.org/api'),
                                           (False, 'https://overpass-api.de/api/interpreter?')])
 @pytest.mark.parametrize("osm_type", ['node', 'way', 'relation'])
 @pytest.mark.parametrize("main_api,url", [(True, 'https://www.openstreetmap.org/api'),
                                           (False, 'https://overpass-api.de/api/interpreter?')])

-def test_import_osm_object_main_api(osm2pgsql_options, monkeypatch, capfd,
-                                    osm_type, main_api, url):
+def test_import_osm_object_main_api(dsn, osm2pgsql_options, monkeypatch,
+                                    capfd, osm_type, main_api, url):

     get_url_mock = CaptureGetUrl(monkeypatch)
 
     get_url_mock = CaptureGetUrl(monkeypatch)
 

-    add_osm_data.add_osm_object(osm_type, 4536, main_api, osm2pgsql_options)
+    add_osm_data.add_osm_object(dsn, osm_type, 4536, main_api, osm2pgsql_options)

     captured = capfd.readouterr()
 
     assert get_url_mock.url.startswith(url)
     captured = capfd.readouterr()
 
     assert get_url_mock.url.startswith(url)