close DB connection when waiting for next update cycle

diff --git a/nominatim/clicmd/replication.py b/nominatim/clicmd/replication.py
index ad2016633775818c4efc624a3c40265a7fcb5395..b795650694d207ed73b32db5f566ae8289ffad58 100644 (file)
--- a/nominatim/clicmd/replication.py
+++ b/nominatim/clicmd/replication.py
@@ -147,10 +147,13 @@ class UpdateReplication:
         tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
         indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1)
 
         tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
         indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1)
 

+        dsn = args.config.get_libpq_dsn()
+

         while True:
         while True:

-            with connect(args.config.get_libpq_dsn()) as conn:
-                start = dt.datetime.now(dt.timezone.utc)
-                state = replication.update(conn, params, socket_timeout=args.socket_timeout)
+            start = dt.datetime.now(dt.timezone.utc)
+            state = replication.update(dsn, params, socket_timeout=args.socket_timeout)
+
+            with connect(dsn) as conn:

                 if state is not replication.UpdateState.NO_CHANGES:
                     status.log_status(conn, start, 'import')
                 batchdate, _, _ = status.get_status(conn)
                 if state is not replication.UpdateState.NO_CHANGES:
                     status.log_status(conn, start, 'import')
                 batchdate, _, _ = status.get_status(conn)


@@ -160,7 +163,7 @@ class UpdateReplication:
                 index_start = dt.datetime.now(dt.timezone.utc)
                 indexer.index_full(analyse=False)
 
                 index_start = dt.datetime.now(dt.timezone.utc)
                 indexer.index_full(analyse=False)
 

-                with connect(args.config.get_libpq_dsn()) as conn:
+                with connect(dsn) as conn:

                     status.set_indexed(conn, True)
                     status.log_status(conn, index_start, 'index')
                     conn.commit()
                     status.set_indexed(conn, True)
                     status.log_status(conn, index_start, 'index')
                     conn.commit()

diff --git a/nominatim/tools/replication.py b/nominatim/tools/replication.py
index 846b9c34dd301644080b6aa8b7c240e81739531f..edd63e49a15931d289b2fd488737ad8d105dc532 100644 (file)
--- a/nominatim/tools/replication.py
+++ b/nominatim/tools/replication.py
@@ -18,7 +18,7 @@ import urllib.request as urlrequest
 
 import requests
 from nominatim.db import status
 
 import requests
 from nominatim.db import status

-from nominatim.db.connection import Connection
+from nominatim.db.connection import Connection, connect

 from nominatim.tools.exec_utils import run_osm2pgsql
 from nominatim.errors import UsageError
 
 from nominatim.tools.exec_utils import run_osm2pgsql
 from nominatim.errors import UsageError
 


@@ -92,12 +92,14 @@ class UpdateState(Enum):
     NO_CHANGES = 3
 
 
     NO_CHANGES = 3
 
 

-def update(conn: Connection, options: MutableMapping[str, Any],
+def update(dsn: str, options: MutableMapping[str, Any],

            socket_timeout: int = 60) -> UpdateState:
     """ Update database from the next batch of data. Returns the state of
         updates according to `UpdateState`.
     """
            socket_timeout: int = 60) -> UpdateState:
     """ Update database from the next batch of data. Returns the state of
         updates according to `UpdateState`.
     """

-    startdate, startseq, indexed = status.get_status(conn)
+    with connect(dsn) as conn:
+        startdate, startseq, indexed = status.get_status(conn)
+        conn.commit()

 
     if startseq is None:
         LOG.error("Replication not set up. "
 
     if startseq is None:
         LOG.error("Replication not set up. "


@@ -130,12 +132,14 @@ def update(conn: Connection, options: MutableMapping[str, Any],
         if endseq is None:
             return UpdateState.NO_CHANGES
 
         if endseq is None:
             return UpdateState.NO_CHANGES
 

-        run_osm2pgsql_updates(conn, options)
+        with connect(dsn) as conn:
+            run_osm2pgsql_updates(conn, options)

 
 

-        # Write the current status to the file
-        endstate = repl.get_state_info(endseq)
-        status.set_status(conn, endstate.timestamp if endstate else None,
-                          seq=endseq, indexed=False)
+            # Write the current status to the file
+            endstate = repl.get_state_info(endseq)
+            status.set_status(conn, endstate.timestamp if endstate else None,
+                              seq=endseq, indexed=False)
+            conn.commit()

 
     return UpdateState.UP_TO_DATE
 
 
     return UpdateState.UP_TO_DATE
 

diff --git a/test/python/tools/test_replication.py b/test/python/tools/test_replication.py
index 2f9c7e866b75da75becfda701f7c0dbb50fff265..7a7340ae41125f52bf7857ec24d06d3a6e50a4dc 100644 (file)
--- a/test/python/tools/test_replication.py
+++ b/test/python/tools/test_replication.py
@@ -100,19 +100,19 @@ def update_options(tmpdir):
                 import_file=tmpdir / 'foo.osm',
                 max_diff_size=1)
 
                 import_file=tmpdir / 'foo.osm',
                 max_diff_size=1)
 

-def test_update_empty_status_table(temp_db_conn):
+def test_update_empty_status_table(dsn):

     with pytest.raises(UsageError):
     with pytest.raises(UsageError):

-        nominatim.tools.replication.update(temp_db_conn, {})
+        nominatim.tools.replication.update(dsn, {})

 
 
 
 

-def test_update_already_indexed(temp_db_conn):
+def test_update_already_indexed(temp_db_conn, dsn):

     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
 
     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
 

-    assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
+    assert nominatim.tools.replication.update(dsn, dict(indexed_only=True)) \

              == nominatim.tools.replication.UpdateState.MORE_PENDING
 
 
              == nominatim.tools.replication.UpdateState.MORE_PENDING
 
 

-def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
+def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, dsn, update_options):

     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
     status.set_status(temp_db_conn, date, seq=34)
 
     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
     status.set_status(temp_db_conn, date, seq=34)
 


@@ -123,13 +123,13 @@ def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
     sleeptime = []
     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 
     sleeptime = []
     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 

-    assert nominatim.tools.replication.update(temp_db_conn, update_options) \
+    assert nominatim.tools.replication.update(dsn, update_options) \

              == nominatim.tools.replication.UpdateState.NO_CHANGES
 
     assert not sleeptime
 
 
              == nominatim.tools.replication.UpdateState.NO_CHANGES
 
     assert not sleeptime
 
 

-def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
+def test_update_no_data_sleep(monkeypatch, temp_db_conn, dsn, update_options):

     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
     status.set_status(temp_db_conn, date, seq=34)
 
     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
     status.set_status(temp_db_conn, date, seq=34)
 


@@ -140,7 +140,7 @@ def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
     sleeptime = []
     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 
     sleeptime = []
     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 

-    assert nominatim.tools.replication.update(temp_db_conn, update_options) \
+    assert nominatim.tools.replication.update(dsn, update_options) \

              == nominatim.tools.replication.UpdateState.NO_CHANGES
 
     assert len(sleeptime) == 1
              == nominatim.tools.replication.UpdateState.NO_CHANGES
 
     assert len(sleeptime) == 1