From 6f3339cc498592850218f8fc714dd45d66cd1815 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 19 Jun 2023 12:02:51 +0200 Subject: [PATCH] close DB connection when waiting for next update cycle --- nominatim/clicmd/replication.py | 11 +++++++---- nominatim/tools/replication.py | 20 ++++++++++++-------- test/python/tools/test_replication.py | 16 ++++++++-------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/nominatim/clicmd/replication.py b/nominatim/clicmd/replication.py index ad201663..b7956506 100644 --- a/nominatim/clicmd/replication.py +++ b/nominatim/clicmd/replication.py @@ -147,10 +147,13 @@ class UpdateReplication: tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config) indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1) + dsn = args.config.get_libpq_dsn() + while True: - with connect(args.config.get_libpq_dsn()) as conn: - start = dt.datetime.now(dt.timezone.utc) - state = replication.update(conn, params, socket_timeout=args.socket_timeout) + start = dt.datetime.now(dt.timezone.utc) + state = replication.update(dsn, params, socket_timeout=args.socket_timeout) + + with connect(dsn) as conn: if state is not replication.UpdateState.NO_CHANGES: status.log_status(conn, start, 'import') batchdate, _, _ = status.get_status(conn) @@ -160,7 +163,7 @@ class UpdateReplication: index_start = dt.datetime.now(dt.timezone.utc) indexer.index_full(analyse=False) - with connect(args.config.get_libpq_dsn()) as conn: + with connect(dsn) as conn: status.set_indexed(conn, True) status.log_status(conn, index_start, 'index') conn.commit() diff --git a/nominatim/tools/replication.py b/nominatim/tools/replication.py index 846b9c34..edd63e49 100644 --- a/nominatim/tools/replication.py +++ b/nominatim/tools/replication.py @@ -18,7 +18,7 @@ import urllib.request as urlrequest import requests from nominatim.db import status -from nominatim.db.connection import Connection +from nominatim.db.connection import Connection, connect from nominatim.tools.exec_utils import run_osm2pgsql from nominatim.errors import UsageError @@ -92,12 +92,14 @@ class UpdateState(Enum): NO_CHANGES = 3 -def update(conn: Connection, options: MutableMapping[str, Any], +def update(dsn: str, options: MutableMapping[str, Any], socket_timeout: int = 60) -> UpdateState: """ Update database from the next batch of data. Returns the state of updates according to `UpdateState`. """ - startdate, startseq, indexed = status.get_status(conn) + with connect(dsn) as conn: + startdate, startseq, indexed = status.get_status(conn) + conn.commit() if startseq is None: LOG.error("Replication not set up. " @@ -130,12 +132,14 @@ def update(conn: Connection, options: MutableMapping[str, Any], if endseq is None: return UpdateState.NO_CHANGES - run_osm2pgsql_updates(conn, options) + with connect(dsn) as conn: + run_osm2pgsql_updates(conn, options) - # Write the current status to the file - endstate = repl.get_state_info(endseq) - status.set_status(conn, endstate.timestamp if endstate else None, - seq=endseq, indexed=False) + # Write the current status to the file + endstate = repl.get_state_info(endseq) + status.set_status(conn, endstate.timestamp if endstate else None, + seq=endseq, indexed=False) + conn.commit() return UpdateState.UP_TO_DATE diff --git a/test/python/tools/test_replication.py b/test/python/tools/test_replication.py index 2f9c7e86..7a7340ae 100644 --- a/test/python/tools/test_replication.py +++ b/test/python/tools/test_replication.py @@ -100,19 +100,19 @@ def update_options(tmpdir): import_file=tmpdir / 'foo.osm', max_diff_size=1) -def test_update_empty_status_table(temp_db_conn): +def test_update_empty_status_table(dsn): with pytest.raises(UsageError): - nominatim.tools.replication.update(temp_db_conn, {}) + nominatim.tools.replication.update(dsn, {}) -def test_update_already_indexed(temp_db_conn): +def test_update_already_indexed(temp_db_conn, dsn): status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False) - assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \ + assert nominatim.tools.replication.update(dsn, dict(indexed_only=True)) \ == nominatim.tools.replication.UpdateState.MORE_PENDING -def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options): +def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, dsn, update_options): date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1) status.set_status(temp_db_conn, date, seq=34) @@ -123,13 +123,13 @@ def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options): sleeptime = [] monkeypatch.setattr(time, 'sleep', sleeptime.append) - assert nominatim.tools.replication.update(temp_db_conn, update_options) \ + assert nominatim.tools.replication.update(dsn, update_options) \ == nominatim.tools.replication.UpdateState.NO_CHANGES assert not sleeptime -def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options): +def test_update_no_data_sleep(monkeypatch, temp_db_conn, dsn, update_options): date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30) status.set_status(temp_db_conn, date, seq=34) @@ -140,7 +140,7 @@ def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options): sleeptime = [] monkeypatch.setattr(time, 'sleep', sleeptime.append) - assert nominatim.tools.replication.update(temp_db_conn, update_options) \ + assert nominatim.tools.replication.update(dsn, update_options) \ == nominatim.tools.replication.UpdateState.NO_CHANGES assert len(sleeptime) == 1 -- 2.39.5