update those github action packages still using node12

[nominatim.git] / nominatim / tools / replication.py

diff --git a/nominatim/tools/replication.py b/nominatim/tools/replication.py
index 04f1c45b9cd728b7b0abe7cbb5ff892eb0fd7bef..db706bf67563d795b863da9b0d025412b8554c0f 100644 (file)
--- a/nominatim/tools/replication.py
+++ b/nominatim/tools/replication.py
@@ -1,20 +1,35 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Functions for updating a database from a replication source.
 """
 """
 Functions for updating a database from a replication source.
 """

+from typing import ContextManager, MutableMapping, Any, Generator, cast
+from contextlib import contextmanager

 import datetime as dt
 from enum import Enum
 import logging
 import time
 
 import datetime as dt
 from enum import Enum
 import logging
 import time
 

-from osmium.replication.server import ReplicationServer
-from osmium import WriteHandler
+from nominatim.db import status
+from nominatim.db.connection import Connection
+from nominatim.tools.exec_utils import run_osm2pgsql
+from nominatim.errors import UsageError

 
 

-from ..db import status
-from .exec_utils import run_osm2pgsql
+try:
+    from osmium.replication.server import ReplicationServer
+    from osmium import WriteHandler
+except ImportError as exc:
+    logging.getLogger().critical("pyosmium not installed. Replication functions not available.\n"
+                                 "To install pyosmium via pip: pip3 install osmium")
+    raise UsageError("replication tools not available") from exc

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 

-def init_replication(conn, base_url):
+def init_replication(conn: Connection, base_url: str) -> None:

     """ Set up replication for the server at the given base URL.
     """
     LOG.info("Using replication source: %s", base_url)
     """ Set up replication for the server at the given base URL.
     """
     LOG.info("Using replication source: %s", base_url)


@@ -31,14 +46,14 @@ def init_replication(conn, base_url):
         LOG.fatal("Cannot reach the configured replication service '%s'.\n"
                   "Does the URL point to a directory containing OSM update data?",
                   base_url)
         LOG.fatal("Cannot reach the configured replication service '%s'.\n"
                   "Does the URL point to a directory containing OSM update data?",
                   base_url)

-        raise RuntimeError("Failed to reach replication service")
+        raise UsageError("Failed to reach replication service")

 
     status.set_status(conn, date=date, seq=seq)
 
 
     status.set_status(conn, date=date, seq=seq)
 

-    LOG.warning("Updates intialised at sequence %s (%s)", seq, date)
+    LOG.warning("Updates initialised at sequence %s (%s)", seq, date)

 
 
 
 

-def check_for_updates(conn, base_url):
+def check_for_updates(conn: Connection, base_url: str) -> int:

     """ Check if new data is available from the replication service at the
         given base URL.
     """
     """ Check if new data is available from the replication service at the
         given base URL.
     """


@@ -71,7 +86,7 @@ class UpdateState(Enum):
     NO_CHANGES = 3
 
 
     NO_CHANGES = 3
 
 

-def update(conn, options):
+def update(conn: Connection, options: MutableMapping[str, Any]) -> UpdateState:

     """ Update database from the next batch of data. Returns the state of
         updates according to `UpdateState`.
     """
     """ Update database from the next batch of data. Returns the state of
         updates according to `UpdateState`.
     """


@@ -80,7 +95,9 @@ def update(conn, options):
     if startseq is None:
         LOG.error("Replication not set up. "
                   "Please run 'nominatim replication --init' first.")
     if startseq is None:
         LOG.error("Replication not set up. "
                   "Please run 'nominatim replication --init' first.")

-        raise RuntimeError("Replication not set up.")
+        raise UsageError("Replication not set up.")
+
+    assert startdate is not None

 
     if not indexed and options['indexed_only']:
         LOG.info("Skipping update. There is data that needs indexing.")
 
     if not indexed and options['indexed_only']:
         LOG.info("Skipping update. There is data that needs indexing.")


@@ -97,22 +114,39 @@ def update(conn, options):
         options['import_file'].unlink()
 
     # Read updates into file.
         options['import_file'].unlink()
 
     # Read updates into file.

-    repl = ReplicationServer(options['base_url'])
-
-    outhandler = WriteHandler(str(options['import_file']))
-    endseq = repl.apply_diffs(outhandler, startseq,
-                              max_size=options['max_diff_size'] * 1024)
-    outhandler.close()
+    with _make_replication_server(options['base_url']) as repl:
+        outhandler = WriteHandler(str(options['import_file']))
+        endseq = repl.apply_diffs(outhandler, startseq + 1,
+                                  max_size=options['max_diff_size'] * 1024)
+        outhandler.close()

 
 

-    if endseq is None:
-        return UpdateState.NO_CHANGES
+        if endseq is None:
+            return UpdateState.NO_CHANGES

 
 

-    # Consume updates with osm2pgsql.
-    options['append'] = True
-    run_osm2pgsql(options)
+        # Consume updates with osm2pgsql.
+        options['append'] = True
+        options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
+        run_osm2pgsql(options)

 
 

-    # Write the current status to the file
-    endstate = repl.get_state_info(endseq)
-    status.set_status(conn, endstate.timestamp, seq=endseq, indexed=False)
+        # Write the current status to the file
+        endstate = repl.get_state_info(endseq)
+        status.set_status(conn, endstate.timestamp if endstate else None,
+                          seq=endseq, indexed=False)

 
     return UpdateState.UP_TO_DATE
 
     return UpdateState.UP_TO_DATE

+
+
+def _make_replication_server(url: str) -> ContextManager[ReplicationServer]:
+    """ Returns a ReplicationServer in form of a context manager.
+
+        Creates a light wrapper around older versions of pyosmium that did
+        not support the context manager interface.
+    """
+    if hasattr(ReplicationServer, '__enter__'):
+        return cast(ContextManager[ReplicationServer], ReplicationServer(url))
+
+    @contextmanager
+    def get_cm() -> Generator[ReplicationServer, None, None]:
+        yield ReplicationServer(url)
+
+    return get_cm()