test/python/tools/test_replication.py

   1 # SPDX-License-Identifier: GPL-3.0-or-later
   2 #
   3 # This file is part of Nominatim. (https://nominatim.org)
   4 #
   5 # Copyright (C) 2025 by the Nominatim developer community.
   6 # For a full list of authors see the git log.
   7 """
   8 Tests for replication functionality.
   9 """
  10 import datetime as dt
  11 import time
  12
  13 import pytest
  14 from osmium.replication.server import OsmosisState
  15
  16 import nominatim_db.tools.replication
  17 import nominatim_db.db.status as status
  18 from nominatim_db.errors import UsageError
  19
  20 OSM_NODE_DATA = """\
  21 <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
  22 <node id="100" visible="true" version="1" changeset="2047" timestamp="2006-01-27T22:09:10Z" user="Foo" uid="111" lat="48.7586670" lon="8.1343060">
  23 </node>
  24 </osm>
  25 """  # noqa
  26
  27
  28 @pytest.fixture(autouse=True)
  29 def setup_status_table(status_table):
  30     pass
  31
  32
  33 # init replication
  34
  35 def test_init_replication_bad_base_url(monkeypatch, place_row, temp_db_conn):
  36     place_row(osm_type='N', osm_id=100)
  37
  38     monkeypatch.setattr(status, "get_url", lambda u: OSM_NODE_DATA)
  39
  40     with pytest.raises(UsageError, match="Failed to reach replication service"):
  41         nominatim_db.tools.replication.init_replication(temp_db_conn, 'https://test.io')
  42
  43
  44 def test_init_replication_success(monkeypatch, place_row, temp_db_conn, temp_db_cursor):
  45     place_row(osm_type='N', osm_id=100)
  46
  47     monkeypatch.setattr(status, "get_url", lambda u: OSM_NODE_DATA)
  48     monkeypatch.setattr(nominatim_db.tools.replication.ReplicationServer,
  49                         "timestamp_to_sequence",
  50                         lambda self, date: 234)
  51
  52     nominatim_db.tools.replication.init_replication(temp_db_conn, 'https://test.io')
  53
  54     expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\
  55                       .replace(tzinfo=dt.timezone.utc)
  56
  57     assert temp_db_cursor.row_set("SELECT * FROM import_status") \
  58         == {(expected_date, 234, True)}
  59
  60
  61 # checking for updates
  62
  63 def test_check_for_updates_empty_status_table(temp_db_conn):
  64     assert nominatim_db.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
  65
  66
  67 def test_check_for_updates_seq_not_set(temp_db_conn):
  68     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc))
  69
  70     assert nominatim_db.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
  71
  72
  73 def test_check_for_updates_no_state(monkeypatch, temp_db_conn):
  74     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=345)
  75
  76     monkeypatch.setattr(nominatim_db.tools.replication.ReplicationServer,
  77                         "get_state_info", lambda self: None)
  78
  79     assert nominatim_db.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 253
  80
  81
  82 @pytest.mark.parametrize("server_sequence,result", [(344, 2), (345, 2), (346, 0)])
  83 def test_check_for_updates_no_new_data(monkeypatch, temp_db_conn,
  84                                        server_sequence, result):
  85     date = dt.datetime.now(dt.timezone.utc)
  86     status.set_status(temp_db_conn, date, seq=345)
  87
  88     monkeypatch.setattr(nominatim_db.tools.replication.ReplicationServer,
  89                         "get_state_info",
  90                         lambda self: OsmosisState(server_sequence, date))
  91
  92     assert result == \
  93         nominatim_db.tools.replication.check_for_updates(temp_db_conn, 'https://test.io')
  94
  95
  96 # updating
  97
  98 @pytest.fixture
  99 def update_options(tmpdir):
 100     return dict(base_url='https://test.io',
 101                 indexed_only=False,
 102                 update_interval=3600,
 103                 import_file=tmpdir / 'foo.osm',
 104                 max_diff_size=1)
 105
 106
 107 def test_update_empty_status_table(dsn):
 108     with pytest.raises(UsageError):
 109         nominatim_db.tools.replication.update(dsn, {})
 110
 111
 112 def test_update_already_indexed(temp_db_conn, dsn):
 113     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
 114
 115     assert nominatim_db.tools.replication.update(dsn, dict(indexed_only=True)) \
 116         == nominatim_db.tools.replication.UpdateState.MORE_PENDING
 117
 118
 119 def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, dsn, update_options):
 120     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
 121     status.set_status(temp_db_conn, date, seq=34)
 122
 123     monkeypatch.setattr(nominatim_db.tools.replication.ReplicationServer,
 124                         "apply_diffs",
 125                         lambda *args, **kwargs: None)
 126
 127     sleeptime = []
 128     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 129
 130     assert nominatim_db.tools.replication.update(dsn, update_options) \
 131         == nominatim_db.tools.replication.UpdateState.NO_CHANGES
 132
 133     assert not sleeptime
 134
 135
 136 def test_update_no_data_sleep(monkeypatch, temp_db_conn, dsn, update_options):
 137     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
 138     status.set_status(temp_db_conn, date, seq=34)
 139
 140     monkeypatch.setattr(nominatim_db.tools.replication.ReplicationServer,
 141                         "apply_diffs",
 142                         lambda *args, **kwargs: None)
 143
 144     sleeptime = []
 145     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 146
 147     assert nominatim_db.tools.replication.update(dsn, update_options) \
 148         == nominatim_db.tools.replication.UpdateState.NO_CHANGES
 149
 150     assert len(sleeptime) == 1
 151     assert sleeptime[0] < 3600
 152     assert sleeptime[0] > 0