test/python/tools/test_replication.py

   1 """
   2 Tests for replication functionality.
   3 """
   4 import datetime as dt
   5 import time
   6
   7 import pytest
   8 from osmium.replication.server import OsmosisState
   9
  10 import nominatim.tools.replication
  11 import nominatim.db.status as status
  12 from nominatim.errors import UsageError
  13
  14 OSM_NODE_DATA = """\
  15 <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
  16 <node id="100" visible="true" version="1" changeset="2047" timestamp="2006-01-27T22:09:10Z" user="Foo" uid="111" lat="48.7586670" lon="8.1343060">
  17 </node>
  18 </osm>
  19 """
  20
  21 @pytest.fixture(autouse=True)
  22 def setup_status_table(status_table):
  23     pass
  24
  25 ### init replication
  26
  27 def test_init_replication_bad_base_url(monkeypatch, place_row, temp_db_conn):
  28     place_row(osm_type='N', osm_id=100)
  29
  30     monkeypatch.setattr(nominatim.db.status, "get_url", lambda u: OSM_NODE_DATA)
  31
  32     with pytest.raises(UsageError, match="Failed to reach replication service"):
  33         nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
  34
  35
  36 def test_init_replication_success(monkeypatch, place_row, temp_db_conn, temp_db_cursor):
  37     place_row(osm_type='N', osm_id=100)
  38
  39     monkeypatch.setattr(nominatim.db.status, "get_url", lambda u: OSM_NODE_DATA)
  40     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
  41                         "timestamp_to_sequence",
  42                         lambda self, date: 234)
  43
  44     nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
  45
  46     expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\
  47                         .replace(tzinfo=dt.timezone.utc)
  48
  49     assert temp_db_cursor.row_set("SELECT * FROM import_status") \
  50              == {(expected_date, 234, True)}
  51
  52
  53 ### checking for updates
  54
  55 def test_check_for_updates_empty_status_table(temp_db_conn):
  56     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
  57
  58
  59 def test_check_for_updates_seq_not_set(temp_db_conn):
  60     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc))
  61
  62     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
  63
  64
  65 def test_check_for_updates_no_state(monkeypatch, temp_db_conn):
  66     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=345)
  67
  68     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
  69                         "get_state_info", lambda self: None)
  70
  71     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 253
  72
  73
  74 @pytest.mark.parametrize("server_sequence,result", [(344, 2), (345, 2), (346, 0)])
  75 def test_check_for_updates_no_new_data(monkeypatch, temp_db_conn,
  76                                        server_sequence, result):
  77     date = dt.datetime.now(dt.timezone.utc)
  78     status.set_status(temp_db_conn, date, seq=345)
  79
  80     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
  81                         "get_state_info",
  82                         lambda self: OsmosisState(server_sequence, date))
  83
  84     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == result
  85
  86
  87 ### updating
  88
  89 @pytest.fixture
  90 def update_options(tmpdir):
  91     return dict(base_url='https://test.io',
  92                 indexed_only=False,
  93                 update_interval=3600,
  94                 import_file=tmpdir / 'foo.osm',
  95                 max_diff_size=1)
  96
  97 def test_update_empty_status_table(temp_db_conn):
  98     with pytest.raises(UsageError):
  99         nominatim.tools.replication.update(temp_db_conn, {})
 100
 101
 102 def test_update_already_indexed(temp_db_conn):
 103     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
 104
 105     assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
 106              == nominatim.tools.replication.UpdateState.MORE_PENDING
 107
 108
 109 def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
 110     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
 111     status.set_status(temp_db_conn, date, seq=34)
 112
 113     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
 114                         "apply_diffs",
 115                         lambda *args, **kwargs: None)
 116
 117     sleeptime = []
 118     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 119
 120     assert nominatim.tools.replication.update(temp_db_conn, update_options) \
 121              == nominatim.tools.replication.UpdateState.NO_CHANGES
 122
 123     assert not sleeptime
 124
 125
 126 def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
 127     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
 128     status.set_status(temp_db_conn, date, seq=34)
 129
 130     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
 131                         "apply_diffs",
 132                         lambda *args, **kwargs: None)
 133
 134     sleeptime = []
 135     monkeypatch.setattr(time, 'sleep', sleeptime.append)
 136
 137     assert nominatim.tools.replication.update(temp_db_conn, update_options) \
 138              == nominatim.tools.replication.UpdateState.NO_CHANGES
 139
 140     assert len(sleeptime) == 1
 141     assert sleeptime[0] < 3600
 142     assert sleeptime[0] > 0