test/python/test_tools_replication.py

   1 """
   2 Tests for replication functionality.
   3 """
   4 import datetime as dt
   5 import time
   6
   7 import pytest
   8 from osmium.replication.server import OsmosisState
   9
  10 import nominatim.tools.replication
  11 import nominatim.db.status as status
  12
  13 OSM_NODE_DATA = """\
  14 <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
  15 <node id="100" visible="true" version="1" changeset="2047" timestamp="2006-01-27T22:09:10Z" user="Foo" uid="111" lat="48.7586670" lon="8.1343060">
  16 </node>
  17 </osm>
  18 """
  19
  20 ### init replication
  21
  22 def test_init_replication_bad_base_url(monkeypatch, status_table, place_row, temp_db_conn, temp_db_cursor):
  23     place_row(osm_type='N', osm_id=100)
  24
  25     monkeypatch.setattr(nominatim.db.status, "get_url", lambda u : OSM_NODE_DATA)
  26
  27     with pytest.raises(RuntimeError, match="Failed to reach replication service"):
  28         nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
  29
  30
  31 def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_conn, temp_db_cursor):
  32     place_row(osm_type='N', osm_id=100)
  33
  34     monkeypatch.setattr(nominatim.db.status, "get_url", lambda u : OSM_NODE_DATA)
  35     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
  36                         "timestamp_to_sequence",
  37                         lambda self, date: 234)
  38
  39     nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
  40
  41     temp_db_cursor.execute("SELECT * FROM import_status")
  42
  43     expected_date = dt.datetime.fromisoformat('2006-01-27T19:09:10').replace(tzinfo=dt.timezone.utc)
  44     assert temp_db_cursor.rowcount == 1
  45     assert temp_db_cursor.fetchone() == [expected_date, 234, True]
  46
  47
  48 ### checking for updates
  49
  50 def test_check_for_updates_empty_status_table(status_table, temp_db_conn):
  51     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
  52
  53
  54 def test_check_for_updates_seq_not_set(status_table, temp_db_conn):
  55     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc))
  56
  57     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
  58
  59
  60 def test_check_for_updates_no_state(monkeypatch, status_table, temp_db_conn):
  61     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=345)
  62
  63     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
  64                         "get_state_info", lambda self: None)
  65
  66     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 253
  67
  68
  69 @pytest.mark.parametrize("server_sequence,result", [(344, 2), (345, 2), (346, 0)])
  70 def test_check_for_updates_no_new_data(monkeypatch, status_table, temp_db_conn,
  71                                        server_sequence, result):
  72     date = dt.datetime.now(dt.timezone.utc)
  73     status.set_status(temp_db_conn, date, seq=345)
  74
  75     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
  76                         "get_state_info",
  77                         lambda self: OsmosisState(server_sequence, date))
  78
  79     assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == result
  80
  81
  82 ### updating
  83
  84 @pytest.fixture
  85 def update_options(tmpdir):
  86     return dict(base_url='https://test.io',
  87                    indexed_only=False,
  88                    update_interval=3600,
  89                    import_file=tmpdir / 'foo.osm',
  90                    max_diff_size=1)
  91
  92 def test_update_empty_status_table(status_table, temp_db_conn):
  93     with pytest.raises(RuntimeError):
  94         nominatim.tools.replication.update(temp_db_conn, {})
  95
  96
  97 def test_update_already_indexed(status_table, temp_db_conn):
  98     status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
  99
 100     assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
 101              == nominatim.tools.replication.UpdateState.MORE_PENDING
 102
 103
 104 def test_update_no_data_no_sleep(monkeypatch, status_table, temp_db_conn, update_options):
 105     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
 106     status.set_status(temp_db_conn, date, seq=34)
 107
 108     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
 109                         "apply_diffs",
 110                         lambda *args, **kwargs: None)
 111
 112     sleeptime = []
 113     monkeypatch.setattr(time, 'sleep', lambda s: sleeptime.append(s))
 114
 115     assert nominatim.tools.replication.update(temp_db_conn, update_options) \
 116              == nominatim.tools.replication.UpdateState.NO_CHANGES
 117
 118     assert not sleeptime
 119
 120
 121 def test_update_no_data_sleep(monkeypatch, status_table, temp_db_conn, update_options):
 122     date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
 123     status.set_status(temp_db_conn, date, seq=34)
 124
 125     monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
 126                         "apply_diffs",
 127                         lambda *args, **kwargs: None)
 128
 129     sleeptime = []
 130     monkeypatch.setattr(time, 'sleep', lambda s: sleeptime.append(s))
 131
 132     assert nominatim.tools.replication.update(temp_db_conn, update_options) \
 133              == nominatim.tools.replication.UpdateState.NO_CHANGES
 134
 135     assert len(sleeptime) == 1
 136     assert sleeptime[0] < 3600
 137     assert sleeptime[0] > 0