test/python/test_tokenizer_factory.py

   1 """
   2 Tests for creating new tokenizers.
   3 """
   4 import importlib
   5 import pytest
   6
   7 from nominatim.db import properties
   8 from nominatim.tokenizer import factory
   9 from nominatim.errors import UsageError
  10 import dummy_tokenizer
  11
  12 @pytest.fixture
  13 def test_config(def_config, tmp_path):
  14     def_config.project_dir = tmp_path
  15     return def_config
  16
  17
  18 @pytest.fixture
  19 def tokenizer_import(monkeypatch):
  20     monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy')
  21
  22     def _import_dummy(module, *args, **kwargs):
  23         return dummy_tokenizer
  24
  25     monkeypatch.setattr(importlib, "import_module", _import_dummy)
  26
  27
  28 def test_setup_dummy_tokenizer(temp_db_conn, test_config,
  29                                tokenizer_import, property_table):
  30     tokenizer = factory.create_tokenizer(test_config)
  31
  32     assert isinstance(tokenizer, dummy_tokenizer.DummyTokenizer)
  33     assert tokenizer.init_state == "new"
  34     assert (test_config.project_dir / 'tokenizer').is_dir()
  35
  36     assert properties.get_property(temp_db_conn, 'tokenizer') == 'dummy'
  37
  38
  39 def test_setup_tokenizer_dir_exists(test_config, tokenizer_import, property_table):
  40     (test_config.project_dir / 'tokenizer').mkdir()
  41
  42     tokenizer = factory.create_tokenizer(test_config)
  43
  44     assert isinstance(tokenizer, dummy_tokenizer.DummyTokenizer)
  45     assert tokenizer.init_state == "new"
  46
  47
  48 def test_setup_tokenizer_dir_failure(test_config, tokenizer_import, property_table):
  49     (test_config.project_dir / 'tokenizer').write_text("foo")
  50
  51     with pytest.raises(UsageError):
  52         factory.create_tokenizer(test_config)
  53
  54
  55 def test_setup_bad_tokenizer_name(test_config, monkeypatch):
  56     monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy')
  57
  58     with pytest.raises(UsageError):
  59         factory.create_tokenizer(test_config)
  60
  61 def test_load_tokenizer(temp_db_conn, test_config,
  62                         tokenizer_import, property_table):
  63     factory.create_tokenizer(test_config)
  64
  65     tokenizer = factory.get_tokenizer_for_db(test_config)
  66
  67     assert isinstance(tokenizer, dummy_tokenizer.DummyTokenizer)
  68     assert tokenizer.init_state == "loaded"
  69
  70
  71 def test_load_no_tokenizer_dir(test_config, tokenizer_import, property_table):
  72     factory.create_tokenizer(test_config)
  73
  74     test_config.project_dir = test_config.project_dir / 'foo'
  75
  76     with pytest.raises(UsageError):
  77         factory.get_tokenizer_for_db(test_config)
  78
  79
  80 def test_load_missing_propoerty(temp_db_cursor, test_config, tokenizer_import, property_table):
  81     factory.create_tokenizer(test_config)
  82
  83     temp_db_cursor.execute("TRUNCATE TABLE nominatim_properties")
  84
  85     with pytest.raises(UsageError):
  86         factory.get_tokenizer_for_db(test_config)
  87