test/python/test_tokenizer_factory.py

   1 """
   2 Tests for creating new tokenizers.
   3 """
   4 import importlib
   5 import pytest
   6
   7 from nominatim.db import properties
   8 from nominatim.tokenizer import factory
   9 from nominatim.errors import UsageError
  10 from dummy_tokenizer import DummyTokenizer
  11
  12 @pytest.fixture
  13 def test_config(def_config, tmp_path):
  14     def_config.project_dir = tmp_path
  15     return def_config
  16
  17
  18 def test_setup_dummy_tokenizer(temp_db_conn, test_config,
  19                                tokenizer_mock, property_table):
  20     tokenizer = factory.create_tokenizer(test_config)
  21
  22     assert isinstance(tokenizer, DummyTokenizer)
  23     assert tokenizer.init_state == "new"
  24     assert (test_config.project_dir / 'tokenizer').is_dir()
  25
  26     assert properties.get_property(temp_db_conn, 'tokenizer') == 'dummy'
  27
  28
  29 def test_setup_tokenizer_dir_exists(test_config, tokenizer_mock, property_table):
  30     (test_config.project_dir / 'tokenizer').mkdir()
  31
  32     tokenizer = factory.create_tokenizer(test_config)
  33
  34     assert isinstance(tokenizer, DummyTokenizer)
  35     assert tokenizer.init_state == "new"
  36
  37
  38 def test_setup_tokenizer_dir_failure(test_config, tokenizer_mock, property_table):
  39     (test_config.project_dir / 'tokenizer').write_text("foo")
  40
  41     with pytest.raises(UsageError):
  42         factory.create_tokenizer(test_config)
  43
  44
  45 def test_setup_bad_tokenizer_name(test_config, monkeypatch):
  46     monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy')
  47
  48     with pytest.raises(UsageError):
  49         factory.create_tokenizer(test_config)
  50
  51 def test_load_tokenizer(temp_db_conn, test_config,
  52                         tokenizer_mock, property_table):
  53     factory.create_tokenizer(test_config)
  54
  55     tokenizer = factory.get_tokenizer_for_db(test_config)
  56
  57     assert isinstance(tokenizer, DummyTokenizer)
  58     assert tokenizer.init_state == "loaded"
  59
  60
  61 def test_load_no_tokenizer_dir(test_config, tokenizer_mock, property_table):
  62     factory.create_tokenizer(test_config)
  63
  64     test_config.project_dir = test_config.project_dir / 'foo'
  65
  66     with pytest.raises(UsageError):
  67         factory.get_tokenizer_for_db(test_config)
  68
  69
  70 def test_load_missing_propoerty(temp_db_cursor, test_config, tokenizer_mock, property_table):
  71     factory.create_tokenizer(test_config)
  72
  73     temp_db_cursor.execute("TRUNCATE TABLE nominatim_properties")
  74
  75     with pytest.raises(UsageError):
  76         factory.get_tokenizer_for_db(test_config)
  77