1 # SPDX-License-Identifier: GPL-3.0-or-later
3 # This file is part of Nominatim. (https://nominatim.org)
5 # Copyright (C) 2024 by the Nominatim developer community.
6 # For a full list of authors see the git log.
8 Tests for creating new tokenizers.
12 from nominatim_db.db import properties
13 from nominatim_db.tokenizer import factory
14 from nominatim_db.errors import UsageError
15 from dummy_tokenizer import DummyTokenizer
18 def test_setup_bad_tokenizer_name(project_env, monkeypatch):
19 monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy')
21 with pytest.raises(UsageError):
22 factory.create_tokenizer(project_env)
26 @pytest.fixture(autouse=True)
27 def init_env(self, project_env, property_table, tokenizer_mock):
28 self.config = project_env
31 def test_setup_dummy_tokenizer(self, temp_db_conn):
32 tokenizer = factory.create_tokenizer(self.config)
34 assert isinstance(tokenizer, DummyTokenizer)
35 assert tokenizer.init_state == "new"
36 assert (self.config.project_dir / 'tokenizer').is_dir()
38 assert properties.get_property(temp_db_conn, 'tokenizer') == 'dummy'
41 def test_setup_tokenizer_dir_exists(self):
42 (self.config.project_dir / 'tokenizer').mkdir()
44 tokenizer = factory.create_tokenizer(self.config)
46 assert isinstance(tokenizer, DummyTokenizer)
47 assert tokenizer.init_state == "new"
50 def test_setup_tokenizer_dir_failure(self):
51 (self.config.project_dir / 'tokenizer').write_text("foo")
53 with pytest.raises(UsageError):
54 factory.create_tokenizer(self.config)
57 def test_load_tokenizer(self):
58 factory.create_tokenizer(self.config)
60 tokenizer = factory.get_tokenizer_for_db(self.config)
62 assert isinstance(tokenizer, DummyTokenizer)
63 assert tokenizer.init_state == "loaded"
66 def test_load_repopulate_tokenizer_dir(self):
67 factory.create_tokenizer(self.config)
69 self.config.project_dir = self.config.project_dir
71 factory.get_tokenizer_for_db(self.config)
72 assert (self.config.project_dir / 'tokenizer').exists()
75 def test_load_missing_property(self, temp_db_cursor):
76 factory.create_tokenizer(self.config)
78 temp_db_cursor.execute("TRUNCATE TABLE nominatim_properties")
80 with pytest.raises(UsageError):
81 factory.get_tokenizer_for_db(self.config)