add tests for sanitizer tagging language

[nominatim.git] / test / python / tokenizer / token_analysis / test_generic.py

diff --git a/test/python/tokenizer/token_analysis/test_generic.py b/test/python/tokenizer/token_analysis/test_generic.py
index f0ce4208e288afbe9fdce86f255abd6220c69de2..a9b09ea43fecd51a60d15a83cc0e7cfdd764b675 100644 (file)
--- a/test/python/tokenizer/token_analysis/test_generic.py
+++ b/test/python/tokenizer/token_analysis/test_generic.py
@@ -40,7 +40,7 @@ def cfgfile(def_config, tmp_path):
 
 
 def get_normalized_variants(proc, name):
 
 
 def get_normalized_variants(proc, name):

-    return proc.get_variants_ascii(proc.get_normalized(name))
+    return proc.analysis[None].get_variants_ascii(proc.normalizer.transliterate(name).strip())

 
 
 def test_variants_empty(cfgfile):
 
 
 def test_variants_empty(cfgfile):


@@ -99,6 +99,6 @@ def test_search_normalized(cfgfile):
     config = cfgfile('~street => s,st', 'master => mstr')
     proc = ICURuleLoader(config).make_token_analysis()
 
     config = cfgfile('~street => s,st', 'master => mstr')
     proc = ICURuleLoader(config).make_token_analysis()
 

-    assert proc.get_search_normalized('Master Street') == 'master street'
-    assert proc.get_search_normalized('Earnes St') == 'earnes st'
-    assert proc.get_search_normalized('Nostreet') == 'nostreet'
+    assert proc.search.transliterate('Master Street').strip() == 'master street'
+    assert proc.search.transliterate('Earnes St').strip() == 'earnes st'
+    assert proc.search.transliterate('Nostreet').strip() == 'nostreet'