use word_token length when penalizing against postcodes

[nominatim.git] / test / python / tokenizer / test_icu.py

diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py
index d85a5b65e565d83187b2688839afe72c1f175fcb..ce00281cff7e09e850fb4d5ee957fb687f65d809 100644 (file)
--- a/test/python/tokenizer/test_icu.py
+++ b/test/python/tokenizer/test_icu.py
@@ -1,26 +1,26 @@
-# SPDX-License-Identifier: GPL-2.0-only
+# SPDX-License-Identifier: GPL-3.0-or-later

 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #

-# Copyright (C) 2022 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.

 # For a full list of authors see the git log.
 """
 Tests for ICU tokenizer.
 """
 # For a full list of authors see the git log.
 """
 Tests for ICU tokenizer.
 """

-import shutil

 import yaml
 import itertools
 
 import pytest
 
 import yaml
 import itertools
 
 import pytest
 

-from nominatim.tokenizer import icu_tokenizer
-import nominatim.tokenizer.icu_rule_loader
-from nominatim.db import properties
-from nominatim.db.sql_preprocessor import SQLPreprocessor
-from nominatim.indexer.place_info import PlaceInfo
+from nominatim_db.tokenizer import icu_tokenizer
+import nominatim_db.tokenizer.icu_rule_loader
+from nominatim_db.db import properties
+from nominatim_db.db.sql_preprocessor import SQLPreprocessor
+from nominatim_db.data.place_info import PlaceInfo

 
 from mock_icu_word_table import MockIcuWordTable
 
 
 from mock_icu_word_table import MockIcuWordTable
 

+

 @pytest.fixture
 def word_table(temp_db_conn):
     return MockIcuWordTable(temp_db_conn)
 @pytest.fixture
 def word_table(temp_db_conn):
     return MockIcuWordTable(temp_db_conn)


@@ -32,8 +32,6 @@ def test_config(project_env, tmp_path):
     sqldir.mkdir()
     (sqldir / 'tokenizer').mkdir()
     (sqldir / 'tokenizer' / 'icu_tokenizer.sql').write_text("SELECT 'a'")
     sqldir.mkdir()
     (sqldir / 'tokenizer').mkdir()
     (sqldir / 'tokenizer' / 'icu_tokenizer.sql').write_text("SELECT 'a'")

-    shutil.copy(str(project_env.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'),
-                str(sqldir / 'tokenizer' / 'icu_tokenizer_tables.sql'))

 
     project_env.lib_dir.sql = sqldir
 
 
     project_env.lib_dir.sql = sqldir
 


@@ -72,7 +70,8 @@ def analyzer(tokenizer_factory, test_config, monkeypatch,
 
     def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',),
                      variants=('~gasse -> gasse', 'street => st', ),
 
     def _mk_analyser(norm=("[[:Punctuation:][:Space:]]+ > ' '",), trans=(':: upper()',),
                      variants=('~gasse -> gasse', 'street => st', ),

-                     sanitizers=[], with_housenumber=False):
+                     sanitizers=[], with_housenumber=False,
+                     with_postcode=False):

         cfgstr = {'normalization': list(norm),
                   'sanitizers': sanitizers,
                   'transliteration': list(trans),
         cfgstr = {'normalization': list(norm),
                   'sanitizers': sanitizers,
                   'transliteration': list(trans),


@@ -81,13 +80,17 @@ def analyzer(tokenizer_factory, test_config, monkeypatch,
         if with_housenumber:
             cfgstr['token-analysis'].append({'id': '@housenumber',
                                              'analyzer': 'housenumbers'})
         if with_housenumber:
             cfgstr['token-analysis'].append({'id': '@housenumber',
                                              'analyzer': 'housenumbers'})

+        if with_postcode:
+            cfgstr['token-analysis'].append({'id': '@postcode',
+                                             'analyzer': 'postcodes'})

         (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr))
         (test_config.project_dir / 'icu_tokenizer.yaml').write_text(yaml.dump(cfgstr))

-        tok.loader = nominatim.tokenizer.icu_rule_loader.ICURuleLoader(test_config)
+        tok.loader = nominatim_db.tokenizer.icu_rule_loader.ICURuleLoader(test_config)

 
         return tok.name_analyzer()
 
     return _mk_analyser
 
 
         return tok.name_analyzer()
 
     return _mk_analyser
 

+

 @pytest.fixture
 def sql_functions(temp_db_conn, def_config, src_dir):
     orig_sql = def_config.lib_dir.sql
 @pytest.fixture
 def sql_functions(temp_db_conn, def_config, src_dir):
     orig_sql = def_config.lib_dir.sql


@@ -151,19 +154,19 @@ LANGUAGE plpgsql;
                               """)
 
 
                               """)
 
 

-

 def test_init_new(tokenizer_factory, test_config, db_prop):
     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 
 def test_init_new(tokenizer_factory, test_config, db_prop):
     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 

-    assert db_prop(nominatim.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES) \
-            .startswith(':: lower ();')
+    prop = db_prop(nominatim_db.tokenizer.icu_rule_loader.DBCFG_IMPORT_NORM_RULES)
+
+    assert prop.startswith(':: lower ();')

 
 
 def test_init_word_table(tokenizer_factory, test_config, place_row, temp_db_cursor):
 
 
 def test_init_word_table(tokenizer_factory, test_config, place_row, temp_db_cursor):

-    place_row(names={'name' : 'Test Area', 'ref' : '52'})
-    place_row(names={'name' : 'No Area'})
-    place_row(names={'name' : 'Holzstrasse'})
+    place_row(names={'name': 'Test Area', 'ref': '52'})
+    place_row(names={'name': 'No Area'})
+    place_row(names={'name': 'Holzstrasse'})

 
     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 
     tok = tokenizer_factory()
     tok.init_new_db(test_config)


@@ -198,18 +201,16 @@ def test_update_sql_functions(db_prop, temp_db_cursor,
     assert test_content == set((('1133', ), ))
 
 
     assert test_content == set((('1133', ), ))
 
 

-def test_finalize_import(tokenizer_factory, temp_db_conn,
-                         temp_db_cursor, test_config, sql_preprocessor_cfg):
-    func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql'
-    func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT
-                            AS $$ SELECT 'b'::text $$ LANGUAGE SQL""")
-
+def test_finalize_import(tokenizer_factory, temp_db_cursor,
+                         test_config, sql_preprocessor_cfg):

     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 
     tok = tokenizer_factory()
     tok.init_new_db(test_config)
 

+    assert not temp_db_cursor.index_exists('word', 'idx_word_word_id')
+

     tok.finalize_import(test_config)
 
     tok.finalize_import(test_config)
 

-    temp_db_cursor.scalar('SELECT test()') == 'b'
+    assert temp_db_cursor.index_exists('word', 'idx_word_word_id')

 
 
 def test_check_database(test_config, tokenizer_factory,
 
 
 def test_check_database(test_config, tokenizer_factory,


@@ -220,23 +221,28 @@ def test_check_database(test_config, tokenizer_factory,
     assert tok.check_database(test_config) is None
 
 
     assert tok.check_database(test_config) is None
 
 

-def test_update_statistics_reverse_only(word_table, tokenizer_factory):
+def test_update_statistics_reverse_only(word_table, tokenizer_factory, test_config):

     tok = tokenizer_factory()
     tok = tokenizer_factory()

-    tok.update_statistics()
+    tok.update_statistics(test_config)

 
 
 
 

-def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory):
+def test_update_statistics(word_table, table_factory, temp_db_cursor,
+                           tokenizer_factory, test_config):

     word_table.add_full_word(1000, 'hello')
     word_table.add_full_word(1000, 'hello')

+    word_table.add_full_word(1001, 'bye')

     table_factory('search_name',
     table_factory('search_name',

-                  'place_id BIGINT, name_vector INT[]',
-                  [(12, [1000])])
+                  'place_id BIGINT, name_vector INT[], nameaddress_vector INT[]',
+                  [(12, [1000], [1001])])

     tok = tokenizer_factory()
 
     tok = tokenizer_factory()
 

-    tok.update_statistics()
+    tok.update_statistics(test_config)

 
     assert temp_db_cursor.scalar("""SELECT count(*) FROM word
 
     assert temp_db_cursor.scalar("""SELECT count(*) FROM word

-                                    WHERE type = 'W' and
-                                          (info->>'count')::int > 0""") > 0
+                                    WHERE type = 'W' and word_id = 1000 and
+                                          (info->>'count')::int > 0""") == 1
+    assert temp_db_cursor.scalar("""SELECT count(*) FROM word
+                                    WHERE type = 'W' and word_id = 1001 and
+                                          (info->>'addr_count')::int > 0""") == 1

 
 
 def test_normalize_postcode(analyzer):
 
 
 def test_normalize_postcode(analyzer):


@@ -246,28 +252,36 @@ def test_normalize_postcode(analyzer):
         anl.normalize_postcode('38 Б') == '38 Б'
 
 
         anl.normalize_postcode('38 Б') == '38 Б'
 
 

-def test_update_postcodes_from_db_empty(analyzer, table_factory, word_table):
-    table_factory('location_postcode', 'postcode TEXT',
-                  content=(('1234',), ('12 34',), ('AB23',), ('1234',)))
+class TestPostcodes:

 
 

-    with analyzer() as anl:
-        anl.update_postcodes_from_db()
+    @pytest.fixture(autouse=True)
+    def setup(self, analyzer, sql_functions):
+        sanitizers = [{'step': 'clean-postcodes'}]
+        with analyzer(sanitizers=sanitizers, with_postcode=True) as anl:
+            self.analyzer = anl
+            yield anl

 
 

-    assert word_table.count() == 3
-    assert word_table.get_postcodes() == {'1234', '12 34', 'AB23'}
+    def process_postcode(self, cc, postcode):
+        return self.analyzer.process_place(PlaceInfo({'country_code': cc,
+                                                      'address': {'postcode': postcode}}))

 
 

+    def test_update_postcodes_deleted(self, word_table):
+        word_table.add_postcode(' 1234', '1234')
+        word_table.add_postcode(' 5678', '5678')

 
 

-def test_update_postcodes_from_db_add_and_remove(analyzer, table_factory, word_table):
-    table_factory('location_postcode', 'postcode TEXT',
-                  content=(('1234',), ('45BC', ), ('XX45', )))
-    word_table.add_postcode(' 1234', '1234')
-    word_table.add_postcode(' 5678', '5678')
+        self.analyzer.update_postcodes_from_db()

 
 

-    with analyzer() as anl:
-        anl.update_postcodes_from_db()
+        assert word_table.count() == 0
+
+    def test_process_place_postcode_simple(self, word_table):
+        info = self.process_postcode('de', '12345')
+
+        assert info['postcode'] == '12345'

 
 

-    assert word_table.count() == 3
-    assert word_table.get_postcodes() == {'1234', '45BC', 'XX45'}
+    def test_process_place_postcode_with_space(self, word_table):
+        info = self.process_postcode('in', '123 567')
+
+        assert info['postcode'] == '123567'

 
 
 def test_update_special_phrase_empty_table(analyzer, word_table):
 
 
 def test_update_special_phrase_empty_table(analyzer, word_table):


@@ -279,9 +293,9 @@ def test_update_special_phrase_empty_table(analyzer, word_table):
         ], True)
 
     assert word_table.get_special() \
         ], True)
 
     assert word_table.get_special() \

-               == {('KÖNIG BEI', 'König bei', 'amenity', 'royal', 'near'),
-                   ('KÖNIGE', 'Könige', 'amenity', 'royal', None),
-                   ('STREET', 'street', 'highway', 'primary', 'in')}
+        == {('KÖNIG BEI', 'König bei', 'amenity', 'royal', 'near'),
+            ('KÖNIGE', 'Könige', 'amenity', 'royal', None),
+            ('STREET', 'street', 'highway', 'primary', 'in')}

 
 
 def test_update_special_phrase_delete_all(analyzer, word_table):
 
 
 def test_update_special_phrase_delete_all(analyzer, word_table):


@@ -322,9 +336,9 @@ def test_update_special_phrase_modify(analyzer, word_table):
         ], True)
 
     assert word_table.get_special() \
         ], True)
 
     assert word_table.get_special() \

-               == {('PRISON', 'prison', 'amenity', 'prison', 'in'),
-                   ('BAR', 'bar', 'highway', 'road', None),
-                   ('GARDEN', 'garden', 'leisure', 'garden', 'near')}
+        == {('PRISON', 'prison', 'amenity', 'prison', 'in'),
+            ('BAR', 'bar', 'highway', 'road', None),
+            ('GARDEN', 'garden', 'leisure', 'garden', 'near')}

 
 
 def test_add_country_names_new(analyzer, word_table):
 
 
 def test_add_country_names_new(analyzer, word_table):


@@ -353,7 +367,6 @@ class TestPlaceNames:
             self.analyzer = anl
             yield anl
 
             self.analyzer = anl
             yield anl
 

-

     def expect_name_terms(self, info, *expected_terms):
         tokens = self.analyzer.get_word_token_info(expected_terms)
         for token in tokens:
     def expect_name_terms(self, info, *expected_terms):
         tokens = self.analyzer.get_word_token_info(expected_terms)
         for token in tokens:


@@ -361,34 +374,29 @@ class TestPlaceNames:
 
         assert eval(info['names']) == set((t[2] for t in tokens))
 
 
         assert eval(info['names']) == set((t[2] for t in tokens))
 

-

     def process_named_place(self, names):
         return self.analyzer.process_place(PlaceInfo({'name': names}))
 
     def process_named_place(self, names):
         return self.analyzer.process_place(PlaceInfo({'name': names}))
 

-

     def test_simple_names(self):
         info = self.process_named_place({'name': 'Soft bAr', 'ref': '34'})
 
         self.expect_name_terms(info, '#Soft bAr', '#34', 'Soft', 'bAr', '34')
 
     def test_simple_names(self):
         info = self.process_named_place({'name': 'Soft bAr', 'ref': '34'})
 
         self.expect_name_terms(info, '#Soft bAr', '#34', 'Soft', 'bAr', '34')
 

-
-    @pytest.mark.parametrize('sep', [',' , ';'])
+    @pytest.mark.parametrize('sep', [',', ';'])

     def test_names_with_separator(self, sep):
         info = self.process_named_place({'name': sep.join(('New York', 'Big Apple'))})
 
         self.expect_name_terms(info, '#New York', '#Big Apple',
                                      'new', 'york', 'big', 'apple')
 
     def test_names_with_separator(self, sep):
         info = self.process_named_place({'name': sep.join(('New York', 'Big Apple'))})
 
         self.expect_name_terms(info, '#New York', '#Big Apple',
                                      'new', 'york', 'big', 'apple')
 

-

     def test_full_names_with_bracket(self):
         info = self.process_named_place({'name': 'Houseboat (left)'})
 
         self.expect_name_terms(info, '#Houseboat (left)', '#Houseboat',
                                      'houseboat', 'left')
 
     def test_full_names_with_bracket(self):
         info = self.process_named_place({'name': 'Houseboat (left)'})
 
         self.expect_name_terms(info, '#Houseboat (left)', '#Houseboat',
                                      'houseboat', 'left')
 

-

     def test_country_name(self, word_table):
     def test_country_name(self, word_table):

-        place = PlaceInfo({'name' : {'name': 'Norge'},
+        place = PlaceInfo({'name': {'name': 'Norge'},

                            'country_code': 'no',
                            'rank_address': 4,
                            'class': 'boundary',
                            'country_code': 'no',
                            'rank_address': 4,
                            'class': 'boundary',


@@ -410,18 +418,15 @@ class TestPlaceAddress:
             self.analyzer = anl
             yield anl
 
             self.analyzer = anl
             yield anl
 

-

     @pytest.fixture
     def getorcreate_hnr_id(self, temp_db_cursor):
         temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT)
                                   RETURNS INTEGER AS $$
                                     SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
 
     @pytest.fixture
     def getorcreate_hnr_id(self, temp_db_cursor):
         temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT)
                                   RETURNS INTEGER AS $$
                                     SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
 

-

     def process_address(self, **kwargs):
         return self.analyzer.process_place(PlaceInfo({'address': kwargs}))
 
     def process_address(self, **kwargs):
         return self.analyzer.process_place(PlaceInfo({'address': kwargs}))
 

-

     def name_token_set(self, *expected_terms):
         tokens = self.analyzer.get_word_token_info(expected_terms)
         for token in tokens:
     def name_token_set(self, *expected_terms):
         tokens = self.analyzer.get_word_token_info(expected_terms)
         for token in tokens:


@@ -429,20 +434,11 @@ class TestPlaceAddress:
 
         return set((t[2] for t in tokens))
 
 
         return set((t[2] for t in tokens))
 

-

     @pytest.mark.parametrize('pcode', ['12345', 'AB 123', '34-345'])
     def test_process_place_postcode(self, word_table, pcode):
     @pytest.mark.parametrize('pcode', ['12345', 'AB 123', '34-345'])
     def test_process_place_postcode(self, word_table, pcode):

-        self.process_address(postcode=pcode)
-
-        assert word_table.get_postcodes() == {pcode, }
-
-
-    @pytest.mark.parametrize('pcode', ['12:23', 'ab;cd;f', '123;836'])
-    def test_process_place_bad_postcode(self, word_table, pcode):
-        self.process_address(postcode=pcode)
-
-        assert not word_table.get_postcodes()
+        info = self.process_address(postcode=pcode)

 
 

+        assert info['postcode'] == pcode

 
     @pytest.mark.parametrize('hnr', ['123a', '1', '101'])
     def test_process_place_housenumbers_simple(self, hnr, getorcreate_hnr_id):
 
     @pytest.mark.parametrize('hnr', ['123a', '1', '101'])
     def test_process_place_housenumbers_simple(self, hnr, getorcreate_hnr_id):


@@ -451,7 +447,6 @@ class TestPlaceAddress:
         assert info['hnr'] == hnr.upper()
         assert info['hnr_tokens'] == "{-1}"
 
         assert info['hnr'] == hnr.upper()
         assert info['hnr_tokens'] == "{-1}"
 

-

     def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id):
         info = self.process_address(housenumber='134',
                                     conscriptionnumber='134',
     def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id):
         info = self.process_address(housenumber='134',
                                     conscriptionnumber='134',


@@ -460,7 +455,6 @@ class TestPlaceAddress:
         assert set(info['hnr'].split(';')) == set(('134', '99A'))
         assert info['hnr_tokens'] == "{-1,-2}"
 
         assert set(info['hnr'].split(';')) == set(('134', '99A'))
         assert info['hnr_tokens'] == "{-1,-2}"
 

-

     def test_process_place_housenumbers_cached(self, getorcreate_hnr_id):
         info = self.process_address(housenumber="45")
         assert info['hnr_tokens'] == "{-1}"
     def test_process_place_housenumbers_cached(self, getorcreate_hnr_id):
         info = self.process_address(housenumber="45")
         assert info['hnr_tokens'] == "{-1}"


@@ -474,37 +468,32 @@ class TestPlaceAddress:
         info = self.process_address(housenumber="41")
         assert eval(info['hnr_tokens']) == {-3}
 
         info = self.process_address(housenumber="41")
         assert eval(info['hnr_tokens']) == {-3}
 

-

     def test_process_place_street(self):
     def test_process_place_street(self):

-        self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road'}}))
+        self.analyzer.process_place(PlaceInfo({'name': {'name': 'Grand Road'}}))

         info = self.process_address(street='Grand Road')
 
         assert eval(info['street']) == self.name_token_set('#Grand Road')
 
         info = self.process_address(street='Grand Road')
 
         assert eval(info['street']) == self.name_token_set('#Grand Road')
 

-

     def test_process_place_nonexisting_street(self):
         info = self.process_address(street='Grand Road')
 
     def test_process_place_nonexisting_street(self):
         info = self.process_address(street='Grand Road')
 

-        assert 'street' not in info
-
+        assert info['street'] == '{}'

 
     def test_process_place_multiple_street_tags(self):
 
     def test_process_place_multiple_street_tags(self):

-        self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road',
+        self.analyzer.process_place(PlaceInfo({'name': {'name': 'Grand Road',

                                                         'ref': '05989'}}))
         info = self.process_address(**{'street': 'Grand Road',
                                                         'ref': '05989'}}))
         info = self.process_address(**{'street': 'Grand Road',

-                                      'street:sym_ul': '05989'})
+                                       'street:sym_ul': '05989'})

 
         assert eval(info['street']) == self.name_token_set('#Grand Road', '#05989')
 
 
         assert eval(info['street']) == self.name_token_set('#Grand Road', '#05989')
 

-

     def test_process_place_street_empty(self):
         info = self.process_address(street='🜵')
 
     def test_process_place_street_empty(self):
         info = self.process_address(street='🜵')
 

-        assert 'street' not in info
-
+        assert info['street'] == '{}'

 
     def test_process_place_street_from_cache(self):
 
     def test_process_place_street_from_cache(self):

-        self.analyzer.process_place(PlaceInfo({'name': {'name' : 'Grand Road'}}))
+        self.analyzer.process_place(PlaceInfo({'name': {'name': 'Grand Road'}}))

         self.process_address(street='Grand Road')
 
         # request address again
         self.process_address(street='Grand Road')
 
         # request address again


@@ -512,45 +501,39 @@ class TestPlaceAddress:
 
         assert eval(info['street']) == self.name_token_set('#Grand Road')
 
 
         assert eval(info['street']) == self.name_token_set('#Grand Road')
 

-

     def test_process_place_place(self):
         info = self.process_address(place='Honu Lulu')
 
     def test_process_place_place(self):
         info = self.process_address(place='Honu Lulu')
 

-        assert eval(info['place']) == self.name_token_set('HONU', 'LULU')
-
+        assert eval(info['place']) == self.name_token_set('HONU', 'LULU', '#HONU LULU')

 
     def test_process_place_place_extra(self):
         info = self.process_address(**{'place:en': 'Honu Lulu'})
 
         assert 'place' not in info
 
 
     def test_process_place_place_extra(self):
         info = self.process_address(**{'place:en': 'Honu Lulu'})
 
         assert 'place' not in info
 

-

     def test_process_place_place_empty(self):
         info = self.process_address(place='🜵')
 
         assert 'place' not in info
 
     def test_process_place_place_empty(self):
         info = self.process_address(place='🜵')
 
         assert 'place' not in info
 

-

     def test_process_place_address_terms(self):
         info = self.process_address(country='de', city='Zwickau', state='Sachsen',
                                     suburb='Zwickau', street='Hauptstr',
                                     full='right behind the church')
 
     def test_process_place_address_terms(self):
         info = self.process_address(country='de', city='Zwickau', state='Sachsen',
                                     suburb='Zwickau', street='Hauptstr',
                                     full='right behind the church')
 

-        city = self.name_token_set('ZWICKAU')
-        state = self.name_token_set('SACHSEN')
+        city = self.name_token_set('ZWICKAU', '#ZWICKAU')
+        state = self.name_token_set('SACHSEN', '#SACHSEN')

 
 

-        result = {k: eval(v) for k,v in info['addr'].items()}
+        result = {k: eval(v) for k, v in info['addr'].items()}

 
         assert result == {'city': city, 'suburb': city, 'state': state}
 
 
         assert result == {'city': city, 'suburb': city, 'state': state}
 

-

     def test_process_place_multiple_address_terms(self):
         info = self.process_address(**{'city': 'Bruxelles', 'city:de': 'Brüssel'})
 
     def test_process_place_multiple_address_terms(self):
         info = self.process_address(**{'city': 'Bruxelles', 'city:de': 'Brüssel'})
 

-        result = {k: eval(v) for k,v in info['addr'].items()}
-
-        assert result == {'city': self.name_token_set('Bruxelles')}
+        result = {k: eval(v) for k, v in info['addr'].items()}

 
 

+        assert result == {'city': self.name_token_set('Bruxelles', '#Bruxelles')}

 
     def test_process_place_address_terms_empty(self):
         info = self.process_address(country='de', city=' ', street='Hauptstr',
 
     def test_process_place_address_terms_empty(self):
         info = self.process_address(country='de', city=' ', street='Hauptstr',


@@ -565,22 +548,21 @@ class TestPlaceHousenumberWithAnalyser:
     def setup(self, analyzer, sql_functions):
         hnr = {'step': 'clean-housenumbers',
                'filter-kind': ['housenumber', 'conscriptionnumber', 'streetnumber']}
     def setup(self, analyzer, sql_functions):
         hnr = {'step': 'clean-housenumbers',
                'filter-kind': ['housenumber', 'conscriptionnumber', 'streetnumber']}

-        with analyzer(trans=(":: upper()", "'🜵' > ' '"), sanitizers=[hnr], with_housenumber=True) as anl:
+        with analyzer(trans=(":: upper()", "'🜵' > ' '"), sanitizers=[hnr],
+                      with_housenumber=True) as anl:

             self.analyzer = anl
             yield anl
 
             self.analyzer = anl
             yield anl
 

-

     @pytest.fixture
     def getorcreate_hnr_id(self, temp_db_cursor):
     @pytest.fixture
     def getorcreate_hnr_id(self, temp_db_cursor):

-        temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION create_analyzed_hnr_id(norm_term TEXT, lookup_terms TEXT[])
-                                  RETURNS INTEGER AS $$
-                                    SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
-
+        temp_db_cursor.execute("""
+            CREATE OR REPLACE FUNCTION create_analyzed_hnr_id(norm_term TEXT, lookup_terms TEXT[])
+            RETURNS INTEGER AS $$
+                SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")

 
     def process_address(self, **kwargs):
         return self.analyzer.process_place(PlaceInfo({'address': kwargs}))
 
 
     def process_address(self, **kwargs):
         return self.analyzer.process_place(PlaceInfo({'address': kwargs}))
 

-

     def name_token_set(self, *expected_terms):
         tokens = self.analyzer.get_word_token_info(expected_terms)
         for token in tokens:
     def name_token_set(self, *expected_terms):
         tokens = self.analyzer.get_word_token_info(expected_terms)
         for token in tokens:


@@ -588,7 +570,6 @@ class TestPlaceHousenumberWithAnalyser:
 
         return set((t[2] for t in tokens))
 
 
         return set((t[2] for t in tokens))
 

-

     @pytest.mark.parametrize('hnr', ['123 a', '1', '101'])
     def test_process_place_housenumbers_simple(self, hnr, getorcreate_hnr_id):
         info = self.process_address(housenumber=hnr)
     @pytest.mark.parametrize('hnr', ['123 a', '1', '101'])
     def test_process_place_housenumbers_simple(self, hnr, getorcreate_hnr_id):
         info = self.process_address(housenumber=hnr)


@@ -596,7 +577,6 @@ class TestPlaceHousenumberWithAnalyser:
         assert info['hnr'] == hnr.upper()
         assert info['hnr_tokens'] == "{-1}"
 
         assert info['hnr'] == hnr.upper()
         assert info['hnr_tokens'] == "{-1}"
 

-

     def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id):
         info = self.process_address(housenumber='134',
                                     conscriptionnumber='134',
     def test_process_place_housenumbers_duplicates(self, getorcreate_hnr_id):
         info = self.process_address(housenumber='134',
                                     conscriptionnumber='134',


@@ -605,7 +585,6 @@ class TestPlaceHousenumberWithAnalyser:
         assert set(info['hnr'].split(';')) == set(('134', '99 A'))
         assert info['hnr_tokens'] == "{-1,-2}"
 
         assert set(info['hnr'].split(';')) == set(('134', '99 A'))
         assert info['hnr_tokens'] == "{-1,-2}"
 

-

     def test_process_place_housenumbers_cached(self, getorcreate_hnr_id):
         info = self.process_address(housenumber="45")
         assert info['hnr_tokens'] == "{-1}"
     def test_process_place_housenumbers_cached(self, getorcreate_hnr_id):
         info = self.process_address(housenumber="45")
         assert info['hnr_tokens'] == "{-1}"


@@ -627,7 +606,6 @@ class TestUpdateWordTokens:
         table_factory('search_name', 'place_id BIGINT, name_vector INT[]')
         self.tok = tokenizer_factory()
 
         table_factory('search_name', 'place_id BIGINT, name_vector INT[]')
         self.tok = tokenizer_factory()
 

-

     @pytest.fixture
     def search_entry(self, temp_db_cursor):
         place_id = itertools.count(1000)
     @pytest.fixture
     def search_entry(self, temp_db_cursor):
         place_id = itertools.count(1000)


@@ -638,7 +616,6 @@ class TestUpdateWordTokens:
 
         return _insert
 
 
         return _insert
 

-

     @pytest.fixture(params=['simple', 'analyzed'])
     def add_housenumber(self, request, word_table):
         if request.param == 'simple':
     @pytest.fixture(params=['simple', 'analyzed'])
     def add_housenumber(self, request, word_table):
         if request.param == 'simple':


@@ -650,7 +627,6 @@ class TestUpdateWordTokens:
 
         return _make
 
 
         return _make
 

-

     @pytest.mark.parametrize('hnr', ('1a', '1234567', '34 5'))
     def test_remove_unused_housenumbers(self, add_housenumber, word_table, hnr):
         word_table.add_housenumber(1000, hnr)
     @pytest.mark.parametrize('hnr', ('1a', '1234567', '34 5'))
     def test_remove_unused_housenumbers(self, add_housenumber, word_table, hnr):
         word_table.add_housenumber(1000, hnr)


@@ -659,7 +635,6 @@ class TestUpdateWordTokens:
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 0
 
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 0
 

-

     def test_keep_unused_numeral_housenumbers(self, add_housenumber, word_table):
         add_housenumber(1000, '5432')
 
     def test_keep_unused_numeral_housenumbers(self, add_housenumber, word_table):
         add_housenumber(1000, '5432')
 


@@ -667,8 +642,8 @@ class TestUpdateWordTokens:
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 1
 
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 1
 

-
-    def test_keep_housenumbers_from_search_name_table(self, add_housenumber, word_table, search_entry):
+    def test_keep_housenumbers_from_search_name_table(self, add_housenumber,
+                                                      word_table, search_entry):

         add_housenumber(9999, '5432a')
         add_housenumber(9991, '9 a')
         search_entry(123, 9999, 34)
         add_housenumber(9999, '5432a')
         add_housenumber(9991, '9 a')
         search_entry(123, 9999, 34)


@@ -677,8 +652,8 @@ class TestUpdateWordTokens:
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 1
 
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 1
 

-
-    def test_keep_housenumbers_from_placex_table(self, add_housenumber, word_table, placex_table):
+    def test_keep_housenumbers_from_placex_table(self, add_housenumber, word_table,
+                                                 placex_table):

         add_housenumber(9999, '5432a')
         add_housenumber(9990, '34z')
         placex_table.add(housenumber='34z')
         add_housenumber(9999, '5432a')
         add_housenumber(9990, '34z')
         placex_table.add(housenumber='34z')


@@ -688,8 +663,8 @@ class TestUpdateWordTokens:
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 1
 
         self.tok.update_word_tokens()
         assert word_table.count_housenumbers() == 1
 

-
-    def test_keep_housenumbers_from_placex_table_hnr_list(self, add_housenumber, word_table, placex_table):
+    def test_keep_housenumbers_from_placex_table_hnr_list(self, add_housenumber,
+                                                          word_table, placex_table):

         add_housenumber(9991, '9 b')
         add_housenumber(9990, '34z')
         placex_table.add(housenumber='9 a;9 b;9 c')
         add_housenumber(9991, '9 b')
         add_housenumber(9990, '34z')
         placex_table.add(housenumber='9 a;9 b;9 c')