Merge pull request #2655 from lonvia/migration-internal-country-name

[nominatim.git] / nominatim / tokenizer / sanitizers / clean_housenumbers.py

diff --git a/nominatim/tokenizer/sanitizers/clean_housenumbers.py b/nominatim/tokenizer/sanitizers/clean_housenumbers.py
index 5b592bcf8d76ae37db53196c85107d23097d5dd7..c229716f8e8353d8d8788670b767ea0c9cebc22f 100644 (file)
--- a/nominatim/tokenizer/sanitizers/clean_housenumbers.py
+++ b/nominatim/tokenizer/sanitizers/clean_housenumbers.py
@@ -5,14 +5,36 @@
 # Copyright (C) 2022 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 # Copyright (C) 2022 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """

-Sanitizer that cleans and normalizes housenumbers.
+Sanitizer that preprocesses address tags for house numbers. The sanitizer
+allows to
+
+* define which tags are to be considered house numbers (see 'filter-kind')
+* split house number lists into individual numbers (see 'delimiters')
+
+Arguments:
+    delimiters: Define the set of characters to be used for
+                splitting a list of house numbers into parts. (default: ',;')
+    filter-kind: Define the address tags that are considered to be a
+                 house number. Either takes a single string or a list of strings,
+                 where each string is a regular expression. An address item
+                 is considered a house number if the 'kind' fully matches any
+                 of the given regular expressions. (default: 'housenumber')
+    convert-to-name: Define house numbers that should be treated as a name
+                     instead of a house number. Either takes a single string
+                     or a list of strings, where each string is a regular
+                     expression that must match the full house number value.

 """
 import re
 
 class _HousenumberSanitizer:
 
     def __init__(self, config):
 """
 import re
 
 class _HousenumberSanitizer:
 
     def __init__(self, config):

-        pass
+        self.filter_kind = config.get_filter_kind('housenumber')
+        self.split_regexp = config.get_delimiter()
+
+        nameregexps = config.get_string_list('convert-to-name', [])
+        self.is_name_regexp = [re.compile(r) for r in nameregexps]
+

 
 
     def __call__(self, obj):
 
 
     def __call__(self, obj):


@@ -21,8 +43,12 @@ class _HousenumberSanitizer:
 
         new_address = []
         for item in obj.address:
 
         new_address = []
         for item in obj.address:

-            if item.kind in ('housenumber', 'streetnumber', 'conscriptionnumber'):
-                new_address.extend(item.clone(kind='housenumber', name=n) for n in self.sanitize(item.name))
+            if self.filter_kind(item):
+                if self._treat_as_name(item.name):
+                    obj.names.append(item.clone(kind='housenumber'))
+                else:
+                    new_address.extend(item.clone(kind='housenumber', name=n)
+                                       for n in self.sanitize(item.name))

             else:
                 # Don't touch other address items.
                 new_address.append(item)
             else:
                 # Don't touch other address items.
                 new_address.append(item)


@@ -36,17 +62,18 @@ class _HousenumberSanitizer:
             The function works as a generator that yields all valid housenumbers
             that can be created from the value.
         """
             The function works as a generator that yields all valid housenumbers
             that can be created from the value.
         """

-        for hnr in self._split_number(value):
-            yield from self._regularize(hnr)
+        for hnr in self.split_regexp.split(value):
+            if hnr:
+                yield from self._regularize(hnr)

 
 
 
 

-    def _split_number(self, hnr):
-        for part in re.split(r'[;,]', hnr):
-            yield part.strip()
+    @staticmethod
+    def _regularize(hnr):
+        yield hnr

 
 
 
 

-    def _regularize(self, hnr):
-        yield hnr
+    def _treat_as_name(self, housenumber):
+        return any(r.fullmatch(housenumber) is not None for r in self.is_name_regexp)

 
 
 def create(config):
 
 
 def create(config):