Merge pull request #2458 from lonvia/add-tokenizer-preprocessing

[nominatim.git] / nominatim / tokenizer / sanitizers / split_name_list.py

diff --git a/nominatim/tokenizer/sanitizers/split_name_list.py b/nominatim/tokenizer/sanitizers/split_name_list.py
index 93651f3e8b3283d1095c58d9b357332f24b6d7e6..f151420396db9825a00067e81c507b0908ef835d 100644 (file)
--- a/nominatim/tokenizer/sanitizers/split_name_list.py
+++ b/nominatim/tokenizer/sanitizers/split_name_list.py
@@ -3,13 +3,19 @@ Name processor that splits name values with multiple values into their component
 """
 import re
 
+from nominatim.errors import UsageError
+
 def create(func):
     """ Create a name processing function that splits name values with
         multiple values into their components. The optional parameter
         'delimiters' can be used to define the characters that should be used
         for splitting. The default is ',;'.
     """
-    regexp = re.compile('[{}]'.format(func.get('delimiters', ',;')))
+    delimiter_set = set(func.get('delimiters', ',;'))
+    if not delimiter_set:
+        raise UsageError("Set of delimiters in split-name-list sanitizer is empty.")
+
+    regexp = re.compile('\\s*[{}]\\s*'.format(''.join('\\' + d for d in delimiter_set)))
 
     def _process(obj):
         if not obj.names:
@@ -18,10 +24,11 @@ def create(func):
         new_names = []
         for name in obj.names:
             split_names = regexp.split(name.name)
+            print(split_names)
             if len(split_names) == 1:
                 new_names.append(name)
             else:
-                new_names.extend(name.clone(name=n) for n in split_names)
+                new_names.extend(name.clone(name=n) for n in split_names if n)
 
         obj.names = new_names