Merge remote-tracking branch 'upstream/master'

[nominatim.git] / nominatim / tokenizer / legacy_icu_tokenizer.py

diff --git a/nominatim/tokenizer/legacy_icu_tokenizer.py b/nominatim/tokenizer/legacy_icu_tokenizer.py
index 156e99ece67f156d463f5d7e84858dcb7b80c027..2bd22c7207cb3f3cbf2de920a0a5887d67afd04d 100644 (file)
--- a/nominatim/tokenizer/legacy_icu_tokenizer.py
+++ b/nominatim/tokenizer/legacy_icu_tokenizer.py
@@ -423,8 +423,7 @@ class LegacyICUNameAnalyzer:
         names = place.get('name')
 
         if names:
         names = place.get('name')
 
         if names:

-            full_names = set((self.make_standard_word(name) for name in names.values()))
-            full_names.discard('')
+            full_names = self._compute_full_names(names)

 
             token_info.add_names(self.conn, full_names)
 
 
             token_info.add_names(self.conn, full_names)
 


@@ -461,6 +460,25 @@ class LegacyICUNameAnalyzer:
         return token_info.data
 
 
         return token_info.data
 
 

+    def _compute_full_names(self, names):
+        """ Return the set of all full name word ids to be used with the
+            given dictionary of names.
+        """
+        full_names = set()
+        for name in (n for ns in names.values() for n in re.split('[;,]', ns)):
+            word = self.make_standard_word(name)
+            if word:
+                full_names.add(word)
+
+                brace_split = name.split('(', 2)
+                if len(brace_split) > 1:
+                    word = self.make_standard_word(brace_split[0])
+                    if word:
+                        full_names.add(word)
+
+        return full_names
+
+

     def _add_postcode(self, postcode):
         """ Make sure the normalized postcode is present in the word table.
         """
     def _add_postcode(self, postcode):
         """ Make sure the normalized postcode is present in the word table.
         """


@@ -519,8 +537,6 @@ class _TokenInfo:
         """
         # Start with all partial names
         terms = set((part for ns in names for part in ns.split()))
         """
         # Start with all partial names
         terms = set((part for ns in names for part in ns.split()))

-        # Add partials for the full terms (TO BE REMOVED)
-        terms.update((n for n in names))

         # Add the full names
         terms.update((' ' + n for n in names))
 
         # Add the full names
         terms.update((' ' + n for n in names))