introduce mutation variants to generic token analyser

[nominatim.git] / nominatim / tokenizer / icu_tokenizer.py

diff --git a/nominatim/tokenizer/icu_tokenizer.py b/nominatim/tokenizer/icu_tokenizer.py
index 33f05cc4b21d463d22dfc675bab665c804fbc276..7b820c9da04173b4e585b816c828b597d775f87b 100644 (file)
--- a/nominatim/tokenizer/icu_tokenizer.py
+++ b/nominatim/tokenizer/icu_tokenizer.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Tokenizer implementing normalisation as used before Nominatim 4 but using
 libICU instead of the PostgreSQL module.
 """
 Tokenizer implementing normalisation as used before Nominatim 4 but using
 libICU instead of the PostgreSQL module.


@@ -416,9 +422,7 @@ class LegacyICUNameAnalyzer(AbstractAnalyzer):
             elif item.kind in ('housenumber', 'streetnumber', 'conscriptionnumber'):
                 hnrs.append(item.name)
             elif item.kind == 'street':
             elif item.kind in ('housenumber', 'streetnumber', 'conscriptionnumber'):
                 hnrs.append(item.name)
             elif item.kind == 'street':

-                token = self._retrieve_full_token(item.name)
-                if token:
-                    streets.append(token)
+                streets.extend(self._retrieve_full_tokens(item.name))

             elif item.kind == 'place':
                 if not item.suffix:
                     token_info.add_place(self._compute_partial_tokens(item.name))
             elif item.kind == 'place':
                 if not item.suffix:
                     token_info.add_place(self._compute_partial_tokens(item.name))


@@ -465,25 +469,20 @@ class LegacyICUNameAnalyzer(AbstractAnalyzer):
         return tokens
 
 
         return tokens
 
 

-    def _retrieve_full_token(self, name):
+    def _retrieve_full_tokens(self, name):

         """ Get the full name token for the given name, if it exists.
             The name is only retrived for the standard analyser.
         """
         """ Get the full name token for the given name, if it exists.
             The name is only retrived for the standard analyser.
         """

-        norm_name = self._normalized(name)
+        norm_name = self._search_normalized(name)

 
         # return cached if possible
         if norm_name in self._cache.fulls:
             return self._cache.fulls[norm_name]
 
 
         # return cached if possible
         if norm_name in self._cache.fulls:
             return self._cache.fulls[norm_name]
 

-        # otherwise compute
-        full, _ = self._cache.names.get(norm_name, (None, None))
-
-        if full is None:
-            with self.conn.cursor() as cur:
-                cur.execute("SELECT word_id FROM word WHERE word = %s and type = 'W' LIMIT 1",
-                            (norm_name, ))
-                if cur.rowcount > 0:
-                    full = cur.fetchone()[0]
+        with self.conn.cursor() as cur:
+            cur.execute("SELECT word_id FROM word WHERE word_token = %s and type = 'W'",
+                        (norm_name, ))
+            full = [row[0] for row in cur]

 
         self._cache.fulls[norm_name] = full
 
 
         self._cache.fulls[norm_name] = full