Merge remote-tracking branch 'upstream/master'

[nominatim.git] / nominatim / tokenizer / legacy_tokenizer.py

diff --git a/nominatim/tokenizer/legacy_tokenizer.py b/nominatim/tokenizer/legacy_tokenizer.py
index 24af1c3a1a4013f035cac265bb116f644c53c8a0..7ce6b24250f4d303ed229216fa29ba92dd6dd095 100644 (file)
--- a/nominatim/tokenizer/legacy_tokenizer.py
+++ b/nominatim/tokenizer/legacy_tokenizer.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Tokenizer implementing normalisation as used before Nominatim 4.
 """
 """
 Tokenizer implementing normalisation as used before Nominatim 4.
 """


@@ -16,6 +22,7 @@ from nominatim.db import properties
 from nominatim.db import utils as db_utils
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.errors import UsageError
 from nominatim.db import utils as db_utils
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.errors import UsageError

+from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer

 
 DBCFG_NORMALIZATION = "tokenizer_normalization"
 DBCFG_MAXWORDFREQ = "tokenizer_maxwordfreq"
 
 DBCFG_NORMALIZATION = "tokenizer_normalization"
 DBCFG_MAXWORDFREQ = "tokenizer_maxwordfreq"


@@ -76,7 +83,7 @@ def _check_module(module_dir, conn):
             raise UsageError("Database module cannot be accessed.") from err
 
 
             raise UsageError("Database module cannot be accessed.") from err
 
 

-class LegacyTokenizer:
+class LegacyTokenizer(AbstractTokenizer):

     """ The legacy tokenizer uses a special PostgreSQL module to normalize
         names and queries. The tokenizer thus implements normalization through
         calls to the database.
     """ The legacy tokenizer uses a special PostgreSQL module to normalize
         names and queries. The tokenizer thus implements normalization through
         calls to the database.


@@ -112,7 +119,7 @@ class LegacyTokenizer:
             self._init_db_tables(config)
 
 
             self._init_db_tables(config)
 
 

-    def init_from_project(self):
+    def init_from_project(self, _):

         """ Initialise the tokenizer from the project directory.
         """
         with connect(self.dsn) as conn:
         """ Initialise the tokenizer from the project directory.
         """
         with connect(self.dsn) as conn:


@@ -141,7 +148,7 @@ class LegacyTokenizer:
                               modulepath=modulepath)
 
 
                               modulepath=modulepath)
 
 

-    def check_database(self):
+    def check_database(self, _):

         """ Check that the tokenizer is set up correctly.
         """
         hint = """\
         """ Check that the tokenizer is set up correctly.
         """
         hint = """\


@@ -185,6 +192,32 @@ class LegacyTokenizer:
             self._save_config(conn, config)
 
 
             self._save_config(conn, config)
 
 

+    def update_statistics(self):
+        """ Recompute the frequency of full words.
+        """
+        with connect(self.dsn) as conn:
+            if conn.table_exists('search_name'):
+                with conn.cursor() as cur:
+                    cur.drop_table("word_frequencies")
+                    LOG.info("Computing word frequencies")
+                    cur.execute("""CREATE TEMP TABLE word_frequencies AS
+                                     SELECT unnest(name_vector) as id, count(*)
+                                     FROM search_name GROUP BY id""")
+                    cur.execute("CREATE INDEX ON word_frequencies(id)")
+                    LOG.info("Update word table with recomputed frequencies")
+                    cur.execute("""UPDATE word SET search_name_count = count
+                                   FROM word_frequencies
+                                   WHERE word_token like ' %' and word_id = id""")
+                    cur.drop_table("word_frequencies")
+            conn.commit()
+
+
+    def update_word_tokens(self):
+        """ No house-keeping implemented for the legacy tokenizer.
+        """
+        LOG.info("No tokenizer clean-up available.")
+
+

     def name_analyzer(self):
         """ Create a new analyzer for tokenizing names and queries
             using this tokinzer. Analyzers are context managers and should
     def name_analyzer(self):
         """ Create a new analyzer for tokenizing names and queries
             using this tokinzer. Analyzers are context managers and should


@@ -238,7 +271,7 @@ class LegacyTokenizer:
         properties.set_property(conn, DBCFG_MAXWORDFREQ, config.MAX_WORD_FREQUENCY)
 
 
         properties.set_property(conn, DBCFG_MAXWORDFREQ, config.MAX_WORD_FREQUENCY)
 
 

-class LegacyNameAnalyzer:
+class LegacyNameAnalyzer(AbstractAnalyzer):

     """ The legacy analyzer uses the special Postgresql module for
         splitting names.
 
     """ The legacy analyzer uses the special Postgresql module for
         splitting names.
 


@@ -255,14 +288,6 @@ class LegacyNameAnalyzer:
         self._cache = _TokenCache(self.conn)
 
 
         self._cache = _TokenCache(self.conn)
 
 

-    def __enter__(self):
-        return self
-
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-
-

     def close(self):
         """ Free all resources used by the analyzer.
         """
     def close(self):
         """ Free all resources used by the analyzer.
         """


@@ -370,8 +395,7 @@ class LegacyNameAnalyzer:
             to_delete = existing_phrases - norm_phrases
 
             if to_add:
             to_delete = existing_phrases - norm_phrases
 
             if to_add:

-                psycopg2.extras.execute_values(
-                    cur,
+                cur.execute_values(

                     """ INSERT INTO word (word_id, word_token, word, class, type,
                                           search_name_count, operator)
                         (SELECT nextval('seq_word'), ' ' || make_standard_name(name), name,
                     """ INSERT INTO word (word_id, word_token, word, class, type,
                                           search_name_count, operator)
                         (SELECT nextval('seq_word'), ' ' || make_standard_name(name), name,


@@ -381,8 +405,7 @@ class LegacyNameAnalyzer:
                     to_add)
 
             if to_delete and should_replace:
                     to_add)
 
             if to_delete and should_replace:

-                psycopg2.extras.execute_values(
-                    cur,
+                cur.execute_values(

                     """ DELETE FROM word USING (VALUES %s) as v(name, in_class, in_type, op)
                         WHERE word = name and class = in_class and type = in_type
                               and ((op = '-' and operator is null) or op = operator)""",
                     """ DELETE FROM word USING (VALUES %s) as v(name, in_class, in_type, op)
                         WHERE word = name and class = in_class and type = in_type
                               and ((op = '-' and operator is null) or op = operator)""",


@@ -414,16 +437,15 @@ class LegacyNameAnalyzer:
         """
         token_info = _TokenInfo(self._cache)
 
         """
         token_info = _TokenInfo(self._cache)
 

-        names = place.get('name')
+        names = place.name

 
         if names:
             token_info.add_names(self.conn, names)
 
 
         if names:
             token_info.add_names(self.conn, names)
 

-            country_feature = place.get('country_feature')
-            if country_feature and re.fullmatch(r'[A-Za-z][A-Za-z]', country_feature):
-                self.add_country_names(country_feature.lower(), names)
+            if place.is_country():
+                self.add_country_names(place.country_code, names)

 
 

-        address = place.get('address')
+        address = place.address

         if address:
             self._process_place_address(token_info, address)
 
         if address:
             self._process_place_address(token_info, address)
 


@@ -504,7 +526,9 @@ class _TokenInfo:
             with conn.cursor() as cur:
                 return cur.scalar("SELECT word_ids_from_name(%s)::text", (name, ))
 
             with conn.cursor() as cur:
                 return cur.scalar("SELECT word_ids_from_name(%s)::text", (name, ))
 

-        self.data['street'] = self.cache.streets.get(street, _get_street)
+        tokens = self.cache.streets.get(street, _get_street)
+        if tokens:
+            self.data['street'] = tokens

 
 
     def add_place(self, conn, place):
 
 
     def add_place(self, conn, place):


@@ -533,9 +557,12 @@ class _TokenInfo:
 
         tokens = {}
         for key, value in terms:
 
         tokens = {}
         for key, value in terms:

-            tokens[key] = self.cache.address_terms.get(value, _get_address_term)
+            items = self.cache.address_terms.get(value, _get_address_term)
+            if items[0] or items[1]:
+                tokens[key] = items

 
 

-        self.data['addr'] = tokens
+        if tokens:
+            self.data['addr'] = tokens

 
 
 class _LRU:
 
 
 class _LRU:


@@ -582,7 +609,7 @@ class _TokenCache:
         with conn.cursor() as cur:
             cur.execute("""SELECT i, ARRAY[getorcreate_housenumber_id(i::text)]::text
                            FROM generate_series(1, 100) as i""")
         with conn.cursor() as cur:
             cur.execute("""SELECT i, ARRAY[getorcreate_housenumber_id(i::text)]::text
                            FROM generate_series(1, 100) as i""")

-            self._cached_housenumbers = {str(r[0]) : r[1] for r in cur}
+            self._cached_housenumbers = {str(r[0]): r[1] for r in cur}

 
         # For postcodes remember the ones that have already been added
         self.postcodes = set()
 
         # For postcodes remember the ones that have already been added
         self.postcodes = set()