Merge pull request #3122 from miku0/sanitizer-final

[nominatim.git] / nominatim / tokenizer / base.py

diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py
index 00ecae447c5843eb0fc772960a421e271a82cb1d..f0fd9dd0e178c5a1319ce6e79a560cb438567f12 100644 (file)
--- a/nominatim/tokenizer/base.py
+++ b/nominatim/tokenizer/base.py
@@ -1,13 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 """

-Abstract class defintions for tokenizers. These base classes are here
+Abstract class definitions for tokenizers. These base classes are here

 mainly for documentation purposes.
 """
 from abc import ABC, abstractmethod
 mainly for documentation purposes.
 """
 from abc import ABC, abstractmethod

-from typing import List, Tuple, Dict, Any
+from typing import List, Tuple, Dict, Any, Optional, Iterable
+from pathlib import Path

 
 from nominatim.config import Configuration
 
 from nominatim.config import Configuration

-
-# pylint: disable=unnecessary-pass
+from nominatim.db.connection import Connection
+from nominatim.data.place_info import PlaceInfo
+from nominatim.typing import Protocol

 
 class AbstractAnalyzer(ABC):
     """ The analyzer provides the functions for analysing names and building
 
 class AbstractAnalyzer(ABC):
     """ The analyzer provides the functions for analysing names and building


@@ -21,7 +29,7 @@ class AbstractAnalyzer(ABC):
         return self
 
 
         return self
 
 

-    def __exit__(self, exc_type, exc_value, traceback) -> None:
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:

         self.close()
 
 
         self.close()
 
 


@@ -29,7 +37,6 @@ class AbstractAnalyzer(ABC):
     def close(self) -> None:
         """ Free all resources used by the analyzer.
         """
     def close(self) -> None:
         """ Free all resources used by the analyzer.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -49,7 +56,6 @@ class AbstractAnalyzer(ABC):
                 found for the given words. Each list entry is a tuple of
                 (original word, word token, word id).
         """
                 found for the given words. Each list entry is a tuple of
                 (original word, word token, word id).
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -65,7 +71,6 @@ class AbstractAnalyzer(ABC):
             Returns:
                 The given postcode after normalization.
         """
             Returns:
                 The given postcode after normalization.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -73,11 +78,11 @@ class AbstractAnalyzer(ABC):
         """ Update the tokenizer's postcode tokens from the current content
             of the `location_postcode` table.
         """
         """ Update the tokenizer's postcode tokens from the current content
             of the `location_postcode` table.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod

-    def update_special_phrases(self, phrases: List[Tuple[str, str, str, str]],
+    def update_special_phrases(self,
+                               phrases: Iterable[Tuple[str, str, str, str]],

                                should_replace: bool) -> None:
         """ Update the tokenizer's special phrase tokens from the given
             list of special phrases.
                                should_replace: bool) -> None:
         """ Update the tokenizer's special phrase tokens from the given
             list of special phrases.


@@ -89,11 +94,10 @@ class AbstractAnalyzer(ABC):
                                 When false, just add the given phrases to the
                                 ones that already exist.
         """
                                 When false, just add the given phrases to the
                                 ones that already exist.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod

-    def add_country_names(self, country_code: str, names: Dict[str, str]):
+    def add_country_names(self, country_code: str, names: Dict[str, str]) -> None:

         """ Add the given names to the tokenizer's list of country tokens.
 
             Arguments:
         """ Add the given names to the tokenizer's list of country tokens.
 
             Arguments:


@@ -101,24 +105,16 @@ class AbstractAnalyzer(ABC):
                               refer to.
                 names: Dictionary of name type to name.
         """
                               refer to.
                 names: Dictionary of name type to name.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod

-    def process_place(self, place: Dict) -> Any:
+    def process_place(self, place: PlaceInfo) -> Any:

         """ Extract tokens for the given place and compute the
             information to be handed to the PL/pgSQL processor for building
             the search index.
 
             Arguments:
         """ Extract tokens for the given place and compute the
             information to be handed to the PL/pgSQL processor for building
             the search index.
 
             Arguments:

-                place: Dictionary with the information about the place. Currently
-                       the following fields may be present:
-
-                       - *name* is a dictionary of names for the place together
-                         with the designation of the name.
-                       - *address* is a dictionary of address terms.
-                       - *country_feature* is set to a country code when the
-                         place describes a country.
+                place: Place information retrieved from the database.

 
             Returns:
                 A JSON-serialisable structure that will be handed into
 
             Returns:
                 A JSON-serialisable structure that will be handed into


@@ -142,26 +138,27 @@ class AbstractTokenizer(ABC):
             the tokenizer remains stable over updates.
 
             Arguments:
             the tokenizer remains stable over updates.
 
             Arguments:

-              config: Read-only object with configuration obtions.
+              config: Read-only object with configuration options.

 
               init_db: When set to False, then initialisation of database
                 tables should be skipped. This option is only required for
 
               init_db: When set to False, then initialisation of database
                 tables should be skipped. This option is only required for

-                migration purposes and can be savely ignored by custom
+                migration purposes and can be safely ignored by custom

                 tokenizers.
 
             TODO: can we move the init_db parameter somewhere else?
         """
                 tokenizers.
 
             TODO: can we move the init_db parameter somewhere else?
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod

-    def init_from_project(self) -> None:
+    def init_from_project(self, config: Configuration) -> None:

         """ Initialise the tokenizer from an existing database setup.
 
             The function should load all previously saved configuration from
             the project directory and/or the property table.
         """ Initialise the tokenizer from an existing database setup.
 
             The function should load all previously saved configuration from
             the project directory and/or the property table.

+
+            Arguments:
+              config: Read-only object with configuration options.

         """
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -172,9 +169,8 @@ class AbstractTokenizer(ABC):
             during query time.
 
             Arguments:
             during query time.
 
             Arguments:

-              config: Read-only object with configuration obtions.
+              config: Read-only object with configuration options.

         """
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -187,24 +183,39 @@ class AbstractTokenizer(ABC):
             data structures or data itself must not be changed by this function.
 
             Arguments:
             data structures or data itself must not be changed by this function.
 
             Arguments:

-              config: Read-only object with configuration obtions.
+              config: Read-only object with configuration options.

         """
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod

-    def check_database(self) -> str:
+    def check_database(self, config: Configuration) -> Optional[str]:

         """ Check that the database is set up correctly and ready for being
             queried.
 
         """ Check that the database is set up correctly and ready for being
             queried.
 

+            Arguments:
+              config: Read-only object with configuration options.
+

             Returns:
               If an issue was found, return an error message with the
               description of the issue as well as hints for the user on
             Returns:
               If an issue was found, return an error message with the
               description of the issue as well as hints for the user on

-              how to resolve the issue.
+              how to resolve the issue. If everything is okay, return `None`.
+        """
+
+
+    @abstractmethod
+    def update_statistics(self) -> None:
+        """ Recompute any tokenizer statistics necessary for efficient lookup.
+            This function is meant to be called from time to time by the user
+            to improve performance. However, the tokenizer must not depend on
+            it to be called in order to work.
+        """

 
 

-              Return `None`, if no issue was found.
+
+    @abstractmethod
+    def update_word_tokens(self) -> None:
+        """ Do house-keeping on the tokenizers internal data structures.
+            Remove unused word tokens, resort data etc.

         """
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -221,4 +232,20 @@ class AbstractTokenizer(ABC):
             When used outside the with construct, the caller must ensure to
             call the close() function before destructing the analyzer.
         """
             When used outside the with construct, the caller must ensure to
             call the close() function before destructing the analyzer.
         """

-        pass
+
+
+    @abstractmethod
+    def most_frequent_words(self, conn: Connection, num: int) -> List[str]:
+        """ Return a list of the `num` most frequent full words
+            in the database.
+        """
+
+
+class TokenizerModule(Protocol):
+    """ Interface that must be exported by modules that implement their
+        own tokenizer.
+    """
+
+    def create(self, dsn: str, data_dir: Path) -> AbstractTokenizer:
+        """ Factory for new tokenizers.
+        """