add type hints for sanitizers

[nominatim.git] / nominatim / tokenizer / base.py

diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py
index 53289c7816e9a32ca671c2205d35f19b217d5d68..5a3d3b1276aa7c6bf3bb36787967d519fe70aa79 100644 (file)
--- a/nominatim/tokenizer/base.py
+++ b/nominatim/tokenizer/base.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Abstract class defintions for tokenizers. These base classes are here
 mainly for documentation purposes.
 """
 Abstract class defintions for tokenizers. These base classes are here
 mainly for documentation purposes.


@@ -6,7 +12,7 @@ from abc import ABC, abstractmethod
 from typing import List, Tuple, Dict, Any
 
 from nominatim.config import Configuration
 from typing import List, Tuple, Dict, Any
 
 from nominatim.config import Configuration

-from nominatim.indexer.place_info import PlaceInfo
+from nominatim.data.place_info import PlaceInfo

 
 # pylint: disable=unnecessary-pass
 
 
 # pylint: disable=unnecessary-pass
 


@@ -22,7 +28,7 @@ class AbstractAnalyzer(ABC):
         return self
 
 
         return self
 
 

-    def __exit__(self, exc_type, exc_value, traceback) -> None:
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:

         self.close()
 
 
         self.close()
 
 


@@ -30,7 +36,6 @@ class AbstractAnalyzer(ABC):
     def close(self) -> None:
         """ Free all resources used by the analyzer.
         """
     def close(self) -> None:
         """ Free all resources used by the analyzer.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -50,7 +55,6 @@ class AbstractAnalyzer(ABC):
                 found for the given words. Each list entry is a tuple of
                 (original word, word token, word id).
         """
                 found for the given words. Each list entry is a tuple of
                 (original word, word token, word id).
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -66,7 +70,6 @@ class AbstractAnalyzer(ABC):
             Returns:
                 The given postcode after normalization.
         """
             Returns:
                 The given postcode after normalization.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -74,7 +77,6 @@ class AbstractAnalyzer(ABC):
         """ Update the tokenizer's postcode tokens from the current content
             of the `location_postcode` table.
         """
         """ Update the tokenizer's postcode tokens from the current content
             of the `location_postcode` table.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -90,11 +92,10 @@ class AbstractAnalyzer(ABC):
                                 When false, just add the given phrases to the
                                 ones that already exist.
         """
                                 When false, just add the given phrases to the
                                 ones that already exist.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod

-    def add_country_names(self, country_code: str, names: Dict[str, str]):
+    def add_country_names(self, country_code: str, names: Dict[str, str]) -> None:

         """ Add the given names to the tokenizer's list of country tokens.
 
             Arguments:
         """ Add the given names to the tokenizer's list of country tokens.
 
             Arguments:


@@ -102,7 +103,6 @@ class AbstractAnalyzer(ABC):
                               refer to.
                 names: Dictionary of name type to name.
         """
                               refer to.
                 names: Dictionary of name type to name.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -145,7 +145,6 @@ class AbstractTokenizer(ABC):
 
             TODO: can we move the init_db parameter somewhere else?
         """
 
             TODO: can we move the init_db parameter somewhere else?
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -158,7 +157,6 @@ class AbstractTokenizer(ABC):
             Arguments:
               config: Read-only object with configuration options.
         """
             Arguments:
               config: Read-only object with configuration options.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -171,7 +169,6 @@ class AbstractTokenizer(ABC):
             Arguments:
               config: Read-only object with configuration options.
         """
             Arguments:
               config: Read-only object with configuration options.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -186,7 +183,6 @@ class AbstractTokenizer(ABC):
             Arguments:
               config: Read-only object with configuration options.
         """
             Arguments:
               config: Read-only object with configuration options.
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -194,17 +190,30 @@ class AbstractTokenizer(ABC):
         """ Check that the database is set up correctly and ready for being
             queried.
 
         """ Check that the database is set up correctly and ready for being
             queried.
 

+            Arguments:
+              config: Read-only object with configuration options.
+

             Returns:
               If an issue was found, return an error message with the
               description of the issue as well as hints for the user on
             Returns:
               If an issue was found, return an error message with the
               description of the issue as well as hints for the user on

-              how to resolve the issue.
+              how to resolve the issue. If everything is okay, return `None`.
+        """

 
 

-            Arguments:
-              config: Read-only object with configuration options.

 
 

-              Return `None`, if no issue was found.
+    @abstractmethod
+    def update_statistics(self) -> None:
+        """ Recompute any tokenizer statistics necessary for efficient lookup.
+            This function is meant to be called from time to time by the user
+            to improve performance. However, the tokenizer must not depend on
+            it to be called in order to work.
+        """
+
+
+    @abstractmethod
+    def update_word_tokens(self) -> None:
+        """ Do house-keeping on the tokenizers internal data structures.
+            Remove unused word tokens, resort data etc.

         """
         """

-        pass

 
 
     @abstractmethod
 
 
     @abstractmethod


@@ -221,4 +230,3 @@ class AbstractTokenizer(ABC):
             When used outside the with construct, the caller must ensure to
             call the close() function before destructing the analyzer.
         """
             When used outside the with construct, the caller must ensure to
             call the close() function before destructing the analyzer.
         """

-        pass