fix uses of config.get_path() to expect None

[nominatim.git] / nominatim / tokenizer / base.py

diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py
index c88c39f028e3f8626ec4f2d4f4188b9471447e55..70a54bfdc28141e62e0e5e63e5e52342044b5a14 100644 (file)
--- a/nominatim/tokenizer/base.py
+++ b/nominatim/tokenizer/base.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Abstract class defintions for tokenizers. These base classes are here
 mainly for documentation purposes.
 """
 Abstract class defintions for tokenizers. These base classes are here
 mainly for documentation purposes.


@@ -6,7 +12,7 @@ from abc import ABC, abstractmethod
 from typing import List, Tuple, Dict, Any
 
 from nominatim.config import Configuration
 from typing import List, Tuple, Dict, Any
 
 from nominatim.config import Configuration

-from nominatim.indexer.place_info import PlaceInfo
+from nominatim.data.place_info import PlaceInfo

 
 # pylint: disable=unnecessary-pass
 
 
 # pylint: disable=unnecessary-pass
 


@@ -203,6 +209,13 @@ class AbstractTokenizer(ABC):
         """
 
 
         """
 
 

+    @abstractmethod
+    def update_word_tokens(self) -> None:
+        """ Do house-keeping on the tokenizers internal data structures.
+            Remove unused word tokens, resort data etc.
+        """
+
+

     @abstractmethod
     def name_analyzer(self) -> AbstractAnalyzer:
         """ Create a new analyzer for tokenizing names and queries
     @abstractmethod
     def name_analyzer(self) -> AbstractAnalyzer:
         """ Create a new analyzer for tokenizing names and queries