add documentation for new refresh command

[nominatim.git] / nominatim / tokenizer / base.py

diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py
index c88c39f028e3f8626ec4f2d4f4188b9471447e55..f81b3bc262ed9f690b0eb6bc794e2cab076de822 100644 (file)
--- a/nominatim/tokenizer/base.py
+++ b/nominatim/tokenizer/base.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Abstract class defintions for tokenizers. These base classes are here
 mainly for documentation purposes.
 """
 Abstract class defintions for tokenizers. These base classes are here
 mainly for documentation purposes.


@@ -203,6 +209,13 @@ class AbstractTokenizer(ABC):
         """
 
 
         """
 
 

+    @abstractmethod
+    def update_word_tokens(self) -> None:
+        """ Do house-keeping on the tokenizers internal data structures.
+            Remove unused word tokens, resort data etc.
+        """
+
+

     @abstractmethod
     def name_analyzer(self) -> AbstractAnalyzer:
         """ Create a new analyzer for tokenizing names and queries
     @abstractmethod
     def name_analyzer(self) -> AbstractAnalyzer:
         """ Create a new analyzer for tokenizing names and queries