X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/bb175cc95898de420781867973b38d033c187e81..aaf2b6032eb3297aeb20b5c98223e9da734f56d4:/nominatim/tokenizer/base.py

diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py
index c88c39f0..70a54bfd 100644
--- a/nominatim/tokenizer/base.py
+++ b/nominatim/tokenizer/base.py
@@ -1,3 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
 """
 Abstract class defintions for tokenizers. These base classes are here
 mainly for documentation purposes.
@@ -6,7 +12,7 @@ from abc import ABC, abstractmethod
 from typing import List, Tuple, Dict, Any
 
 from nominatim.config import Configuration
-from nominatim.indexer.place_info import PlaceInfo
+from nominatim.data.place_info import PlaceInfo
 
 # pylint: disable=unnecessary-pass
 
@@ -203,6 +209,13 @@ class AbstractTokenizer(ABC):
         """
 
 
+    @abstractmethod
+    def update_word_tokens(self) -> None:
+        """ Do house-keeping on the tokenizers internal data structures.
+            Remove unused word tokens, resort data etc.
+        """
+
+
     @abstractmethod
     def name_analyzer(self) -> AbstractAnalyzer:
         """ Create a new analyzer for tokenizing names and queries