X-Git-Url: https://git.openstreetmap.org./nominatim.git/blobdiff_plain/4ae5ba7fc4a3451c806d89a583158412ea26bd01..7b0e0dfd883b268b58a721e2967303fa368e563f:/nominatim/tokenizer/base.py diff --git a/nominatim/tokenizer/base.py b/nominatim/tokenizer/base.py index 00ecae44..061cff36 100644 --- a/nominatim/tokenizer/base.py +++ b/nominatim/tokenizer/base.py @@ -1,13 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2022 by the Nominatim developer community. +# For a full list of authors see the git log. """ -Abstract class defintions for tokenizers. These base classes are here +Abstract class definitions for tokenizers. These base classes are here mainly for documentation purposes. """ from abc import ABC, abstractmethod -from typing import List, Tuple, Dict, Any +from typing import List, Tuple, Dict, Any, Optional, Iterable +from pathlib import Path from nominatim.config import Configuration - -# pylint: disable=unnecessary-pass +from nominatim.db.connection import Connection +from nominatim.data.place_info import PlaceInfo +from nominatim.typing import Protocol class AbstractAnalyzer(ABC): """ The analyzer provides the functions for analysing names and building @@ -21,7 +29,7 @@ class AbstractAnalyzer(ABC): return self - def __exit__(self, exc_type, exc_value, traceback) -> None: + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: self.close() @@ -29,7 +37,6 @@ class AbstractAnalyzer(ABC): def close(self) -> None: """ Free all resources used by the analyzer. """ - pass @abstractmethod @@ -46,10 +53,9 @@ class AbstractAnalyzer(ABC): Returns: The function returns the list of all tuples that could be - found for the given words. Each list entry is a tuple of - (original word, word token, word id). + found for the given words. Each list entry is a tuple of + (original word, word token, word id). """ - pass @abstractmethod @@ -65,7 +71,6 @@ class AbstractAnalyzer(ABC): Returns: The given postcode after normalization. """ - pass @abstractmethod @@ -73,11 +78,11 @@ class AbstractAnalyzer(ABC): """ Update the tokenizer's postcode tokens from the current content of the `location_postcode` table. """ - pass @abstractmethod - def update_special_phrases(self, phrases: List[Tuple[str, str, str, str]], + def update_special_phrases(self, + phrases: Iterable[Tuple[str, str, str, str]], should_replace: bool) -> None: """ Update the tokenizer's special phrase tokens from the given list of special phrases. @@ -89,11 +94,10 @@ class AbstractAnalyzer(ABC): When false, just add the given phrases to the ones that already exist. """ - pass @abstractmethod - def add_country_names(self, country_code: str, names: Dict[str, str]): + def add_country_names(self, country_code: str, names: Dict[str, str]) -> None: """ Add the given names to the tokenizer's list of country tokens. Arguments: @@ -101,28 +105,20 @@ class AbstractAnalyzer(ABC): refer to. names: Dictionary of name type to name. """ - pass @abstractmethod - def process_place(self, place: Dict) -> Any: + def process_place(self, place: PlaceInfo) -> Any: """ Extract tokens for the given place and compute the information to be handed to the PL/pgSQL processor for building the search index. Arguments: - place: Dictionary with the information about the place. Currently - the following fields may be present: - - - *name* is a dictionary of names for the place together - with the designation of the name. - - *address* is a dictionary of address terms. - - *country_feature* is set to a country code when the - place describes a country. + place: Place information retrieved from the database. Returns: A JSON-serialisable structure that will be handed into - the database via the `token_info` field. + the database via the `token_info` field. """ @@ -142,26 +138,25 @@ class AbstractTokenizer(ABC): the tokenizer remains stable over updates. Arguments: - config: Read-only object with configuration obtions. + config: Read-only object with configuration options. init_db: When set to False, then initialisation of database tables should be skipped. This option is only required for - migration purposes and can be savely ignored by custom + migration purposes and can be safely ignored by custom tokenizers. - - TODO: can we move the init_db parameter somewhere else? """ - pass @abstractmethod - def init_from_project(self) -> None: + def init_from_project(self, config: Configuration) -> None: """ Initialise the tokenizer from an existing database setup. The function should load all previously saved configuration from the project directory and/or the property table. + + Arguments: + config: Read-only object with configuration options. """ - pass @abstractmethod @@ -172,9 +167,8 @@ class AbstractTokenizer(ABC): during query time. Arguments: - config: Read-only object with configuration obtions. + config: Read-only object with configuration options. """ - pass @abstractmethod @@ -187,24 +181,39 @@ class AbstractTokenizer(ABC): data structures or data itself must not be changed by this function. Arguments: - config: Read-only object with configuration obtions. + config: Read-only object with configuration options. """ - pass @abstractmethod - def check_database(self) -> str: + def check_database(self, config: Configuration) -> Optional[str]: """ Check that the database is set up correctly and ready for being queried. + Arguments: + config: Read-only object with configuration options. + Returns: If an issue was found, return an error message with the - description of the issue as well as hints for the user on - how to resolve the issue. + description of the issue as well as hints for the user on + how to resolve the issue. If everything is okay, return `None`. + """ + - Return `None`, if no issue was found. + @abstractmethod + def update_statistics(self) -> None: + """ Recompute any tokenizer statistics necessary for efficient lookup. + This function is meant to be called from time to time by the user + to improve performance. However, the tokenizer must not depend on + it to be called in order to work. + """ + + + @abstractmethod + def update_word_tokens(self) -> None: + """ Do house-keeping on the tokenizers internal data structures. + Remove unused word tokens, resort data etc. """ - pass @abstractmethod @@ -221,4 +230,24 @@ class AbstractTokenizer(ABC): When used outside the with construct, the caller must ensure to call the close() function before destructing the analyzer. """ - pass + + + @abstractmethod + def most_frequent_words(self, conn: Connection, num: int) -> List[str]: + """ Return a list of the most frequent full words in the database. + + Arguments: + conn: Open connection to the database which may be used to + retrieve the words. + num: Maximum number of words to return. + """ + + +class TokenizerModule(Protocol): + """ Interface that must be exported by modules that implement their + own tokenizer. + """ + + def create(self, dsn: str, data_dir: Path) -> AbstractTokenizer: + """ Factory for new tokenizers. + """