remove tests that differ between lua and gazetteer versions

[nominatim.git] / nominatim / tokenizer / factory.py

diff --git a/nominatim/tokenizer/factory.py b/nominatim/tokenizer/factory.py
index 1079c935985fd4ed92a24e6f30db63461b832906..f5159fa00fb64b3c64a36a95fd855a9aab61b847 100644 (file)
--- a/nominatim/tokenizer/factory.py
+++ b/nominatim/tokenizer/factory.py
@@ -1,44 +1,59 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.

 """
 Functions for creating a tokenizer or initialising the right one for an
 existing database.
 
 A tokenizer is something that is bound to the lifetime of a database. It
 """
 Functions for creating a tokenizer or initialising the right one for an
 existing database.
 
 A tokenizer is something that is bound to the lifetime of a database. It

-can be choosen and configured before the intial import but then needs to
+can be chosen and configured before the initial import but then needs to

 be used consistently when querying and updating the database.
 
 This module provides the functions to create and configure a new tokenizer
 be used consistently when querying and updating the database.
 
 This module provides the functions to create and configure a new tokenizer

-as well as instanciating the appropriate tokenizer for updating an existing
+as well as instantiating the appropriate tokenizer for updating an existing

 database.
 
 A tokenizer usually also includes PHP code for querying. The appropriate PHP
 normalizer module is installed, when the tokenizer is created.
 """
 database.
 
 A tokenizer usually also includes PHP code for querying. The appropriate PHP
 normalizer module is installed, when the tokenizer is created.
 """

+from typing import Optional

 import logging
 import importlib
 import logging
 import importlib

+from pathlib import Path

 
 

-from ..errors import UsageError
-from ..db import properties
-from ..db.connection import connect
+from nominatim.errors import UsageError
+from nominatim.db import properties
+from nominatim.db.connection import connect
+from nominatim.config import Configuration
+from nominatim.tokenizer.base import AbstractTokenizer, TokenizerModule

 
 LOG = logging.getLogger()
 
 
 LOG = logging.getLogger()
 

-def _import_tokenizer(name):
+def _import_tokenizer(name: str) -> TokenizerModule:

     """ Load the tokenizer.py module from project directory.
     """
     """ Load the tokenizer.py module from project directory.
     """

-    try:
-        return importlib.import_module('nominatim.tokenizer.' + name + '_tokenizer')
-    except ModuleNotFoundError as exp:
+    src_file = Path(__file__).parent / (name + '_tokenizer.py')
+    if not src_file.is_file():

         LOG.fatal("No tokenizer named '%s' available. "
                   "Check the setting of NOMINATIM_TOKENIZER.", name)
         LOG.fatal("No tokenizer named '%s' available. "
                   "Check the setting of NOMINATIM_TOKENIZER.", name)

-        raise UsageError('Tokenizer not found') from exp
+        raise UsageError('Tokenizer not found')

 
 

+    return importlib.import_module('nominatim.tokenizer.' + name + '_tokenizer')

 
 

-def create_tokenizer(config):
+
+def create_tokenizer(config: Configuration, init_db: bool = True,
+                     module_name: Optional[str] = None) -> AbstractTokenizer:

     """ Create a new tokenizer as defined by the given configuration.
 
         The tokenizer data and code is copied into the 'tokenizer' directory
         of the project directory and the tokenizer loaded from its new location.
     """
     """ Create a new tokenizer as defined by the given configuration.
 
         The tokenizer data and code is copied into the 'tokenizer' directory
         of the project directory and the tokenizer loaded from its new location.
     """

+    if module_name is None:
+        module_name = config.TOKENIZER
+

     # Create the directory for the tokenizer data
     basedir = config.project_dir / 'tokenizer'
     if not basedir.exists():
     # Create the directory for the tokenizer data
     basedir = config.project_dir / 'tokenizer'
     if not basedir.exists():


@@ -47,18 +62,19 @@ def create_tokenizer(config):
         LOG.fatal("Tokenizer directory '%s' cannot be created.", basedir)
         raise UsageError("Tokenizer setup failed.")
 
         LOG.fatal("Tokenizer directory '%s' cannot be created.", basedir)
         raise UsageError("Tokenizer setup failed.")
 

-    tokenizer_module = _import_tokenizer(config.TOKENIZER)
+    # Import and initialize the tokenizer.
+    tokenizer_module = _import_tokenizer(module_name)

 
     tokenizer = tokenizer_module.create(config.get_libpq_dsn(), basedir)
 
     tokenizer = tokenizer_module.create(config.get_libpq_dsn(), basedir)

-    tokenizer.init_new_db(config)
+    tokenizer.init_new_db(config, init_db=init_db)

 
     with connect(config.get_libpq_dsn()) as conn:
 
     with connect(config.get_libpq_dsn()) as conn:

-        properties.set_property(conn, 'tokenizer', config.TOKENIZER)
+        properties.set_property(conn, 'tokenizer', module_name)

 
     return tokenizer
 
 
 
     return tokenizer
 
 

-def get_tokenizer_for_db(config):
+def get_tokenizer_for_db(config: Configuration) -> AbstractTokenizer:

     """ Instantiate a tokenizer for an existing database.
 
         The function looks up the appropriate tokenizer in the database
     """ Instantiate a tokenizer for an existing database.
 
         The function looks up the appropriate tokenizer in the database


@@ -66,8 +82,8 @@ def get_tokenizer_for_db(config):
     """
     basedir = config.project_dir / 'tokenizer'
     if not basedir.is_dir():
     """
     basedir = config.project_dir / 'tokenizer'
     if not basedir.is_dir():

-        LOG.fatal("Cannot find tokenizer data in '%s'.", basedir)
-        raise UsageError('Cannot initialize tokenizer.')
+        # Directory will be repopulated by tokenizer below.
+        basedir.mkdir()

 
     with connect(config.get_libpq_dsn()) as conn:
         name = properties.get_property(conn, 'tokenizer')
 
     with connect(config.get_libpq_dsn()) as conn:
         name = properties.get_property(conn, 'tokenizer')


@@ -79,6 +95,6 @@ def get_tokenizer_for_db(config):
     tokenizer_module = _import_tokenizer(name)
 
     tokenizer = tokenizer_module.create(config.get_libpq_dsn(), basedir)
     tokenizer_module = _import_tokenizer(name)
 
     tokenizer = tokenizer_module.create(config.get_libpq_dsn(), basedir)

-    tokenizer.init_from_project()
+    tokenizer.init_from_project(config)

 
     return tokenizer
 
     return tokenizer