add streaming json writer for JSON output

[nominatim.git] / nominatim / tokenizer / factory.py

diff --git a/nominatim/tokenizer/factory.py b/nominatim/tokenizer/factory.py
index fbda246238f16bebb6b75806a735975e564ca815..d6bc51638019cd31281260a68402348f03954a13 100644 (file)
--- a/nominatim/tokenizer/factory.py
+++ b/nominatim/tokenizer/factory.py
@@ -9,27 +9,30 @@ Functions for creating a tokenizer or initialising the right one for an
 existing database.
 
 A tokenizer is something that is bound to the lifetime of a database. It
-can be choosen and configured before the intial import but then needs to
+can be chosen and configured before the initial import but then needs to
 be used consistently when querying and updating the database.
 
 This module provides the functions to create and configure a new tokenizer
-as well as instanciating the appropriate tokenizer for updating an existing
+as well as instantiating the appropriate tokenizer for updating an existing
 database.
 
 A tokenizer usually also includes PHP code for querying. The appropriate PHP
 normalizer module is installed, when the tokenizer is created.
 """
+from typing import Optional
 import logging
 import importlib
 from pathlib import Path
 
-from ..errors import UsageError
-from ..db import properties
-from ..db.connection import connect
+from nominatim.errors import UsageError
+from nominatim.db import properties
+from nominatim.db.connection import connect
+from nominatim.config import Configuration
+from nominatim.tokenizer.base import AbstractTokenizer, TokenizerModule
 
 LOG = logging.getLogger()
 
-def _import_tokenizer(name):
+def _import_tokenizer(name: str) -> TokenizerModule:
     """ Load the tokenizer.py module from project directory.
     """
     src_file = Path(__file__).parent / (name + '_tokenizer.py')
@@ -41,7 +44,8 @@ def _import_tokenizer(name):
     return importlib.import_module('nominatim.tokenizer.' + name + '_tokenizer')
 
 
-def create_tokenizer(config, init_db=True, module_name=None):
+def create_tokenizer(config: Configuration, init_db: bool = True,
+                     module_name: Optional[str] = None) -> AbstractTokenizer:
     """ Create a new tokenizer as defined by the given configuration.
 
         The tokenizer data and code is copied into the 'tokenizer' directory
@@ -51,6 +55,7 @@ def create_tokenizer(config, init_db=True, module_name=None):
         module_name = config.TOKENIZER
 
     # Create the directory for the tokenizer data
+    assert config.project_dir is not None
     basedir = config.project_dir / 'tokenizer'
     if not basedir.exists():
         basedir.mkdir()
@@ -70,16 +75,17 @@ def create_tokenizer(config, init_db=True, module_name=None):
     return tokenizer
 
 
-def get_tokenizer_for_db(config):
+def get_tokenizer_for_db(config: Configuration) -> AbstractTokenizer:
     """ Instantiate a tokenizer for an existing database.
 
         The function looks up the appropriate tokenizer in the database
         and initialises it.
     """
+    assert config.project_dir is not None
     basedir = config.project_dir / 'tokenizer'
     if not basedir.is_dir():
-        LOG.fatal("Cannot find tokenizer data in '%s'.", basedir)
-        raise UsageError('Cannot initialize tokenizer.')
+        # Directory will be repopulated by tokenizer below.
+        basedir.mkdir()
 
     with connect(config.get_libpq_dsn()) as conn:
         name = properties.get_property(conn, 'tokenizer')