2 Tokenizer implementing normalisation as used before Nominatim 4.
9 from nominatim.db.connection import connect
10 from nominatim.db import properties
11 from nominatim.errors import UsageError
13 DBCFG_NORMALIZATION = "tokenizer_normalization"
15 LOG = logging.getLogger()
17 def create(dsn, data_dir):
18 """ Create a new instance of the tokenizer provided by this module.
20 return LegacyTokenizer(dsn, data_dir)
23 def _install_module(src_dir, module_dir):
24 """ Copies the PostgreSQL normalisation module into the project
25 directory if necessary. For historical reasons the module is
26 saved in the '/module' subdirectory and not with the other tokenizer
29 The function detects when the installation is run from the
30 build directory. It doesn't touch the module in that case.
32 if module_dir.exists() and src_dir.samefile(module_dir):
33 LOG.info('Running from build directory. Leaving database module as is.')
36 if not module_dir.exists():
39 destfile = module_dir / 'nominatim.so'
40 shutil.copy(str(src_dir / 'nominatim.so'), str(destfile))
43 LOG.info('Database module installed at %s', str(destfile))
46 def _check_module(module_dir, conn):
47 with conn.cursor() as cur:
49 cur.execute("""CREATE FUNCTION nominatim_test_import_func(text)
50 RETURNS text AS '{}/nominatim.so', 'transliteration'
51 LANGUAGE c IMMUTABLE STRICT;
52 DROP FUNCTION nominatim_test_import_func(text)
53 """.format(module_dir))
54 except psycopg2.DatabaseError as err:
55 LOG.fatal("Error accessing database module: %s", err)
56 raise UsageError("Database module cannot be accessed.") from err
59 class LegacyTokenizer:
60 """ The legacy tokenizer uses a special PostgreSQL module to normalize
61 names and queries. The tokenizer thus implements normalization through
62 calls to the database.
65 def __init__(self, dsn, data_dir):
67 self.data_dir = data_dir
68 self.normalization = None
71 def init_new_db(self, config):
72 """ Set up a new tokenizer for the database.
74 This copies all necessary data in the project directory to make
75 sure the tokenizer remains stable even over updates.
77 # Find and optionally install the PsotgreSQL normalization module.
78 if config.DATABASE_MODULE_PATH:
79 LOG.info("Using custom path for database module at '%s'",
80 config.DATABASE_MODULE_PATH)
81 module_dir = config.DATABASE_MODULE_PATH
83 _install_module(config.lib_dir.module, config.project_dir / 'module')
84 module_dir = config.project_dir / 'module'
86 self.normalization = config.TERM_NORMALIZATION
88 with connect(self.dsn) as conn:
89 _check_module(module_dir, conn)
91 # Stable configuration is saved in the database.
92 properties.set_property(conn, DBCFG_NORMALIZATION, self.normalization)
97 def init_from_project(self):
98 """ Initialise the tokenizer from the project directory.
100 with connect(self.dsn) as conn:
101 self.normalization = properties.get_property(conn, DBCFG_NORMALIZATION)