+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
"""
Functions for setting up and importing a new Nominatim database.
"""
from pathlib import Path
import psutil
-import psycopg2.extras
from psycopg2 import sql as pysql
from nominatim.db.connection import connect, get_pg_env
-from nominatim.db import utils as db_utils
from nominatim.db.async_connection import DBConnection
from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.tools.exec_utils import run_osm2pgsql
LOG = logging.getLogger()
-def setup_database_skeleton(dsn, data_dir, no_partitions, rouser=None):
- """ Create a new database for Nominatim and populate it with the
- essential extensions and data.
+def _require_version(module, actual, expected):
+ """ Compares the version for the given module and raises an exception
+ if the actual version is too old.
"""
- LOG.warning('Creating database')
- create_db(dsn, rouser)
+ if actual < expected:
+ LOG.fatal('Minimum supported version of %s is %d.%d. '
+ 'Found version %d.%d.',
+ module, expected[0], expected[1], actual[0], actual[1])
+ raise UsageError(f'{module} is too old.')
- LOG.warning('Setting up database')
- with connect(dsn) as conn:
- setup_extensions(conn)
- LOG.warning('Loading basic data')
- import_base_data(dsn, data_dir, no_partitions)
+def setup_database_skeleton(dsn, rouser=None):
+ """ Create a new database for Nominatim and populate it with the
+ essential extensions.
+ The function fails when the database already exists or Postgresql or
+ PostGIS versions are too old.
-def create_db(dsn, rouser=None):
- """ Create a new database for the given DSN. Fails when the database
- already exists or the PostgreSQL version is too old.
Uses `createdb` to create the database.
If 'rouser' is given, then the function also checks that the user
raise UsageError('Creating new database failed.')
with connect(dsn) as conn:
- postgres_version = conn.server_version_tuple()
- if postgres_version < POSTGRESQL_REQUIRED_VERSION:
- LOG.fatal('Minimum supported version of Postgresql is %d.%d. '
- 'Found version %d.%d.',
- POSTGRESQL_REQUIRED_VERSION[0], POSTGRESQL_REQUIRED_VERSION[1],
- postgres_version[0], postgres_version[1])
- raise UsageError('PostgreSQL server is too old.')
+ _require_version('PostgreSQL server',
+ conn.server_version_tuple(),
+ POSTGRESQL_REQUIRED_VERSION)
if rouser is not None:
with conn.cursor() as cur:
"\n createuser %s", rouser, rouser)
raise UsageError('Missing read-only user.')
+ # Create extensions.
+ with conn.cursor() as cur:
+ cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
+ cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
+ conn.commit()
-
-def setup_extensions(conn):
- """ Set up all extensions needed for Nominatim. Also checks that the
- versions of the extensions are sufficient.
- """
- with conn.cursor() as cur:
- cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
- cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
- conn.commit()
-
- postgis_version = conn.postgis_version_tuple()
- if postgis_version < POSTGIS_REQUIRED_VERSION:
- LOG.fatal('Minimum supported version of PostGIS is %d.%d. '
- 'Found version %d.%d.',
- POSTGIS_REQUIRED_VERSION[0], POSTGIS_REQUIRED_VERSION[1],
- postgis_version[0], postgis_version[1])
- raise UsageError('PostGIS version is too old.')
-
-
-def import_base_data(dsn, sql_dir, ignore_partitions=False):
- """ Create and populate the tables with basic static data that provides
- the background for geocoding. Data is assumed to not yet exist.
- """
- db_utils.execute_file(dsn, sql_dir / 'country_name.sql')
- db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')
-
- if ignore_partitions:
- with connect(dsn) as conn:
- with conn.cursor() as cur:
- cur.execute('UPDATE country_name SET partition = 0')
- conn.commit()
+ _require_version('PostGIS',
+ conn.postgis_version_tuple(),
+ POSTGIS_REQUIRED_VERSION)
def import_osm_data(osm_files, options, drop=False, ignore_errors=False):
sql = SQLPreprocessor(conn, config)
sql.run_sql_file(conn, 'indices.sql', drop=drop)
-
-
-def create_country_names(conn, tokenizer, languages=None):
- """ Add default country names to search index. `languages` is a comma-
- separated list of language codes as used in OSM. If `languages` is not
- empty then only name translations for the given languages are added
- to the index.
- """
- if languages:
- languages = languages.split(',')
-
- def _include_key(key):
- return key == 'name' or \
- (key.startswith('name:') and (not languages or key[5:] in languages))
-
- with conn.cursor() as cur:
- psycopg2.extras.register_hstore(cur)
- cur.execute("""SELECT country_code, name FROM country_name
- WHERE country_code is not null""")
-
- with tokenizer.name_analyzer() as analyzer:
- for code, name in cur:
- names = {'countrycode': code}
- if code == 'gb':
- names['short_name'] = 'UK'
- if code == 'us':
- names['short_name'] = 'United States'
-
- # country names (only in languages as provided)
- if name:
- names.update(((k, v) for k, v in name.items() if _include_key(k)))
-
- analyzer.add_country_names(code, names)
-
- conn.commit()
projects / nominatim.git / blobdiff