return join(' ', $aEscaped);
}
- public function run()
+ public function run($bExitOnFail = False)
{
$sCmd = $this->escapedCmd();
// $aEnv does not need escaping, proc_open seems to handle it fine
fclose($aPipes[0]); // no stdin
$iStat = proc_close($hProc);
+
+ if ($iStat != 0 && $bExitOnFail) {
+ exit($iStat);
+ }
+
return $iStat;
}
$bDidSomething = false;
+$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
+if (isset($aCMDResult['quiet']) && $aCMDResult['quiet']) {
+ $oNominatimCmd->addParams('--quiet');
+}
+if ($aCMDResult['verbose']) {
+ $oNominatimCmd->addParams('--verbose');
+}
+
+
//*******************************************************
// Making some sanity check:
// Check if osm-file is set and points to a valid file
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
- $oSetup->createDB();
+ (clone($oNominatimCmd))->addParams('transition', '--create-db')->run(true);
}
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$bDidSomething = true;
- $oSetup->setupDB();
+ (clone($oNominatimCmd))->addParams('transition', '--setup-db')->run(true);
}
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
}
}
- public function createDB()
- {
- info('Create DB');
- $oDB = new \Nominatim\DB;
-
- if ($oDB->checkConnection()) {
- fail('database already exists ('.getSetting('DATABASE_DSN').')');
- }
-
- $oCmd = (new \Nominatim\Shell('createdb'))
- ->addParams('-E', 'UTF-8')
- ->addParams('-p', $this->aDSNInfo['port']);
-
- if (isset($this->aDSNInfo['username'])) {
- $oCmd->addParams('-U', $this->aDSNInfo['username']);
- }
- if (isset($this->aDSNInfo['password'])) {
- $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
- }
- if (isset($this->aDSNInfo['hostspec'])) {
- $oCmd->addParams('-h', $this->aDSNInfo['hostspec']);
- }
- $oCmd->addParams($this->aDSNInfo['database']);
-
- $result = $oCmd->run();
- if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
- }
-
- public function setupDB()
- {
- info('Setup DB');
-
- $fPostgresVersion = $this->db()->getPostgresVersion();
- echo 'Postgres version found: '.$fPostgresVersion."\n";
-
- if ($fPostgresVersion < 9.03) {
- fail('Minimum supported version of Postgresql is 9.3.');
- }
-
- $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
- $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
-
- $fPostgisVersion = $this->db()->getPostgisVersion();
- echo 'Postgis version found: '.$fPostgisVersion."\n";
-
- if ($fPostgisVersion < 2.2) {
- echo "Minimum required Postgis version 2.2\n";
- exit(1);
- }
-
- $sPgUser = getSetting('DATABASE_WEBUSER');
- $i = $this->db()->getOne("select count(*) from pg_user where usename = '$sPgUser'");
- if ($i == 0) {
- echo "\nERROR: Web user '".$sPgUser."' does not exist. Create it with:\n";
- echo "\n createuser ".$sPgUser."\n\n";
- exit(1);
- }
-
- if (!getSetting('DATABASE_MODULE_PATH')) {
- // If no custom module path is set then copy the module into the
- // project directory, but only if it is not the same file already
- // (aka we are running from the build dir).
- $sDest = CONST_InstallDir.'/module';
- if ($sDest != CONST_Default_ModulePath) {
- if (!file_exists($sDest)) {
- mkdir($sDest);
- }
- if (!copy(CONST_Default_ModulePath.'/nominatim.so', $sDest.'/nominatim.so')) {
- echo "Failed to copy database module to $sDest.";
- exit(1);
- }
- chmod($sDest.'/nominatim.so', 0755);
- info("Database module installed at $sDest.");
- } else {
- info('Running from build directory. Leaving database module as is.');
- }
- } else {
- info('Using database module from DATABASE_MODULE_PATH ('.getSetting('DATABASE_MODULE_PATH').').');
- }
- // Try accessing the C module, so we know early if something is wrong
- $this->checkModulePresence(); // raises exception on failure
-
- $this->pgsqlRunScriptFile(CONST_DataDir.'/country_name.sql');
- $this->pgsqlRunScriptFile(CONST_DataDir.'/country_osm_grid.sql.gz');
-
- if ($this->bNoPartitions) {
- $this->pgsqlRunScript('update country_name set partition = 0');
- }
- }
-
public function importData($sOSMFile)
{
info('Import data');
else:
parser.parser.epilog = 'php-cgi not found. Query commands not available.'
+ parser.add_subcommand('transition', clicmd.AdminTransition)
+
return parser.run(**kwargs)
from .refresh import UpdateRefresh
from .admin import AdminFuncs
from .freeze import SetupFreeze
+from .transition import AdminTransition
if args.postcodes:
LOG.warning("Update postcodes centroid")
- with connect(args.config.get_libpq_dsn()) as conn:
- refresh.update_postcodes(conn, args.sqllib_dir)
+ refresh.update_postcodes(args.config.get_libpq_dsn(), args.sqllib_dir)
if args.word_counts:
LOG.warning('Recompute frequency of full-word search terms')
- with connect(args.config.get_libpq_dsn()) as conn:
- refresh.recompute_word_counts(conn, args.sqllib_dir)
+ refresh.recompute_word_counts(args.config.get_libpq_dsn(), args.sqllib_dir)
if args.address_levels:
cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
--- /dev/null
+"""
+Implementation of the 'transition' subcommand.
+
+This subcommand provides standins for functions that were available
+through the PHP scripts but are now no longer directly accessible.
+This module will be removed as soon as the transition phase is over.
+"""
+import logging
+
+from ..db.connection import connect
+
+# Do not repeat documentation of subcommand classes.
+# pylint: disable=C0111
+# Using non-top-level imports to avoid eventually unused imports.
+# pylint: disable=E0012,C0415
+
+LOG = logging.getLogger()
+
+class AdminTransition:
+ """\
+ Internal functions for code transition. Do not use.
+ """
+
+ @staticmethod
+ def add_args(parser):
+ group = parser.add_argument_group('Sub-functions')
+ group.add_argument('--create-db', action='store_true',
+ help='Create nominatim db')
+ group.add_argument('--setup-db', action='store_true',
+ help='Build a blank nominatim db')
+ group = parser.add_argument_group('Options')
+ group.add_argument('--no-partitions', action='store_true',
+ help='Do not partition search indices')
+
+ @staticmethod
+ def run(args):
+ from ..tools import database_import
+
+ if args.create_db:
+ LOG.warning('Create DB')
+ database_import.create_db(args.config.get_libpq_dsn())
+
+ if args.setup_db:
+ LOG.warning('Setup DB')
+ mpath = database_import.install_module(args.module_dir, args.project_dir,
+ args.config.DATABASE_MODULE_PATH)
+
+ with connect(args.config.get_libpq_dsn()) as conn:
+ database_import.setup_extensions(conn)
+ database_import.check_module_dir_path(conn, mpath)
+
+ database_import.import_base_data(args.config.get_libpq_dsn(),
+ args.data_dir, args.no_partitions)
"""
version = self.server_version
if version < 100000:
- return (version / 10000, (version % 10000) / 100)
+ return (int(version / 10000), (version % 10000) / 100)
+
+ return (int(version / 10000), version % 10000)
+
+
+ def postgis_version_tuple(self):
+ """ Return the postgis version installed in the database as a
+ tuple of (major, minor). Assumes that the PostGIS extension
+ has been installed already.
+ """
+ with self.cursor() as cur:
+ version = cur.scalar('SELECT postgis_lib_version()')
+
+ return tuple((int(x) for x in version.split('.')[:2]))
- return (version / 10000, version % 10000)
def connect(dsn):
""" Open a connection to the database using the specialised connection
'sslcrl': 'PGSSLCRL',
'requirepeer': 'PGREQUIREPEER',
'ssl_min_protocol_version': 'PGSSLMINPROTOCOLVERSION',
- 'ssl_min_protocol_version': 'PGSSLMAXPROTOCOLVERSION',
+ 'ssl_max_protocol_version': 'PGSSLMAXPROTOCOLVERSION',
'gssencmode': 'PGGSSENCMODE',
'krbsrvname': 'PGKRBSRVNAME',
'gsslib': 'PGGSSLIB',
If `base_env` is None, then the OS environment is used as a base
environment.
"""
- env = base_env if base_env is not None else os.environ
+ env = dict(base_env if base_env is not None else os.environ)
for param, value in psycopg2.extensions.parse_dsn(dsn).items():
if param in _PG_CONNECTION_STRINGS:
"""
import subprocess
import logging
+import gzip
from .connection import get_pg_env
from ..errors import UsageError
LOG = logging.getLogger()
+def _pipe_to_proc(proc, fdesc):
+ chunk = fdesc.read(2048)
+ while chunk and proc.poll() is None:
+ try:
+ proc.stdin.write(chunk)
+ except BrokenPipeError as exc:
+ raise UsageError("Failed to execute SQL file.") from exc
+ chunk = fdesc.read(2048)
+
+ return len(chunk)
+
def execute_file(dsn, fname, ignore_errors=False):
""" Read an SQL file and run its contents against the given database
using psql.
if not LOG.isEnabledFor(logging.INFO):
proc.stdin.write('set client_min_messages to WARNING;'.encode('utf-8'))
- with fname.open('rb') as fdesc:
- chunk = fdesc.read(2048)
- while chunk and proc.poll() is None:
- proc.stdin.write(chunk)
- chunk = fdesc.read(2048)
+ if fname.suffix == '.gz':
+ with gzip.open(str(fname), 'rb') as fdesc:
+ remain = _pipe_to_proc(proc, fdesc)
+ else:
+ with fname.open('rb') as fdesc:
+ remain = _pipe_to_proc(proc, fdesc)
proc.stdin.close()
ret = proc.wait()
- print(ret, chunk)
- if ret != 0 or chunk:
+ if ret != 0 or remain > 0:
raise UsageError("Failed to execute SQL file.")
--- /dev/null
+"""
+Functions for setting up and importing a new Nominatim database.
+"""
+import logging
+import subprocess
+import shutil
+
+from ..db.connection import connect, get_pg_env
+from ..db import utils as db_utils
+from ..errors import UsageError
+from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
+
+LOG = logging.getLogger()
+
+def create_db(dsn, rouser=None):
+ """ Create a new database for the given DSN. Fails when the database
+ already exists or the PostgreSQL version is too old.
+ Uses `createdb` to create the database.
+
+ If 'rouser' is given, then the function also checks that the user
+ with that given name exists.
+
+ Requires superuser rights by the caller.
+ """
+ proc = subprocess.run(['createdb'], env=get_pg_env(dsn), check=False)
+
+ if proc.returncode != 0:
+ raise UsageError('Creating new database failed.')
+
+ with connect(dsn) as conn:
+ postgres_version = conn.server_version_tuple() # pylint: disable=E1101
+ if postgres_version < POSTGRESQL_REQUIRED_VERSION:
+ LOG.fatal('Minimum supported version of Postgresql is %d.%d. '
+ 'Found version %d.%d.',
+ POSTGRESQL_REQUIRED_VERSION[0], POSTGRESQL_REQUIRED_VERSION[1],
+ postgres_version[0], postgres_version[1])
+ raise UsageError('PostgreSQL server is too old.')
+
+ if rouser is not None:
+ with conn.cursor() as cur: # pylint: disable=E1101
+ cnt = cur.scalar('SELECT count(*) FROM pg_user where usename = %s',
+ (rouser, ))
+ if cnt == 0:
+ LOG.fatal("Web user '%s' does not exists. Create it with:\n"
+ "\n createuser %s", rouser, rouser)
+ raise UsageError('Missing read-only user.')
+
+
+
+def setup_extensions(conn):
+ """ Set up all extensions needed for Nominatim. Also checks that the
+ versions of the extensions are sufficient.
+ """
+ with conn.cursor() as cur:
+ cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
+ cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
+ conn.commit()
+
+ postgis_version = conn.postgis_version_tuple()
+ if postgis_version < POSTGIS_REQUIRED_VERSION:
+ LOG.fatal('Minimum supported version of PostGIS is %d.%d. '
+ 'Found version %d.%d.',
+ POSTGIS_REQUIRED_VERSION[0], POSTGIS_REQUIRED_VERSION[1],
+ postgis_version[0], postgis_version[1])
+ raise UsageError('PostGIS version is too old.')
+
+
+def install_module(src_dir, project_dir, module_dir):
+ """ Copy the normalization module from src_dir into the project
+ directory under the '/module' directory. If 'module_dir' is set, then
+ use the module from there instead and check that it is accessible
+ for Postgresql.
+
+ The function detects when the installation is run from the
+ build directory. It doesn't touch the module in that case.
+ """
+ if not module_dir:
+ module_dir = project_dir / 'module'
+
+ if not module_dir.exists() or not src_dir.samefile(module_dir):
+
+ if not module_dir.exists():
+ module_dir.mkdir()
+
+ destfile = module_dir / 'nominatim.so'
+ shutil.copy(str(src_dir / 'nominatim.so'), str(destfile))
+ destfile.chmod(0o755)
+
+ LOG.info('Database module installed at %s', str(destfile))
+ else:
+ LOG.info('Running from build directory. Leaving database module as is.')
+ else:
+ LOG.info("Using custom path for database module at '%s'", module_dir)
+
+ return module_dir
+
+
+def check_module_dir_path(conn, path):
+ """ Check that the normalisation module can be found and executed
+ from the given path.
+ """
+ with conn.cursor() as cur:
+ cur.execute("""CREATE FUNCTION nominatim_test_import_func(text)
+ RETURNS text AS '{}/nominatim.so', 'transliteration'
+ LANGUAGE c IMMUTABLE STRICT;
+ DROP FUNCTION nominatim_test_import_func(text)
+ """.format(path))
+
+
+def import_base_data(dsn, sql_dir, ignore_partitions=False):
+ """ Create and populate the tables with basic static data that provides
+ the background for geocoding.
+ """
+ db_utils.execute_file(dsn, sql_dir / 'country_name.sql')
+ db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')
+
+ if ignore_partitions:
+ with connect(dsn) as conn:
+ with conn.cursor() as cur: # pylint: disable=E1101
+ cur.execute('UPDATE country_name SET partition = 0')
+ conn.commit() # pylint: disable=E1101
Helper functions for executing external programs.
"""
import logging
-import os
import subprocess
import urllib.request as urlrequest
from urllib.parse import urlencode
-from psycopg2.extensions import parse_dsn
-
from ..version import NOMINATIM_VERSION
from ..db.connection import get_pg_env
"""
NOMINATIM_VERSION = "3.6.0"
+
+POSTGRESQL_REQUIRED_VERSION = (9, 3)
+POSTGIS_REQUIRED_VERSION = (2, 2)
php:
cd php && phpunit ./
+python:
+ pytest python
+
.PHONY: bdd php no-test-db
def def_config():
return Configuration(None, SRC_DIR.resolve() / 'settings')
+@pytest.fixture
+def src_dir():
+ return SRC_DIR.resolve()
@pytest.fixture
def status_table(temp_db_conn):
the actual functions.
"""
import datetime as dt
+import time
+from pathlib import Path
+
import psycopg2
import pytest
-import time
import nominatim.cli
import nominatim.clicmd.api
from nominatim.errors import UsageError
from nominatim.db import status
+SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
+
def call_nominatim(*args):
return nominatim.cli.nominatim(module_dir='build/module',
osm2pgsql_path='build/osm2pgsql/osm2pgsql',
- phplib_dir='lib-php',
- data_dir='.',
+ phplib_dir=str(SRC_DIR / 'lib-php'),
+ data_dir=str(SRC_DIR / 'data'),
phpcgi_path='/usr/bin/php-cgi',
- sqllib_dir='lib-sql',
- config_dir='settings',
+ sqllib_dir=str(SRC_DIR / 'lib-sql'),
+ config_dir=str(SRC_DIR / 'settings'),
cli_args=args)
class MockParamCapture:
assert len(ver) == 2
assert ver[0] > 8
+
+def test_connection_postgis_version_tuple(db, temp_db_cursor):
+ temp_db_cursor.execute('CREATE EXTENSION postgis')
+
+ ver = db.postgis_version_tuple()
+
+ assert isinstance(ver, tuple)
+ assert len(ver) == 2
+ assert ver[0] >= 2
+
+
def test_cursor_scalar(db, temp_db_cursor):
temp_db_cursor.execute('CREATE TABLE dummy (id INT)')
with pytest.raises(FileNotFoundError):
db_utils.execute_file(dsn, tmp_path / 'test2.sql')
+
def test_execute_file_bad_sql(dsn, tmp_path):
tmpfile = tmp_path / 'test.sql'
tmpfile.write_text('CREATE STABLE test (id INT)')
--- /dev/null
+"""
+Tests for functions to import a new database.
+"""
+import pytest
+import psycopg2
+import sys
+
+from nominatim.tools import database_import
+from nominatim.errors import UsageError
+
+@pytest.fixture
+def nonexistant_db():
+ dbname = 'test_nominatim_python_unittest'
+
+ conn = psycopg2.connect(database='postgres')
+
+ conn.set_isolation_level(0)
+ with conn.cursor() as cur:
+ cur.execute('DROP DATABASE IF EXISTS {}'.format(dbname))
+
+ yield dbname
+
+ with conn.cursor() as cur:
+ cur.execute('DROP DATABASE IF EXISTS {}'.format(dbname))
+
+
+def test_create_db_success(nonexistant_db):
+ database_import.create_db('dbname=' + nonexistant_db, rouser='www-data')
+
+ conn = psycopg2.connect(database=nonexistant_db)
+ conn.close()
+
+
+def test_create_db_already_exists(temp_db):
+ with pytest.raises(UsageError):
+ database_import.create_db('dbname=' + temp_db)
+
+
+def test_create_db_unsupported_version(nonexistant_db, monkeypatch):
+ monkeypatch.setattr(database_import, 'POSTGRESQL_REQUIRED_VERSION', (100, 4))
+
+ with pytest.raises(UsageError, match='PostgreSQL server is too old.'):
+ database_import.create_db('dbname=' + nonexistant_db)
+
+
+def test_create_db_missing_ro_user(nonexistant_db):
+ with pytest.raises(UsageError, match='Missing read-only user.'):
+ database_import.create_db('dbname=' + nonexistant_db, rouser='sdfwkjkjgdugu2;jgsafkljas;')
+
+
+def test_setup_extensions(temp_db_conn, temp_db_cursor):
+ database_import.setup_extensions(temp_db_conn)
+
+ temp_db_cursor.execute('CREATE TABLE t (h HSTORE, geom GEOMETRY(Geometry, 4326))')
+
+
+def test_setup_extensions_old_postgis(temp_db_conn, monkeypatch):
+ monkeypatch.setattr(database_import, 'POSTGIS_REQUIRED_VERSION', (50, 50))
+
+ with pytest.raises(UsageError, match='PostGIS version is too old.'):
+ database_import.setup_extensions(temp_db_conn)
+
+
+def test_install_module(tmp_path):
+ src_dir = tmp_path / 'source'
+ src_dir.mkdir()
+ (src_dir / 'nominatim.so').write_text('TEST nomiantim.so')
+
+ project_dir = tmp_path / 'project'
+ project_dir.mkdir()
+
+ database_import.install_module(src_dir, project_dir, '')
+
+ outfile = project_dir / 'module' / 'nominatim.so'
+
+ assert outfile.exists()
+ assert outfile.read_text() == 'TEST nomiantim.so'
+ assert outfile.stat().st_mode == 33261
+
+
+def test_import_base_data(src_dir, temp_db, temp_db_cursor):
+ temp_db_cursor.execute('CREATE EXTENSION hstore')
+ temp_db_cursor.execute('CREATE EXTENSION postgis')
+ database_import.import_base_data('dbname=' + temp_db, src_dir / 'data')
+
+ assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0
+
+
+def test_import_base_data_ignore_partitions(src_dir, temp_db, temp_db_cursor):
+ temp_db_cursor.execute('CREATE EXTENSION hstore')
+ temp_db_cursor.execute('CREATE EXTENSION postgis')
+ database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
+ ignore_partitions=True)
+
+ assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0
+ assert temp_db_cursor.scalar('SELECT count(*) FROM country_name WHERE partition != 0') == 0
projects / nominatim.git / commitdiff