From 62747c934d3445b000810f8a7737ea86777995e3 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 17 Jul 2018 17:18:33 -0600 Subject: [PATCH] Work on setup/update scripts, unit tests, and documentation to enable Postgres server to be optionally configured on a remote host --- data/words.sql | 1 + docs/admin/Faq.md | 2 +- lib/cmd.php | 12 ++- nominatim/index.c | 43 ++++++--- sql/partition-tables.src.sql | 1 + test/README.md | 5 ++ test/bdd/environment.py | 39 +++++++-- utils/setup.php | 164 +++++++++++++++++++++++++---------- utils/update.php | 73 +++++++++++++--- 9 files changed, 263 insertions(+), 77 deletions(-) diff --git a/data/words.sql b/data/words.sql index 85578f20..73251b04 100644 --- a/data/words.sql +++ b/data/words.sql @@ -23,6 +23,7 @@ CREATE TABLE word_frequencies ( count bigint ); +TRUNCATE TABLE word_frequencies; -- -- Data for Name: word_frequencies; Type: TABLE DATA; Schema: public; Owner: - diff --git a/docs/admin/Faq.md b/docs/admin/Faq.md index f3ad670a..5fb0d9ae 100644 --- a/docs/admin/Faq.md +++ b/docs/admin/Faq.md @@ -112,7 +112,7 @@ to get the full error message. On CentOS v7 the PostgreSQL server is started with `systemd`. Check if `/usr/lib/systemd/system/httpd.service` contains a line `PrivateTmp=true`. If so then Apache cannot see the `/tmp/.s.PGSQL.5432` file. It's a good security feature, -so use the [preferred solution](../appendix/Install-on-Centos-7/#adding-selinux-security-settings). +so use the [[#PostgreSQL_UNIX_Socket_Location_on_CentOS|preferred solution]] However, you can solve this the quick and dirty way by commenting out that line and then run diff --git a/lib/cmd.php b/lib/cmd.php index 28d56f2e..8ccbb036 100644 --- a/lib/cmd.php +++ b/lib/cmd.php @@ -158,6 +158,16 @@ function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreError $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $procenv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $procenv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } if (!$bVerbose) { $sCMD .= ' -q'; } @@ -170,7 +180,7 @@ function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreError 2 => STDERR ); $ahPipes = null; - $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes); + $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes, null, $procenv); if (!is_resource($hProcess)) { fail('unable to start pgsql'); } diff --git a/nominatim/index.c b/nominatim/index.c index c16aba9e..bb553f7e 100644 --- a/nominatim/index.c +++ b/nominatim/index.c @@ -262,13 +262,16 @@ struct index_thread_data * thread_data, const char *structuredoutputfile) void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile) { - struct index_thread_data * thread_data; + struct index_thread_data *thread_data; PGconn *conn; - PGresult * res; + PGresult *res; + int num_rows = 0, status_code = 0; + int db_has_locale = 0; + char *result_string = NULL; int rank; - + int i; xmlTextWriterPtr writer; @@ -283,6 +286,23 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co exit(EXIT_FAILURE); } + res = PQexec(conn, "SHOW lc_messages"); + status_code = PQresultStatus(res); + if (status_code != PGRES_TUPLES_OK && status_code != PGRES_SINGLE_TUPLE) { + fprintf(stderr, "Failed determining database locale: %s\n", PQerrorMessage(conn)); + exit(EXIT_FAILURE); + } + num_rows = PQntuples(res); + if (num_rows > 0) + { + result_string = PQgetvalue(res, 0, 0); + if (result_string && (strlen(result_string) > 0) && (strcasecmp(result_string, "C") != 0)) + { + // non-default locale if the result exists, is non-empty, and is not "C" + db_has_locale = 1; + } + } + pg_prepare_params[0] = PG_OID_INT4; res = PQprepare(conn, "index_sectors", "select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status > 0 group by geometry_sector order by geometry_sector", @@ -392,19 +412,20 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co } PQclear(res); - // Make sure the error message is not localized as we parse it later. - res = PQexec(thread_data[i].conn, "SET lc_messages TO 'C'"); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + if (db_has_locale) { - fprintf(stderr, "Failed to set langauge: %s\n", PQerrorMessage(thread_data[i].conn)); - exit(EXIT_FAILURE); + // Make sure the error message is not localized as we parse it later. + res = PQexec(thread_data[i].conn, "SET lc_messages TO 'C'"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "Failed to set langauge: %s\n", PQerrorMessage(thread_data[i].conn)); + exit(EXIT_FAILURE); + } + PQclear(res); } - PQclear(res); - nominatim_exportCreatePreparedQueries(thread_data[i].conn); } - fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads); for (rank = rank_min; rank <= rank_max; rank++) diff --git a/sql/partition-tables.src.sql b/sql/partition-tables.src.sql index 61ed5281..20dafcd7 100644 --- a/sql/partition-tables.src.sql +++ b/sql/partition-tables.src.sql @@ -48,6 +48,7 @@ CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USI CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid) {ts:address-index}; CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off) {ts:address-index}; +DROP TABLE IF EXISTS location_road_-partition-; CREATE TABLE location_road_-partition- ( place_id BIGINT, partition SMALLINT, diff --git a/test/README.md b/test/README.md index 2a357e40..0487fd40 100644 --- a/test/README.md +++ b/test/README.md @@ -73,6 +73,11 @@ The tests can be configured with a set of environment variables: the test databases (db tests) * `TEST_DB` - name of test database (db tests) * `ABI_TEST_DB` - name of the database containing the API test data (api tests) + * `DB_HOST` - (optional) hostname of database host + * `DB_USER` - (optional) username of database login + * `DB_PASS` - (optional) password for database login + * `SERVER_MODULE_PATH` - (optional) path on the Postgres server to Nominatim + * module shared library file * `TEST_SETTINGS_TEMPLATE` - file to write temporary Nominatim settings to * `REMOVE_TEMPLATE` - if true, the template database will not be reused during the next run. Reusing the base templates speeds up tests diff --git a/test/bdd/environment.py b/test/bdd/environment.py index 162346de..fdc65a5e 100644 --- a/test/bdd/environment.py +++ b/test/bdd/environment.py @@ -14,10 +14,14 @@ userconfig = { 'BUILDDIR' : os.path.join(os.path.split(__file__)[0], "../../build"), 'REMOVE_TEMPLATE' : False, 'KEEP_TEST_DB' : False, + 'DB_HOST' : None, + 'DB_USER' : None, + 'DB_PASS' : None, 'TEMPLATE_DB' : 'test_template_nominatim', 'TEST_DB' : 'test_nominatim', 'API_TEST_DB' : 'test_api_nominatim', 'TEST_SETTINGS_FILE' : '/tmp/nominatim_settings.php', + 'SERVER_MODULE_PATH' : None, 'PHPCOV' : False, # set to output directory to enable code coverage } @@ -30,9 +34,13 @@ class NominatimEnvironment(object): def __init__(self, config): self.build_dir = os.path.abspath(config['BUILDDIR']) self.src_dir = os.path.abspath(os.path.join(os.path.split(__file__)[0], "../..")) + self.db_host = config['DB_HOST'] + self.db_user = config['DB_USER'] + self.db_pass = config['DB_PASS'] self.template_db = config['TEMPLATE_DB'] self.test_db = config['TEST_DB'] self.api_test_db = config['API_TEST_DB'] + self.server_module_path = config['SERVER_MODULE_PATH'] self.local_settings_file = config['TEST_SETTINGS_FILE'] self.reuse_template = not config['REMOVE_TEMPLATE'] self.keep_scenario_db = config['KEEP_TEST_DB'] @@ -42,6 +50,17 @@ class NominatimEnvironment(object): self.template_db_done = False + def connect_database(self, dbname): + dbargs = {'database': dbname} + if self.db_host: + dbargs['host'] = self.db_host + if self.db_user: + dbargs['user'] = self.db_user + if self.db_pass: + dbargs['password'] = self.db_pass + conn = psycopg2.connect(**dbargs) + return conn + def next_code_coverage_file(self): fn = os.path.join(self.code_coverage_path, "%06d.cov" % self.code_coverage_id) self.code_coverage_id += 1 @@ -50,7 +69,11 @@ class NominatimEnvironment(object): def write_nominatim_config(self, dbname): f = open(self.local_settings_file, 'w') - f.write(" getProcessorCount()) { - $iInstances = getProcessorCount(); - warn("resetting threads to $iInstances"); -} // Assume we can steal all the cache memory in the box (unless told otherwise) if (isset($aCMDResult['osm2pgsql-cache'])) { @@ -82,6 +80,12 @@ if (isset($aCMDResult['osm2pgsql-cache'])) { $iCacheMemory = getCacheMemoryMB(); } +$modulePath = CONST_InstallPath . '/module'; +if (isset($aCMDResult['module-path'])) { + $modulePath = $aCMDResult['module-path']; + echo 'module path: ' . $modulePath . '\n'; +} + $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; @@ -92,7 +96,22 @@ if ($aCMDResult['create-db'] || $aCMDResult['all']) { if (!PEAR::isError($oDB)) { fail('database already exists ('.CONST_Database_DSN.')'); } - passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']); + + $createdbCmd = 'createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']; + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $createdbCmd .= ' -U ' . $aDSNInfo['username']; + } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $createdbCmd .= ' -h ' . $aDSNInfo['hostspec']; + } + + $procenv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $procenv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + + $result = runWithEnv($createdbCmd, $procenv); + if ($result != 0) fail('Error executing external command: '.$createdbCmd); } if ($aCMDResult['setup-db'] || $aCMDResult['all']) { @@ -143,7 +162,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) { // Try accessing the C module, so we know early if something is wrong // and can simply error out. $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; - $sSQL .= CONST_InstallPath."/module/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; + $sSQL .= $modulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; $oResult = $oDB->query($sSQL); @@ -180,8 +199,8 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) { // is only defined in the subsequently called create_tables. // Create dummies here that will be overwritten by the proper // versions in create-tables. - pgsqlRunScript('CREATE TABLE place_boundingbox ()'); - pgsqlRunScript('create type wikipedia_article_match as ()'); + pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); + pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); } if ($aCMDResult['import-data'] || $aCMDResult['all']) { @@ -209,8 +228,20 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) { $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; $osm2pgsql .= ' -C '.$iCacheMemory; $osm2pgsql .= ' -P '.$aDSNInfo['port']; + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $osm2pgsql .= ' -U ' . $aDSNInfo['username']; + } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $osm2pgsql .= ' -H ' . $aDSNInfo['hostspec']; + } + + $procenv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $procenv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; - passthruCheckReturn($osm2pgsql); + runWithEnv($osm2pgsql, $procenv); $oDB =& getDB(); if (!$aCMDResult['ignore-errors'] && !chksql($oDB->getRow('select * from place limit 1'))) { @@ -221,9 +252,6 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) { if ($aCMDResult['create-functions'] || $aCMDResult['all']) { info('Create Functions'); $bDidSomething = true; - if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) { - fail('nominatim module not built'); - } create_sql_functions($aCMDResult); } @@ -413,14 +441,23 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) { fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); } - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; + $failed = false; + for ($i = 0; $i <= $iLoadThreads; $i++) { + while (($pgresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { + $resultStatus = pg_result_status($pgresult); + // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, + // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, + // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR + echo 'Query result ' . $i . ' is: ' . $resultStatus . '\n'; + if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { + $resultError = pg_result_error($pgresult); + echo '-- error text ' . $i . ': ' . $resultError . '\n'; + $failed = true; + } } - sleep(1); - echo '.'; + } + if ($failed) { + fail('SQL errors loading placex and/or location_property_osmline tables'); } echo "\n"; info('Reanalysing database'); @@ -579,14 +616,34 @@ if ($aCMDResult['index'] || $aCMDResult['all']) { $bDidSomething = true; $sOutputFile = ''; $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sBaseCmd .= ' -H ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sBaseCmd .= ' -U ' . $aDSNInfo['username']; + } + $procenv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $procenv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + info('Index ranks 0 - 4'); - passthruCheckReturn($sBaseCmd.' -R 4'); + $status = runWithEnv($sBaseCmd.' -R 4', $procenv); + if ($status != 0) { + fail('error status ' . $status . ' running nominatim!'); + } if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); info('Index ranks 5 - 25'); - passthruCheckReturn($sBaseCmd.' -r 5 -R 25'); + $status = runWithEnv($sBaseCmd.' -r 5 -R 25', $procenv); + if ($status != 0) { + fail('error status ' . $status . ' running nominatim!'); + } if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); info('Index ranks 26 - 30'); - passthruCheckReturn($sBaseCmd.' -r 26'); + $status = runWithEnv($sBaseCmd.' -r 26', $procenv); + if ($status != 0) { + fail('error status ' . $status . ' running nominatim!'); + } info('Index postcodes'); $oDB =& getDB(); @@ -722,6 +779,16 @@ function pgsqlRunScriptFile($sFilename) if (!$aCMDResult['verbose']) { $sCMD .= ' -q'; } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $procenv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $procenv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } $ahGzipPipes = null; if (preg_match('/\\.gz$/', $sFilename)) { @@ -745,10 +812,9 @@ function pgsqlRunScriptFile($sFilename) 2 => array('file', '/dev/null', 'a') ); $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $procenv); if (!is_resource($hProcess)) fail('unable to start pgsql'); - // TODO: error checking while (!feof($ahPipes[1])) { echo fread($ahPipes[1], 4096); @@ -830,32 +896,41 @@ function pgsqlRunDropAndRestore($sDumpFile) $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile; - - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); - if (!is_resource($hProcess)) fail('unable to start pg_restore'); - - fclose($ahPipes[0]); - - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $procenv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $procenv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); } - fclose($ahPipes[1]); - $iReturn = proc_close($hProcess); + $iReturn = runWithEnv($sCMD, $procenv); } function passthruCheckReturn($cmd) { $result = -1; passthru($cmd, $result); - if ($result != 0) fail('Error executing external command: '.$cmd); +} + +function runWithEnv($cmd, $env) +{ + $fds = array(0 => array('pipe', 'r'), + 1 => STDOUT, + 2 => STDERR); + $pipes = null; + $proc = @proc_open($cmd, $fds, $pipes, null, $env); + if (!is_resource($proc)) { + fail('unable to run command:' . $cmd); + } + + fclose($pipes[0]); // no stdin + + $stat = proc_close($proc); + return $stat; } function replace_tablespace($sTemplate, $sTablespace, $sSql) @@ -871,8 +946,9 @@ function replace_tablespace($sTemplate, $sTablespace, $sSql) function create_sql_functions($aCMDResult) { + global $modulePath; $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate); + $sTemplate = str_replace('{modulepath}', $modulePath, $sTemplate); if ($aCMDResult['enable-diff-updates']) { $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); } diff --git a/utils/update.php b/utils/update.php index 6caa7e4b..40c72535 100755 --- a/utils/update.php +++ b/utils/update.php @@ -5,6 +5,8 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once(CONST_BasePath.'/lib/init-cmd.php'); ini_set('memory_limit', '800M'); +# (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) + $aCMDOptions = array( 'Import / update / index osm data', @@ -14,6 +16,7 @@ $aCMDOptions array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), + array('update-functions', '', 0, 1, 0, 0, 'bool', 'Update trigger functions to support differential updates'), array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), @@ -56,6 +59,17 @@ if ($iCacheMemory + 500 > getTotalMemoryMB()) { echo "WARNING: resetting cache memory to $iCacheMemory\n"; } $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; +if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; +} +if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sOsm2pgsqlCmd .= ' -H ' . $aDSNInfo['hostspec']; +} +$procenv = null; +if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $procenv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); +} + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } @@ -84,11 +98,13 @@ if ($aResult['init-updates']) { echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n"; fail('pyosmium-get-changes not found or not usable'); } - $sSetup = CONST_InstallPath.'/utils/setup.php'; - $iRet = -1; - passthru($sSetup.' --create-functions --enable-diff-updates', $iRet); - if ($iRet != 0) { - fail('Error running setup script'); + if ($aResult['update-functions']) { + $sSetup = CONST_InstallPath.'/utils/setup.php'; + $iRet = -1; + passthru($argv[0].' '.$sSetup.' --create-functions --enable-diff-updates', $iRet); + if ($iRet != 0) { + fail('Error running setup script'); + } } $sDatabaseDate = getDatabaseDate($oDB); @@ -106,8 +122,8 @@ if ($aResult['init-updates']) { } pg_query($oDB->connection, 'TRUNCATE import_status'); - $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('"; - $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)'; + $sSQL = 'INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('; + $sSQL .= "'".$sDatabaseDate."',".$aOutput[0].', true)'; if (!pg_query($oDB->connection, $sSQL)) { fail('Could not enter sequence into database.'); } @@ -137,7 +153,7 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { // Import the file $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile; echo $sCMD."\n"; - exec($sCMD, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sCMD, $procenv); if ($iErrorLevel) { fail("Error from osm2pgsql, $iErrorLevel\n"); @@ -189,7 +205,7 @@ if ($bHaveDiff) { // import generated change file $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile; echo $sCMD."\n"; - exec($sCMD, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sCMD, $procenv); if ($iErrorLevel) { fail("osm2pgsql exited with error level $iErrorLevel\n"); } @@ -273,7 +289,15 @@ if ($aResult['recompute-word-counts']) { } if ($aResult['index']) { - passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']); + $cmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $cmd .= ' -H ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $cmd .= ' -U ' . $aDSNInfo['username']; + } + + runWithEnv($cmd, $procenv); } if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { @@ -287,6 +311,12 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size; $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMDIndex .= ' -U ' . $aDSNInfo['username']; + } while (true) { $fStartTime = time(); @@ -354,7 +384,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $fCMDStartTime = time(); echo $sCMDImport."\n"; unset($sJunk); - exec($sCMDImport, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sCMDImport, $procenv); if ($iErrorLevel) { echo "Error executing osm2pgsql: $iErrorLevel\n"; exit($iErrorLevel); @@ -383,7 +413,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $fCMDStartTime = time(); echo "$sThisIndexCmd\n"; - exec($sThisIndexCmd, $sJunk, $iErrorLevel); + $iErrorLevel = runWithEnv($sThisIndexCmd, $procenv); if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; exit($iErrorLevel); @@ -398,7 +428,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $oDB->query($sSQL); echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; - $sSQL = 'update import_status set indexed = true'; + $sSQL = 'UPDATE import_status SET indexed = true'; $oDB->query($sSQL); } @@ -407,3 +437,20 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (!$aResult['import-osmosis-all']) exit(0); } } + +function runWithEnv($cmd, $env) +{ + $fds = array(0 => array('pipe', 'r'), + 1 => STDOUT, + 2 => STDERR); + $pipes = null; + $proc = @proc_open($cmd, $fds, $pipes, null, $env); + if (!is_resource($proc)) { + fail('unable to run command:' . $cmd); + } + + fclose($pipes[0]); // no stdin + + $stat = proc_close($proc); + return $stat; +} -- 2.39.5