#-----------------------------------------------------------------------------
- # python and pyosmium (imports/updates only)
+ # python (imports/updates only)
#-----------------------------------------------------------------------------
if (BUILD_IMPORTER)
- find_package(PythonInterp 3)
-
- find_program(PYOSMIUM pyosmium-get-changes)
- if (NOT EXISTS "${PYOSMIUM}")
- set(PYOSMIUM_PATH "")
- message(WARNING "pyosmium-get-changes not found (required for updates)")
- else()
- set(PYOSMIUM_PATH "${PYOSMIUM}")
- message(STATUS "Using pyosmium-get-changes at ${PYOSMIUM_PATH}")
- endif()
+ find_package(PythonInterp 3.5 REQUIRED)
endif()
#-----------------------------------------------------------------------------
# sanity check if PHP binary exists
if (NOT EXISTS ${PHP_BIN})
message(FATAL_ERROR "PHP binary not found. Install php or provide location with -DPHP_BIN=/path/php ")
+ else()
+ message (STATUS "Using PHP binary " ${PHP_BIN})
+ endif()
+ if (NOT PHPCGI_BIN)
+ find_program (PHPCGI_BIN php-cgi)
+ endif()
+ # sanity check if PHP binary exists
+ if (NOT EXISTS ${PHPCGI_BIN})
+ message(WARNING "php-cgi binary not found. nominatim tool will not provide query functions.")
+ set (PHPCGI_BIN "")
+ else()
+ message (STATUS "Using php-cgi binary " ${PHPCGI_BIN})
endif()
- message (STATUS "Using PHP binary " ${PHP_BIN})
endif()
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
if (BUILD_API)
+ set(WEBSITEFILES
+ 403.html
+ 509.html
+ crossdomain.xml
+ favicon.ico
+ nominatim.xml
+ robots.txt
+ taginfo.json
+ )
+
+ foreach (webfile ${WEBSITEFILES})
+ configure_file(${PROJECT_SOURCE_DIR}/website/${webfile}
+ ${PROJECT_BINARY_DIR}/website/${webfile})
+ endforeach()
+
add_custom_target(serve
php -S 127.0.0.1:8088
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website
set(TEST_BDD db osm2pgsql api)
- foreach (test ${TEST_BDD})
- add_test(NAME bdd_${test}
- COMMAND behave ${test}
- WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/bdd)
- set_tests_properties(bdd_${test}
- PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}")
- endforeach()
-
- add_test(NAME php
- COMMAND phpunit ./
- WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/php)
-
- add_test(NAME phpcs
- COMMAND phpcs --report-width=120 --colors lib website utils
- WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
+ find_program(PYTHON_BEHAVE behave)
+ find_program(PYLINT NAMES pylint3 pylint)
+ find_program(PYTEST NAMES pytest py.test-3 py.test)
+ find_program(PHPCS phpcs)
+ find_program(PHPUNIT phpunit)
+
+ if (PYTHON_BEHAVE)
+ message(STATUS "Using Python behave binary ${PYTHON_BEHAVE}")
+ foreach (test ${TEST_BDD})
+ add_test(NAME bdd_${test}
+ COMMAND ${PYTHON_BEHAVE} ${test}
+ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/bdd)
+ set_tests_properties(bdd_${test}
+ PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}")
+ endforeach()
+ else()
+ message(WARNING "behave not found. BDD tests disabled." )
+ endif()
+
+ if (PHPUNIT)
+ message(STATUS "Using phpunit binary ${PHPUNIT}")
+ add_test(NAME php
+ COMMAND ${PHPUNIT} ./
+ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test/php)
+ else()
+ message(WARNING "phpunit not found. PHP unit tests disabled." )
+ endif()
+
+ if (PHPCS)
+ message(STATUS "Using phpcs binary ${PHPCS}")
+ add_test(NAME phpcs
+ COMMAND ${PHPCS} --report-width=120 --colors lib website utils
+ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
+ else()
+ message(WARNING "phpcs not found. PHP linting tests disabled." )
+ endif()
+
+ if (PYLINT)
+ message(STATUS "Using pylint binary ${PYLINT}")
+ add_test(NAME pylint
+ COMMAND ${PYLINT} --extension-pkg-whitelist=osmium nominatim
+ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
+ else()
+ message(WARNING "pylint not found. Python linting tests disabled.")
+ endif()
+
+ if (PYTEST)
+ message(STATUS "Using pytest binary ${PYTEST}")
+ add_test(NAME pytest
+ COMMAND ${PYTEST} test/python
+ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
+ else()
+ message(WARNING "pytest not found. Python tests disabled." )
+ endif()
endif()
#-----------------------------------------------------------------------------
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/setup_functions.php');
require_once(CONST_LibDir.'/setup/SetupClass.php');
- require_once(CONST_LibDir.'/setup/AddressLevelParser.php');
ini_set('memory_limit', '800M');
setupHTTPProxy();
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
+
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
date_default_timezone_set('Etc/UTC');
);
}
-
- $oIndexCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py'))
- ->addParams('--database', $aDSNInfo['database'])
- ->addParams('--port', $aDSNInfo['port'])
- ->addParams('--threads', $aResult['index-instances']);
- if (!$aResult['quiet']) {
- $oIndexCmd->addParams('--verbose');
+ $oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
+ if ($aResult['quiet']) {
+ $oNominatimCmd->addParams('--quiet');
}
if ($aResult['verbose']) {
- $oIndexCmd->addParams('--verbose');
- }
- if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
- $oIndexCmd->addParams('--host', $aDSNInfo['hostspec']);
- }
- if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
- $oIndexCmd->addParams('--username', $aDSNInfo['username']);
- }
- if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
- $oIndexCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
+ $oNominatimCmd->addParams('--verbose');
}
- $sPyosmiumBin = getSetting('PYOSMIUM_BINARY');
- $sBaseURL = getSetting('REPLICATION_URL');
-
if ($aResult['init-updates']) {
- // sanity check that the replication URL is correct
- $sBaseState = file_get_contents($sBaseURL.'/state.txt');
- if ($sBaseState === false) {
- echo "\nCannot find state.txt file at the configured replication URL.\n";
- echo "Does the URL point to a directory containing OSM update data?\n\n";
- fail('replication URL not reachable.');
- }
- // sanity check for pyosmium-get-changes
- if (!$sPyosmiumBin) {
- echo "\nNOMINATIM_PYOSMIUM_BINARY not configured.\n";
- echo "You need to install pyosmium and set up the path to pyosmium-get-changes\n";
- echo "in your local .env file.\n\n";
- fail('NOMINATIM_PYOSMIUM_BINARY not configured');
- }
-
- $aOutput = 0;
- $oCMD = new \Nominatim\Shell($sPyosmiumBin, '--help');
- exec($oCMD->escapedCmd(), $aOutput, $iRet);
-
- if ($iRet != 0) {
- echo "Cannot execute pyosmium-get-changes.\n";
- echo "Make sure you have pyosmium installed correctly\n";
- echo "and have set up NOMINATIM_PYOSMIUM_BINARY to point to pyosmium-get-changes.\n";
- fail('pyosmium-get-changes not found or not usable');
- }
-
- if (!$aResult['no-update-functions']) {
- // instantiate setupClass to use the function therein
- $cSetup = new SetupFunctions(array(
- 'enable-diff-updates' => true,
- 'verbose' => $aResult['verbose']
- ));
- $cSetup->createFunctions();
- }
-
- $sDatabaseDate = getDatabaseDate($oDB);
- if (!$sDatabaseDate) {
- fail('Cannot determine date of database.');
- }
- $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
-
- // get the appropriate state id
- $aOutput = 0;
- $oCMD = (new \Nominatim\Shell($sPyosmiumBin))
- ->addParams('--start-date', $sWindBack)
- ->addParams('--server', $sBaseURL);
-
- exec($oCMD->escapedCmd(), $aOutput, $iRet);
- if ($iRet != 0 || $aOutput[0] == 'None') {
- fail('Error running pyosmium tools');
- }
+ $oCmd = (clone($oNominatimCmd))->addParams('replication', '--init');
- $oDB->exec('TRUNCATE import_status');
- $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
- $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';
-
- try {
- $oDB->exec($sSQL);
- } catch (\Nominatim\DatabaseError $e) {
- fail('Could not enter sequence into database.');
+ if ($aResult['no-update-functions']) {
+ $oCmd->addParams('--no-update-functions');
}
- echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";
+ $oCmd->run();
}
if ($aResult['check-for-updates']) {
- $aLastState = $oDB->getRow('SELECT sequence_id FROM import_status');
-
- if (!$aLastState['sequence_id']) {
- fail('Updates not set up. Please run ./utils/update.php --init-updates.');
- }
-
- $oCmd = (new \Nominatim\Shell(CONST_BinDir.'/check_server_for_updates.py'))
- ->addParams($sBaseURL)
- ->addParams($aLastState['sequence_id']);
- $iRet = $oCmd->run();
-
- exit($iRet);
+ exit((clone($oNominatimCmd))->addParams('replication', '--check-for-updates')->run());
}
if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
}
if ($aResult['calculate-postcodes']) {
- info('Update postcodes centroids');
- $sTemplate = file_get_contents(CONST_DataDir.'/sql/update-postcodes.sql');
- runSQLScript($sTemplate, true, true);
+ (clone($oNominatimCmd))->addParams('refresh', '--postcodes')->run();
}
$sTemporaryFile = CONST_InstallDir.'/osmosischange.osc';
}
if ($aResult['recompute-word-counts']) {
- info('Recompute frequency of full-word search terms');
- $sTemplate = file_get_contents(CONST_DataDir.'/sql/words_from_search_name.sql');
- runSQLScript($sTemplate, true, true);
+ (clone($oNominatimCmd))->addParams('refresh', '--word-counts')->run();
}
if ($aResult['index']) {
- $oCmd = (clone $oIndexCmd)
- ->addParams('--minrank', $aResult['index-rank'], '-b');
- $oCmd->run();
-
- $oCmd = (clone $oIndexCmd)
- ->addParams('--minrank', $aResult['index-rank']);
- $oCmd->run();
-
- $oDB->exec('update import_status set indexed = true');
+ (clone $oNominatimCmd)
+ ->addParams('index', '--minrank', $aResult['index-rank'])
+ ->addParams('--threads', $aResult['index-instances'])
+ ->run();
}
if ($aResult['update-address-levels']) {
- $sAddressLevelConfig = getSettingConfig('ADDRESS_LEVEL_CONFIG', 'address-levels.json');
- echo 'Updating address levels from '.$sAddressLevelConfig.".\n";
- $oAlParser = new \Nominatim\Setup\AddressLevelParser($sAddressLevelConfig);
- $oAlParser->createTable($oDB, 'address_levels');
+ (clone($oNominatimCmd))->addParams('refresh', '--address-levels')->run();
}
if ($aResult['recompute-importance']) {
}
if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
- //
- if (strpos($sBaseURL, 'download.geofabrik.de') !== false && getSetting('REPLICATION_UPDATE_INTERVAL') < 86400) {
- fail('Error: Update interval too low for download.geofabrik.de. ' .
- "Please check install documentation (https://nominatim.org/release-docs/latest/admin/Import-and-Update#setting-up-the-update-process)\n");
+ $oCmd = (clone($oNominatimCmd))
+ ->addParams('replication')
+ ->addParams('--threads', $aResult['index-instances']);
+
+ if (!$aResult['import-osmosis-all']) {
+ $oCmd->addParams('--once');
}
- $sImportFile = CONST_InstallDir.'/osmosischange.osc';
-
- $oCMDDownload = (new \Nominatim\Shell($sPyosmiumBin))
- ->addParams('--server', $sBaseURL)
- ->addParams('--outfile', $sImportFile)
- ->addParams('--size', getSetting('REPLICATION_MAX_DIFF'));
-
- $oCMDImport = (clone $oOsm2pgsqlCmd)->addParams($sImportFile);
-
- while (true) {
- $fStartTime = time();
- $aLastState = $oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status');
-
- if (!$aLastState['sequence_id']) {
- echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
- exit(1);
- }
-
- echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n";
-
- $sBatchEnd = $aLastState['lastimportdate'];
- $iEndSequence = $aLastState['sequence_id'];
-
- if ($aLastState['indexed']) {
- // Sleep if the update interval has not yet been reached.
- $fNextUpdate = $aLastState['unix_ts'] + getSetting('REPLICATION_UPDATE_INTERVAL');
- if ($fNextUpdate > $fStartTime) {
- $iSleepTime = $fNextUpdate - $fStartTime;
- echo "Waiting for next update for $iSleepTime sec.";
- sleep($iSleepTime);
- }
-
- // Download the next batch of changes.
- do {
- $fCMDStartTime = time();
- $iNextSeq = (int) $aLastState['sequence_id'];
- unset($aOutput);
-
- $oCMD = (clone $oCMDDownload)->addParams('--start-id', $iNextSeq);
- echo $oCMD->escapedCmd()."\n";
- if (file_exists($sImportFile)) {
- unlink($sImportFile);
- }
- exec($oCMD->escapedCmd(), $aOutput, $iResult);
-
- if ($iResult == 3) {
- $sSleep = getSetting('REPLICATION_RECHECK_INTERVAL');
- echo 'No new updates. Sleeping for '.$sSleep." sec.\n";
- sleep($sSleep);
- } elseif ($iResult != 0) {
- echo 'ERROR: updates failed.';
- exit($iResult);
- } else {
- $iEndSequence = (int)$aOutput[0];
- }
- } while ($iResult);
-
- // get the newest object from the diff file
- $sBatchEnd = 0;
- $iRet = 0;
- $oCMD = new \Nominatim\Shell(CONST_BinDir.'/osm_file_date.py', $sImportFile);
- exec($oCMD->escapedCmd(), $sBatchEnd, $iRet);
- if ($iRet == 5) {
- echo "Diff file is empty. skipping import.\n";
- if (!$aResult['import-osmosis-all']) {
- exit(0);
- } else {
- continue;
- }
- }
- if ($iRet != 0) {
- fail('Error getting date from diff file.');
- }
- $sBatchEnd = $sBatchEnd[0];
-
- // Import the file
- $fCMDStartTime = time();
-
-
- echo $oCMDImport->escapedCmd()."\n";
- unset($sJunk);
- $iErrorLevel = $oCMDImport->run();
- if ($iErrorLevel) {
- echo "Error executing osm2pgsql: $iErrorLevel\n";
- exit($iErrorLevel);
- }
-
- // write the update logs
- $iFileSize = filesize($sImportFile);
- $sSQL = 'INSERT INTO import_osmosis_log';
- $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
- $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
- $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
- $sSQL .= date('Y-m-d H:i:s')."','import')";
- var_Dump($sSQL);
- $oDB->exec($sSQL);
-
- // update the status
- $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence";
- var_Dump($sSQL);
- $oDB->exec($sSQL);
- echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
- }
-
- // Index file
- if (!$aResult['no-index']) {
- $fCMDStartTime = time();
-
- $oThisIndexCmd = clone($oIndexCmd);
- $oThisIndexCmd->addParams('-b');
- echo $oThisIndexCmd->escapedCmd()."\n";
- $iErrorLevel = $oThisIndexCmd->run();
- if ($iErrorLevel) {
- echo "Error: $iErrorLevel\n";
- exit($iErrorLevel);
- }
-
- $oThisIndexCmd = clone($oIndexCmd);
- echo $oThisIndexCmd->escapedCmd()."\n";
- $iErrorLevel = $oThisIndexCmd->run();
- if ($iErrorLevel) {
- echo "Error: $iErrorLevel\n";
- exit($iErrorLevel);
- }
-
- $sSQL = 'INSERT INTO import_osmosis_log';
- $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
- $sSQL .= " values ('$sBatchEnd',$iEndSequence,NULL,'";
- $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
- $sSQL .= date('Y-m-d H:i:s')."','index')";
- var_Dump($sSQL);
- $oDB->exec($sSQL);
- echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
-
- $sSQL = 'update import_status set indexed = true';
- $oDB->exec($sSQL);
- } else {
- if ($aResult['import-osmosis-all']) {
- echo "Error: --no-index cannot be used with continuous imports (--import-osmosis-all).\n";
- exit(1);
- }
- }
-
- $fDuration = time() - $fStartTime;
- echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";
- if (!$aResult['import-osmosis-all']) exit(0);
+ if ($aResult['no-index']) {
+ $oCmd->addParams('--no-index');
}
+
+ exit($oCmd->run());
}
projects / nominatim.git / commitdiff