From: Sarah Hoffmann Date: Wed, 30 Dec 2020 10:30:58 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~188 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/c3238682a8867edfb3c052ca4d2d27b3926ad17a?hp=-c Merge remote-tracking branch 'upstream/master' --- c3238682a8867edfb3c052ca4d2d27b3926ad17a diff --combined CMakeLists.txt index 816ce647,b4f4ddb8..3fad37d7 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@@ -19,7 -19,7 +19,7 @@@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_ project(nominatim) set(NOMINATIM_VERSION_MAJOR 3) - set(NOMINATIM_VERSION_MINOR 5) + set(NOMINATIM_VERSION_MINOR 6) set(NOMINATIM_VERSION_PATCH 0) set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}") @@@ -96,66 -96,28 +96,43 @@@ endif( if (BUILD_IMPORTER) set(CUSTOMSCRIPTS - utils/check_import_finished.php - utils/country_languages.php - utils/importWikipedia.php - utils/export.php - utils/query.php - utils/setup.php - utils/specialphrases.php - utils/update.php - utils/warm.php + check_import_finished.php + country_languages.php + importWikipedia.php + export.php + query.php + setup.php + specialphrases.php + update.php + warm.php ) foreach (script_source ${CUSTOMSCRIPTS}) configure_file(${PROJECT_SOURCE_DIR}/cmake/script.tmpl - ${PROJECT_BINARY_DIR}/${script_source}) + ${PROJECT_BINARY_DIR}/utils/${script_source}) endforeach() endif() #----------------------------------------------------------------------------- - # webserver scripts (API only) + # Targets for running a development webserver from the build directory. #----------------------------------------------------------------------------- if (BUILD_API) - set(WEBSITESCRIPTS - website/deletable.php - website/details.php - website/lookup.php - website/polygons.php - website/reverse.php - website/search.php - website/status.php - ) - + set(WEBSITEFILES + 403.html + 509.html + crossdomain.xml + favicon.ico + nominatim.xml + robots.txt + taginfo.json + ) + - foreach (script_source ${WEBSITESCRIPTS}) - configure_file(${PROJECT_SOURCE_DIR}/cmake/website.tmpl - ${PROJECT_BINARY_DIR}/${script_source}) - endforeach() - - set(WEBPATHS css images js) - + foreach (webfile ${WEBSITEFILES}) + configure_file(${PROJECT_SOURCE_DIR}/website/${webfile} + ${PROJECT_BINARY_DIR}/website/${webfile}) + endforeach() + - foreach (wp ${WEBPATHS}) - execute_process( - COMMAND ln -sf ${PROJECT_SOURCE_DIR}/website/${wp} ${PROJECT_BINARY_DIR}/website/ - ) - endforeach() - add_custom_target(serve php -S 127.0.0.1:8088 WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/website @@@ -167,13 -129,6 +144,6 @@@ ) endif() - #----------------------------------------------------------------------------- - # default settings - #----------------------------------------------------------------------------- - - configure_file(${PROJECT_SOURCE_DIR}/settings/defaults.php - ${PROJECT_BINARY_DIR}/settings/settings.php) - #----------------------------------------------------------------------------- # Tests #----------------------------------------------------------------------------- diff --combined sql/functions/address_lookup.sql index a2b7c90a,4d7cc789..b832aed8 --- a/sql/functions/address_lookup.sql +++ b/sql/functions/address_lookup.sql @@@ -79,6 -79,18 +79,18 @@@ END $$ LANGUAGE plpgsql STABLE; + DROP TYPE IF EXISTS addressdata_place; + CREATE TYPE addressdata_place AS ( + place_id BIGINT, + country_code VARCHAR(2), + housenumber TEXT, + postcode TEXT, + class TEXT, + type TEXT, + name HSTORE, + address HSTORE, + centroid GEOMETRY + ); -- Compute the list of address parts for the given place. -- @@@ -87,7 -99,7 +99,7 @@@ CREATE OR REPLACE FUNCTION get_addressd RETURNS setof addressline AS $$ DECLARE - place RECORD; + place addressdata_place; location RECORD; current_rank_address INTEGER; location_isaddress BOOLEAN; @@@ -98,9 -110,9 +110,9 @@@ BEGI -- first query osmline (interpolation lines) IF in_housenumber >= 0 THEN SELECT parent_place_id as place_id, country_code, - in_housenumber::text as housenumber, postcode, + in_housenumber as housenumber, postcode, 'place' as class, 'house' as type, - null::hstore as name, null::hstore as address, + null as name, null as address, ST_Centroid(linegeo) as centroid INTO place FROM location_property_osmline @@@ -111,10 -123,10 +123,10 @@@ --then query tiger data -- %NOTIGERDATA% IF 0 THEN IF place IS NULL AND in_housenumber >= 0 THEN - SELECT parent_place_id as place_id, 'us'::varchar(2) as country_code, - in_housenumber::text as housenumber, postcode, + SELECT parent_place_id as place_id, 'us' as country_code, + in_housenumber as housenumber, postcode, 'place' as class, 'house' as type, - null::hstore as name, null::hstore as address, + null as name, null as address, ST_Centroid(linegeo) as centroid INTO place FROM location_property_tiger @@@ -125,10 -137,10 +137,10 @@@ -- %NOAUXDATA% IF 0 THEN IF place IS NULL THEN - SELECT parent_place_id as place_id, 'us'::varchar(2) as country_code, + SELECT parent_place_id as place_id, 'us' as country_code, housenumber, postcode, 'place' as class, 'house' as type, - null::hstore as name, null::hstore as address, + null as name, null as address, centroid INTO place FROM location_property_aux @@@ -141,8 -153,8 +153,8 @@@ SELECT parent_place_id as place_id, country_code, null::text as housenumber, postcode, 'place' as class, 'postcode' as type, - null::hstore as name, null::hstore as address, - null::geometry as centroid + null as name, null as address, + null as centroid INTO place FROM location_postcode WHERE place_id = in_place_id; @@@ -167,8 -179,8 +179,8 @@@ select coalesce(linked_place_id, place_id) as place_id, country_code, housenumber, postcode, class, type, - null::hstore as name, address, - null::geometry as centroid + null as name, address, + null as centroid INTO place FROM placex where place_id = in_place_id; END IF; @@@ -211,7 -223,7 +223,7 @@@ FOR location IN SELECT placex.place_id, osm_type, osm_id, name, class, type, coalesce(extratags->'linked_place', extratags->'place') as place_type, - admin_level, fromarea, isaddress, + admin_level, fromarea, isaddress and linked_place_id is NULL as isaddress, CASE WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, distance, country_code, postcode FROM place_addressline join placex on (address_place_id = placex.place_id) diff --combined utils/update.php index 6e6a99ab,931ca179..5bc2d73d --- a/utils/update.php +++ b/utils/update.php @@@ -1,9 -1,9 +1,9 @@@ connect(); $fPostgresVersion = $oDB->getPostgresVersion(); - $aDSNInfo = Nominatim\DB::parseDSN(CONST_Database_DSN); + $aDSNInfo = Nominatim\DB::parseDSN(getSetting('DATABASE_DSN')); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; // cache memory to be used by osm2pgsql, should not be more than the available memory @@@ -66,7 -70,7 +71,7 @@@ if ($iCacheMemory + 500 > getTotalMemor echo "WARNING: resetting cache memory to $iCacheMemory\n"; } - $oOsm2pgsqlCmd = (new \Nominatim\Shell(CONST_Osm2pgsql_Binary)) + $oOsm2pgsqlCmd = (new \Nominatim\Shell(getOsm2pgsqlBinary())) ->addParams('--hstore') ->addParams('--latlong') ->addParams('--append') @@@ -76,7 -80,7 +81,7 @@@ ->addParams('--number-processes', 1) ->addParams('--cache', $iCacheMemory) ->addParams('--output', 'gazetteer') - ->addParams('--style', CONST_Import_Style) + ->addParams('--style', getImportStyle()) ->addParams('--database', $aDSNInfo['database']) ->addParams('--port', $aDSNInfo['port']); @@@ -89,8 -93,8 +94,8 @@@ if (isset($aDSNInfo['username']) && $aD if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { $oOsm2pgsqlCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']); } - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $oOsm2pgsqlCmd->addParams('--flat-nodes', CONST_Osm2pgsql_Flatnode_File); + if (getSetting('FLATNODE_FILE')) { + $oOsm2pgsqlCmd->addParams('--flat-nodes', getSetting('FLATNODE_FILE')); } if ($fPostgresVersion >= 11.0) { $oOsm2pgsqlCmd->addEnvPair( @@@ -100,7 -104,7 +105,7 @@@ } - $oIndexCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py')) + $oIndexCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py')) ->addParams('--database', $aDSNInfo['database']) ->addParams('--port', $aDSNInfo['port']) ->addParams('--threads', $aResult['index-instances']); @@@ -120,31 -124,34 +125,34 @@@ if (isset($aDSNInfo['password']) && $aD $oIndexCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']); } + $sPyosmiumBin = getSetting('PYOSMIUM_BINARY'); + $sBaseURL = getSetting('REPLICATION_URL'); + if ($aResult['init-updates']) { // sanity check that the replication URL is correct - $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt'); + $sBaseState = file_get_contents($sBaseURL.'/state.txt'); if ($sBaseState === false) { echo "\nCannot find state.txt file at the configured replication URL.\n"; echo "Does the URL point to a directory containing OSM update data?\n\n"; fail('replication URL not reachable.'); } // sanity check for pyosmium-get-changes - if (!CONST_Pyosmium_Binary) { - echo "\nCONST_Pyosmium_Binary not configured.\n"; + if (!$sPyosmiumBin) { + echo "\nNOMINATIM_PYOSMIUM_BINARY not configured.\n"; echo "You need to install pyosmium and set up the path to pyosmium-get-changes\n"; - echo "in your local settings file.\n\n"; - fail('CONST_Pyosmium_Binary not configured'); + echo "in your local .env file.\n\n"; + fail('NOMINATIM_PYOSMIUM_BINARY not configured'); } $aOutput = 0; - $oCMD = new \Nominatim\Shell(CONST_Pyosmium_Binary, '--help'); + $oCMD = new \Nominatim\Shell($sPyosmiumBin, '--help'); exec($oCMD->escapedCmd(), $aOutput, $iRet); if ($iRet != 0) { echo "Cannot execute pyosmium-get-changes.\n"; echo "Make sure you have pyosmium installed correctly\n"; - echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n"; + echo "and have set up NOMINATIM_PYOSMIUM_BINARY to point to pyosmium-get-changes.\n"; fail('pyosmium-get-changes not found or not usable'); } @@@ -165,9 -172,9 +173,9 @@@ // get the appropriate state id $aOutput = 0; - $oCMD = (new \Nominatim\Shell(CONST_Pyosmium_Binary)) + $oCMD = (new \Nominatim\Shell($sPyosmiumBin)) ->addParams('--start-date', $sWindBack) - ->addParams('--server', CONST_Replication_Url); + ->addParams('--server', $sBaseURL); exec($oCMD->escapedCmd(), $aOutput, $iRet); if ($iRet != 0 || $aOutput[0] == 'None') { @@@ -194,8 -201,8 +202,8 @@@ if ($aResult['check-for-updates']) fail('Updates not set up. Please run ./utils/update.php --init-updates.'); } - $oCmd = (new \Nominatim\Shell(CONST_BasePath.'/utils/check_server_for_updates.py')) - ->addParams(CONST_Replication_Url) + $oCmd = (new \Nominatim\Shell(CONST_BinDir.'/check_server_for_updates.py')) + ->addParams($sBaseURL) ->addParams($aLastState['sequence_id']); $iRet = $oCmd->run(); @@@ -224,11 -231,11 +232,11 @@@ if (isset($aResult['import-diff']) || i if ($aResult['calculate-postcodes']) { info('Update postcodes centroids'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql'); + $sTemplate = file_get_contents(CONST_DataDir.'/sql/update-postcodes.sql'); runSQLScript($sTemplate, true, true); } - $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc'; + $sTemporaryFile = CONST_InstallDir.'/osmosischange.osc'; $bHaveDiff = false; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; $sContentURL = ''; @@@ -275,7 -282,7 +283,7 @@@ if ($bHaveDiff) if ($aResult['recompute-word-counts']) { info('Recompute frequency of full-word search terms'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); + $sTemplate = file_get_contents(CONST_DataDir.'/sql/words_from_search_name.sql'); runSQLScript($sTemplate, true, true); } @@@ -292,8 -299,9 +300,9 @@@ if ($aResult['index']) } if ($aResult['update-address-levels']) { - echo 'Updating address levels from '.CONST_Address_Level_Config.".\n"; - $oAlParser = new \Nominatim\Setup\AddressLevelParser(CONST_Address_Level_Config); + $sAddressLevelConfig = getSettingConfig('ADDRESS_LEVEL_CONFIG', 'address-levels.json'); + echo 'Updating address levels from '.$sAddressLevelConfig.".\n"; + $oAlParser = new \Nominatim\Setup\AddressLevelParser($sAddressLevelConfig); $oAlParser->createTable($oDB, 'address_levels'); } @@@ -316,17 -324,17 +325,17 @@@ if ($aResult['recompute-importance']) if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // - if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { + if (strpos($sBaseURL, 'download.geofabrik.de') !== false && getSetting('REPLICATION_UPDATE_INTERVAL') < 86400) { fail('Error: Update interval too low for download.geofabrik.de. ' . "Please check install documentation (https://nominatim.org/release-docs/latest/admin/Import-and-Update#setting-up-the-update-process)\n"); } - $sImportFile = CONST_InstallPath.'/osmosischange.osc'; + $sImportFile = CONST_InstallDir.'/osmosischange.osc'; - $oCMDDownload = (new \Nominatim\Shell(CONST_Pyosmium_Binary)) - ->addParams('--server', CONST_Replication_Url) + $oCMDDownload = (new \Nominatim\Shell($sPyosmiumBin)) + ->addParams('--server', $sBaseURL) ->addParams('--outfile', $sImportFile) - ->addParams('--size', CONST_Replication_Max_Diff_size); + ->addParams('--size', getSetting('REPLICATION_MAX_DIFF')); $oCMDImport = (clone $oOsm2pgsqlCmd)->addParams($sImportFile); @@@ -346,7 -354,7 +355,7 @@@ if ($aLastState['indexed']) { // Sleep if the update interval has not yet been reached. - $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval; + $fNextUpdate = $aLastState['unix_ts'] + getSetting('REPLICATION_UPDATE_INTERVAL'); if ($fNextUpdate > $fStartTime) { $iSleepTime = $fNextUpdate - $fStartTime; echo "Waiting for next update for $iSleepTime sec."; @@@ -367,8 -375,9 +376,9 @@@ exec($oCMD->escapedCmd(), $aOutput, $iResult); if ($iResult == 3) { - echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n"; - sleep(CONST_Replication_Recheck_Interval); + $sSleep = getSetting('REPLICATION_RECHECK_INTERVAL'); + echo 'No new updates. Sleeping for '.$sSleep." sec.\n"; + sleep($sSleep); } elseif ($iResult != 0) { echo 'ERROR: updates failed.'; exit($iResult); @@@ -380,7 -389,7 +390,7 @@@ // get the newest object from the diff file $sBatchEnd = 0; $iRet = 0; - $oCMD = new \Nominatim\Shell(CONST_BasePath.'/utils/osm_file_date.py', $sImportFile); + $oCMD = new \Nominatim\Shell(CONST_BinDir.'/osm_file_date.py', $sImportFile); exec($oCMD->escapedCmd(), $sBatchEnd, $iRet); if ($iRet == 5) { echo "Diff file is empty. skipping import.\n";