From: Sarah Hoffmann Date: Fri, 22 Jan 2016 22:34:00 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~433 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/521d6294993f8d8d46d93d514c08556d0c924786?ds=sidebyside;hp=-c Merge remote-tracking branch 'upstream/master' --- 521d6294993f8d8d46d93d514c08556d0c924786 diff --combined lib/lib.php index dfd7008d,50cf3dc4..d1afedbd --- a/lib/lib.php +++ b/lib/lib.php @@@ -184,7 -184,7 +184,7 @@@ { $aResult = array(array(join(' ',$aWords))); $sFirstToken = ''; - if ($iDepth < 8) { + if ($iDepth < 7) { while(sizeof($aWords) > 1) { $sWord = array_shift($aWords); @@@ -970,7 -970,7 +970,7 @@@ } // degrees decimal seconds // N 40 26 46 W 79 58 56 - // N 40° 26′ 46″ W, 79° 58′ 56″ + // N 40° 26′ 46″, W 79° 58′ 56″ // 1 2 3 4 5 6 7 8 elseif (preg_match('/\\b([NS])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*[, ]+([EW])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*\\b/', $sQuery, $aData)) { diff --combined sql/functions.sql index 1efb1899,bd64697a..18e8bdbc --- a/sql/functions.sql +++ b/sql/functions.sql @@@ -2055,12 -2055,6 +2055,12 @@@ BEGI END IF; + -- refuse to update multiplpoygons with too many objects, too much of a performance hit + IF ST_NumGeometries(NEW.geometry) > 2000 THEN + RAISE WARNING 'Dropping update of % % because of geometry complexity.', NEW.osm_type, NEW.osm_id; + RETURN NULL; + END IF; + IF coalesce(existing.name::text, '') != coalesce(NEW.name::text, '') OR coalesce(existing.extratags::text, '') != coalesce(NEW.extratags::text, '') OR coalesce(existing.housenumber, '') != coalesce(NEW.housenumber, '') @@@ -2124,28 -2118,27 +2124,27 @@@ END; $$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION get_name_by_language(name hstore, languagepref TEXT[]) RETURNS TEXT AS $$ DECLARE - search TEXT[]; - found BOOLEAN; + result TEXT; BEGIN - IF name is null THEN RETURN null; END IF; - search := languagepref; - - FOR j IN 1..array_upper(search, 1) LOOP - IF name ? search[j] AND trim(name->search[j]) != '' THEN - return trim(name->search[j]); + FOR j IN 1..array_upper(languagepref,1) LOOP + IF name ? languagepref[j] THEN + result := trim(name->languagepref[j]); + IF result != '' THEN + return result; + END IF; END IF; END LOOP; -- anything will do as a fallback - just take the first name type thing there is - search := avals(name); - RETURN search[1]; + RETURN trim((avals(name))[1]); END; $$ LANGUAGE plpgsql IMMUTABLE; diff --combined utils/setup.php index 0ecf5e6d,eff7b71b..01bf1134 --- a/utils/setup.php +++ b/utils/setup.php @@@ -39,6 -39,7 +39,7 @@@ array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'), array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); @@@ -133,7 -134,7 +134,7 @@@ pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql'); } else { - pgsqlRunScript('CREATE EXTENSION postgis'); + pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); } if ($fPostgisVersion < 2.1) { // Function was renamed in 2.1 and throws an annoying deprecation warning @@@ -201,7 -202,7 +202,7 @@@ if (CONST_Tablespace_Place_Index) $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.$iCacheMemory; + $osm2pgsql .= ' -C 25000'; $osm2pgsql .= ' -P '.$aDSNInfo['port']; $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; passthruCheckReturn($osm2pgsql); @@@ -548,7 -549,7 +549,7 @@@ $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,"; $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,"; $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y "; - $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; + $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x"; if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; @@@ -557,7 -558,7 +558,7 @@@ if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); } - if ($aCMDResult['osmosis-init'] || $aCMDResult['all']) + if ($aCMDResult['osmosis-init'] || ($aCMDResult['all'] && !$aCMDResult['drop'])) // no use doing osmosis-init when dropping update tables { $bDidSomething = true; $oDB =& getDB(); @@@ -729,6 -730,69 +730,69 @@@ } } + if ($aCMDResult['drop']) + { + // The implementation is potentially a bit dangerous because it uses + // a positive selection of tables to keep, and deletes everything else. + // Including any tables that the unsuspecting user might have manually + // created. USE AT YOUR OWN PERIL. + $bDidSomething = true; + + // tables we want to keep. everything else goes. + $aKeepTables = array( + "*columns", + "import_polygon_*", + "import_status", + "place_addressline", + "location_property*", + "placex", + "search_name", + "seq_*", + "word", + "query_log", + "new_query_log", + "gb_postcode", + "spatial_ref_sys", + "country_name", + "place_classtype_*" + ); + + $oDB =& getDB(); + $aDropTables = array(); + $aHaveTables = $oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"); + if (PEAR::isError($aHaveTables)) + { + fail($aPartitions->getMessage()); + } + foreach($aHaveTables as $sTable) + { + $bFound = false; + foreach ($aKeepTables as $sKeep) + { + if (fnmatch($sKeep, $sTable)) + { + $bFound = true; + break; + } + } + if (!$bFound) array_push($aDropTables, $sTable); + } + + foreach ($aDropTables as $sDrop) + { + if ($aCMDResult['verbose']) echo "dropping table $sDrop\n"; + @pg_query($oDB->connection, "DROP TABLE $sDrop CASCADE"); + // ignore warnings/errors as they might be caused by a table having + // been deleted already by CASCADE + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File)) + { + if ($aCMDResult['verbose']) echo "deleting ".CONST_Osm2pgsql_Flatnode_File."\n"; + unlink(CONST_Osm2pgsql_Flatnode_File); + } + } + if (!$bDidSomething) { showUsage($aCMDOptions, true); diff --combined utils/update.php index 9ce0f334,cc275463..c6de7af6 --- a/utils/update.php +++ b/utils/update.php @@@ -47,21 -47,20 +47,19 @@@ showUsage($aCMDOptions, true, 'Select either import of hourly or daily'); } - if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1; if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0; /* // Lock to prevent multiple copies running if (exec('/bin/ps uww | grep '.basename(__FILE__).' | grep -v /dev/null | grep -v grep -c', $aOutput2, $iResult) > 1) { - echo "Copy already running\n"; - exit; + fail("Copy already running\n"); } if (!isset($aResult['max-load'])) $aResult['max-load'] = 1.9; if (!isset($aResult['max-blocking'])) $aResult['max-blocking'] = 3; if (getBlockingProcesses() > $aResult['max-blocking']) { - echo "Too many blocking processes for import\n"; - exit; + fail("Too many blocking processes for import\n"); } */ @@@ -119,8 -118,7 +117,7 @@@ $sNextFile = $aResult['import-diff']; if (!file_exists($sNextFile)) { - echo "Cannot open $sNextFile\n"; - exit; + fail("Cannot open $sNextFile\n"); } // Don't update the import status - we don't know what this file contains $sUpdateSQL = 'update import_status set lastimportdate = now() where false'; @@@ -136,8 -134,7 +133,7 @@@ if ($iErrorLevel) { - echo "Error from osm2pgsql, $iErrorLevel\n"; - exit; + fail("Error from osm2pgsql, $iErrorLevel\n"); } // Move the date onwards @@@ -204,8 -201,7 +200,7 @@@ exec($sCMD, $sJunk, $iErrorLevel); if ($iErrorLevel) { - echo "Error converting osm to osc, osmosis returned: $iErrorLevel\n"; - exit; + fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n"); } } else @@@ -220,8 -216,7 +215,7 @@@ $hProc = proc_open($sCMD, $aSpec, $aPipes); if (!is_resource($hProc)) { - echo "Error converting osm to osc, osmosis failed\n"; - exit; + fail("Error converting osm to osc, osmosis failed\n"); } fwrite($aPipes[0], $sModifyXMLstr); fclose($aPipes[0]); @@@ -236,7 -231,7 +230,7 @@@ echo "Error converting osm to osc, osmosis returned: $iError\n"; echo $sOut; echo $sErrors; - exit; + exit(-1); } } @@@ -246,8 -241,7 +240,7 @@@ exec($sCMD, $sJunk, $iErrorLevel); if ($iErrorLevel) { - echo "osm2pgsql exited with error level $iErrorLevel\n"; - exit; + fail("osm2pgsql exited with error level $iErrorLevel\n"); } } @@@ -256,8 -250,7 +249,7 @@@ $pgver = (float) CONST_Postgresql_Version; if ($pgver < 9.3) { - echo "ERROR: deduplicate is only currently supported in postgresql 9.3"; - exit; + fail("ERROR: deduplicate is only currently supported in postgresql 9.3"); } $oDB =& getDB(); @@@ -280,7 -273,7 +272,7 @@@ if (PEAR::isError($aTokenSet)) { var_dump($aTokenSet, $sSQL); - exit; + exit(1); } $aKeep = array_shift($aTokenSet); @@@ -296,7 -289,7 +288,7 @@@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } $sSQL = "update search_name set"; @@@ -306,7 -299,7 +298,7 @@@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } $sSQL = "update location_area_country set"; @@@ -316,7 -309,7 +308,7 @@@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } foreach ($aPartitions as $sPartition) @@@ -328,7 -321,7 +320,7 @@@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } $sSQL = "update location_area_country set"; @@@ -338,7 -331,7 +330,7 @@@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } } @@@ -347,7 -340,7 +339,7 @@@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } } @@@ -356,7 -349,6 +348,7 @@@ if ($aResult['index']) { + if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1; passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']); } @@@ -364,8 -356,7 +356,7 @@@ { if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { - echo "Error: Update interval too low for download.geofabrik.de. Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n"; - exit; + fail("Error: Update interval too low for download.geofabrik.de. Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n"); } $sImportFile = CONST_BasePath.'/data/osmosischange.osc'; @@@ -374,7 -365,7 +365,7 @@@ $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile; $sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory; $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; - $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; + $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; if (!$aResult['no-npi']) { $sCMDIndex .= '-F '; } @@@ -459,16 -450,7 +450,16 @@@ $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory); // Index file - $sThisIndexCmd = $sCMDIndex; + if (!isset($aResult['index-instances'])) + { + if (getLoadAverage() < 24) + $iIndexInstances = 2; + else + $iIndexInstances = 1; + } else + $iIndexInstances = $aResult['index-instances']; + + $sThisIndexCmd = $sCMDIndex.' -t '.$iIndexInstances; $fCMDStartTime = time(); if (!$aResult['no-npi']) @@@ -536,7 -518,7 +527,7 @@@ $fDuration = time() - $fStartTime; echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n"; - if (!$aResult['import-osmosis-all']) exit; + if (!$aResult['import-osmosis-all']) exit(0); if ( CONST_Replication_Update_Interval > 60 ) { @@@ -558,7 -540,7 +549,7 @@@ if (PEAR::isError($iNPIID)) { var_dump($iNPIID); - exit; + exit(1); } $sConfigDirectory = CONST_BasePath.'/settings'; $sCMDImportTemplate = $sBasePath.'/nominatim/nominatim -d gazetteer -P 5433 -I -T '.$sBasePath.'/nominatim/partitionedtags.def -F '; @@@ -589,8 -571,7 +580,7 @@@ exec($sCMDImport, $sJunk, $iErrorLevel); if ($iErrorLevel) { - echo "Error: $iErrorLevel\n"; - exit; + fail("Error: $iErrorLevel\n"); } $sBatchEnd = $iNPIID; echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; diff --combined website/reverse.php index ee2fa475,5217f734..daf0a72d --- a/website/reverse.php +++ b/website/reverse.php @@@ -6,12 -6,24 +6,12 @@@ require_once(CONST_BasePath.'/lib/PlaceLookup.php'); require_once(CONST_BasePath.'/lib/ReverseGeocode.php'); - if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) - { - $fLoadAvg = getLoadAverage(); - if ($fLoadAvg > 2) sleep(60); - if ($fLoadAvg > 4) sleep(120); - if ($fLoadAvg > 6) - { - echo "Bulk User: Temporary block due to high server load\n"; - exit; - } - } - $oDB =& getDB(); ini_set('memory_limit', '200M'); // Format for output $sOutputFormat = 'xml'; - if (isset($_GET['format']) && ($_GET['format'] == 'xml' || $_GET['format'] == 'json' || $_GET['format'] == 'jsonv2')) + if (isset($_GET['format']) && ( $_GET['format'] == 'html' || $_GET['format'] == 'xml' || $_GET['format'] == 'json' || $_GET['format'] == 'jsonv2')) { $sOutputFormat = $_GET['format']; } @@@ -57,7 -69,6 +57,7 @@@ $aPlace = null; } + logEnd($oDB, $hLog, sizeof($aPlace)?1:0); if (CONST_Debug) {