From: Sarah Hoffmann Date: Wed, 23 Apr 2014 19:46:08 +0000 (+0200) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~541 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/ed65ad2f5abc5915c6b6ce4e8fdb876979cf151b?ds=inline;hp=-c Merge remote-tracking branch 'upstream/master' --- ed65ad2f5abc5915c6b6ce4e8fdb876979cf151b diff --combined lib/log.php index 501259d0,37d83c47..59e04d17 --- a/lib/log.php +++ b/lib/log.php @@@ -3,7 -3,7 +3,7 @@@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array()) { $aStartTime = explode('.',microtime(true)); - if (!$aStartTime[1]) $aStartTime[1] = '0'; + if (!isset($aStartTime[1])) $aStartTime[1] = '0'; $sOutputFormat = ''; if (isset($_GET['format'])) $sOutputFormat = $_GET['format']; @@@ -17,15 -17,15 +17,15 @@@ if (CONST_Log_DB) { - // Log - if ($sType == 'search') - { - $oDB->query('insert into query_log values ('.getDBQuoted($hLog[0]).','.getDBQuoted($hLog[3]).','.getDBQuoted($hLog[1]).')'); - } - - $sSQL = 'insert into new_query_log (type,starttime,query,ipaddress,useragent,language,format)'; + if (isset($_GET['email'])) + $sUserAgent = $_GET['email']; + elseif (isset($_SERVER['HTTP_REFERER'])) + $sUserAgent = $_SERVER['HTTP_REFERER']; + else + $sUserAgent = $_SERVER['HTTP_USER_AGENT']; + $sSQL = 'insert into new_query_log (type,starttime,query,ipaddress,useragent,language,format,searchterm)'; $sSQL .= ' values ('.getDBQuoted($sType).','.getDBQuoted($hLog[0]).','.getDBQuoted($hLog[2]); - $sSQL .= ','.getDBQuoted($hLog[1]).','.getDBQuoted($_SERVER['HTTP_USER_AGENT']).','.getDBQuoted(join(',',$aLanguageList)).','.getDBQuoted($sOutputFormat).')'; + $sSQL .= ','.getDBQuoted($hLog[1]).','.getDBQuoted($sUserAgent).','.getDBQuoted(join(',',$aLanguageList)).','.getDBQuoted($sOutputFormat).','.getDBQuoted($hLog[3]).')'; $oDB->query($sSQL); } @@@ -59,6 -59,12 +59,6 @@@ if (CONST_Log_DB) { - $sSQL = 'update query_log set endtime = '.getDBQuoted($sEndTime).', results = '.$iNumResults; - $sSQL .= ' where starttime = '.getDBQuoted($hLog[0]); - $sSQL .= ' and ipaddress = '.getDBQuoted($hLog[1]); - $sSQL .= ' and query = '.getDBQuoted($hLog[3]); - $oDB->query($sSQL); - $sSQL = 'update new_query_log set endtime = '.getDBQuoted($sEndTime).', results = '.$iNumResults; $sSQL .= ' where starttime = '.getDBQuoted($hLog[0]); $sSQL .= ' and ipaddress = '.getDBQuoted($hLog[1]); diff --combined sql/functions.sql index 092aa8ac,79ba1a40..980528ae --- a/sql/functions.sql +++ b/sql/functions.sql @@@ -922,11 -922,6 +922,11 @@@ DECLAR BEGIN --DEBUG: RAISE WARNING '% %',NEW.osm_type,NEW.osm_id; + -- remove operator tag for most places, messes too much with search_name indexes + IF NEW.class not in ('amenity', 'shop') THEN + NEW.name := delete(NEW.name, 'operator'); + END IF; + -- just block these IF NEW.class in ('landuse','natural') and NEW.name is null THEN -- RAISE WARNING 'empty landuse %',NEW.osm_id; @@@ -1605,17 -1600,13 +1605,13 @@@ BEGI END IF; -- merge in extra tags - IF NOT linkedPlacex.extratags IS NULL THEN - NEW.extratags := linkedPlacex.extratags || NEW.extratags; - END IF; - - IF NOT NEW.extratags ? linkedPlacex.class THEN - NEW.extratags := NEW.extratags || hstore(linkedPlacex.class, linkedPlacex.type); - END IF; + NEW.extratags := hstore(linkedPlacex.class, linkedPlacex.type) || coalesce(linkedPlacex.extratags, ''::hstore) || coalesce(NEW.extratags, ''::hstore); -- mark the linked place (excludes from search results) UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id; + -- keep a note of the node id in case we need it for wikipedia in a bit + linked_node_id := linkedPlacex.osm_id; END LOOP; END LOOP; @@@ -1644,13 -1635,7 +1640,7 @@@ END IF; -- merge in extra tags - IF NOT linkedPlacex.extratags IS NULL THEN - NEW.extratags := linkedPlacex.extratags || NEW.extratags; - END IF; - - IF NOT NEW.extratags ? linkedPlacex.class THEN - NEW.extratags := NEW.extratags || hstore(linkedPlacex.class, linkedPlacex.type); - END IF; + NEW.extratags := hstore(linkedPlacex.class, linkedPlacex.type) || coalesce(linkedPlacex.extratags, ''::hstore) || coalesce(NEW.extratags, ''::hstore); -- mark the linked place (excludes from search results) UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id; @@@ -1692,11 -1677,7 +1682,7 @@@ name_vector := make_keywords(NEW.name); -- merge in extra tags - NEW.extratags := linkedPlacex.extratags || NEW.extratags; - - IF NOT NEW.extratags ? linkedPlacex.class THEN - NEW.extratags := NEW.extratags || hstore(linkedPlacex.class, linkedPlacex.type); - END IF; + NEW.extratags := hstore(linkedPlacex.class, linkedPlacex.type) || coalesce(linkedPlacex.extratags, ''::hstore) || coalesce(NEW.extratags, ''::hstore); -- mark the linked place (excludes from search results) UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id; @@@ -2036,11 -2017,6 +2022,11 @@@ BEGI --DEBUG: RAISE WARNING '%', existingplacex; END IF; + -- remove operator tag for most places, messes too much with search_name indexes + IF NEW.class not in ('amenity', 'shop') THEN + NEW.name := delete(NEW.name, 'operator'); + END IF; + -- Just block these - lots and pointless IF NEW.class in ('landuse','natural') and NEW.name is null THEN -- if the name tag was removed, older versions might still be lurking in the place table @@@ -2231,12 -2207,6 +2217,12 @@@ END IF; + -- refuse to update multiplpoygons with too many objects, too much of a performance hit + IF ST_NumGeometries(NEW.geometry) > 2000 THEN + RAISE WARNING 'Dropping update of % % because of geometry complexity.', NEW.osm_type, NEW.osm_id; + RETURN NULL; + END IF; + IF coalesce(existing.name::text, '') != coalesce(NEW.name::text, '') OR coalesce(existing.extratags::text, '') != coalesce(NEW.extratags::text, '') OR coalesce(existing.housenumber, '') != coalesce(NEW.housenumber, '') @@@ -3096,6 -3066,7 +3082,7 @@@ DECLAR diameter FLOAT; rank INTEGER; BEGIN + UPDATE placex SET indexed_status = 2 WHERE place_id = placeid; SELECT geometry, rank_search FROM placex WHERE place_id = placeid INTO placegeom, rank; IF placegeom IS NOT NULL AND ST_IsValid(placegeom) THEN IF ST_GeometryType(placegeom) in ('ST_Polygon','ST_MultiPolygon') THEN diff --combined utils/setup.php index b96e4e7e,9e946035..edf0b14d --- a/utils/setup.php +++ b/utils/setup.php @@@ -164,6 -164,13 +164,13 @@@ { pgsqlRunScript('update country_name set partition = 0'); } + + // the following will be needed by create_functions later but + // is only defined in the subsequently called create_tables. + // Create dummies here that will be overwritten by the proper + // versions in create-tables. + pgsqlRunScript('CREATE TABLE place_boundingbox ()'); + pgsqlRunScript('create type wikipedia_article_match as ()'); } if ($aCMDResult['import-data'] || $aCMDResult['all']) @@@ -182,9 -189,8 +189,9 @@@ { $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } + $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data'; $osm2pgsql .= ' -lsc -O gazetteer --hstore'; - $osm2pgsql .= ' -C '.$iCacheMemory; + $osm2pgsql .= ' -C 18000'; $osm2pgsql .= ' -P '.$aDSNInfo['port']; $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; passthruCheckReturn($osm2pgsql); @@@ -510,7 -516,7 +517,7 @@@ $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,"; $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,"; $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y "; - $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; + $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x"; if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; @@@ -724,7 -730,11 +731,11 @@@ } fclose($ahPipes[1]); - proc_close($hProcess); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) + { + fail("pgsql returned with error code ($iReturn)"); + } if ($ahGzipPipes) { fclose($ahGzipPipes[1]); @@@ -739,6 -749,7 +750,7 @@@ $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; + $sCMD .= ' -v ON_ERROR_STOP=1'; $aDescriptors = array( 0 => array('pipe', 'r'), 1 => STDOUT, @@@ -754,7 -765,11 +766,11 @@@ $sScript = substr($sScript, $written); } fclose($ahPipes[0]); - proc_close($hProcess); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) + { + fail("pgsql returned with error code ($iReturn)"); + } } function pgsqlRunRestoreData($sDumpFile) @@@ -782,7 -797,11 +798,11 @@@ } fclose($ahPipes[1]); - proc_close($hProcess); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) + { + fail("pgsql returned with error code ($iReturn)"); + } } function pgsqlRunDropAndRestore($sDumpFile) @@@ -810,7 -829,11 +830,11 @@@ } fclose($ahPipes[1]); - proc_close($hProcess); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) + { + fail("pgsql returned with error code ($iReturn)"); + } } function passthruCheckReturn($cmd)