From: Sarah Hoffmann Date: Sun, 2 Mar 2014 16:53:45 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~549 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/aade8c75cb6036c85f4df36c810fbbcc07f43907?hp=-c Merge remote-tracking branch 'upstream/master' --- aade8c75cb6036c85f4df36c810fbbcc07f43907 diff --combined lib/Geocode.php index 532a7720,9b7b9d6c..b65cc277 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@@ -15,7 -15,7 +15,7 @@@ protected $aExcludePlaceIDs = array(); protected $bDeDupe = true; - protected $bReverseInPlan = false; + protected $bReverseInPlan = true; protected $iLimit = 20; protected $iFinalLimit = 10; @@@ -27,8 -27,6 +27,8 @@@ protected $bBoundedSearch = false; protected $aViewBox = false; + protected $sViewboxSmallSQL = false; + protected $sViewboxLargeSQL = false; protected $aRoutePoints = false; protected $iMaxRank = 20; @@@ -280,16 -278,12 +280,16 @@@ // Get the details for display (is this a redundant extra step?) $sPlaceIDs = join(',',$aPlaceIDs); + $sImportanceSQL = ''; + if ($this->sViewboxSmallSQL) $sImportanceSQL .= " case when ST_Contains($this->sViewboxSmallSQL, ST_Collect(centroid)) THEN 1 ELSE 0.75 END * "; + if ($this->sViewboxLargeSQL) $sImportanceSQL .= " case when ST_Contains($this->sViewboxLargeSQL, ST_Collect(centroid)) THEN 1 ELSE 0.75 END * "; + $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; - $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance, "; + $sSQL .= $sImportanceSQL."coalesce(importance,0.75-(rank_search::float/40)) as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "(extratags->'place') as extra_place "; $sSQL .= "from placex where place_id in ($sPlaceIDs) "; @@@ -314,21 -308,20 +314,21 @@@ $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; - $sSQL .= "-0.15 as importance, "; + $sSQL .= $sImportanceSQL."-1.15 as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "null as extra_place "; $sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) "; $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank "; $sSQL .= "group by place_id"; - if (!$this->bDeDupe) $sSQL .= ",place_id"; + if (!$this->bDeDupe) $sSQL .= ",place_id "; + /* $sSQL .= " union "; $sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; - $sSQL .= "-0.10 as importance, "; + $sSQL .= $sImportanceSQL."-1.10 as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "null as extra_place "; $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) "; @@@ -336,10 -329,9 +336,10 @@@ $sSQL .= "group by place_id"; if (!$this->bDeDupe) $sSQL .= ",place_id"; $sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) "; + */ } - $sSQL .= "order by importance desc"; + $sSQL .= " order by importance desc"; if (CONST_Debug) { echo "
"; var_dump($sSQL); } $aSearchResults = $this->oDB->getAll($sSQL); @@@ -394,8 -386,7 +394,8 @@@ } // Hack to make it handle "new york, ny" (and variants) correctly - $sQuery = str_ireplace(array('New York, ny','new york, new york', 'New York ny','new york new york'), 'new york city, ny', $this->sQuery); + //$sQuery = str_ireplace(array('New York, ny','new york, new york', 'New York ny','new york new york'), 'new york city, ny', $this->sQuery); + $sQuery = $this->sQuery; // Conflicts between US state abreviations and various words for 'the' in different languages if (isset($this->aLangPrefOrder['name:en'])) @@@ -406,7 -397,7 +406,7 @@@ } // View Box SQL - $sViewboxCentreSQL = $sViewboxSmallSQL = $sViewboxLargeSQL = false; + $sViewboxCentreSQL; $bBoundingBoxSearch = false; if ($this->aViewBox) { @@@ -417,8 -408,8 +417,8 @@@ $aBigViewBox[1] = $this->aViewBox[1] + $fWidth; $aBigViewBox[3] = $this->aViewBox[3] - $fWidth; - $sViewboxSmallSQL = "ST_SetSRID(ST_MakeBox2D(ST_Point(".(float)$this->aViewBox[0].",".(float)$this->aViewBox[1]."),ST_Point(".(float)$this->aViewBox[2].",".(float)$this->aViewBox[3].")),4326)"; - $sViewboxLargeSQL = "ST_SetSRID(ST_MakeBox2D(ST_Point(".(float)$aBigViewBox[0].",".(float)$aBigViewBox[1]."),ST_Point(".(float)$aBigViewBox[2].",".(float)$aBigViewBox[3].")),4326)"; + $this->sViewboxSmallSQL = "ST_SetSRID(ST_MakeBox2D(ST_Point(".(float)$this->aViewBox[0].",".(float)$this->aViewBox[1]."),ST_Point(".(float)$this->aViewBox[2].",".(float)$this->aViewBox[3].")),4326)"; + $this->sViewboxLargeSQL = "ST_SetSRID(ST_MakeBox2D(ST_Point(".(float)$aBigViewBox[0].",".(float)$aBigViewBox[1]."),ST_Point(".(float)$aBigViewBox[2].",".(float)$aBigViewBox[3].")),4326)"; $bBoundingBoxSearch = $this->bBoundedSearch; } @@@ -436,20 -427,20 +436,20 @@@ $sViewboxCentreSQL .= ")'::geometry,4326)"; $sSQL = "select st_buffer(".$sViewboxCentreSQL.",".(float)($_GET['routewidth']/69).")"; - $sViewboxSmallSQL = $this->oDB->getOne($sSQL); - if (PEAR::isError($sViewboxSmallSQL)) + $this->sViewboxSmallSQL = $this->oDB->getOne($sSQL); + if (PEAR::isError($this->sViewboxSmallSQL)) { - failInternalError("Could not get small viewbox.", $sSQL, $sViewboxSmallSQL); + failInternalError("Could not get small viewbox.", $sSQL, $this->sViewboxSmallSQL); } - $sViewboxSmallSQL = "'".$sViewboxSmallSQL."'::geometry"; + $this->sViewboxSmallSQL = "'".$this->sViewboxSmallSQL."'::geometry"; $sSQL = "select st_buffer(".$sViewboxCentreSQL.",".(float)($_GET['routewidth']/30).")"; - $sViewboxLargeSQL = $this->oDB->getOne($sSQL); - if (PEAR::isError($sViewboxLargeSQL)) + $this->sViewboxLargeSQL = $this->oDB->getOne($sSQL); + if (PEAR::isError($this->sViewboxLargeSQL)) { - failInternalError("Could not get large viewbox.", $sSQL, $sViewboxLargeSQL); + failInternalError("Could not get large viewbox.", $sSQL, $this->sViewboxLargeSQL); } - $sViewboxLargeSQL = "'".$sViewboxLargeSQL."'::geometry"; + $this->sViewboxLargeSQL = "'".$this->sViewboxLargeSQL."'::geometry"; $bBoundingBoxSearch = $this->bBoundedSearch; } @@@ -820,7 -811,6 +820,7 @@@ $aSearch['sType'] = $aSearchTerm['type']; if (sizeof($aSearch['aName'])) $aSearch['sOperator'] = 'name'; else $aSearch['sOperator'] = 'near'; // near = in for the moment + if (strlen($aSearchTerm['operator']) == 0) $aSearch['iSearchRank'] += 1; // Do we have a shortcut id? if ($aSearch['sOperator'] == 'name') @@@ -1100,7 -1090,7 +1100,7 @@@ { $sSQL = "select place_id from place_classtype_".$aSearch['sClass']."_".$aSearch['sType']." ct"; if ($sCountryCodesSQL) $sSQL .= " join placex using (place_id)"; - $sSQL .= " where st_contains($sViewboxSmallSQL, ct.centroid)"; + $sSQL .= " where st_contains($this->sViewboxSmallSQL, ct.centroid)"; if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; if (sizeof($this->aExcludePlaceIDs)) { @@@ -1118,7 -1108,7 +1118,7 @@@ { $sSQL = "select place_id from place_classtype_".$aSearch['sClass']."_".$aSearch['sType']." ct"; if ($sCountryCodesSQL) $sSQL .= " join placex using (place_id)"; - $sSQL .= " where st_contains($sViewboxLargeSQL, ct.centroid)"; + $sSQL .= " where st_contains($this->sViewboxLargeSQL, ct.centroid)"; if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; if ($sViewboxCentreSQL) $sSQL .= " order by st_distance($sViewboxCentreSQL, ct.centroid) asc"; $sSQL .= " limit $this->iLimit"; @@@ -1129,7 -1119,7 +1129,7 @@@ else { $sSQL = "select place_id from placex where class='".$aSearch['sClass']."' and type='".$aSearch['sType']."'"; - $sSQL .= " and st_contains($sViewboxSmallSQL, geometry) and linked_place_id is null"; + $sSQL .= " and st_contains($this->sViewboxSmallSQL, geometry) and linked_place_id is null"; if ($sCountryCodesSQL) $sSQL .= " and calculated_country_code in ($sCountryCodesSQL)"; if ($sViewboxCentreSQL) $sSQL .= " order by st_distance($sViewboxCentreSQL, centroid) asc"; $sSQL .= " limit $this->iLimit"; @@@ -1149,7 -1139,7 +1149,7 @@@ // TODO: filter out the pointless search terms (2 letter name tokens and less) // they might be right - but they are just too darned expensive to run if (sizeof($aSearch['aName'])) $aTerms[] = "name_vector @> ARRAY[".join($aSearch['aName'],",")."]"; - if (sizeof($aSearch['aNameNonSearch'])) $aTerms[] = "array_cat(name_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aNameNonSearch'],",")."]"; + //if (sizeof($aSearch['aNameNonSearch'])) $aTerms[] = "array_cat(name_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aNameNonSearch'],",")."]"; if (sizeof($aSearch['aAddress']) && $aSearch['aName'] != $aSearch['aAddress']) { // For infrequent name terms disable index usage for address @@@ -1157,13 -1147,12 +1157,13 @@@ sizeof($aSearch['aName']) == 1 && $aWordFrequencyScores[$aSearch['aName'][reset($aSearch['aName'])]] < CONST_Search_NameOnlySearchFrequencyThreshold) { - $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join(array_merge($aSearch['aAddress'],$aSearch['aAddressNonSearch']),",")."]"; + //$aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join(array_merge($aSearch['aAddress'],$aSearch['aAddressNonSearch']),",")."]"; + $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddress'],",")."]"; } else { $aTerms[] = "nameaddress_vector @> ARRAY[".join($aSearch['aAddress'],",")."]"; - if (sizeof($aSearch['aAddressNonSearch'])) $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddressNonSearch'],",")."]"; + //if (sizeof($aSearch['aAddressNonSearch'])) $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddressNonSearch'],",")."]"; } } if ($aSearch['sCountryCode']) $aTerms[] = "country_code = '".pg_escape_string($aSearch['sCountryCode'])."'"; @@@ -1182,12 -1171,12 +1182,12 @@@ $aTerms[] = "country_code in ($sCountryCodesSQL)"; } - if ($bBoundingBoxSearch) $aTerms[] = "centroid && $sViewboxSmallSQL"; + if ($bBoundingBoxSearch) $aTerms[] = "centroid && $this->sViewboxSmallSQL"; if ($sNearPointSQL) $aOrder[] = "ST_Distance($sNearPointSQL, centroid) asc"; $sImportanceSQL = '(case when importance = 0 OR importance IS NULL then 0.75-(search_rank::float/40) else importance end)'; - if ($sViewboxSmallSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxSmallSQL, centroid) THEN 1 ELSE 0.5 END"; - if ($sViewboxLargeSQL) $sImportanceSQL .= " * case when ST_Contains($sViewboxLargeSQL, centroid) THEN 1 ELSE 0.5 END"; + if ($this->sViewboxSmallSQL) $sImportanceSQL .= " * case when ST_Contains($this->sViewboxSmallSQL, centroid) THEN 1 ELSE 0.5 END"; + if ($this->sViewboxLargeSQL) $sImportanceSQL .= " * case when ST_Contains($this->sViewboxLargeSQL, centroid) THEN 1 ELSE 0.5 END"; $aOrder[] = "$sImportanceSQL DESC"; if (sizeof($aSearch['aFullNameAddress'])) { @@@ -1251,7 -1240,6 +1251,7 @@@ $aPlaceIDs = $this->oDB->getCol($sSQL); // If not try the aux fallback table + /* if (!sizeof($aPlaceIDs)) { $sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'"; @@@ -1263,7 -1251,6 +1263,7 @@@ if (CONST_Debug) var_dump($sSQL); $aPlaceIDs = $this->oDB->getCol($sSQL); } + */ if (!sizeof($aPlaceIDs)) { @@@ -1515,12 -1502,10 +1515,12 @@@ { preg_match_all('/(-?[0-9.]+) (-?[0-9.]+)/',$aMatch[1],$aPolyPoints,PREG_SET_ORDER); } + /* elseif (preg_match('#MULTIPOLYGON\\(\\(\\(([- 0-9.,]+)#',$aPointPolygon['astext'],$aMatch)) { preg_match_all('/(-?[0-9.]+) (-?[0-9.]+)/',$aMatch[1],$aPolyPoints,PREG_SET_ORDER); } + */ elseif (preg_match('#POINT\\((-?[0-9.]+) (-?[0-9.]+)\\)#',$aPointPolygon['astext'],$aMatch)) { $fRadius = 0.01; @@@ -1596,7 -1581,7 +1596,7 @@@ $aResult['aPolyPoints'][] = array($aPoint[1], $aPoint[2]); } } - $aResult['aBoundingBox'] = array($aPointPolygon['minlat'],$aPointPolygon['maxlat'],$aPointPolygon['minlon'],$aPointPolygon['maxlon']); + $aResult['aBoundingBox'] = array((string)$aPointPolygon['minlat'],(string)$aPointPolygon['maxlat'],(string)$aPointPolygon['minlon'],(string)$aPointPolygon['maxlon']); } // Is there an icon set for this type of result? diff --combined utils/setup.php index e23a2211,8d0a20b8..b96e4e7e --- a/utils/setup.php +++ b/utils/setup.php @@@ -147,7 -147,15 +147,15 @@@ pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql'); - pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql'); + pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); + if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) + { + pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); + } + else + { + echo "WARNING: external UK postcode table not found.\n"; + } pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); @@@ -174,9 -182,8 +182,9 @@@ { $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } + $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data'; $osm2pgsql .= ' -lsc -O gazetteer --hstore'; - $osm2pgsql .= ' -C '.$iCacheMemory; + $osm2pgsql .= ' -C 18000'; $osm2pgsql .= ' -P '.$aDSNInfo['port']; $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; passthruCheckReturn($osm2pgsql); @@@ -502,7 -509,7 +510,7 @@@ $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,"; $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,"; $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y "; - $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; + $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x"; if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; @@@ -678,10 -685,29 +686,29 @@@ // Convert database DSN to psql parameters $aDSNInfo = DB::parseDSN(CONST_Database_DSN); if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename; + $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; + + $ahGzipPipes = null; + if (preg_match('/\\.gz$/', $sFilename)) + { + $aDescriptors = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); + if (!is_resource($hGzipProcess)) fail('unable to start zcat'); + $aReadPipe = $ahGzipPipes[1]; + fclose($ahGzipPipes[0]); + } + else + { + $sCMD .= ' -f '.$sFilename; + $aReadPipe = array('pipe', 'r'); + } $aDescriptors = array( - 0 => array('pipe', 'r'), + 0 => $aReadPipe, 1 => array('pipe', 'w'), 2 => array('file', '/dev/null', 'a') ); @@@ -689,7 -715,6 +716,6 @@@ $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); if (!is_resource($hProcess)) fail('unable to start pgsql'); - fclose($ahPipes[0]); // TODO: error checking while(!feof($ahPipes[1])) @@@ -699,6 -724,12 +725,12 @@@ fclose($ahPipes[1]); proc_close($hProcess); + if ($ahGzipPipes) + { + fclose($ahGzipPipes[1]); + proc_close($hGzipProcess); + } + } function pgsqlRunScript($sScript)