From: Sarah Hoffmann Date: Tue, 11 Aug 2015 21:19:51 +0000 (+0200) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: deploy~438 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/fa0cf215e723401fec62a3b336a4ed8908c2bfed?ds=inline;hp=-c Merge remote-tracking branch 'upstream/master' Conflicts: website/reverse.php website/search.php --- fa0cf215e723401fec62a3b336a4ed8908c2bfed diff --combined lib/Geocode.php index 38f76142,8b94e549..a3ac4f0e --- a/lib/Geocode.php +++ b/lib/Geocode.php @@@ -6,6 -6,8 +6,8 @@@ protected $aLangPrefOrder = array(); protected $bIncludeAddressDetails = false; + protected $bIncludeExtraTags = false; + protected $bIncludeNameDetails = false; protected $bIncludePolygonAsPoints = false; protected $bIncludePolygonAsText = false; @@@ -16,7 -18,7 +18,7 @@@ protected $aExcludePlaceIDs = array(); protected $bDeDupe = true; - protected $bReverseInPlan = false; + protected $bReverseInPlan = true; protected $iLimit = 20; protected $iFinalLimit = 10; @@@ -68,6 -70,16 +70,16 @@@ return $this->bIncludeAddressDetails; } + function getIncludeExtraTags() + { + return $this->bIncludeExtraTags; + } + + function getIncludeNameDetails() + { + return $this->bIncludeNameDetails; + } + function setIncludePolygonAsPoints($b = true) { $this->bIncludePolygonAsPoints = $b; @@@ -214,6 -226,11 +226,11 @@@ function loadParamArray($aParams) { if (isset($aParams['addressdetails'])) $this->bIncludeAddressDetails = (bool)$aParams['addressdetails']; + if ((float) CONST_Postgresql_Version > 9.2) + { + if (isset($aParams['extratags'])) $this->bIncludeExtraTags = (bool)$aParams['extratags']; + if (isset($aParams['namedetails'])) $this->bIncludeNameDetails = (bool)$aParams['namedetails']; + } if (isset($aParams['bounded'])) $this->bBoundedSearch = (bool)$aParams['bounded']; if (isset($aParams['dedupe'])) $this->bDeDupe = (bool)$aParams['dedupe']; @@@ -389,6 -406,8 +406,8 @@@ $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; + if ($this->bIncludeExtraTags) $sSQL .= "hstore_to_json(extratags)::text as extra,"; + if ($this->bIncludeNameDetails) $sSQL .= "hstore_to_json(name)::text as names,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= $sImportanceSQL."coalesce(importance,0.75-(rank_search::float/40)) as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; @@@ -405,6 -424,8 +424,8 @@@ $sSQL .= ",langaddress "; $sSQL .= ",placename "; $sSQL .= ",ref "; + if ($this->bIncludeExtraTags) $sSQL .= ",extratags"; + if ($this->bIncludeNameDetails) $sSQL .= ",name"; $sSQL .= ",extratags->'place' "; if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank) @@@ -414,6 -435,8 +435,8 @@@ $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; + if ($this->bIncludeExtraTags) $sSQL .= "null as extra,"; + if ($this->bIncludeNameDetails) $sSQL .= "null as names,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= $sImportanceSQL."-1.15 as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; @@@ -422,12 -445,13 +445,14 @@@ $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank "; $sSQL .= "group by place_id"; if (!$this->bDeDupe) $sSQL .= ",place_id "; + /* $sSQL .= " union "; $sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; + if ($this->bIncludeExtraTags) $sSQL .= "null as extra,"; + if ($this->bIncludeNameDetails) $sSQL .= "null as names,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= $sImportanceSQL."-1.10 as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; @@@ -437,7 -461,6 +462,7 @@@ $sSQL .= "group by place_id"; if (!$this->bDeDupe) $sSQL .= ",place_id"; $sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) "; + */ } $sSQL .= " order by importance desc"; @@@ -1247,7 -1270,7 +1272,7 @@@ // TODO: filter out the pointless search terms (2 letter name tokens and less) // they might be right - but they are just too darned expensive to run if (sizeof($aSearch['aName'])) $aTerms[] = "name_vector @> ARRAY[".join($aSearch['aName'],",")."]"; - if (sizeof($aSearch['aNameNonSearch'])) $aTerms[] = "array_cat(name_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aNameNonSearch'],",")."]"; + //if (sizeof($aSearch['aNameNonSearch'])) $aTerms[] = "array_cat(name_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aNameNonSearch'],",")."]"; if (sizeof($aSearch['aAddress']) && $aSearch['aName'] != $aSearch['aAddress']) { // For infrequent name terms disable index usage for address @@@ -1255,13 -1278,12 +1280,13 @@@ sizeof($aSearch['aName']) == 1 && $aWordFrequencyScores[$aSearch['aName'][reset($aSearch['aName'])]] < CONST_Search_NameOnlySearchFrequencyThreshold) { - $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join(array_merge($aSearch['aAddress'],$aSearch['aAddressNonSearch']),",")."]"; + //$aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join(array_merge($aSearch['aAddress'],$aSearch['aAddressNonSearch']),",")."]"; + $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddress'],",")."]"; } else { $aTerms[] = "nameaddress_vector @> ARRAY[".join($aSearch['aAddress'],",")."]"; - if (sizeof($aSearch['aAddressNonSearch'])) $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddressNonSearch'],",")."]"; + //if (sizeof($aSearch['aAddressNonSearch'])) $aTerms[] = "array_cat(nameaddress_vector,ARRAY[]::integer[]) @> ARRAY[".join($aSearch['aAddressNonSearch'],",")."]"; } } if ($aSearch['sCountryCode']) $aTerms[] = "country_code = '".pg_escape_string($aSearch['sCountryCode'])."'"; @@@ -1371,7 -1393,6 +1396,7 @@@ $aPlaceIDs = $this->oDB->getCol($sSQL); // If not try the aux fallback table + /* if (!sizeof($aPlaceIDs)) { $sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'"; @@@ -1383,7 -1404,6 +1408,7 @@@ if (CONST_Debug) var_dump($sSQL); $aPlaceIDs = $this->oDB->getCol($sSQL); } + */ if (!sizeof($aPlaceIDs)) { @@@ -1661,12 -1681,10 +1686,12 @@@ { preg_match_all('/(-?[0-9.]+) (-?[0-9.]+)/',$aMatch[1],$aPolyPoints,PREG_SET_ORDER); } + /* elseif (preg_match('#MULTIPOLYGON\\(\\(\\(([- 0-9.,]+)#',$aPointPolygon['astext'],$aMatch)) { preg_match_all('/(-?[0-9.]+) (-?[0-9.]+)/',$aMatch[1],$aPolyPoints,PREG_SET_ORDER); } + */ elseif (preg_match('#POINT\\((-?[0-9.]+) (-?[0-9.]+)\\)#',$aPointPolygon['astext'],$aMatch)) { $iSteps = max(8, min(100, ($fRadius * 40000)^2)); @@@ -1763,6 -1781,30 +1788,30 @@@ } } + if ($this->bIncludeExtraTags) + { + if ($aResult['extra']) + { + $aResult['sExtraTags'] = json_decode($aResult['extra']); + } + else + { + $aResult['sExtraTags'] = array(); + } + } + + if ($this->bIncludeNameDetails) + { + if ($aResult['names']) + { + $aResult['sNameDetails'] = json_decode($aResult['names']); + } + else + { + $aResult['sNameDetails'] = array(); + } + } + // Adjust importance for the number of exact string matches in the result $aResult['importance'] = max(0.001,$aResult['importance']); $iCountWords = 0; @@@ -1797,6 -1839,7 +1846,7 @@@ { $aResult['foundorder'] += 0.01; } + if (CONST_Debug) { var_dump($aResult); } $aSearchResults[$iResNum] = $aResult; } uasort($aSearchResults, 'byImportance'); diff --combined lib/lib.php index e548dc3a,57bd4723..dfd7008d --- a/lib/lib.php +++ b/lib/lib.php @@@ -39,6 -39,12 +39,12 @@@ exit; } + function getParamBool($name, $default=false) + { + if (!isset($_GET[$name])) return $default; + + return (bool) $_GET[$name]; + } function fail($sError, $sUserError = false) { @@@ -178,7 -184,7 +184,7 @@@ { $aResult = array(array(join(' ',$aWords))); $sFirstToken = ''; - if ($iDepth < 8) { + if ($iDepth < 7) { while(sizeof($aWords) > 1) { $sWord = array_shift($aWords); @@@ -668,12 -674,11 +674,11 @@@ } - function javascript_renderData($xVal) + function javascript_renderData($xVal, $iOptions = 0) { header("Access-Control-Allow-Origin: *"); - $iOptions = 0; if (defined('PHP_VERSION_ID') && PHP_VERSION_ID > 50400) - $iOptions = JSON_UNESCAPED_UNICODE; + $iOptions |= JSON_UNESCAPED_UNICODE; $jsonout = json_encode($xVal, $iOptions); if( ! isset($_GET['json_callback'])) diff --combined settings/settings.php index 62892ed5,cc81e7a9..3deda462 --- a/settings/settings.php +++ b/settings/settings.php @@@ -78,15 -78,13 +78,15 @@@ // Website settings @define('CONST_NoAccessControl', true); - @define('CONST_ClosedForIndexing', false); - @define('CONST_ClosedForIndexingExceptionIPs', ''); @define('CONST_BlockedIPs', ''); + @define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks'); + @define('CONST_WhitelistedIPs', ''); + @define('CONST_BlockedUserAgents', ''); + @define('CONST_BlockReverseMaxLoad', 15); @define('CONST_BulkUserIPs', ''); @define('CONST_BlockMessage', ''); // additional info to show for blocked IPs - @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); + @define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/'); @define('CONST_Tile_Default', 'Mapnik'); @define('CONST_Default_Language', false); @@@ -101,6 -99,9 +101,9 @@@ @define('CONST_Search_TryDroppedAddressTerms', false); @define('CONST_Search_NameOnlySearchFrequencyThreshold', 500); + // If set to true, then reverse order of queries will be tried by default. + // When set to false only selected languages alloow reverse search. + @define('CONST_Search_ReversePlanForAll', true); @define('CONST_Places_Max_ID_count', 50); diff --combined tests/features/api/search_params.feature index 92ca8c39,7099c72f..a4885a15 --- a/tests/features/api/search_params.feature +++ b/tests/features/api/search_params.feature @@@ -70,7 -70,7 +70,7 @@@ Feature: Search querie Then result addresses contain | ID | city | 0 | Chicago - + Scenario: No POI search with unbounded viewbox Given the request parameters | viewbox @@@ -83,7 -83,7 +83,7 @@@ Scenario: bounded search remains within viewbox, even with no results Given the request parameters | bounded | viewbox - | 1 | 43.54285,-5.662003,43.5403125,-5.6563282 + | 1 | 43.5403125,-5.6563282,43.54285,-5.662003 When sending json search query "restaurant" Then less than 1 result is returned @@@ -202,3 -202,31 +202,31 @@@ | 0.5 | 999 | nan + + Scenario Outline: Search with extratags + Given the request parameters + | extratags + | 1 + When sending search query "Hauptstr" + Then result 0 has attributes extratags + And result 1 has attributes extratags + + Examples: + | format + | xml + | json + | jsonv2 + + Scenario Outline: Search with namedetails + Given the request parameters + | namedetails + | 1 + When sending search query "Hauptstr" + Then result 0 has attributes namedetails + And result 1 has attributes namedetails + + Examples: + | format + | xml + | json + | jsonv2 diff --combined tests/features/api/search_simple.feature index 747d4004,2cb27b7c..43f46098 --- a/tests/features/api/search_simple.feature +++ b/tests/features/api/search_simple.feature @@@ -51,6 -51,10 +51,10 @@@ Feature: Simple Test | limit | 1000 | dedupe | 1 | dedupe | 0 + | extratags | 1 + | extratags | 0 + | namedetails | 1 + | namedetails | 0 Scenario: Search with invalid output format Given the request parameters @@@ -104,35 -108,35 +108,35 @@@ Scenario: Empty XML search with viewbox Given the request parameters | viewbox - | 12,45.13,77,33 + | 12,45.13,13,44 When sending xml search query "xnznxvcx" Then result header contains | attr | value | querystring | xnznxvcx | polygon | false - | viewbox | 12,45.13,77,33 + | viewbox | 12,45.13,13,44 Scenario: Empty XML search with viewboxlbrt Given the request parameters | viewboxlbrt - | 12,34.13,77,45 + | 12,34.13,13,35 When sending xml search query "xnznxvcx" Then result header contains | attr | value | querystring | xnznxvcx | polygon | false - | viewbox | 12,45.13,77,33 + | viewbox | 12,34.13,13,35 Scenario: Empty XML search with viewboxlbrt and viewbox Given the request parameters - | viewbox | viewboxblrt - | 12,45.13,77,33 | 1,2,3,4 + | viewbox | viewboxblrt + | 12,45.13,13.5,44 | 1,0,2,1 When sending xml search query "pub" Then result header contains | attr | value | querystring | pub | polygon | false - | viewbox | 12,45.13,77,33 + | viewbox | 12,45.13,13.5,44 Scenario Outline: Empty XML search with polygon values diff --combined utils/setup.php index 3bbe18d4,14fe675e..be422949 --- a/utils/setup.php +++ b/utils/setup.php @@@ -78,11 -78,13 +78,13 @@@ } // Assume we can steal all the cache memory in the box (unless told otherwise) - $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB()); - if ($iCacheMemory > getTotalMemoryMB()) + if (isset($aCMDResult['osm2pgsql-cache'])) + { + $iCacheMemory = $aCMDResult['osm2pgsql-cache']; + } + else { $iCacheMemory = getCacheMemoryMB(); - echo "WARNING: resetting cache memory to $iCacheMemory\n"; } $aDSNInfo = DB::parseDSN(CONST_Database_DSN); @@@ -199,7 -201,7 +201,7 @@@ if (CONST_Tablespace_Place_Index) $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; $osm2pgsql .= ' -lsc -O gazetteer --hstore'; - $osm2pgsql .= ' -C '.$iCacheMemory; + $osm2pgsql .= ' -C 25000'; $osm2pgsql .= ' -P '.$aDSNInfo['port']; $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; passthruCheckReturn($osm2pgsql); @@@ -546,7 -548,7 +548,7 @@@ $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,"; $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,"; $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y "; - $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; + $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x"; if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; diff --combined website/reverse.php index d70a11e9,abc33a09..ee2fa475 --- a/website/reverse.php +++ b/website/reverse.php @@@ -6,6 -6,18 +6,6 @@@ require_once(CONST_BasePath.'/lib/PlaceLookup.php'); require_once(CONST_BasePath.'/lib/ReverseGeocode.php'); - if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false) - { - $fLoadAvg = getLoadAverage(); - if ($fLoadAvg > 2) sleep(60); - if ($fLoadAvg > 4) sleep(120); - if ($fLoadAvg > 6) - { - echo "Bulk User: Temporary block due to high server load\n"; - exit; - } - } - $oDB =& getDB(); ini_set('memory_limit', '200M'); @@@ -16,42 -28,52 +16,53 @@@ $sOutputFormat = $_GET['format']; } - // Show address breakdown - $bShowAddressDetails = true; - if (isset($_GET['addressdetails'])) $bShowAddressDetails = (bool)$_GET['addressdetails']; - // Preferred language $aLangPrefOrder = getPreferredLanguages(); $hLog = logStart($oDB, 'reverse', $_SERVER['QUERY_STRING'], $aLangPrefOrder); + if (isset($_GET['osm_type']) && isset($_GET['osm_id']) && (int)$_GET['osm_id'] && ($_GET['osm_type'] == 'N' || $_GET['osm_type'] == 'W' || $_GET['osm_type'] == 'R')) { - $oPlaceLookup = new PlaceLookup($oDB); - $oPlaceLookup->setLanguagePreference($aLangPrefOrder); - $oPlaceLookup->setIncludeAddressDetails($bShowAddressDetails); - $oPlaceLookup->setOSMID($_GET['osm_type'], $_GET['osm_id']); - - $aPlace = $oPlaceLookup->lookup(); + $aLookup = array('osm_type' => $_GET['osm_type'], 'osm_id' => $_GET['osm_id']); } else if (isset($_GET['lat']) && isset($_GET['lon']) && preg_match('/^[+-]?[0-9]*\.?[0-9]+$/', $_GET['lat']) && preg_match('/^[+-]?[0-9]*\.?[0-9]+$/', $_GET['lon'])) { $oReverseGeocode = new ReverseGeocode($oDB); $oReverseGeocode->setLanguagePreference($aLangPrefOrder); - $oReverseGeocode->setIncludeAddressDetails($bShowAddressDetails); $oReverseGeocode->setLatLon($_GET['lat'], $_GET['lon']); $oReverseGeocode->setZoom(@$_GET['zoom']); - $aPlace = $oReverseGeocode->lookup(); + $aLookup = $oReverseGeocode->lookup(); + if (CONST_Debug) var_dump($aLookup); + } + else + { + $aLookup = null; + } + + if ($aLookup) + { + $oPlaceLookup = new PlaceLookup($oDB); + $oPlaceLookup->setLanguagePreference($aLangPrefOrder); + $oPlaceLookup->setIncludeAddressDetails(getParamBool('addressdetails', true)); + $oPlaceLookup->setIncludeExtraTags(getParamBool('extratags', false)); + $oPlaceLookup->setIncludeNameDetails(getParamBool('namedetails', false)); + + $aPlace = $oPlaceLookup->lookupPlace($aLookup); } else { $aPlace = null; } + logEnd($oDB, $hLog, sizeof($aPlace)?1:0); - if (CONST_Debug) exit; + if (CONST_Debug) + { + var_dump($aPlace); + exit; + } include(CONST_BasePath.'/lib/template/address-'.$sOutputFormat.'.php');