From: Marc Tobias Metten Date: Thu, 15 Oct 2015 00:30:51 +0000 (+0200) Subject: more tests for libphp - incorporate lonvias feedback X-Git-Tag: v.2.5.0~18^2 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/2338d04e9f0cd8f0ca294a1f6664d641d1522b25?hp=1e4c22a03708968ec011edaf72a55a931a4204e0 more tests for libphp - incorporate lonvias feedback --- diff --git a/VAGRANT.md b/VAGRANT.md index d97fb3f7..c662d5e4 100644 --- a/VAGRANT.md +++ b/VAGRANT.md @@ -38,7 +38,8 @@ is. 3. Import a small country (Monaco) - You need to give the virtual machine more memory (2GB) for an import, see `Vagrantfile`. + You need to give the virtual machine more memory (2GB) for an import, + see `Vagrantfile`. Otherwise 1GB is enough. See the FAQ how to skip this step and point Nominatim to an existing database. @@ -46,7 +47,7 @@ is. # inside the virtual machine: cd Nominatim wget --no-verbose --output-document=data/monaco.osm.pbf http://download.geofabrik.de/europe/monaco-latest.osm.pbf - utils/setup.php --osm-file data/monaco.osm.pbf --osm2pgsql-cache 1000 --all | tee monaco.$$.log + ./utils/setup.php --osm-file data/monaco.osm.pbf --osm2pgsql-cache 1000 --all 2>&1 | tee monaco.$$.log ./utils/specialphrases.php --countries > data/specialphrases_countries.sql psql -d nominatim -f data/specialphrases_countries.sql ``` diff --git a/Vagrantfile b/Vagrantfile index 1b165327..843dd668 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -30,7 +30,7 @@ Vagrant.configure("2") do |config| config.vm.provider "virtualbox" do |vb| vb.gui = false - vb.customize ["modifyvm", :id, "--memory", "1024"] + vb.customize ["modifyvm", :id, "--memory", "2048"] end diff --git a/configure.ac b/configure.ac index badcb5b2..53a8e871 100644 --- a/configure.ac +++ b/configure.ac @@ -9,6 +9,7 @@ AC_PREREQ(2.61) AM_INIT_AUTOMAKE([1.9.6 dist-bzip2 std-options check-news]) dnl Additional macro definitions are in here +m4_include([m4/ax_lib_postgresql_svr.m4]) AC_CONFIG_MACRO_DIR([osm2pgsql/m4]) dnl Generate configuration header file @@ -45,6 +46,7 @@ if test "x$POSTGRESQL_VERSION" = "x" then AC_MSG_ERROR([postgresql client library not found]) fi +AX_LIB_POSTGRESQL_SVR(9.0) if test ! -f "$POSTGRESQL_PGXS" then AC_MSG_ERROR([postgresql server development library not found]) diff --git a/lib/Geocode.php b/lib/Geocode.php index c89cc48c..80561d27 100644 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@ -6,6 +6,8 @@ protected $aLangPrefOrder = array(); protected $bIncludeAddressDetails = false; + protected $bIncludeExtraTags = false; + protected $bIncludeNameDetails = false; protected $bIncludePolygonAsPoints = false; protected $bIncludePolygonAsText = false; @@ -68,6 +70,16 @@ return $this->bIncludeAddressDetails; } + function getIncludeExtraTags() + { + return $this->bIncludeExtraTags; + } + + function getIncludeNameDetails() + { + return $this->bIncludeNameDetails; + } + function setIncludePolygonAsPoints($b = true) { $this->bIncludePolygonAsPoints = $b; @@ -214,6 +226,11 @@ function loadParamArray($aParams) { if (isset($aParams['addressdetails'])) $this->bIncludeAddressDetails = (bool)$aParams['addressdetails']; + if ((float) CONST_Postgresql_Version > 9.2) + { + if (isset($aParams['extratags'])) $this->bIncludeExtraTags = (bool)$aParams['extratags']; + if (isset($aParams['namedetails'])) $this->bIncludeNameDetails = (bool)$aParams['namedetails']; + } if (isset($aParams['bounded'])) $this->bBoundedSearch = (bool)$aParams['bounded']; if (isset($aParams['dedupe'])) $this->bDeDupe = (bool)$aParams['dedupe']; @@ -389,6 +406,8 @@ $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; + if ($this->bIncludeExtraTags) $sSQL .= "hstore_to_json(extratags)::text as extra,"; + if ($this->bIncludeNameDetails) $sSQL .= "hstore_to_json(name)::text as names,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= $sImportanceSQL."coalesce(importance,0.75-(rank_search::float/40)) as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; @@ -405,6 +424,8 @@ $sSQL .= ",langaddress "; $sSQL .= ",placename "; $sSQL .= ",ref "; + if ($this->bIncludeExtraTags) $sSQL .= ",extratags"; + if ($this->bIncludeNameDetails) $sSQL .= ",name"; $sSQL .= ",extratags->'place' "; if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank) @@ -414,6 +435,8 @@ $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; + if ($this->bIncludeExtraTags) $sSQL .= "null as extra,"; + if ($this->bIncludeNameDetails) $sSQL .= "null as names,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= $sImportanceSQL."-1.15 as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; @@ -427,6 +450,8 @@ $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; + if ($this->bIncludeExtraTags) $sSQL .= "null as extra,"; + if ($this->bIncludeNameDetails) $sSQL .= "null as names,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= $sImportanceSQL."-1.10 as importance, "; $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; @@ -1756,6 +1781,30 @@ } } + if ($this->bIncludeExtraTags) + { + if ($aResult['extra']) + { + $aResult['sExtraTags'] = json_decode($aResult['extra']); + } + else + { + $aResult['sExtraTags'] = (object) array(); + } + } + + if ($this->bIncludeNameDetails) + { + if ($aResult['names']) + { + $aResult['sNameDetails'] = json_decode($aResult['names']); + } + else + { + $aResult['sNameDetails'] = (object) array(); + } + } + // Adjust importance for the number of exact string matches in the result $aResult['importance'] = max(0.001,$aResult['importance']); $iCountWords = 0; @@ -1790,6 +1839,7 @@ { $aResult['foundorder'] += 0.01; } + if (CONST_Debug) { var_dump($aResult); } $aSearchResults[$iResNum] = $aResult; } uasort($aSearchResults, 'byImportance'); diff --git a/lib/PlaceLookup.php b/lib/PlaceLookup.php index 84c68a99..c5129fee 100644 --- a/lib/PlaceLookup.php +++ b/lib/PlaceLookup.php @@ -5,12 +5,16 @@ protected $iPlaceID; - protected $bIsTiger = false; + protected $sType = false; protected $aLangPrefOrder = array(); protected $bAddressDetails = false; + protected $bExtraTags = false; + + protected $bNameDetails = false; + function PlaceLookup(&$oDB) { $this->oDB =& $oDB; @@ -26,6 +30,22 @@ $this->bAddressDetails = $bAddressDetails; } + function setIncludeExtraTags($bExtraTags = false) + { + if ((float) CONST_Postgresql_Version > 9.2) + { + $this->bExtraTags = $bExtraTags; + } + } + + function setIncludeNameDetails($bNameDetails = false) + { + if ((float) CONST_Postgresql_Version > 9.2) + { + $this->bNameDetails = $bNameDetails; + } + } + function setPlaceID($iPlaceID) { $this->iPlaceID = $iPlaceID; @@ -37,9 +57,16 @@ $this->iPlaceID = $this->oDB->getOne($sSQL); } - function setIsTiger($b = false) + function lookupPlace($details) { - $this->bIsTiger = $b; + if (isset($details['place_id'])) $this->iPlaceID = $details['place_id']; + if (isset($details['type'])) $this->sType = $details['type']; + if (isset($details['osm_type']) && isset($details['osm_id'])) + { + $this->setOSMID($details['osm_type'], $details['osm_id']); + } + + return $this->lookup(); } function lookup() @@ -48,28 +75,33 @@ $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]"; - $sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, extratags, parent_place_id, linked_place_id, rank_address, rank_search, "; - $sSQL .= " coalesce(importance,0.75-(rank_search::float/40)) as importance, indexed_status, indexed_date, wikipedia, calculated_country_code, "; - $sSQL .= " get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; - $sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; - $sSQL .= " get_name_by_language(name, ARRAY['ref']) as ref,"; - $sSQL .= " (case when centroid is null then st_y(st_centroid(geometry)) else st_y(centroid) end) as lat,"; - $sSQL .= " (case when centroid is null then st_x(st_centroid(geometry)) else st_x(centroid) end) as lon"; - $sSQL .= " from placex where place_id = ".(int)$this->iPlaceID; - - - if ($this->bIsTiger) + if ($this->sType == 'tiger') { $sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,"; - $sSQL .= " 'us' as country_code, null as extratags, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,"; + $sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,"; $sSQL .= " coalesce(null,0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, 'us' as calculated_country_code, "; $sSQL .= " get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= " null as placename,"; $sSQL .= " null as ref,"; + if ($this->bExtraTags) $sSQL .= " null as extra,"; + if ($this->bNameDetails) $sSQL .= " null as names,"; $sSQL .= " st_y(centroid) as lat,"; $sSQL .= " st_x(centroid) as lon"; $sSQL .= " from location_property_tiger where place_id = ".(int)$this->iPlaceID; } + else + { + $sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, parent_place_id, linked_place_id, rank_address, rank_search, "; + $sSQL .= " coalesce(importance,0.75-(rank_search::float/40)) as importance, indexed_status, indexed_date, wikipedia, calculated_country_code, "; + $sSQL .= " get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; + $sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; + $sSQL .= " get_name_by_language(name, ARRAY['ref']) as ref,"; + if ($this->bExtraTags) $sSQL .= " hstore_to_json(extratags) as extra,"; + if ($this->bNameDetails) $sSQL .= " hstore_to_json(name) as names,"; + $sSQL .= " (case when centroid is null then st_y(st_centroid(geometry)) else st_y(centroid) end) as lat,"; + $sSQL .= " (case when centroid is null then st_x(st_centroid(geometry)) else st_x(centroid) end) as lon"; + $sSQL .= " from placex where place_id = ".(int)$this->iPlaceID; + } $aPlace = $this->oDB->getRow($sSQL); @@ -87,6 +119,29 @@ $aPlace['aAddress'] = $aAddress; } + if ($this->bExtraTags) + { + if ($aPlace['extra']) + { + $aPlace['sExtraTags'] = json_decode($aPlace['extra']); + } + else + { + $aPlace['sExtraTags'] = (object) array(); + } + } + + if ($this->bNameDetails) + { + if ($aPlace['names']) + { + $aPlace['sNameDetails'] = json_decode($aPlace['names']); + } + else + { + $aPlace['sNameDetails'] = (object) array(); + } + } $aClassType = getClassTypes(); $sAddressType = ''; @@ -129,7 +184,7 @@ function getAddressNames() { - $aAddressLines = $this->getAddressDetails(false);; + $aAddressLines = $this->getAddressDetails(false); $aAddress = array(); $aFallback = array(); diff --git a/lib/ReverseGeocode.php b/lib/ReverseGeocode.php index ae83af1e..e40ce6cc 100644 --- a/lib/ReverseGeocode.php +++ b/lib/ReverseGeocode.php @@ -9,8 +9,6 @@ protected $aLangPrefOrder = array(); - protected $bShowAddressDetails = true; - function ReverseGeocode(&$oDB) { $this->oDB =& $oDB; @@ -21,11 +19,6 @@ $this->aLangPrefOrder = $aLangPref; } - function setIncludeAddressDetails($bAddressDetails = true) - { - $this->bAddressDetails = $bAddressDetails; - } - function setLatLon($fLat, $fLon) { $this->fLat = (float)$fLat; @@ -97,7 +90,7 @@ $sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')'; $sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank; $sSQL .= ' and (name is not null or housenumber is not null)'; - $sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')'; + $sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\',\'man_made\')'; $sSQL .= ' and indexed_status = 0 '; $sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') '; $sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))'; @@ -171,13 +164,8 @@ } } - $oPlaceLookup = new PlaceLookup($this->oDB); - $oPlaceLookup->setLanguagePreference($this->aLangPrefOrder); - $oPlaceLookup->setIncludeAddressDetails($this->bAddressDetails); - $oPlaceLookup->setPlaceId($iPlaceID); - $oPlaceLookup->setIsTiger($bPlaceIsTiger); - - return $oPlaceLookup->lookup(); + return array('place_id' => $iPlaceID, + 'type' => $bPlaceIsTiger ? 'tiger' : 'osm'); } } ?> diff --git a/lib/lib.php b/lib/lib.php index c68f04eb..50cf3dc4 100644 --- a/lib/lib.php +++ b/lib/lib.php @@ -39,6 +39,12 @@ exit; } + function getParamBool($name, $default=false) + { + if (!isset($_GET[$name])) return $default; + + return (bool) $_GET[$name]; + } function fail($sError, $sUserError = false) { @@ -668,12 +674,11 @@ } - function javascript_renderData($xVal) + function javascript_renderData($xVal, $iOptions = 0) { header("Access-Control-Allow-Origin: *"); - $iOptions = 0; if (defined('PHP_VERSION_ID') && PHP_VERSION_ID > 50400) - $iOptions = JSON_UNESCAPED_UNICODE; + $iOptions |= JSON_UNESCAPED_UNICODE; $jsonout = json_encode($xVal, $iOptions); if( ! isset($_GET['json_callback'])) @@ -965,7 +970,7 @@ } // degrees decimal seconds // N 40 26 46 W 79 58 56 - // N 40° 26′ 46″ W, 79° 58′ 56″ + // N 40° 26′ 46″, W 79° 58′ 56″ // 1 2 3 4 5 6 7 8 elseif (preg_match('/\\b([NS])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*[, ]+([EW])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*\\b/', $sQuery, $aData)) { diff --git a/lib/template/address-json.php b/lib/template/address-json.php index 2f077818..ff245e14 100644 --- a/lib/template/address-json.php +++ b/lib/template/address-json.php @@ -10,7 +10,7 @@ } else { - if ($aPlace['place_id']) $aFilteredPlaces['place_id'] = $aPlace['place_id']; + if (isset($aPlace['place_id'])) $aFilteredPlaces['place_id'] = $aPlace['place_id']; $aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright"; $sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':''))); if ($sOSMType) @@ -21,7 +21,9 @@ if (isset($aPlace['lat'])) $aFilteredPlaces['lat'] = $aPlace['lat']; if (isset($aPlace['lon'])) $aFilteredPlaces['lon'] = $aPlace['lon']; $aFilteredPlaces['display_name'] = $aPlace['langaddress']; - if ($bShowAddressDetails) $aFilteredPlaces['address'] = $aPlace['aAddress']; + if (isset($aPlace['aAddress'])) $aFilteredPlaces['address'] = $aPlace['aAddress']; + if (isset($aPlace['sExtraTags'])) $aFilteredPlaces['extratags'] = $aPlace['sExtraTags']; + if (isset($aPlace['sNameDetails'])) $aFilteredPlaces['namedetails'] = $aPlace['sNameDetails']; } javascript_renderData($aFilteredPlaces); diff --git a/lib/template/address-jsonv2.php b/lib/template/address-jsonv2.php index 92cf4f1c..a8a05dd4 100644 --- a/lib/template/address-jsonv2.php +++ b/lib/template/address-jsonv2.php @@ -23,16 +23,19 @@ $aFilteredPlaces['place_rank'] = $aPlace['rank_search']; - $aFilteredPlaces['category'] = $aPlace['class']; - $aFilteredPlaces['type'] = $aPlace['type']; + $aFilteredPlaces['category'] = $aPlace['class']; + $aFilteredPlaces['type'] = $aPlace['type']; $aFilteredPlaces['importance'] = $aPlace['importance']; - $aFilteredPlaces['addresstype'] = strtolower($aPlace['addresstype']); + $aFilteredPlaces['addresstype'] = strtolower($aPlace['addresstype']); $aFilteredPlaces['display_name'] = $aPlace['langaddress']; - $aFilteredPlaces['name'] = $aPlace['placename']; - if ($bShowAddressDetails && $aPlace['aAddress'] && sizeof($aPlace['aAddress'])) $aFilteredPlaces['address'] = $aPlace['aAddress']; + $aFilteredPlaces['name'] = $aPlace['placename']; + + if (isset($aPlace['aAddress'])) $aFilteredPlaces['address'] = $aPlace['aAddress']; + if (isset($aPlace['sExtraTags'])) $aFilteredPlaces['extratags'] = $aPlace['sExtraTags']; + if (isset($aPlace['sNameDetails'])) $aFilteredPlaces['namedetails'] = $aPlace['sNameDetails']; } javascript_renderData($aFilteredPlaces); diff --git a/lib/template/address-xml.php b/lib/template/address-xml.php index 9eeb3b77..39d9a147 100644 --- a/lib/template/address-xml.php +++ b/lib/template/address-xml.php @@ -29,7 +29,8 @@ if (isset($aPlace['lon'])) echo ' lon="'.htmlspecialchars($aPlace['lon']).'"'; echo ">".htmlspecialchars($aPlace['langaddress']).""; - if ($bShowAddressDetails) { + if (isset($aPlace['aAddress'])) + { echo ""; foreach($aPlace['aAddress'] as $sKey => $sValue) { @@ -40,6 +41,28 @@ } echo ""; } + + if (isset($aPlace['sExtraTags'])) + { + echo ""; + foreach ($aPlace['sExtraTags'] as $sKey => $sValue) + { + echo ''; + } + echo ""; + } + + if (isset($aPlace['sNameDetails'])) + { + echo ""; + foreach ($aPlace['sNameDetails'] as $sKey => $sValue) + { + echo ''; + echo htmlspecialchars($sValue); + echo ""; + } + echo ""; + } } echo ""; diff --git a/lib/template/search-json.php b/lib/template/search-json.php index 57586fb9..3dcaabdb 100644 --- a/lib/template/search-json.php +++ b/lib/template/search-json.php @@ -74,6 +74,9 @@ $aPlace['geokml'] = $aPointDetails['askml']; } + if (isset($aPointDetails['sExtraTags'])) $aPlace['extratags'] = $aPointDetails['sExtraTags']; + if (isset($aPointDetails['sNameDetails'])) $aPlace['namedetails'] = $aPointDetails['sNameDetails']; + $aFilteredPlaces[] = $aPlace; } diff --git a/lib/template/search-jsonv2.php b/lib/template/search-jsonv2.php index 126f7866..e9747720 100644 --- a/lib/template/search-jsonv2.php +++ b/lib/template/search-jsonv2.php @@ -73,7 +73,10 @@ $aPlace['geokml'] = $aPointDetails['askml']; } + if (isset($aPointDetails['sExtraTags'])) $aPlace['extratags'] = $aPointDetails['sExtraTags']; + if (isset($aPointDetails['sNameDetails'])) $aPlace['namedetails'] = $aPointDetails['sNameDetails']; + $aFilteredPlaces[] = $aPlace; } - javascript_renderData($aFilteredPlaces, array('geojson')); + javascript_renderData($aFilteredPlaces); diff --git a/lib/template/search-xml.php b/lib/template/search-xml.php index 693330bb..b61ff22f 100644 --- a/lib/template/search-xml.php +++ b/lib/template/search-xml.php @@ -5,7 +5,8 @@ echo "?xml version=\"1.0\" encoding=\"UTF-8\" ?"; echo ">\n"; - echo ""; - } + $bHasDelim = false; if (isset($aResult['askml'])) { + if (!$bHasDelim) + { + $bHasDelim = true; + echo ">"; + } echo "\n"; echo $aResult['askml']; echo ""; } + if (isset($aResult['sExtraTags'])) + { + if (!$bHasDelim) + { + $bHasDelim = true; + echo ">"; + } + echo "\n"; + foreach ($aResult['sExtraTags'] as $sKey => $sValue) + { + echo ''; + } + echo ""; + } + + if (isset($aResult['sNameDetails'])) + { + if (!$bHasDelim) + { + $bHasDelim = true; + echo ">"; + } + echo "\n"; + foreach ($aResult['sNameDetails'] as $sKey => $sValue) + { + echo ''; + echo htmlspecialchars($sValue); + echo ""; + } + echo ""; + } + if (isset($aResult['address'])) { + if (!$bHasDelim) + { + $bHasDelim = true; + echo ">"; + } echo "\n"; foreach($aResult['address'] as $sKey => $sValue) { @@ -111,7 +151,7 @@ } } - if (isset($aResult['address']) || isset($aResult['askml'])) + if ($bHasDelim) { echo ""; } @@ -121,4 +161,4 @@ } } - echo ""; + echo ""; diff --git a/m4/ax_lib_postgresql_svr.m4 b/m4/ax_lib_postgresql_svr.m4 new file mode 100644 index 00000000..56ecb5b4 --- /dev/null +++ b/m4/ax_lib_postgresql_svr.m4 @@ -0,0 +1,125 @@ +# SYNOPSIS +# +# AX_LIB_POSTGRESQL_SVR([MINIMUM-VERSION]) +# +# DESCRIPTION +# +# This macro provides tests of availability of PostgreSQL server library +# +# This macro calls: +# +# AC_SUBST(POSTGRESQL_PGXS) +# AC_SUBST(POSTGRESQL_SERVER_CFLAGS) +# +# LICENSE +# +# Copyright (c) 2008 Mateusz Loskot +# Copyright (c) 2015 Sarah Hoffmann +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. + +AC_DEFUN([AX_LIB_POSTGRESQL_SVR], +[ + AC_ARG_WITH([postgresql], + AC_HELP_STRING([--with-postgresql-svr=@<:@ARG@:>@], + [use PostgreSQL server library @<:@default=yes@:>@, optionally specify path to pg_config] + ), + [ + if test "$withval" = "no"; then + want_postgresql="no" + elif test "$withval" = "yes"; then + want_postgresql="yes" + else + want_postgresql="yes" + PG_CONFIG="$withval" + fi + ], + [want_postgresql="yes"] + ) + + dnl + dnl Check PostgreSQL server libraries + dnl + + if test "$want_postgresql" = "yes"; then + + if test -z "$PG_CONFIG" -o test; then + AC_PATH_PROG([PG_CONFIG], [pg_config], []) + fi + + if test ! -x "$PG_CONFIG"; then + AC_MSG_ERROR([$PG_CONFIG does not exist or it is not an exectuable file]) + PG_CONFIG="no" + found_postgresql="no" + fi + + if test "$PG_CONFIG" != "no"; then + AC_MSG_CHECKING([for PostgreSQL server libraries]) + + POSTGRESQL_SERVER_CFLAGS="-I`$PG_CONFIG --includedir-server`" + + POSTGRESQL_VERSION=`$PG_CONFIG --version | sed -e 's#PostgreSQL ##'` + + POSTGRESQL_PGXS=`$PG_CONFIG --pgxs` + if test -f "$POSTGRESQL_PGXS" + then + found_postgresql="yes" + AC_MSG_RESULT([yes]) + fi + else + found_postgresql="no" + AC_MSG_RESULT([no]) + fi + fi + + dnl + dnl Check if required version of PostgreSQL is available + dnl + + + postgresql_version_req=ifelse([$1], [], [], [$1]) + + if test "$found_postgresql" = "yes" -a -n "$postgresql_version_req"; then + + AC_MSG_CHECKING([if PostgreSQL version is >= $postgresql_version_req]) + + dnl Decompose required version string of PostgreSQL + dnl and calculate its number representation + postgresql_version_req_major=`expr $postgresql_version_req : '\([[0-9]]*\)'` + postgresql_version_req_minor=`expr $postgresql_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + postgresql_version_req_micro=`expr $postgresql_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$postgresql_version_req_micro" = "x"; then + postgresql_version_req_micro="0" + fi + + postgresql_version_req_number=`expr $postgresql_version_req_major \* 1000000 \ + \+ $postgresql_version_req_minor \* 1000 \ + \+ $postgresql_version_req_micro` + + dnl Decompose version string of installed PostgreSQL + dnl and calculate its number representation + postgresql_version_major=`expr $POSTGRESQL_VERSION : '\([[0-9]]*\)'` + postgresql_version_minor=`expr $POSTGRESQL_VERSION : '[[0-9]]*\.\([[0-9]]*\)'` + postgresql_version_micro=`expr $POSTGRESQL_VERSION : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$postgresql_version_micro" = "x"; then + postgresql_version_micro="0" + fi + + postgresql_version_number=`expr $postgresql_version_major \* 1000000 \ + \+ $postgresql_version_minor \* 1000 \ + \+ $postgresql_version_micro` + + postgresql_version_check=`expr $postgresql_version_number \>\= $postgresql_version_req_number` + if test "$postgresql_version_check" = "1"; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + fi + + AC_SUBST([POSTGRESQL_PGXS]) + AC_SUBST([POSTGRESQL_SERVER_CFLAGS]) +]) + diff --git a/nominatim/Makefile.am b/nominatim/Makefile.am index 3f65398a..d27ae554 100644 --- a/nominatim/Makefile.am +++ b/nominatim/Makefile.am @@ -2,8 +2,8 @@ bin_PROGRAMS = nominatim nominatim_SOURCES = export.c geometry.cpp import.c index.c input.c nominatim.c postgresql.c sprompt.c -AM_CFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML_CPPFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DVERSION='"@PACKAGE_VERSION@"' -AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML_CPPFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ +AM_CFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @XML_CPPFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DVERSION='"@PACKAGE_VERSION@"' +AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @XML_CPPFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ nominatim_LDADD = @PTHREAD_CFLAGS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML_LIBS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ -lz diff --git a/osm2pgsql b/osm2pgsql index 07db15e5..8179cdb6 160000 --- a/osm2pgsql +++ b/osm2pgsql @@ -1 +1 @@ -Subproject commit 07db15e5e55eca63971a014ddb08bc47cd89f762 +Subproject commit 8179cdb67e70d7fb5605ab6ddedfd0bd3347db47 diff --git a/settings/settings.php b/settings/settings.php index a1fbc9f8..cc81e7a9 100644 --- a/settings/settings.php +++ b/settings/settings.php @@ -99,6 +99,11 @@ @define('CONST_Search_TryDroppedAddressTerms', false); @define('CONST_Search_NameOnlySearchFrequencyThreshold', 500); + // If set to true, then reverse order of queries will be tried by default. + // When set to false only selected languages alloow reverse search. + @define('CONST_Search_ReversePlanForAll', true); + + @define('CONST_Places_Max_ID_count', 50); // Set to zero to disable polygon output @define('CONST_PolygonOutput_MaximumTypes', 1); diff --git a/sql/functions.sql b/sql/functions.sql index 3c8f16e0..bd64697a 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -2118,28 +2118,27 @@ BEGIN END; $$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION get_name_by_language(name hstore, languagepref TEXT[]) RETURNS TEXT AS $$ DECLARE - search TEXT[]; - found BOOLEAN; + result TEXT; BEGIN - IF name is null THEN RETURN null; END IF; - search := languagepref; - - FOR j IN 1..array_upper(search, 1) LOOP - IF name ? search[j] AND trim(name->search[j]) != '' THEN - return trim(name->search[j]); + FOR j IN 1..array_upper(languagepref,1) LOOP + IF name ? languagepref[j] THEN + result := trim(name->languagepref[j]); + IF result != '' THEN + return result; + END IF; END IF; END LOOP; -- anything will do as a fallback - just take the first name type thing there is - search := avals(name); - RETURN search[1]; + RETURN trim((avals(name))[1]); END; $$ LANGUAGE plpgsql IMMUTABLE; diff --git a/sql/indices.src.sql b/sql/indices.src.sql index 9a9c3793..73ba662b 100644 --- a/sql/indices.src.sql +++ b/sql/indices.src.sql @@ -14,7 +14,7 @@ CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:sear CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index}; CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) {ts:address-index} where indexed_status > 0; CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL; -CREATE INDEX idx_placex_reverse_geometry ON placex USING gist (geometry) {ts:search-index} where rank_search != 28 and (name is not null or housenumber is not null) and class not in ('waterway','railway','tunnel','bridge'); +CREATE INDEX idx_placex_reverse_geometry ON placex USING gist (geometry) {ts:search-index} where rank_search != 28 and (name is not null or housenumber is not null) and class not in ('waterway','railway','tunnel','bridge','man_made'); CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index}; CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid) {ts:address-index}; diff --git a/tests-php/Nominatim/NominatimTest.php b/tests-php/Nominatim/NominatimTest.php index a36c029e..4c5638b1 100644 --- a/tests-php/Nominatim/NominatimTest.php +++ b/tests-php/Nominatim/NominatimTest.php @@ -128,8 +128,10 @@ class NominatimTest extends \PHPUnit_Framework_TestCase // 4 words => 8 sets // 10 words => 511 sets // 15 words => 12911 sets + // 18 words => 65536 sets // 20 words => 169766 sets - // 28 words => 397594 sets + // 22 words => 401930 sets + // 28 words => 3505699 sets (needs more than 4GB via 'phpunit -d memory_limit=' to run) $this->assertEquals( 8, count( getWordSets(array_fill( 0, 4, 'a'),0) ) @@ -137,8 +139,8 @@ class NominatimTest extends \PHPUnit_Framework_TestCase $this->assertEquals( - 8, - count( getWordSets(array_fill( 0, 28, 'a'),0) ) + 65536, + count( getWordSets(array_fill( 0, 18, 'a'),0) ) ); diff --git a/tests-php/README.txt b/tests-php/README.txt index 31fd21ed..d551c1da 100644 --- a/tests-php/README.txt +++ b/tests-php/README.txt @@ -6,7 +6,8 @@ https://phpunit.de/manual/4.2/en/ installed. To execute the test suite run -$ phpunit +$ cd tests-php +$ phpunit ./ It will read phpunit.xml which points to the library, test path, bootstrap strip and set other parameters. diff --git a/tests/features/api/lookup.feature b/tests/features/api/lookup.feature new file mode 100644 index 00000000..7b86fb49 --- /dev/null +++ b/tests/features/api/lookup.feature @@ -0,0 +1,15 @@ +Feature: Places by osm_type and osm_id Tests + Simple tests for internal server errors and response format. + + Scenario: address lookup for existing node, way, relation + When looking up xml places N158845944,W72493656,,R62422,X99,N0 + Then the result is valid xml + exactly 3 results are returned + When looking up json places N158845944,W72493656,,R62422,X99,N0 + Then the result is valid json + exactly 3 results are returned + + Scenario: address lookup for non-existing or invalid node, way, relation + When looking up xml places X99,,N0,nN158845944,ABC,,W9 + Then the result is valid xml + exactly 0 results are returned \ No newline at end of file diff --git a/tests/features/api/reverse.feature b/tests/features/api/reverse.feature index f8c4f388..fa636acf 100644 --- a/tests/features/api/reverse.feature +++ b/tests/features/api/reverse.feature @@ -36,3 +36,28 @@ Feature: Reverse geocoding | 0 | Kings Estate Drive | 84128 | us And result 0 has attributes osm_id,osm_type + Scenario Outline: Reverse Geocoding with extratags + Given the request parameters + | extratags + | 1 + When looking up coordinates 48.86093,2.2978 + Then result 0 has attributes extratags + + Examples: + | format + | xml + | json + | jsonv2 + + Scenario Outline: Reverse Geocoding with namedetails + Given the request parameters + | namedetails + | 1 + When looking up coordinates 48.86093,2.2978 + Then result 0 has attributes namedetails + + Examples: + | format + | xml + | json + | jsonv2 diff --git a/tests/features/api/reverse_simple.feature b/tests/features/api/reverse_simple.feature index 7d564dde..8621ec65 100644 --- a/tests/features/api/reverse_simple.feature +++ b/tests/features/api/reverse_simple.feature @@ -94,4 +94,4 @@ Feature: Simple Reverse Tests | 48.966.0 | 8.4482 | 48.966 | 8.448.2 | Nan | 8.448 - | 48.966 | Nan + | 48.966 | Nan \ No newline at end of file diff --git a/tests/features/api/search_params.feature b/tests/features/api/search_params.feature index b9d06791..7099c72f 100644 --- a/tests/features/api/search_params.feature +++ b/tests/features/api/search_params.feature @@ -70,7 +70,7 @@ Feature: Search queries Then result addresses contain | ID | city | 0 | Chicago - + Scenario: No POI search with unbounded viewbox Given the request parameters | viewbox @@ -202,3 +202,31 @@ Feature: Search queries | 0.5 | 999 | nan + + Scenario Outline: Search with extratags + Given the request parameters + | extratags + | 1 + When sending search query "Hauptstr" + Then result 0 has attributes extratags + And result 1 has attributes extratags + + Examples: + | format + | xml + | json + | jsonv2 + + Scenario Outline: Search with namedetails + Given the request parameters + | namedetails + | 1 + When sending search query "Hauptstr" + Then result 0 has attributes namedetails + And result 1 has attributes namedetails + + Examples: + | format + | xml + | json + | jsonv2 diff --git a/tests/features/api/search_simple.feature b/tests/features/api/search_simple.feature index 3e6b6e35..2cb27b7c 100644 --- a/tests/features/api/search_simple.feature +++ b/tests/features/api/search_simple.feature @@ -51,6 +51,10 @@ Feature: Simple Tests | limit | 1000 | dedupe | 1 | dedupe | 0 + | extratags | 1 + | extratags | 0 + | namedetails | 1 + | namedetails | 0 Scenario: Search with invalid output format Given the request parameters diff --git a/tests/features/db/update/search_terms.feature b/tests/features/db/update/search_terms.feature new file mode 100644 index 00000000..d8c4440a --- /dev/null +++ b/tests/features/db/update/search_terms.feature @@ -0,0 +1,117 @@ +@DB +Feature: Update of search terms + Tests that search_name table is filled correctly + + Scenario: POI-inherited postcode remains when way type is changed + Given the scene roads-with-pois + And the place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 12345 | North St |:p-S1 + And the place ways + | osm_id | class | type | name | geometry + | 1 | highway | residential | North St | :w-north + When importing + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + When updating place ways + | osm_id | class | type | name | geometry + | 1 | highway | unclassified | North St | :w-north + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + + Scenario: POI-inherited postcode remains when way name is changed + Given the scene roads-with-pois + And the place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 12345 | North St |:p-S1 + And the place ways + | osm_id | class | type | name | geometry + | 1 | highway | residential | North St | :w-north + When importing + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + When updating place ways + | osm_id | class | type | name | geometry + | 1 | highway | unclassified | South St | :w-north + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + + Scenario: POI-inherited postcode remains when way geometry is changed + Given the scene roads-with-pois + And the place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 12345 | North St |:p-S1 + And the place ways + | osm_id | class | type | name | geometry + | 1 | highway | residential | North St | :w-north + When importing + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + When updating place ways + | osm_id | class | type | name | geometry + | 1 | highway | unclassified | South St | :w-south + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + + Scenario: POI-inherited postcode is added when POI postcode changes + Given the scene roads-with-pois + And the place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 12345 | North St |:p-S1 + And the place ways + | osm_id | class | type | name | geometry + | 1 | highway | residential | North St | :w-north + When importing + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + When updating place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 54321 | North St |:p-S1 + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 54321 + + Scenario: POI-inherited postcode remains when POI geometry changes + Given the scene roads-with-pois + And the place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 12345 | North St |:p-S1 + And the place ways + | osm_id | class | type | name | geometry + | 1 | highway | residential | North St | :w-north + When importing + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + When updating place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 12345 | North St |:p-S2 + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + + + Scenario: POI-inherited postcode remains when another POI is deleted + Given the scene roads-with-pois + And the place nodes + | osm_id | class | type | housenumber | postcode | street | geometry + | 1 | place | house | 1 | 12345 | North St |:p-S1 + | 2 | place | house | 2 | | North St |:p-S2 + And the place ways + | osm_id | class | type | name | geometry + | 1 | highway | residential | North St | :w-north + When importing + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 + When marking for delete N2 + Then search_name table contains + | place_id | nameaddress_vector + | W1 | 12345 diff --git a/tests/features/osm2pgsql/import/tags.feature b/tests/features/osm2pgsql/import/tags.feature index 759c6b96..149c4813 100644 --- a/tests/features/osm2pgsql/import/tags.feature +++ b/tests/features/osm2pgsql/import/tags.feature @@ -79,19 +79,20 @@ Feature: Tag evaluation | my:ref | br:name | name:prefix + | name:source Scenario: Special character in name tag Given the osm nodes: | id | tags - | 1 | 'highway' : 'yes', 'name: de' : 'Foo', 'name' : 'real' - | 2 | 'highway' : 'yes', 'name:\nde' : 'Foo', 'name' : 'real' - | 3 | 'highway' : 'yes', 'name:\tde' : 'Foo', 'name:\\' : 'real' + | 1 | 'highway' : 'yes', 'name: de' : 'Foo', 'name' : 'real1' + | 2 | 'highway' : 'yes', 'name: de' : 'Foo', 'name' : 'real2' + | 3 | 'highway' : 'yes', 'name: de' : 'Foo', 'name:\\' : 'real3' When loading osm data Then table place contains | object | name - | N1 | 'name:_de' : 'Foo', 'name' : 'real' - | N2 | 'name:_de' : 'Foo', 'name' : 'real' - | N3 | 'name:_de' : 'Foo', 'name:\\\\' : 'real' + | N1 | 'name: de' : 'Foo', 'name' : 'real1' + | N2 | 'name: de' : 'Foo', 'name' : 'real2' + | N3 | 'name: de' : 'Foo', 'name:\\\\' : 'real3' Scenario Outline: Included places Given the osm nodes: @@ -217,6 +218,14 @@ Feature: Tag evaluation | boundary | administrative | waterway | stream + Scenario: Footways are not included if they are sidewalks + Given the osm nodes: + | id | tags + | 2 | 'highway' : 'footway', 'name' : 'To Hell', 'footway' : 'sidewalk' + | 23 | 'highway' : 'footway', 'name' : 'x' + When loading osm data + Then table place has no entry for N2 + Scenario: named junctions are included if there is no other tag Given the osm nodes: | id | tags @@ -393,7 +402,7 @@ Feature: Tag evaluation Then table place contains | object | class | type | isin | N10 | place | village | Feebourgh county - | N11 | place | village | Alabama,Feebourgh county + | N11 | place | village | Feebourgh county,Alabama | N12 | place | village | Feebourgh county Scenario Outline: Import of address tags @@ -508,6 +517,10 @@ Feature: Tag evaluation | locality | wikipedia | wikipedia:de + | wikidata + | name:prefix + | name:botanical + | name:etymology:wikidata Scenario: buildings Given the osm nodes: diff --git a/tests/steps/api_result.py b/tests/steps/api_result.py index e86641fc..91c07296 100644 --- a/tests/steps/api_result.py +++ b/tests/steps/api_result.py @@ -27,17 +27,35 @@ def _parse_xml(): world.results = [] # results - if page.nodeName == 'searchresults': + if page.nodeName == 'searchresults' or page.nodeName == 'lookupresults': for node in page.childNodes: if node.nodeName != "#text": assert_equals(node.nodeName, 'place', msg="Unexpected element '%s'" % node.nodeName) newresult = OrderedDict(node.attributes.items()) assert_not_in('address', newresult) assert_not_in('geokml', newresult) + assert_not_in('extratags', newresult) + assert_not_in('namedetails', newresult) address = OrderedDict() for sub in node.childNodes: if sub.nodeName == 'geokml': newresult['geokml'] = sub.childNodes[0].toxml() + elif sub.nodeName == 'extratags': + newresult['extratags'] = {} + for tag in sub.childNodes: + assert_equals(tag.nodeName, 'tag') + attrs = dict(tag.attributes.items()) + assert_in('key', attrs) + assert_in('value', attrs) + newresult['extratags'][attrs['key']] = attrs['value'] + elif sub.nodeName == 'namedetails': + newresult['namedetails'] = {} + for tag in sub.childNodes: + assert_equals(tag.nodeName, 'name') + attrs = dict(tag.attributes.items()) + assert_in('desc', attrs) + newresult['namedetails'][attrs['desc']] = tag.firstChild.nodeValue.strip() + elif sub.nodeName == '#text': pass else: @@ -65,6 +83,21 @@ def _parse_xml(): for sub in node.childNodes: address[sub.nodeName] = sub.firstChild.nodeValue.strip() world.results[0]['address'] = address + elif node.nodeName == 'extratags': + world.results[0]['extratags'] = {} + for tag in node.childNodes: + assert_equals(tag.nodeName, 'tag') + attrs = dict(tag.attributes.items()) + assert_in('key', attrs) + assert_in('value', attrs) + world.results[0]['extratags'][attrs['key']] = attrs['value'] + elif node.nodeName == 'namedetails': + world.results[0]['namedetails'] = {} + for tag in node.childNodes: + assert_equals(tag.nodeName, 'name') + attrs = dict(tag.attributes.items()) + assert_in('desc', attrs) + world.results[0]['namedetails'][attrs['desc']] = tag.firstChild.nodeValue.strip() elif node.nodeName == "#text": pass else: @@ -92,6 +125,8 @@ def api_result_is_valid(step, fmt): _parse_xml() elif world.response_format == 'json': world.results = json.JSONDecoder(object_pairs_hook=OrderedDict).decode(world.page) + if world.request_type == 'reverse': + world.results = (world.results,) else: assert False, "Unknown page format: %s" % (world.response_format) diff --git a/tests/steps/api_setup.py b/tests/steps/api_setup.py index fcaa39a2..c9a4bac4 100644 --- a/tests/steps/api_setup.py +++ b/tests/steps/api_setup.py @@ -10,6 +10,7 @@ import logging logger = logging.getLogger(__name__) def api_call(requesttype): + world.request_type = requesttype world.json_callback = None data = urllib.urlencode(world.params) url = "%s/%s?%s" % (world.config.base_url, requesttype, data) @@ -123,6 +124,13 @@ def api_setup_details(step, obj): world.params['place_id'] = obj api_call('details') +@step(u'looking up (\w+) places ((?:[a-z]\d+,*)+)') +def api_setup_lookup(step, fmt, ids): + world.params['osm_ids'] = ids + if fmt and fmt.strip(): + world.params['format'] = fmt.strip() + api_call('lookup') + @step(u'sending an API call (\w+)') def api_general_call(step, call): api_call(call) diff --git a/tests/steps/osm2pgsql_setup.py b/tests/steps/osm2pgsql_setup.py index e5198d9d..eaa14573 100644 --- a/tests/steps/osm2pgsql_setup.py +++ b/tests/steps/osm2pgsql_setup.py @@ -140,7 +140,7 @@ def osm2pgsql_load_place(step): world.osm2pgsql.sort(cmp=_sort_xml_entries) # create a OSM file in /tmp - with tempfile.NamedTemporaryFile(dir='/tmp', delete=False) as fd: + with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.osm', delete=False) as fd: fname = fd.name fd.write("\n") fd.write('\n') @@ -188,7 +188,7 @@ def osm2pgsql_update_place(step): world.run_nominatim_script('setup', 'index', 'index-noanalyse') world.run_nominatim_script('setup', 'create-functions', 'create-partition-functions', 'enable-diff-updates') - with tempfile.NamedTemporaryFile(dir='/tmp', delete=False) as fd: + with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.osc', delete=False) as fd: fname = fd.name fd.write("\n") fd.write('\n') diff --git a/utils/setup.php b/utils/setup.php index b49813a8..eff7b71b 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -39,6 +39,7 @@ array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'), array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); @@ -78,11 +79,13 @@ } // Assume we can steal all the cache memory in the box (unless told otherwise) - $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB()); - if ($iCacheMemory > getTotalMemoryMB()) + if (isset($aCMDResult['osm2pgsql-cache'])) + { + $iCacheMemory = $aCMDResult['osm2pgsql-cache']; + } + else { $iCacheMemory = getCacheMemoryMB(); - echo "WARNING: resetting cache memory to $iCacheMemory\n"; } $aDSNInfo = DB::parseDSN(CONST_Database_DSN); @@ -131,7 +134,7 @@ pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql'); } else { - pgsqlRunScript('CREATE EXTENSION postgis'); + pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); } if ($fPostgisVersion < 2.1) { // Function was renamed in 2.1 and throws an annoying deprecation warning @@ -198,7 +201,7 @@ $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; if (CONST_Tablespace_Place_Index) $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; - $osm2pgsql .= ' -lsc -O gazetteer --hstore'; + $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; $osm2pgsql .= ' -C '.$iCacheMemory; $osm2pgsql .= ' -P '.$aDSNInfo['port']; $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; @@ -555,7 +558,7 @@ if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); } - if ($aCMDResult['osmosis-init'] || $aCMDResult['all']) + if ($aCMDResult['osmosis-init'] || ($aCMDResult['all'] && !$aCMDResult['drop'])) // no use doing osmosis-init when dropping update tables { $bDidSomething = true; $oDB =& getDB(); @@ -710,6 +713,7 @@ @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php'); @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php'); @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php'); + @symlink(CONST_BasePath.'/website/lookup.php', $sTargetDir.'/lookup.php'); @symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php'); @symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php'); @symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php'); @@ -726,6 +730,69 @@ } } + if ($aCMDResult['drop']) + { + // The implementation is potentially a bit dangerous because it uses + // a positive selection of tables to keep, and deletes everything else. + // Including any tables that the unsuspecting user might have manually + // created. USE AT YOUR OWN PERIL. + $bDidSomething = true; + + // tables we want to keep. everything else goes. + $aKeepTables = array( + "*columns", + "import_polygon_*", + "import_status", + "place_addressline", + "location_property*", + "placex", + "search_name", + "seq_*", + "word", + "query_log", + "new_query_log", + "gb_postcode", + "spatial_ref_sys", + "country_name", + "place_classtype_*" + ); + + $oDB =& getDB(); + $aDropTables = array(); + $aHaveTables = $oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"); + if (PEAR::isError($aHaveTables)) + { + fail($aPartitions->getMessage()); + } + foreach($aHaveTables as $sTable) + { + $bFound = false; + foreach ($aKeepTables as $sKeep) + { + if (fnmatch($sKeep, $sTable)) + { + $bFound = true; + break; + } + } + if (!$bFound) array_push($aDropTables, $sTable); + } + + foreach ($aDropTables as $sDrop) + { + if ($aCMDResult['verbose']) echo "dropping table $sDrop\n"; + @pg_query($oDB->connection, "DROP TABLE $sDrop CASCADE"); + // ignore warnings/errors as they might be caused by a table having + // been deleted already by CASCADE + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File)) + { + if ($aCMDResult['verbose']) echo "deleting ".CONST_Osm2pgsql_Flatnode_File."\n"; + unlink(CONST_Osm2pgsql_Flatnode_File); + } + } + if (!$bDidSomething) { showUsage($aCMDOptions, true); diff --git a/utils/update.php b/utils/update.php index c5fc14a6..13790450 100755 --- a/utils/update.php +++ b/utils/update.php @@ -83,7 +83,7 @@ $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } - $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; + $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; if (!is_null(CONST_Osm2pgsql_Flatnode_File)) { $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; diff --git a/vagrant-provision.sh b/vagrant-provision.sh index aad75443..68e279c5 100755 --- a/vagrant-provision.sh +++ b/vagrant-provision.sh @@ -84,8 +84,6 @@ sudo apt-get install -y libprotobuf-c0-dev protobuf-c-compiler \ # now ideally login as $USERNAME and continue su $USERNAME -l -pwd -ls -la /home/vagrant cd /home/vagrant/Nominatim # cd ~/Nominatim @@ -96,6 +94,12 @@ chmod +x ./ chmod +x ./module +LOCALSETTINGS_FILE='settings/local.php' +if [[ -e "$LOCALSETTINGS_FILE" ]]; then + echo "$LOCALSETTINGS_FILE already exist, writing to settings/local-vagrant.php instead." + LOCALSETTINGS_FILE='settings/local-vagrant.php' +fi + # IP=`curl -s http://bot.whatismyipaddress.com` IP=localhost echo " settings/local.php +" > $LOCALSETTINGS_FILE @@ -118,7 +122,7 @@ echo " 2) sleep(60); + if ($fLoadAvg > 4) sleep(120); + if ($fLoadAvg > 6) + { + userError("Bulk User: Temporary block due to high server load"); + exit; + } + } + + $oDB =& getDB(); + ini_set('memory_limit', '200M'); + + // Format for output + $sOutputFormat = 'xml'; + if (isset($_GET['format']) && ($_GET['format'] == 'xml' || $_GET['format'] == 'json')) + { + $sOutputFormat = $_GET['format']; + } + + // Preferred language + $aLangPrefOrder = getPreferredLanguages(); + + $hLog = logStart($oDB, 'place', $_SERVER['QUERY_STRING'], $aLangPrefOrder); + + $aSearchResults = array(); + $aCleanedQueryParts = array(); + if (isset($_GET['osm_ids'])) + { + $oPlaceLookup = new PlaceLookup($oDB); + $oPlaceLookup->setLanguagePreference($aLangPrefOrder); + $oPlaceLookup->setIncludeAddressDetails(getParamBool('addressdetails', true)); + $oPlaceLookup->setIncludeExtraTags(getParamBool('extratags', false)); + $oPlaceLookup->setIncludeNameDetails(getParamBool('namedetails', false)); + + $aOsmIds = explode(',', $_GET['osm_ids']); + + if ( count($aOsmIds) > CONST_Places_Max_ID_count ) + { + userError('Bulk User: Only ' . CONST_Places_Max_ID_count . " ids are allowed in one request."); + exit; + } + + foreach ($aOsmIds AS $sItem) + { + // Skip empty sItem + if (empty($sItem)) continue; + + $sType = $sItem[0]; + $iId = (int) substr($sItem, 1); + if ( $iId > 0 && ($sType == 'N' || $sType == 'W' || $sType == 'R') ) + { + $aCleanedQueryParts[] = $sType . $iId; + $oPlaceLookup->setOSMID($sType, $iId); + $oPlace = $oPlaceLookup->lookup(); + if ($oPlace){ + // we want to use the search-* output templates, so we need to fill + // $aSearchResults and slightly change the (reverse search) oPlace + // key names + $oResult = $oPlace; + unset($oResult['aAddress']); + if (isset($oPlace['aAddress'])) $oResult['address'] = $oPlace['aAddress']; + unset($oResult['langaddress']); + $oResult['name'] = $oPlace['langaddress']; + $aSearchResults[] = $oResult; + } + } + } + } + + + if (CONST_Debug) exit; + + $sXmlRootTag = 'lookupresults'; + $sQuery = join(',',$aCleanedQueryParts); + // we initialize these to avoid warnings in our logfile + $sViewBox = ''; + $bShowPolygons = ''; + $aExcludePlaceIDs = []; + $sMoreURL = ''; + + include(CONST_BasePath.'/lib/template/search-'.$sOutputFormat.'.php'); diff --git a/website/reverse.php b/website/reverse.php index 93cb5877..abc33a09 100755 --- a/website/reverse.php +++ b/website/reverse.php @@ -28,40 +28,52 @@ $sOutputFormat = $_GET['format']; } - // Show address breakdown - $bShowAddressDetails = true; - if (isset($_GET['addressdetails'])) $bShowAddressDetails = (bool)$_GET['addressdetails']; - // Preferred language $aLangPrefOrder = getPreferredLanguages(); $hLog = logStart($oDB, 'reverse', $_SERVER['QUERY_STRING'], $aLangPrefOrder); + if (isset($_GET['osm_type']) && isset($_GET['osm_id']) && (int)$_GET['osm_id'] && ($_GET['osm_type'] == 'N' || $_GET['osm_type'] == 'W' || $_GET['osm_type'] == 'R')) { - $oPlaceLookup = new PlaceLookup($oDB); - $oPlaceLookup->setLanguagePreference($aLangPrefOrder); - $oPlaceLookup->setIncludeAddressDetails($bShowAddressDetails); - $oPlaceLookup->setOSMID($_GET['osm_type'], $_GET['osm_id']); - - $aPlace = $oPlaceLookup->lookup(); + $aLookup = array('osm_type' => $_GET['osm_type'], 'osm_id' => $_GET['osm_id']); } else if (isset($_GET['lat']) && isset($_GET['lon']) && preg_match('/^[+-]?[0-9]*\.?[0-9]+$/', $_GET['lat']) && preg_match('/^[+-]?[0-9]*\.?[0-9]+$/', $_GET['lon'])) { $oReverseGeocode = new ReverseGeocode($oDB); $oReverseGeocode->setLanguagePreference($aLangPrefOrder); - $oReverseGeocode->setIncludeAddressDetails($bShowAddressDetails); $oReverseGeocode->setLatLon($_GET['lat'], $_GET['lon']); $oReverseGeocode->setZoom(@$_GET['zoom']); - $aPlace = $oReverseGeocode->lookup(); + $aLookup = $oReverseGeocode->lookup(); + if (CONST_Debug) var_dump($aLookup); + } + else + { + $aLookup = null; + } + + if ($aLookup) + { + $oPlaceLookup = new PlaceLookup($oDB); + $oPlaceLookup->setLanguagePreference($aLangPrefOrder); + $oPlaceLookup->setIncludeAddressDetails(getParamBool('addressdetails', true)); + $oPlaceLookup->setIncludeExtraTags(getParamBool('extratags', false)); + $oPlaceLookup->setIncludeNameDetails(getParamBool('namedetails', false)); + + $aPlace = $oPlaceLookup->lookupPlace($aLookup); } else { $aPlace = null; } - if (CONST_Debug) exit; + + if (CONST_Debug) + { + var_dump($aPlace); + exit; + } include(CONST_BasePath.'/lib/template/address-'.$sOutputFormat.'.php'); diff --git a/website/search.php b/website/search.php index bf27ef9a..b107e120 100755 --- a/website/search.php +++ b/website/search.php @@ -20,10 +20,14 @@ $aLangPrefOrder = getPreferredLanguages(); $oGeocode->setLanguagePreference($aLangPrefOrder); - if (isset($aLangPrefOrder['name:de'])) $oGeocode->setReverseInPlan(true); - if (isset($aLangPrefOrder['name:ru'])) $oGeocode->setReverseInPlan(true); - if (isset($aLangPrefOrder['name:ja'])) $oGeocode->setReverseInPlan(true); - if (isset($aLangPrefOrder['name:pl'])) $oGeocode->setReverseInPlan(true); + if (CONST_Search_ReversePlanForAll + || isset($aLangPrefOrder['name:de']) + || isset($aLangPrefOrder['name:ru']) + || isset($aLangPrefOrder['name:ja']) + || isset($aLangPrefOrder['name:pl'])) + { + $oGeocode->setReverseInPlan(true); + } // Format for output $sOutputFormat = 'html'; @@ -122,10 +126,12 @@ $bShowPolygons = (isset($_GET['polygon']) && $_GET['polygon']); $aExcludePlaceIDs = $oGeocode->getExcludedPlaceIDs(); - $sMoreURL = CONST_Website_BaseURL.'search?format='.urlencode($sOutputFormat).'&exclude_place_ids='.join(',',$oGeocode->getExcludedPlaceIDs()); + $sMoreURL = CONST_Website_BaseURL.'search.php?format='.urlencode($sOutputFormat).'&exclude_place_ids='.join(',',$oGeocode->getExcludedPlaceIDs()); if (isset($_SERVER["HTTP_ACCEPT_LANGUAGE"])) $sMoreURL .= '&accept-language='.$_SERVER["HTTP_ACCEPT_LANGUAGE"]; if ($bShowPolygons) $sMoreURL .= '&polygon=1'; if ($oGeocode->getIncludeAddressDetails()) $sMoreURL .= '&addressdetails=1'; + if ($oGeocode->getIncludeExtraTags()) $sMoreURL .= '&extratags=1'; + if ($oGeocode->getIncludeNameDetails()) $sMoreURL .= '&namedetails=1'; if ($sViewBox) $sMoreURL .= '&viewbox='.urlencode($sViewBox); if (isset($_GET['nearlat']) && isset($_GET['nearlon'])) $sMoreURL .= '&nearlat='.(float)$_GET['nearlat'].'&nearlon='.(float)$_GET['nearlon']; $sMoreURL .= '&q='.urlencode($sQuery); diff --git a/website/status.php b/website/status.php index a876f999..832f4600 100644 --- a/website/status.php +++ b/website/status.php @@ -10,7 +10,7 @@ exit; } - $oDB =& getDB(); + $oDB =& DB::connect(CONST_Database_DSN, false); if (!$oDB || PEAR::isError($oDB)) { statusError("No database");