From 49d0ce1de4106914d90f6a9c959c7c804e960aba Mon Sep 17 00:00:00 2001 From: Markus Gail Date: Mon, 7 Mar 2016 11:07:33 +0100 Subject: [PATCH] change tiger housenumber format to save as lines instead of points and thus new interpolation --- lib/Geocode.php | 80 +++++++++++++++++++++++++------------ settings/settings.php | 2 +- sql/tiger_import_finish.sql | 7 ++-- sql/tiger_import_start.sql | 21 ++++------ utils/tigerAddressImport.py | 6 +-- 5 files changed, 70 insertions(+), 46 deletions(-) diff --git a/lib/Geocode.php b/lib/Geocode.php index a6ff279c..a6f8e0e5 100644 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@ -44,6 +44,10 @@ protected $sQuery = false; protected $aStructuredQuery = false; + + //for Tiger housenumber interpolation + protected $searchedHousenumber=-1; + protected $housenumberFound=false; function Geocode(&$oDB) { @@ -430,21 +434,29 @@ if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank) { - $sSQL .= " union "; - $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; - $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; - $sSQL .= "null as placename,"; - $sSQL .= "null as ref,"; - if ($this->bIncludeExtraTags) $sSQL .= "null as extra,"; - if ($this->bIncludeNameDetails) $sSQL .= "null as names,"; - $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; - $sSQL .= $sImportanceSQL."-1.15 as importance, "; - $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; - $sSQL .= "null as extra_place "; - $sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) "; - $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank "; - $sSQL .= "group by place_id"; - if (!$this->bDeDupe) $sSQL .= ",place_id "; + //query also location_property_tiger_line and location_property_aux + //Tiger search only if it was searched for a housenumber (searchedHousenumber >=0) and if it was found (housenumberFound = true) + //only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points + if($this->searchedHousenumber>=0 && $this->housenumberFound){ + $sSQL .= "union "; + $sSQL .= "select 'T' as osm_type, place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code"; + $sSQL .= ", get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress "; + $sSQL .= ", null as placename"; + $sSQL .= ", null as ref"; + if ($this->bIncludeExtraTags) $sSQL .= ", null as extra"; + if ($this->bIncludeNameDetails) $sSQL .= ", null as names"; + $sSQL .= ", avg(st_x(point)) as lon, avg(st_y(point)) as lat"; + $sSQL .= $sImportanceSQL.", -1.15 as importance "; + $sSQL .= ", 1.0 as addressimportance "; //not sure how the addressimportance is/should be calculated for Tiger data + $sSQL .= ", null as extra_place "; + $sSQL .= " from (select place_id"; + //interpolate the Tiger housenumbers here + $sSQL .= ",ST_LineInterpolatePoint(linegeo, ($this->searchedHousenumber::float-startnumber::float)/(endnumber-startnumber)::float) as point, parent_place_id "; + $sSQL .= "from location_property_tiger_line where place_id in ($sPlaceIDs) "; + $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here + $sSQL .= " group by place_id"; //why group by place_id, isnt place_id unique? + if (!$this->bDeDupe) $sSQL .= ",place_id "; + } $sSQL .= " union "; $sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; @@ -1390,6 +1402,7 @@ //var_Dump($aPlaceIDs); //exit; + //now search for housenumber, if housenumber provided if ($aSearch['sHouseNumber'] && sizeof($aPlaceIDs)) { $aRoadPlaceIDs = $aPlaceIDs; @@ -1406,7 +1419,7 @@ if (CONST_Debug) var_dump($sSQL); $aPlaceIDs = $this->oDB->getCol($sSQL); - // If not try the aux fallback table + // If nothing found try the aux fallback table if (!sizeof($aPlaceIDs)) { $sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'"; @@ -1418,27 +1431,43 @@ if (CONST_Debug) var_dump($sSQL); $aPlaceIDs = $this->oDB->getCol($sSQL); } - + //if nothing was found in placex or location_property_aux, then search in Tiger data for this housenumber(location_property_tiger_line) if (!sizeof($aPlaceIDs)) { - $sSQL = "select place_id from location_property_tiger where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'"; - if (sizeof($this->aExcludePlaceIDs)) + //$sSQL = "select place_id from location_property_tiger where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'"; + //new query for lines, not housenumbers anymore + $this->searchedHousenumber = intval($aSearch['sHouseNumber']); + if($this->searchedHousenumber%2==0){ + //if housenumber is even, look for housenumber in streets with interpolationtype even or all + $sSQL = "select distinct place_id from location_property_tiger_line where parent_place_id in (".$sPlaceIDs.") and (interpolationtype='even' or interpolationtype='all') and ".$this->searchedHousenumber.">=startnumber and ".$this->searchedHousenumber."<=endnumber"; + }else{ + //look for housenumber in streets with interpolationtype odd or all + $sSQL = "select distinct place_id from location_property_tiger_line where parent_place_id in (".$sPlaceIDs.") and (interpolationtype='odd' or interpolationtype='all') and ".$this->searchedHousenumber.">=startnumber and ".$this->searchedHousenumber."<=endnumber"; + } + + if (sizeof($this->aExcludePlaceIDs)) { $sSQL .= " and place_id not in (".join(',',$this->aExcludePlaceIDs).")"; } //$sSQL .= " limit $this->iLimit"; if (CONST_Debug) var_dump($sSQL); - $aPlaceIDs = $this->oDB->getCol($sSQL); + //get place IDs + $aPlaceIDs = $this->oDB->getCol($sSQL,0); } - // Fallback to the road + // Fallback to the road (if no housenumber was found) if (!sizeof($aPlaceIDs) && preg_match('/[0-9]+/', $aSearch['sHouseNumber'])) { $aPlaceIDs = $aRoadPlaceIDs; - } - + //set to false, if no housenumbers were found + $this->housenumberFound=false; + }else{ + //housenumber was found + $this->housenumberFound=true; + } } + if ($aSearch['sClass'] && sizeof($aPlaceIDs)) { $sPlaceIDs = join(',',$aPlaceIDs); @@ -1767,16 +1796,17 @@ { $aResult['label'] = $aClassType[$aResult['class'].':'.$aResult['type']]['label']; } - + /* Implement this function later. if tag '&addressdetails=1' is set in query if ($this->bIncludeAddressDetails) { + * getAddressDetails() is defined in lib.php and uses the SQL function get_addressdata in functions.sql $aResult['address'] = getAddressDetails($this->oDB, $sLanguagePrefArraySQL, $aResult['place_id'], $aResult['country_code']); if ($aResult['extra_place'] == 'city' && !isset($aResult['address']['city'])) { $aResult['address'] = array_merge(array('city' => array_shift(array_values($aResult['address']))), $aResult['address']); } } - + */ if ($this->bIncludeExtraTags) { if ($aResult['extra']) diff --git a/settings/settings.php b/settings/settings.php index 04ffe162..21033ff3 100644 --- a/settings/settings.php +++ b/settings/settings.php @@ -84,7 +84,7 @@ @define('CONST_BulkUserIPs', ''); @define('CONST_BlockMessage', ''); // additional info to show for blocked IPs - @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); + @define('CONST_Website_BaseURL', 'http://localhost/nominatim/'); @define('CONST_Tile_Default', 'Mapnik'); @define('CONST_Default_Language', false); diff --git a/sql/tiger_import_finish.sql b/sql/tiger_import_finish.sql index 4718d502..374c00b3 100644 --- a/sql/tiger_import_finish.sql +++ b/sql/tiger_import_finish.sql @@ -1,4 +1,5 @@ -CREATE INDEX idx_location_property_tiger_housenumber_parent_place_id_imp ON location_property_tiger_import (parent_place_id, housenumber) {ts:aux-index}; +--index only on parent_place_id +CREATE INDEX idx_location_property_tiger_parent_place_id_imp ON location_property_tiger_import (parent_place_id) {ts:aux-index}; CREATE UNIQUE INDEX idx_location_property_tiger_place_id_imp ON location_property_tiger_import (place_id) {ts:aux-index}; GRANT SELECT ON location_property_tiger_import TO "{www-user}"; @@ -6,7 +7,7 @@ GRANT SELECT ON location_property_tiger_import TO "{www-user}"; DROP TABLE IF EXISTS location_property_tiger; ALTER TABLE location_property_tiger_import RENAME TO location_property_tiger; -ALTER INDEX idx_location_property_tiger_housenumber_parent_place_id_imp RENAME TO idx_location_property_tiger_housenumber_parent_place_id; +ALTER INDEX idx_location_property_tiger_parent_place_id_imp RENAME TO idx_location_property_tiger_housenumber_parent_place_id; ALTER INDEX idx_location_property_tiger_place_id_imp RENAME TO idx_location_property_tiger_place_id; -DROP FUNCTION tigger_create_interpolation (linegeo geometry, in_startnumber integer, in_endnumber integer, interpolationtype text, in_street text, in_isin text, in_postcode text); +DROP FUNCTION tiger_line_import (linegeo geometry, in_startnumber integer, in_endnumber integer, interpolationtype text, in_street text, in_isin text, in_postcode text); diff --git a/sql/tiger_import_start.sql b/sql/tiger_import_start.sql index 9846f609..c3e1ed02 100644 --- a/sql/tiger_import_start.sql +++ b/sql/tiger_import_start.sql @@ -1,17 +1,14 @@ DROP TABLE IF EXISTS location_property_tiger_import; -CREATE TABLE location_property_tiger_import () INHERITS (location_property) {ts:aux-data}; +CREATE TABLE location_property_tiger_import (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, street TEXT, isin TEXT, postcode TEXT); -CREATE OR REPLACE FUNCTION tigger_create_interpolation(linegeo GEOMETRY, in_startnumber INTEGER, +CREATE OR REPLACE FUNCTION tiger_line_import(linegeo GEOMETRY, in_startnumber INTEGER, in_endnumber INTEGER, interpolationtype TEXT, in_street TEXT, in_isin TEXT, in_postcode TEXT) RETURNS INTEGER AS $$ DECLARE - startnumber INTEGER; endnumber INTEGER; stepsize INTEGER; - housenum INTEGER; - newpoints INTEGER; numberrange INTEGER; rangestartnumber INTEGER; place_centroid GEOMETRY; @@ -32,7 +29,7 @@ BEGIN numberrange := endnumber - startnumber; rangestartnumber := startnumber; - + IF (interpolationtype = 'odd' AND startnumber%2 = 0) OR (interpolationtype = 'even' AND startnumber%2 = 1) THEN startnumber := startnumber + 1; stepsize := 2; @@ -75,15 +72,11 @@ BEGIN END LOOP; END IF; - newpoints := 0; - FOR housenum IN startnumber..endnumber BY stepsize LOOP - insert into location_property_tiger_import (place_id, partition, parent_place_id, housenumber, postcode, centroid) - values (nextval('seq_place'), out_partition, out_parent_place_id, housenum, in_postcode, - ST_LineInterpolatePoint(linegeo, (housenum::float-rangestartnumber::float)/numberrange::float)); - newpoints := newpoints + 1; - END LOOP; +--insert street(line) into import table +insert into location_property_tiger_import (linegeo, place_id, partition, parent_place_id, startnumber, endnumber, interpolationtype, street, isin, postcode) +values (linegeo, nextval('seq_place'), out_partition, out_parent_place_id, startnumber, endnumber, interpolationtype, in_street, in_isin, in_postcode); - RETURN newpoints; + RETURN 1; END; $$ LANGUAGE plpgsql; diff --git a/utils/tigerAddressImport.py b/utils/tigerAddressImport.py index 1eb10d76..5499f50e 100755 --- a/utils/tigerAddressImport.py +++ b/utils/tigerAddressImport.py @@ -3893,8 +3893,8 @@ def addressways(waylist, nodelist, first_id): # ret.append( "" % (iSource, VERSION, import_guid) ) # ret.append( "" % (iAttrib) ) # ret.append( "" ) - - ret.append( "select tigger_create_interpolation(ST_GeomFromText('LINESTRING(%s)',4326), '%s', '%s', '%s', '%s', '%s', '%s');" % + # call new tiger_line_import function to save the lines in the DB. + ret.append( "select tiger_line_import(ST_GeomFromText('LINESTRING(%s)',4326), '%s', '%s', '%s', '%s', '%s', '%s');" % ( ",".join(rlinestring), rfromadd.replace("'", "''"), rtoadd.replace("'", "''"), interpolationtype.replace("'", "''"), name.replace("'", "''"), county.replace("'", "''"), zipr.replace("'", "''") ) ) if left: @@ -3919,7 +3919,7 @@ def addressways(waylist, nodelist, first_id): interpolationtype = "all"; else: interpolationtype = "all"; - ret.append( "select tigger_create_interpolation(ST_GeomFromText('LINESTRING(%s)',4326), '%s', '%s', '%s', '%s', '%s', '%s');" % + ret.append( "select tiger_line_import(ST_GeomFromText('LINESTRING(%s)',4326), '%s', '%s', '%s', '%s', '%s', '%s');" % ( ",".join(llinestring), lfromadd.replace("'", "''"), ltoadd.replace("'", "''"), interpolationtype.replace("'", "''"), name.replace("'", "''"), county.replace("'", "''"), zipl.replace("'", "''") ) ) return ret -- 2.39.5