]> git.openstreetmap.org Git - nominatim.git/commitdiff
make tables for external data (Tiger and aux) configurable
authorSarah Hoffmann <lonvia@denofr.de>
Thu, 7 Apr 2016 19:47:51 +0000 (21:47 +0200)
committerSarah Hoffmann <lonvia@denofr.de>
Thu, 7 Apr 2016 19:47:51 +0000 (21:47 +0200)
Introduces two new settings CONST_Use_US_Tiger_Data and
CONST_Use_Aux_Location_data, which are disabled by default.
When false the corresponding tables are ignored in queries
and updates.

Aux and tiger tables are no longer created by default. This
has to be done by the corresponding import scripts. The former
aux table creation can be found in sql/aux_tables.sql for
reference.

lib/Geocode.php
lib/PlaceLookup.php
lib/ReverseGeocode.php
settings/settings.php
sql/aux_tables.sql [new file with mode: 0644]
sql/functions.sql
sql/tables.sql
utils/setup.php
website/details.php
website/hierarchy.php

index f2ca5e5008f36d440cc67f8dc81eb5c497f94b61..611ca6de630f4252dad05dba6c4764c64077d336 100644 (file)
 
                        if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
                        {
-                               //query also location_property_tiger and location_property_aux
-                               //Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
-                               //only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
-                               $sHousenumbers = "";
-                               $i = 0;
-                               $length = count($aPlaceIDs);
-                               foreach($aPlaceIDs as $placeID => $housenumber)
-                {
-                                       $i++;
-                                       $sHousenumbers .= "(".$placeID.", ".$housenumber.")";
-                                       if($i<$length)
-                                               $sHousenumbers .= ", ";
+                               if (CONST_Use_US_Tiger_Data)
+                               {
+                                       //query also location_property_tiger and location_property_aux
+                                       //Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
+                                       //only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
+                                       $sHousenumbers = "";
+                                       $i = 0;
+                                       $length = count($aPlaceIDs);
+                                       foreach($aPlaceIDs as $placeID => $housenumber)
+                                       {
+                                               $i++;
+                                               $sHousenumbers .= "(".$placeID.", ".$housenumber.")";
+                                               if($i<$length)
+                                                       $sHousenumbers .= ", ";
+                                       }
+
+                                       $sSQL .= "union ";
+                                       $sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
+                                       $sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
+                                       $sSQL .= ", null as placename";
+                                       $sSQL .= ", null as ref";
+                                       if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
+                                       if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
+                                       $sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
+                                       $sSQL .= $sImportanceSQL."-1.15 as importance ";
+                                       $sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
+                                       $sSQL .= ", null as extra_place ";
+                                       $sSQL .= " from (select place_id";
+                                       //interpolate the Tiger housenumbers here
+                                       $sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
+                                       $sSQL .= "from (location_property_tiger ";
+                                       $sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
+                                       $sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
+                                       $sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
+                                       if (!$this->bDeDupe) $sSQL .= ", place_id ";
                                }
 
-                               $sSQL .= "union ";
-                               $sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
-                               $sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
-                               $sSQL .= ", null as placename";
-                               $sSQL .= ", null as ref";
-                               if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
-                               if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
-                               $sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
-                               $sSQL .= $sImportanceSQL."-1.15 as importance ";
-                               $sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
-                               $sSQL .= ", null as extra_place ";
-                               $sSQL .= " from (select place_id";
-                               //interpolate the Tiger housenumbers here
-                               $sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
-                               $sSQL .= "from (location_property_tiger ";
-                               $sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
-                               $sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
-                               $sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
-                               if (!$this->bDeDupe) $sSQL .= ", place_id ";
-
-                               $sSQL .= " union ";
-                               $sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
-                               $sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
-                               $sSQL .= "null as placename, ";
-                               $sSQL .= "null as ref, ";
-                               if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
-                               if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
-                               $sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
-                               $sSQL .= $sImportanceSQL."-1.10 as importance, ";
-                               $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
-                               $sSQL .= "null as extra_place ";
-                               $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
-                               $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
-                               $sSQL .= "group by place_id";
-                               if (!$this->bDeDupe) $sSQL .= ", place_id";
-                               $sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
+                               if (CONST_Use_Aux_Location_data)
+                               {
+                                       $sSQL .= " union ";
+                                       $sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
+                                       $sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
+                                       $sSQL .= "null as placename, ";
+                                       $sSQL .= "null as ref, ";
+                                       if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
+                                       if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
+                                       $sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
+                                       $sSQL .= $sImportanceSQL."-1.10 as importance, ";
+                                       $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+                                       $sSQL .= "null as extra_place ";
+                                       $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
+                                       $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
+                                       $sSQL .= "group by place_id";
+                                       if (!$this->bDeDupe) $sSQL .= ", place_id";
+                                       $sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
+                               }
                        }
 
                        $sSQL .= " order by importance desc";
                                                                $aPlaceIDs = $this->oDB->getCol($sSQL);
 
                                                                // If nothing found try the aux fallback table
-                                                               if (!sizeof($aPlaceIDs))
+                                                               if (CONST_Use_Aux_Location_data && !sizeof($aPlaceIDs))
                                                                {
                                                                        $sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
                                                                        if (sizeof($this->aExcludePlaceIDs))
                                                                }
                                                                //if nothing was found in placex or location_property_aux, then search in Tiger data for this housenumber(location_property_tiger)
                                                                $searchedHousenumber = intval($aSearch['sHouseNumber']);
-                                                               if (!sizeof($aPlaceIDs))
+                                                               if (CONST_Use_US_Tiger_Data && !sizeof($aPlaceIDs))
                                                                {
                                                                        //new query for lines, not housenumbers anymore
                                                                        if($searchedHousenumber%2 == 0){
                                                $sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
                                                if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'";
                                                if ($this->aAddressRankList) $sSQL .= " OR placex.rank_address in (".join(',',$this->aAddressRankList).")";
-                                               $sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
-                                               $sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
-                                               if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
+                                               if (CONST_Use_US_Tiger_Data)
+                                               {
+                                                       $sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
+                                                       $sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
+                                                       if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
+                                               }
                                                $sSQL .= ")";
                                                if (CONST_Debug) var_dump($sSQL);
                                                $aFilteredPlaceIDs = $this->oDB->getCol($sSQL);
index 94e414dc7d45e930e8d1092f45f1f9e4125860ee..c1fec6496ef66ca3118f7b214b2c7ee6f85a7173 100644 (file)
 
                        $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
 
-                       if ($this->sType == 'tiger')
+                       if (CONST_Use_US_Tiger_Data && $this->sType == 'tiger')
                        {
                                $sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,";
                                $sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
 
                        if ($this->bAddressDetails)
                        {
-                               if($this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
+                               if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
                                        $aAddress = $this->getAddressNames($aPlace['housenumber']);
                                else
                                        $aAddress = $this->getAddressNames();
index 3dff161e4136a443b6362171c719057b8d8f5b39..75a9b71c3fd902c1bbcc8a613afa5e2acd0ce162 100644 (file)
                        }
 
                        // Only street found? If it's in the US we can check TIGER data for nearest housenumber
-                       if ($bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
+                       if (CONST_Use_US_Tiger_Data && $bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
                        {
                                $fSearchDiam = 0.001;
                                $sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';
index e1dd625fddfc2926b879e6f377775249c5abcb3f..266ff356b64773f2229fa4d49834795a7c2e915f 100644 (file)
        @define('CONST_Limit_Reindexing', true);
        // Set to false to avoid importing extra postcodes for the US.
        @define('CONST_Use_Extra_US_Postcodes', true);
+       // Set to true after importing Tiger house number data for the US.
+       // Note: The tables must already exist or queries will throw errors.
+       //       After changing this setting run ./utils/setup --create-functions
+       //       again.
+       @define('CONST_Use_US_Tiger_Data', false);
+       // Set to true after importing other external house number data.
+       // Note: the aux tables must already exist or queries will throw errors.
+       //       After changing this setting run ./utils/setup --create-functions
+       //       again.
+       @define('CONST_Use_Aux_Location_data', false);
 
        // Proxy settings
        @define('CONST_HTTP_Proxy', false);
diff --git a/sql/aux_tables.sql b/sql/aux_tables.sql
new file mode 100644 (file)
index 0000000..8105473
--- /dev/null
@@ -0,0 +1,6 @@
+CREATE TABLE location_property_aux () INHERITS (location_property);
+CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
+CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
+GRANT SELECT ON location_property_aux TO "{www-user}";
+
index 5b068dfe03f5ed7557939c2f99054d5e98b74160..4256490ebcf967a80d82bd16550252acebbfc64e 100644 (file)
@@ -1024,7 +1024,7 @@ BEGIN
 
   --DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
 
-  RETURN NEW; -- @DIFFUPDATES@ The following is not needed until doing diff updates, and slows the main index process down
+  RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
 
   IF NEW.rank_address > 0 THEN
     IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
@@ -1650,6 +1650,7 @@ BEGIN
       END IF;
     END IF;
 
+    -- %NOTIGERDATA% IF 0 THEN
     -- for the USA we have an additional address table.  Merge in zip codes from there too
     IF NEW.rank_search = 26 AND NEW.calculated_country_code = 'us' THEN
       FOR location IN SELECT distinct postcode from location_property_tiger where parent_place_id = NEW.place_id LOOP
@@ -1662,6 +1663,7 @@ BEGIN
         nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
       END LOOP;
     END IF;
+    -- %NOTIGERDATA% END IF;
 
 -- RAISE WARNING 'ISIN: %', isin_tokens;
 
@@ -2257,18 +2259,22 @@ DECLARE
   hadcountry BOOLEAN;
 BEGIN
     --first query tiger data
+  -- %NOTIGERDATA% IF 0 THEN
   select parent_place_id,'us', 30, postcode, null, 'place', 'house' from location_property_tiger 
     WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
     INTO for_place_id,searchcountrycode, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
   IF for_place_id IS NOT NULL THEN
     searchhousenumber = in_housenumber::text;
   END IF;
-  
+  -- %NOTIGERDATA% END IF;
+
+  -- %NOAUXDATA% IF 0 THEN
   IF for_place_id IS NULL THEN
     select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
       WHERE place_id = in_place_id 
       INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
   END IF;
+  -- %NOAUXDATA% END IF;
 
   IF for_place_id IS NULL THEN
     select parent_place_id, calculated_country_code, housenumber, rank_search, postcode, name, class, type from placex 
index 0434e13af8b5494d2bb0fb86f8864e7822cbac97..49338c4a34569530f39c4131e4ee75be75b625ae 100644 (file)
@@ -100,15 +100,6 @@ CREATE TABLE location_property (
   );
 SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
 
-CREATE TABLE location_property_aux () INHERITS (location_property);
-CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
-CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
-CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
-GRANT SELECT ON location_property_aux TO "{www-user}";
-
-CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT);
-GRANT SELECT ON location_property_tiger TO "{www-user}";
-
 drop table IF EXISTS search_name;
 CREATE TABLE search_name (
   place_id BIGINT,
index 0026a8135dc76c03b7d42bbf3e20203388c61e7e..cfe7f560191e353153aff9b5500f96f96141ed7b 100755 (executable)
                if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
                $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
                $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
-               if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
-               if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
-               if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
+               if ($aCMDResult['enable-diff-updates'])
+               {
+                       $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
+               }
+               if ($aCMDResult['enable-debug-statements'])
+               {
+                       $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
+               }
+               if (CONST_Limit_Reindexing)
+               {
+                       $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
+               }
+               if (!CONST_Use_US_Tiger_Data)
+               {
+                       $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate);
+               }
+               if (!CONST_Use_Aux_Location_data)
+               {
+                       $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
+               }
                pgsqlRunScript($sTemplate);
        }
 
index c7a567552c06afe614cf6ae08b2f0246d31b155f..5edef6f5001938505960cf0e6dffe5d35e05d952 100755 (executable)
 
        $iPlaceID = (int)$_GET['place_id'];
 
-       $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
-       if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
-       $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
-       if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+       if (CONST_Use_US_Tiger_Data)
+       {
+               $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
+               if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+       }
+
+       if (CONST_Use_Aux_Location_data)
+       {
+               $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
+               if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+       }
 
        $hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
 
index 5e3d7299db68d8513d5bbdec3e9b348a828dc995..6a281aa871dc01415130c69d87545536aafeb1de 100755 (executable)
 
        $iPlaceID = (int)$_GET['place_id'];
 
-       $sAuxHouseNumber = false;
-       $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
-       if ($iParentPlaceID)
+       if (CONST_Use_US_Tiger_Data)
        {
-               $iPlaceID = $iParentPlaceID;
+               $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
+               if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
        }
-       else
+
+       if (CONST_Use_Aux_Location_data)
        {
                $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
                if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;