]> git.openstreetmap.org Git - nominatim.git/commitdiff
Merge pull request #425 from lonvia/disable-aux-tables
authorSarah Hoffmann <lonvia@denofr.de>
Tue, 12 Apr 2016 21:00:24 +0000 (23:00 +0200)
committerSarah Hoffmann <lonvia@denofr.de>
Tue, 12 Apr 2016 21:00:24 +0000 (23:00 +0200)
make tables for external data (Tiger and aux) configurable

lib/Geocode.php
lib/PlaceLookup.php
lib/ReverseGeocode.php
settings/settings.php
sql/aux_tables.sql [new file with mode: 0644]
sql/functions.sql
sql/tables.sql
utils/setup.php
website/details.php
website/hierarchy.php

index f2ca5e5008f36d440cc67f8dc81eb5c497f94b61..611ca6de630f4252dad05dba6c4764c64077d336 100644 (file)
 
                        if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
                        {
-                               //query also location_property_tiger and location_property_aux
-                               //Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
-                               //only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
-                               $sHousenumbers = "";
-                               $i = 0;
-                               $length = count($aPlaceIDs);
-                               foreach($aPlaceIDs as $placeID => $housenumber)
-                {
-                                       $i++;
-                                       $sHousenumbers .= "(".$placeID.", ".$housenumber.")";
-                                       if($i<$length)
-                                               $sHousenumbers .= ", ";
+                               if (CONST_Use_US_Tiger_Data)
+                               {
+                                       //query also location_property_tiger and location_property_aux
+                                       //Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
+                                       //only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
+                                       $sHousenumbers = "";
+                                       $i = 0;
+                                       $length = count($aPlaceIDs);
+                                       foreach($aPlaceIDs as $placeID => $housenumber)
+                                       {
+                                               $i++;
+                                               $sHousenumbers .= "(".$placeID.", ".$housenumber.")";
+                                               if($i<$length)
+                                                       $sHousenumbers .= ", ";
+                                       }
+
+                                       $sSQL .= "union ";
+                                       $sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
+                                       $sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
+                                       $sSQL .= ", null as placename";
+                                       $sSQL .= ", null as ref";
+                                       if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
+                                       if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
+                                       $sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
+                                       $sSQL .= $sImportanceSQL."-1.15 as importance ";
+                                       $sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
+                                       $sSQL .= ", null as extra_place ";
+                                       $sSQL .= " from (select place_id";
+                                       //interpolate the Tiger housenumbers here
+                                       $sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
+                                       $sSQL .= "from (location_property_tiger ";
+                                       $sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
+                                       $sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
+                                       $sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
+                                       if (!$this->bDeDupe) $sSQL .= ", place_id ";
                                }
 
-                               $sSQL .= "union ";
-                               $sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
-                               $sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
-                               $sSQL .= ", null as placename";
-                               $sSQL .= ", null as ref";
-                               if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
-                               if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
-                               $sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
-                               $sSQL .= $sImportanceSQL."-1.15 as importance ";
-                               $sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
-                               $sSQL .= ", null as extra_place ";
-                               $sSQL .= " from (select place_id";
-                               //interpolate the Tiger housenumbers here
-                               $sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
-                               $sSQL .= "from (location_property_tiger ";
-                               $sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
-                               $sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
-                               $sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
-                               if (!$this->bDeDupe) $sSQL .= ", place_id ";
-
-                               $sSQL .= " union ";
-                               $sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
-                               $sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
-                               $sSQL .= "null as placename, ";
-                               $sSQL .= "null as ref, ";
-                               if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
-                               if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
-                               $sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
-                               $sSQL .= $sImportanceSQL."-1.10 as importance, ";
-                               $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
-                               $sSQL .= "null as extra_place ";
-                               $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
-                               $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
-                               $sSQL .= "group by place_id";
-                               if (!$this->bDeDupe) $sSQL .= ", place_id";
-                               $sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
+                               if (CONST_Use_Aux_Location_data)
+                               {
+                                       $sSQL .= " union ";
+                                       $sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
+                                       $sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
+                                       $sSQL .= "null as placename, ";
+                                       $sSQL .= "null as ref, ";
+                                       if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
+                                       if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
+                                       $sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
+                                       $sSQL .= $sImportanceSQL."-1.10 as importance, ";
+                                       $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+                                       $sSQL .= "null as extra_place ";
+                                       $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
+                                       $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
+                                       $sSQL .= "group by place_id";
+                                       if (!$this->bDeDupe) $sSQL .= ", place_id";
+                                       $sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
+                               }
                        }
 
                        $sSQL .= " order by importance desc";
                                                                $aPlaceIDs = $this->oDB->getCol($sSQL);
 
                                                                // If nothing found try the aux fallback table
-                                                               if (!sizeof($aPlaceIDs))
+                                                               if (CONST_Use_Aux_Location_data && !sizeof($aPlaceIDs))
                                                                {
                                                                        $sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
                                                                        if (sizeof($this->aExcludePlaceIDs))
                                                                }
                                                                //if nothing was found in placex or location_property_aux, then search in Tiger data for this housenumber(location_property_tiger)
                                                                $searchedHousenumber = intval($aSearch['sHouseNumber']);
-                                                               if (!sizeof($aPlaceIDs))
+                                                               if (CONST_Use_US_Tiger_Data && !sizeof($aPlaceIDs))
                                                                {
                                                                        //new query for lines, not housenumbers anymore
                                                                        if($searchedHousenumber%2 == 0){
                                                $sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
                                                if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'";
                                                if ($this->aAddressRankList) $sSQL .= " OR placex.rank_address in (".join(',',$this->aAddressRankList).")";
-                                               $sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
-                                               $sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
-                                               if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
+                                               if (CONST_Use_US_Tiger_Data)
+                                               {
+                                                       $sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
+                                                       $sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
+                                                       if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
+                                               }
                                                $sSQL .= ")";
                                                if (CONST_Debug) var_dump($sSQL);
                                                $aFilteredPlaceIDs = $this->oDB->getCol($sSQL);
index 94e414dc7d45e930e8d1092f45f1f9e4125860ee..c1fec6496ef66ca3118f7b214b2c7ee6f85a7173 100644 (file)
 
                        $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
 
-                       if ($this->sType == 'tiger')
+                       if (CONST_Use_US_Tiger_Data && $this->sType == 'tiger')
                        {
                                $sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,";
                                $sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
 
                        if ($this->bAddressDetails)
                        {
-                               if($this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
+                               if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
                                        $aAddress = $this->getAddressNames($aPlace['housenumber']);
                                else
                                        $aAddress = $this->getAddressNames();
index 3dff161e4136a443b6362171c719057b8d8f5b39..75a9b71c3fd902c1bbcc8a613afa5e2acd0ce162 100644 (file)
                        }
 
                        // Only street found? If it's in the US we can check TIGER data for nearest housenumber
-                       if ($bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
+                       if (CONST_Use_US_Tiger_Data && $bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
                        {
                                $fSearchDiam = 0.001;
                                $sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';
index 33cfa3f201768fbef491b0532738dec505458350..d44bb967d4ab19687bf0a92fb583faf0934bba4b 100644 (file)
        @define('CONST_Limit_Reindexing', true);
        // Set to false to avoid importing extra postcodes for the US.
        @define('CONST_Use_Extra_US_Postcodes', true);
+       // Set to true after importing Tiger house number data for the US.
+       // Note: The tables must already exist or queries will throw errors.
+       //       After changing this setting run ./utils/setup --create-functions
+       //       again.
+       @define('CONST_Use_US_Tiger_Data', false);
+       // Set to true after importing other external house number data.
+       // Note: the aux tables must already exist or queries will throw errors.
+       //       After changing this setting run ./utils/setup --create-functions
+       //       again.
+       @define('CONST_Use_Aux_Location_data', false);
 
        // Proxy settings
        @define('CONST_HTTP_Proxy', false);
diff --git a/sql/aux_tables.sql b/sql/aux_tables.sql
new file mode 100644 (file)
index 0000000..8105473
--- /dev/null
@@ -0,0 +1,6 @@
+CREATE TABLE location_property_aux () INHERITS (location_property);
+CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
+CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
+GRANT SELECT ON location_property_aux TO "{www-user}";
+
index 5b068dfe03f5ed7557939c2f99054d5e98b74160..4256490ebcf967a80d82bd16550252acebbfc64e 100644 (file)
@@ -1024,7 +1024,7 @@ BEGIN
 
   --DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
 
-  RETURN NEW; -- @DIFFUPDATES@ The following is not needed until doing diff updates, and slows the main index process down
+  RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
 
   IF NEW.rank_address > 0 THEN
     IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
@@ -1650,6 +1650,7 @@ BEGIN
       END IF;
     END IF;
 
+    -- %NOTIGERDATA% IF 0 THEN
     -- for the USA we have an additional address table.  Merge in zip codes from there too
     IF NEW.rank_search = 26 AND NEW.calculated_country_code = 'us' THEN
       FOR location IN SELECT distinct postcode from location_property_tiger where parent_place_id = NEW.place_id LOOP
@@ -1662,6 +1663,7 @@ BEGIN
         nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
       END LOOP;
     END IF;
+    -- %NOTIGERDATA% END IF;
 
 -- RAISE WARNING 'ISIN: %', isin_tokens;
 
@@ -2257,18 +2259,22 @@ DECLARE
   hadcountry BOOLEAN;
 BEGIN
     --first query tiger data
+  -- %NOTIGERDATA% IF 0 THEN
   select parent_place_id,'us', 30, postcode, null, 'place', 'house' from location_property_tiger 
     WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
     INTO for_place_id,searchcountrycode, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
   IF for_place_id IS NOT NULL THEN
     searchhousenumber = in_housenumber::text;
   END IF;
-  
+  -- %NOTIGERDATA% END IF;
+
+  -- %NOAUXDATA% IF 0 THEN
   IF for_place_id IS NULL THEN
     select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
       WHERE place_id = in_place_id 
       INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
   END IF;
+  -- %NOAUXDATA% END IF;
 
   IF for_place_id IS NULL THEN
     select parent_place_id, calculated_country_code, housenumber, rank_search, postcode, name, class, type from placex 
index f99e78f5eef9ff7880eb54f4e51cb3efee95af07..bcc3eec1b854aa398406dd0a4738ff09addd2ebd 100644 (file)
@@ -75,15 +75,6 @@ CREATE TABLE location_property (
   );
 SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
 
-CREATE TABLE location_property_aux () INHERITS (location_property);
-CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
-CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
-CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
-GRANT SELECT ON location_property_aux TO "{www-user}";
-
-CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT);
-GRANT SELECT ON location_property_tiger TO "{www-user}";
-
 drop table IF EXISTS search_name;
 CREATE TABLE search_name (
   place_id BIGINT,
index 0026a8135dc76c03b7d42bbf3e20203388c61e7e..cfe7f560191e353153aff9b5500f96f96141ed7b 100755 (executable)
                if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
                $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
                $sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
-               if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
-               if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
-               if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
+               if ($aCMDResult['enable-diff-updates'])
+               {
+                       $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
+               }
+               if ($aCMDResult['enable-debug-statements'])
+               {
+                       $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
+               }
+               if (CONST_Limit_Reindexing)
+               {
+                       $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
+               }
+               if (!CONST_Use_US_Tiger_Data)
+               {
+                       $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate);
+               }
+               if (!CONST_Use_Aux_Location_data)
+               {
+                       $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
+               }
                pgsqlRunScript($sTemplate);
        }
 
index c7a567552c06afe614cf6ae08b2f0246d31b155f..5edef6f5001938505960cf0e6dffe5d35e05d952 100755 (executable)
 
        $iPlaceID = (int)$_GET['place_id'];
 
-       $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
-       if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
-       $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
-       if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+       if (CONST_Use_US_Tiger_Data)
+       {
+               $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
+               if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+       }
+
+       if (CONST_Use_Aux_Location_data)
+       {
+               $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
+               if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+       }
 
        $hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
 
index 5e3d7299db68d8513d5bbdec3e9b348a828dc995..6a281aa871dc01415130c69d87545536aafeb1de 100755 (executable)
 
        $iPlaceID = (int)$_GET['place_id'];
 
-       $sAuxHouseNumber = false;
-       $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
-       if ($iParentPlaceID)
+       if (CONST_Use_US_Tiger_Data)
        {
-               $iPlaceID = $iParentPlaceID;
+               $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
+               if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
        }
-       else
+
+       if (CONST_Use_Aux_Location_data)
        {
                $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
                if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;