during initial setup defer creation of non-vital until after nominatim indexing,

analyse DB during nominatim indexing so the query planer uses indexes correctly

diff --git a/sql/indices.src.sql b/sql/indices.src.sql
new file mode 100644 (file)
index 0000000..1664a96
--- /dev/null
+++ b/sql/indices.src.sql
@@ -0,0 +1,28 @@
+CREATE INDEX idx_word_word_id on word USING BTREE (word_id);
+
+CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector) WITH (fastupdate = off);
+CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off);
+CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid);
+
+CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id);
+
+CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id);
+CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline);
+
+DROP INDEX IF EXISTS idx_placex_rank_search;
+CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search);
+CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address);
+CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) where indexed_status > 0;
+CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL;
+CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) where indexed_status > 0 and class='place' and type='houses';
+CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_address,osm_type,osm_id);
+
+CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id);
+
+CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid);
+CREATE INDEX idx_search_name_country_nameaddress_vector ON search_name_country USING GIN (nameaddress_vector) WITH (fastupdate = off);
+
+-- start
+CREATE INDEX idx_search_name_-partition-_nameaddress_vector ON search_name_-partition- USING GIN (nameaddress_vector) WITH (fastupdate = off);
+CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid);
+-- end

diff --git a/sql/partitions.src.sql b/sql/partitions.src.sql
index b4177a024b23d41923cf0d0e432721842a46adac..df4b3b169338361aee77092a6efaadc9ec5e7d3b 100644 (file)
--- a/sql/partitions.src.sql
+++ b/sql/partitions.src.sql
@@ -14,14 +14,11 @@ create type nearfeature as (
 );
 
 CREATE TABLE location_area_country () INHERITS (location_area_large);
-CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id);
 CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry);
 
 CREATE TABLE search_name_country () INHERITS (search_name_blank);
 CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id);
-CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid);
 CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off);
-CREATE INDEX idx_search_name_country_nameaddress_vector ON search_name_country USING GIN (nameaddress_vector) WITH (fastupdate = off);
 
 -- start
 CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large);
@@ -32,13 +29,11 @@ CREATE TABLE search_name_-partition- () INHERITS (search_name_blank);
 CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id);
 CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid);
 CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off);
-CREATE INDEX idx_search_name_-partition-_nameaddress_vector ON search_name_-partition- USING GIN (nameaddress_vector) WITH (fastupdate = off);
 
 CREATE TABLE location_property_-partition- () INHERITS (location_property);
 CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id);
 CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id);
 CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber);
---CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid);
 
 CREATE TABLE location_road_-partition- (
   partition integer,

diff --git a/sql/tables.sql b/sql/tables.sql
index 78de9e8c38f701382489c65030c71b4a198e5bef..19d8c084d530bfd6dae224a97a980d1e1ab9845d 100644 (file)
--- a/sql/tables.sql
+++ b/sql/tables.sql
@@ -78,7 +78,6 @@ CREATE TABLE word (
   operator TEXT
   );
 SELECT AddGeometryColumn('word', 'location', 4326, 'GEOMETRY', 2);
-CREATE INDEX idx_word_word_id on word USING BTREE (word_id);
 CREATE INDEX idx_word_word_token on word USING BTREE (word_token);
 --CREATE INDEX idx_word_trigram ON word USING gin(word_trigram gin_trgm_ops) WITH (fastupdate = off);
 GRANT SELECT ON word TO "www-data" ;
@@ -138,9 +137,6 @@ SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2);
 
 drop table IF EXISTS search_name;
 CREATE TABLE search_name () INHERITS (search_name_blank);
-CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector) WITH (fastupdate = off);
-CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off);
-CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid);
 CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id);
 
 drop table IF EXISTS place_addressline;
@@ -153,7 +149,6 @@ CREATE TABLE place_addressline (
   cached_rank_address integer
   );
 CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id);
-CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id);
 
 drop table IF EXISTS place_boundingbox CASCADE;
 CREATE TABLE place_boundingbox (
@@ -165,9 +160,7 @@ CREATE TABLE place_boundingbox (
   numfeatures integer,
   area float
   );
-CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id);
 SELECT AddGeometryColumn('place_boundingbox', 'outline', 4326, 'GEOMETRY', 2);
-CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline);
 GRANT SELECT on place_boundingbox to "www-data" ;
 GRANT INSERT on place_boundingbox to "www-data" ;
 
@@ -221,23 +214,13 @@ CREATE TABLE placex (
 SELECT AddGeometryColumn('placex', 'geometry', 4326, 'GEOMETRY', 2);
 CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id);
 CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id);
-CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search);
-CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address);
+CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector);
 CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry);
 
 --CREATE INDEX idx_placex_indexed ON placex USING BTREE (indexed);
 
-CREATE INDEX idx_placex_pending ON placex USING BTREE (rank_search) where indexed_status > 0;
-CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) where indexed_status > 0;
-
 --CREATE INDEX idx_placex_pendingbylatlon ON placex USING BTREE (geometry_index(geometry_sector,indexed,name),rank_search)  where geometry_index(geometry_sector,indexed,name) IS NOT NULL;
 
-CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL;
-CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) where indexed_status > 0 and class='place' and type='houses';
-
-CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_address,osm_type,osm_id);
-CLUSTER placex USING idx_placex_sector;
-
 DROP SEQUENCE seq_place;
 CREATE SEQUENCE seq_place start 1;
 GRANT SELECT on placex to "www-data" ;

diff --git a/utils/setup.php b/utils/setup.php
index b0eecfc8ca3998b3678a5ff7f56cb04f6a4467fd..0521b15d98ea391940658b3f9fd7792feeb7080f 100755 (executable)
--- a/utils/setup.php
+++ b/utils/setup.php
@@ -30,6 +30,7 @@
                array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
                array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
                array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
+               array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
                array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
        );
        getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
@@ -107,7 +108,6 @@
                $osm2pgsql = CONST_Osm2pgsql_Binary;
                if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql");
                passthru($osm2pgsql.' -lsc -O gazetteer -C 12000 --hstore -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']);
-        pgsqlRunScript('ANALYSE');
 
                $oDB =& getDB();
                $x = $oDB->getRow('select * from place limit 1');
@@ -241,6 +241,8 @@
                        echo '.';
                }
                echo "\n";
+               echo "Reanalysing database...\n";
+               pgsqlRunScript('ANALYSE');
        }
 
        if ($aCMDResult['create-roads'])
@@ -344,7 +346,7 @@
                if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
        }
 
-       if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
+       if ($aCMDResult['osmosis-init'] && isset($aCMDResult['osmosis-init-date']))
        {
                $bDidSomething = true;
 
@@ -365,7 +367,40 @@
                $bDidSomething = true;
                $sOutputFile = '';
                if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
-               passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile);
+               $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
+               passthru($sBaseCmd.' -R 4');
+               pgsqlRunScript('ANALYSE');
+               passthru($sBaseCmd.' -r 5 -R 25');
+               pgsqlRunScript('ANALYSE');
+               passthru($sBaseCmd.' -r 26');
+       }
+
+       if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
+       {
+               echo "Search indices\n";
+               $bDidSomething = true;
+               $oDB =& getDB();
+               $sSQL = 'select partition from country_name order by country_code';
+               $aPartitions = $oDB->getCol($sSQL);
+               if (PEAR::isError($aPartitions))
+               {
+                       fail($aPartitions->getMessage());
+               }
+               $aPartitions[] = 0;
+
+               $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
+               preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
+               foreach($aMatches as $aMatch)
+               {
+                       $sResult = '';
+                       foreach($aPartitions as $sPartitionName)
+                       {
+                               $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
+                       }
+                       $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
+               }
+
+               pgsqlRunScript($sTemplate);
        }
 
        if (isset($aCMDResult['create-website']))