From 8c7bd6beabb9b877a155c554147f4a3ce8378e7e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 10 Feb 2012 21:01:35 +0000 Subject: [PATCH] during initial setup defer creation of non-vital until after nominatim indexing, analyse DB during nominatim indexing so the query planer uses indexes correctly --- sql/indices.src.sql | 28 ++++++++++++++++++++++++++++ sql/partitions.src.sql | 5 ----- sql/tables.sql | 19 +------------------ utils/setup.php | 41 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 67 insertions(+), 26 deletions(-) create mode 100644 sql/indices.src.sql diff --git a/sql/indices.src.sql b/sql/indices.src.sql new file mode 100644 index 00000000..1664a96b --- /dev/null +++ b/sql/indices.src.sql @@ -0,0 +1,28 @@ +CREATE INDEX idx_word_word_id on word USING BTREE (word_id); + +CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector) WITH (fastupdate = off); +CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off); +CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid); + +CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id); + +CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id); +CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline); + +DROP INDEX IF EXISTS idx_placex_rank_search; +CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search); +CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address); +CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) where indexed_status > 0; +CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL; +CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) where indexed_status > 0 and class='place' and type='houses'; +CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_address,osm_type,osm_id); + +CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id); + +CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid); +CREATE INDEX idx_search_name_country_nameaddress_vector ON search_name_country USING GIN (nameaddress_vector) WITH (fastupdate = off); + +-- start +CREATE INDEX idx_search_name_-partition-_nameaddress_vector ON search_name_-partition- USING GIN (nameaddress_vector) WITH (fastupdate = off); +CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid); +-- end diff --git a/sql/partitions.src.sql b/sql/partitions.src.sql index b4177a02..df4b3b16 100644 --- a/sql/partitions.src.sql +++ b/sql/partitions.src.sql @@ -14,14 +14,11 @@ create type nearfeature as ( ); CREATE TABLE location_area_country () INHERITS (location_area_large); -CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id); CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry); CREATE TABLE search_name_country () INHERITS (search_name_blank); CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id); -CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid); CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off); -CREATE INDEX idx_search_name_country_nameaddress_vector ON search_name_country USING GIN (nameaddress_vector) WITH (fastupdate = off); -- start CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large); @@ -32,13 +29,11 @@ CREATE TABLE search_name_-partition- () INHERITS (search_name_blank); CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id); CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid); CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off); -CREATE INDEX idx_search_name_-partition-_nameaddress_vector ON search_name_-partition- USING GIN (nameaddress_vector) WITH (fastupdate = off); CREATE TABLE location_property_-partition- () INHERITS (location_property); CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id); CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id); CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber); ---CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid); CREATE TABLE location_road_-partition- ( partition integer, diff --git a/sql/tables.sql b/sql/tables.sql index 78de9e8c..19d8c084 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -78,7 +78,6 @@ CREATE TABLE word ( operator TEXT ); SELECT AddGeometryColumn('word', 'location', 4326, 'GEOMETRY', 2); -CREATE INDEX idx_word_word_id on word USING BTREE (word_id); CREATE INDEX idx_word_word_token on word USING BTREE (word_token); --CREATE INDEX idx_word_trigram ON word USING gin(word_trigram gin_trgm_ops) WITH (fastupdate = off); GRANT SELECT ON word TO "www-data" ; @@ -138,9 +137,6 @@ SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2); drop table IF EXISTS search_name; CREATE TABLE search_name () INHERITS (search_name_blank); -CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector) WITH (fastupdate = off); -CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off); -CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid); CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id); drop table IF EXISTS place_addressline; @@ -153,7 +149,6 @@ CREATE TABLE place_addressline ( cached_rank_address integer ); CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id); -CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id); drop table IF EXISTS place_boundingbox CASCADE; CREATE TABLE place_boundingbox ( @@ -165,9 +160,7 @@ CREATE TABLE place_boundingbox ( numfeatures integer, area float ); -CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id); SELECT AddGeometryColumn('place_boundingbox', 'outline', 4326, 'GEOMETRY', 2); -CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline); GRANT SELECT on place_boundingbox to "www-data" ; GRANT INSERT on place_boundingbox to "www-data" ; @@ -221,23 +214,13 @@ CREATE TABLE placex ( SELECT AddGeometryColumn('placex', 'geometry', 4326, 'GEOMETRY', 2); CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id); CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id); -CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search); -CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address); +CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector); CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry); --CREATE INDEX idx_placex_indexed ON placex USING BTREE (indexed); -CREATE INDEX idx_placex_pending ON placex USING BTREE (rank_search) where indexed_status > 0; -CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) where indexed_status > 0; - --CREATE INDEX idx_placex_pendingbylatlon ON placex USING BTREE (geometry_index(geometry_sector,indexed,name),rank_search) where geometry_index(geometry_sector,indexed,name) IS NOT NULL; -CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL; -CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) where indexed_status > 0 and class='place' and type='houses'; - -CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_address,osm_type,osm_id); -CLUSTER placex USING idx_placex_sector; - DROP SEQUENCE seq_place; CREATE SEQUENCE seq_place start 1; GRANT SELECT on placex to "www-data" ; diff --git a/utils/setup.php b/utils/setup.php index b0eecfc8..0521b15d 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -30,6 +30,7 @@ array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'), array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'), ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); @@ -107,7 +108,6 @@ $osm2pgsql = CONST_Osm2pgsql_Binary; if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql"); passthru($osm2pgsql.' -lsc -O gazetteer -C 12000 --hstore -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']); - pgsqlRunScript('ANALYSE'); $oDB =& getDB(); $x = $oDB->getRow('select * from place limit 1'); @@ -241,6 +241,8 @@ echo '.'; } echo "\n"; + echo "Reanalysing database...\n"; + pgsqlRunScript('ANALYSE'); } if ($aCMDResult['create-roads']) @@ -344,7 +346,7 @@ if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); } - if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date'])) + if ($aCMDResult['osmosis-init'] && isset($aCMDResult['osmosis-init-date'])) { $bDidSomething = true; @@ -365,7 +367,40 @@ $bDidSomething = true; $sOutputFile = ''; if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output']; - passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile); + $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile; + passthru($sBaseCmd.' -R 4'); + pgsqlRunScript('ANALYSE'); + passthru($sBaseCmd.' -r 5 -R 25'); + pgsqlRunScript('ANALYSE'); + passthru($sBaseCmd.' -r 26'); + } + + if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) + { + echo "Search indices\n"; + $bDidSomething = true; + $oDB =& getDB(); + $sSQL = 'select partition from country_name order by country_code'; + $aPartitions = $oDB->getCol($sSQL); + if (PEAR::isError($aPartitions)) + { + fail($aPartitions->getMessage()); + } + $aPartitions[] = 0; + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); + preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); + foreach($aMatches as $aMatch) + { + $sResult = ''; + foreach($aPartitions as $sPartitionName) + { + $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); + } + $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); + } + + pgsqlRunScript($sTemplate); } if (isset($aCMDResult['create-website'])) -- 2.39.5