+
+ if ($fPostgisVersion < 2.0) {
+ echo "Helper functions for postgis < 2.0\n";
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/postgis_15_aux.sql');
+ } else {
+ echo "Helper functions for postgis >= 2.0\n";
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/postgis_20_aux.sql');
+ }
+ pgsqlRunScript($sTemplate);
+ }
+
+ if ($aCMDResult['create-minimal-tables'])
+ {
+ echo "Minimal Tables\n";
+ $bDidSomething = true;
+ pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
+
+ $sScript = '';
+
+ // Backstop the import process - easliest possible import id
+ $sScript .= "insert into import_npi_log values (18022);\n";
+
+ $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
+ if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
+
+ while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
+ {
+ list($sClass, $sType) = explode(' ', trim($sLine));
+ $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
+ $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
+
+ $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
+ $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
+
+ $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
+ $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
+ }
+ fclose($hFile);
+ pgsqlRunScript($sScript);
+ }
+
+ if ($aCMDResult['create-tables'] || $aCMDResult['all'])
+ {
+ $bDidSomething = true;
+
+ echo "Tables\n";
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
+ $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:address-data}',
+ CONST_Tablespace_Address_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:address-index}',
+ CONST_Tablespace_Address_Index, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:search-data}',
+ CONST_Tablespace_Search_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:search-index}',
+ CONST_Tablespace_Search_Index, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-data}',
+ CONST_Tablespace_Aux_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-index}',
+ CONST_Tablespace_Aux_Index, $sTemplate);
+ pgsqlRunScript($sTemplate, false);
+
+ // re-run the functions
+ echo "Functions\n";
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
+ $sTemplate = str_replace('{modulepath}',
+ CONST_BasePath.'/module', $sTemplate);
+ pgsqlRunScript($sTemplate);
+ }
+
+ if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
+ {
+ echo "Partition Tables\n";
+ $bDidSomething = true;
+ $oDB =& getDB();
+ $sSQL = 'select distinct partition from country_name';
+ $aPartitions = $oDB->getCol($sSQL);
+ if (PEAR::isError($aPartitions))
+ {
+ fail($aPartitions->getMessage());
+ }
+ if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
+
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
+ $sTemplate = replace_tablespace('{ts:address-data}',
+ CONST_Tablespace_Address_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:address-index}',
+ CONST_Tablespace_Address_Index, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:search-data}',
+ CONST_Tablespace_Search_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:search-index}',
+ CONST_Tablespace_Search_Index, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-data}',
+ CONST_Tablespace_Aux_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-index}',
+ CONST_Tablespace_Aux_Index, $sTemplate);
+ preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
+ foreach($aMatches as $aMatch)
+ {
+ $sResult = '';
+ foreach($aPartitions as $sPartitionName)
+ {
+ $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
+ }
+ $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
+ }
+
+ pgsqlRunScript($sTemplate);
+ }
+
+
+ if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
+ {
+ echo "Partition Functions\n";
+ $bDidSomething = true;
+ $oDB =& getDB();
+ $sSQL = 'select distinct partition from country_name';
+ $aPartitions = $oDB->getCol($sSQL);
+ if (PEAR::isError($aPartitions))
+ {
+ fail($aPartitions->getMessage());
+ }
+ if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
+
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
+ preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
+ foreach($aMatches as $aMatch)
+ {
+ $sResult = '';
+ foreach($aPartitions as $sPartitionName)
+ {
+ $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
+ }
+ $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
+ }
+
+ pgsqlRunScript($sTemplate);
+ }
+
+ if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
+ {
+ $bDidSomething = true;
+ $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
+ $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
+ if (file_exists($sWikiArticlesFile))
+ {
+ echo "Importing wikipedia articles...";
+ pgsqlRunDropAndRestore($sWikiArticlesFile);
+ echo "...done\n";
+ }
+ else
+ {
+ echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
+ }
+ if (file_exists($sWikiRedirectsFile))
+ {
+ echo "Importing wikipedia redirects...";
+ pgsqlRunDropAndRestore($sWikiRedirectsFile);
+ echo "...done\n";
+ }
+ else
+ {
+ echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
+ }
+ }
+
+
+ if ($aCMDResult['load-data'] || $aCMDResult['all'])
+ {
+ echo "Drop old Data\n";
+ $bDidSomething = true;
+
+ $oDB =& getDB();
+ if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
+ echo '.';
+ if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
+ echo '.';
+
+ $sSQL = 'select distinct partition from country_name';
+ $aPartitions = $oDB->getCol($sSQL);
+ if (PEAR::isError($aPartitions))
+ {
+ fail($aPartitions->getMessage());
+ }
+ if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
+ foreach($aPartitions as $sPartition)
+ {
+ if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
+ echo '.';
+ }
+
+ // used by getorcreate_word_id to ignore frequent partial words
+ if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
+ echo ".\n";
+
+ // pre-create the word list
+ if (!$aCMDResult['disable-token-precalc'])
+ {
+ echo "Loading word list\n";
+ pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
+ }
+
+ echo "Load Data\n";
+ $aDBInstances = array();
+ for($i = 0; $i < $iInstances; $i++)
+ {
+ $aDBInstances[$i] =& getDB(true);
+ $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
+ $sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
+ $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
+ if ($aCMDResult['verbose']) echo "$sSQL\n";
+ if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
+ }
+ $bAnyBusy = true;
+ while($bAnyBusy)
+ {
+ $bAnyBusy = false;
+ for($i = 0; $i < $iInstances; $i++)
+ {
+ if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
+ }
+ sleep(1);
+ echo '.';
+ }
+ echo "\n";
+ echo "Reanalysing database...\n";
+ pgsqlRunScript('ANALYSE');
+ }
+
+ if ($aCMDResult['import-tiger-data'])
+ {
+ $bDidSomething = true;
+
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
+ $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-data}',
+ CONST_Tablespace_Aux_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-index}',
+ CONST_Tablespace_Aux_Index, $sTemplate);
+ pgsqlRunScript($sTemplate, false);
+
+ $aDBInstances = array();
+ for($i = 0; $i < $iInstances; $i++)
+ {
+ $aDBInstances[$i] =& getDB(true);
+ }
+
+ foreach(glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile)
+ {
+ echo $sFile.': ';
+ $hFile = fopen($sFile, "r");
+ $sSQL = fgets($hFile, 100000);
+ $iLines = 0;
+
+ while(true)
+ {
+ for($i = 0; $i < $iInstances; $i++)
+ {
+ if (!pg_connection_busy($aDBInstances[$i]->connection))
+ {
+ while(pg_get_result($aDBInstances[$i]->connection));
+ $sSQL = fgets($hFile, 100000);
+ if (!$sSQL) break 2;
+ if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
+ $iLines++;
+ if ($iLines == 1000)
+ {
+ echo ".";
+ $iLines = 0;
+ }
+ }
+ }
+ usleep(10);
+ }
+
+ fclose($hFile);
+
+ $bAnyBusy = true;
+ while($bAnyBusy)
+ {
+ $bAnyBusy = false;
+ for($i = 0; $i < $iInstances; $i++)
+ {
+ if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
+ }
+ usleep(10);
+ }
+ echo "\n";
+ }
+
+ echo "Creating indexes\n";
+ $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
+ $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-data}',
+ CONST_Tablespace_Aux_Data, $sTemplate);
+ $sTemplate = replace_tablespace('{ts:aux-index}',
+ CONST_Tablespace_Aux_Index, $sTemplate);
+ pgsqlRunScript($sTemplate, false);
projects / nominatim.git / blobdiff