Merge remote-tracking branch 'upstream/master'

[nominatim.git] / lib / setup / SetupClass.php

diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php
index 2fdb3926d59ecc23d2f26ecc75e94776f3b81ca8..34ece8e0f7567d327793a002782c320fb2bcebca 100755 (executable)
--- a/lib/setup/SetupClass.php
+++ b/lib/setup/SetupClass.php
@@ -10,6 +10,7 @@ class SetupFunctions
     protected $iInstances;
     protected $sModulePath;
     protected $aDSNInfo;
     protected $iInstances;
     protected $sModulePath;
     protected $aDSNInfo;

+    protected $bQuiet;

     protected $bVerbose;
     protected $sIgnoreErrors;
     protected $bEnableDiffUpdates;
     protected $bVerbose;
     protected $sIgnoreErrors;
     protected $bEnableDiffUpdates;


@@ -29,10 +30,13 @@ class SetupFunctions
             warn('resetting threads to '.$this->iInstances);
         }
 
             warn('resetting threads to '.$this->iInstances);
         }
 

-        // Assume we can steal all the cache memory in the box (unless told otherwise)

         if (isset($aCMDResult['osm2pgsql-cache'])) {
             $this->iCacheMemory = $aCMDResult['osm2pgsql-cache'];
         if (isset($aCMDResult['osm2pgsql-cache'])) {
             $this->iCacheMemory = $aCMDResult['osm2pgsql-cache'];

+        } elseif (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
+            // When flatnode files are enabled then disable cache per default.
+            $this->iCacheMemory = 0;

         } else {
         } else {

+            // Otherwise: Assume we can steal all the cache memory in the box.

             $this->iCacheMemory = getCacheMemoryMB();
         }
 
             $this->iCacheMemory = getCacheMemoryMB();
         }
 


@@ -46,6 +50,7 @@ class SetupFunctions
         }
 
         // setting member variables based on command line options stored in $aCMDResult
         }
 
         // setting member variables based on command line options stored in $aCMDResult

+        $this->bQuiet = $aCMDResult['quiet'];

         $this->bVerbose = $aCMDResult['verbose'];
 
         //setting default values which are not set by the update.php array
         $this->bVerbose = $aCMDResult['verbose'];
 
         //setting default values which are not set by the update.php array


@@ -220,88 +225,34 @@ class SetupFunctions
         info('Create Tables');
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
         info('Create Tables');
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');

-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-data}',
-            CONST_Tablespace_Address_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-index}',
-            CONST_Tablespace_Address_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-data}',
-            CONST_Tablespace_Search_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-index}',
-            CONST_Tablespace_Search_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);

 
         $this->pgsqlRunScript($sTemplate, false);
 
         if ($bReverseOnly) {
 
         $this->pgsqlRunScript($sTemplate, false);
 
         if ($bReverseOnly) {

-            $this->pgExec('DROP TABLE search_name');
+            $this->dropTable('search_name');

         }
 
         $oAlParser = new AddressLevelParser(CONST_Address_Level_Config);
         $oAlParser->createTable($this->oDB, 'address_levels');
     }
 
         }
 
         $oAlParser = new AddressLevelParser(CONST_Address_Level_Config);
         $oAlParser->createTable($this->oDB, 'address_levels');
     }
 

-    public function createPartitionTables()
+    public function createTableTriggers()

     {
     {

-        info('Create Partition Tables');
-
-        $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-data}',
-            CONST_Tablespace_Address_Data,
-            $sTemplate
-        );
+        info('Create Tables');

 
 

-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-index}',
-            CONST_Tablespace_Address_Index,
-            $sTemplate
-        );
+        $sTemplate = file_get_contents(CONST_BasePath.'/sql/table-triggers.sql');
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);

 
 

-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-data}',
-            CONST_Tablespace_Search_Data,
-            $sTemplate
-        );
-
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-index}',
-            CONST_Tablespace_Search_Index,
-            $sTemplate
-        );
+        $this->pgsqlRunScript($sTemplate, false);
+    }

 
 

-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
+    public function createPartitionTables()
+    {
+        info('Create Partition Tables');

 
 

-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);

 
         $this->pgsqlRunPartitionScript($sTemplate);
     }
 
         $this->pgsqlRunPartitionScript($sTemplate);
     }


@@ -319,8 +270,8 @@ class SetupFunctions
         $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikimedia-importance.sql.gz';
         if (file_exists($sWikiArticlesFile)) {
             info('Importing wikipedia articles and redirects');
         $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikimedia-importance.sql.gz';
         if (file_exists($sWikiArticlesFile)) {
             info('Importing wikipedia articles and redirects');

-            $this->pgExec('DROP TABLE IF EXISTS wikipedia_article');
-            $this->pgExec('DROP TABLE IF EXISTS wikipedia_redirect');
+            $this->dropTable('wikipedia_article');
+            $this->dropTable('wikipedia_redirect');

             $this->pgsqlRunScriptFile($sWikiArticlesFile);
         } else {
             warn('wikipedia importance dump file not found - places will have default importance');
             $this->pgsqlRunScriptFile($sWikiArticlesFile);
         } else {
             warn('wikipedia importance dump file not found - places will have default importance');


@@ -331,25 +282,25 @@ class SetupFunctions
     {
         info('Drop old Data');
 
     {
         info('Drop old Data');
 

-        $this->pgExec('TRUNCATE word');
+        $this->oDB->exec('TRUNCATE word');

         echo '.';
         echo '.';

-        $this->pgExec('TRUNCATE placex');
+        $this->oDB->exec('TRUNCATE placex');

         echo '.';
         echo '.';

-        $this->pgExec('TRUNCATE location_property_osmline');
+        $this->oDB->exec('TRUNCATE location_property_osmline');

         echo '.';
         echo '.';

-        $this->pgExec('TRUNCATE place_addressline');
+        $this->oDB->exec('TRUNCATE place_addressline');

         echo '.';
         echo '.';

-        $this->pgExec('TRUNCATE location_area');
+        $this->oDB->exec('TRUNCATE location_area');

         echo '.';
         if (!$this->dbReverseOnly()) {
         echo '.';
         if (!$this->dbReverseOnly()) {

-            $this->pgExec('TRUNCATE search_name');
+            $this->oDB->exec('TRUNCATE search_name');

             echo '.';
         }
             echo '.';
         }

-        $this->pgExec('TRUNCATE search_name_blank');
+        $this->oDB->exec('TRUNCATE search_name_blank');

         echo '.';
         echo '.';

-        $this->pgExec('DROP SEQUENCE seq_place');
+        $this->oDB->exec('DROP SEQUENCE seq_place');

         echo '.';
         echo '.';

-        $this->pgExec('CREATE SEQUENCE seq_place start 100000');
+        $this->oDB->exec('CREATE SEQUENCE seq_place start 100000');

         echo '.';
 
         $sSQL = 'select distinct partition from country_name';
         echo '.';
 
         $sSQL = 'select distinct partition from country_name';


@@ -357,14 +308,14 @@ class SetupFunctions
 
         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {
 
         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {

-            $this->pgExec('TRUNCATE location_road_'.$sPartition);
+            $this->oDB->exec('TRUNCATE location_road_'.$sPartition);

             echo '.';
         }
 
         // used by getorcreate_word_id to ignore frequent partial words
         $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
         $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
             echo '.';
         }
 
         // used by getorcreate_word_id to ignore frequent partial words
         $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
         $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';

-        $this->pgExec($sSQL);
+        $this->oDB->exec($sSQL);

         echo ".\n";
 
         // pre-create the word list
         echo ".\n";
 
         // pre-create the word list


@@ -461,17 +412,8 @@ class SetupFunctions
         if (empty($aFilenames)) return;
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
         if (empty($aFilenames)) return;
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');

-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
+

         $this->pgsqlRunScript($sTemplate, false);
 
         $aDBInstances = array();
         $this->pgsqlRunScript($sTemplate, false);
 
         $aDBInstances = array();


@@ -524,24 +466,15 @@ class SetupFunctions
 
         info('Creating indexes on Tiger data');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
 
         info('Creating indexes on Tiger data');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');

-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
+

         $this->pgsqlRunScript($sTemplate, false);
     }
 
     public function calculatePostcodes($bCMDResultAll)
     {
         info('Calculate Postcodes');
         $this->pgsqlRunScript($sTemplate, false);
     }
 
     public function calculatePostcodes($bCMDResultAll)
     {
         info('Calculate Postcodes');

-        $this->pgExec('TRUNCATE location_postcode');
+        $this->oDB->exec('TRUNCATE location_postcode');

 
         $sSQL  = 'INSERT INTO location_postcode';
         $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
 
         $sSQL  = 'INSERT INTO location_postcode';
         $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';


@@ -552,7 +485,7 @@ class SetupFunctions
         $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
         $sSQL .= '       AND geometry IS NOT null';
         $sSQL .= ' GROUP BY country_code, pc';
         $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
         $sSQL .= '       AND geometry IS NOT null';
         $sSQL .= ' GROUP BY country_code, pc';

-        $this->pgExec($sSQL);
+        $this->oDB->exec($sSQL);

 
         // only add postcodes that are not yet available in OSM
         $sSQL  = 'INSERT INTO location_postcode';
 
         // only add postcodes that are not yet available in OSM
         $sSQL  = 'INSERT INTO location_postcode';


@@ -562,7 +495,7 @@ class SetupFunctions
         $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
         $sSQL .= '        (SELECT postcode FROM location_postcode';
         $sSQL .= "          WHERE country_code = 'us')";
         $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
         $sSQL .= '        (SELECT postcode FROM location_postcode';
         $sSQL .= "          WHERE country_code = 'us')";

-        $this->pgExec($sSQL);
+        $this->oDB->exec($sSQL);

 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';
 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';


@@ -571,26 +504,32 @@ class SetupFunctions
         $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
         $sSQL .= '           (SELECT postcode FROM location_postcode';
         $sSQL .= "             WHERE country_code = 'gb')";
         $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
         $sSQL .= '           (SELECT postcode FROM location_postcode';
         $sSQL .= "             WHERE country_code = 'gb')";

-        $this->pgExec($sSQL);
+        $this->oDB->exec($sSQL);

 
         if (!$bCMDResultAll) {
             $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
             $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
 
         if (!$bCMDResultAll) {
             $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
             $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';

-            $this->pgExec($sSQL);
+            $this->oDB->exec($sSQL);

         }
 
         $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
         $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
         }
 
         $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
         $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';

-        $this->pgExec($sSQL);
+        $this->oDB->exec($sSQL);

     }
 
     public function index($bIndexNoanalyse)
     {
         $sOutputFile = '';
     }
 
     public function index($bIndexNoanalyse)
     {
         $sOutputFile = '';

-        $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i'
+        $sBaseCmd = CONST_BasePath.'/nominatim/nominatim.py'

             .' -d '.escapeshellarg($this->aDSNInfo['database'])
             .' -P '.escapeshellarg($this->aDSNInfo['port'])
             .' -t '.escapeshellarg($this->iInstances.$sOutputFile);
             .' -d '.escapeshellarg($this->aDSNInfo['database'])
             .' -P '.escapeshellarg($this->aDSNInfo['port'])
             .' -t '.escapeshellarg($this->iInstances.$sOutputFile);

+        if (!$this->bQuiet) {
+            $sBaseCmd .= ' -v';
+        }
+        if ($this->bVerbose) {
+            $sBaseCmd .= ' -v';
+        }

         if (isset($this->aDSNInfo['hostspec'])) {
             $sBaseCmd .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
         }
         if (isset($this->aDSNInfo['hostspec'])) {
             $sBaseCmd .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
         }


@@ -620,7 +559,7 @@ class SetupFunctions
 
         info('Index postcodes');
         $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
 
         info('Index postcodes');
         $sSQL = 'UPDATE location_postcode SET indexed_status = 0';

-        $this->pgExec($sSQL);
+        $this->oDB->exec($sSQL);

     }
 
     public function createSearchIndices()
     }
 
     public function createSearchIndices()


@@ -631,22 +570,8 @@ class SetupFunctions
         if (!$this->dbReverseOnly()) {
             $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql');
         }
         if (!$this->dbReverseOnly()) {
             $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql');
         }

-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-index}',
-            CONST_Tablespace_Address_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-index}',
-            CONST_Tablespace_Search_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
+

         $this->pgsqlRunScript($sTemplate);
     }
 
         $this->pgsqlRunScript($sTemplate);
     }
 


@@ -718,8 +643,7 @@ class SetupFunctions
             if (!$bFound) array_push($aDropTables, $sTable);
         }
         foreach ($aDropTables as $sDrop) {
             if (!$bFound) array_push($aDropTables, $sTable);
         }
         foreach ($aDropTables as $sDrop) {

-            if ($this->bVerbose) echo "Dropping table $sDrop\n";
-            $this->oDB->exec("DROP TABLE IF EXISTS $sDrop CASCADE");
+            $this->dropTable($sDrop);

         }
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
         }
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {


@@ -742,7 +666,21 @@ class SetupFunctions
 
     private function createSqlFunctions()
     {
 
     private function createSqlFunctions()
     {

-        $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
+        $sBasePath = CONST_BasePath.'/sql/functions/';
+        $sTemplate = file_get_contents($sBasePath.'utils.sql');
+        $sTemplate .= file_get_contents($sBasePath.'normalization.sql');
+        $sTemplate .= file_get_contents($sBasePath.'importance.sql');
+        $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql');
+        $sTemplate .= file_get_contents($sBasePath.'interpolation.sql');
+        if ($this->oDB->tableExists('place')) {
+            $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql');
+        }
+        if ($this->oDB->tableExists('placex')) {
+            $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql');
+        }
+        if ($this->oDB->tableExists('location_postcode')) {
+            $sTemplate .= file_get_contents($sBasePath.'postcode_triggers.sql');
+        }

         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
         if ($this->bEnableDiffUpdates) {
             $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
         if ($this->bEnableDiffUpdates) {
             $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);


@@ -842,13 +780,27 @@ class SetupFunctions
         }
     }
 
         }
     }
 

-    private function replaceTablespace($sTemplate, $sTablespace, $sSql)
+    private function replaceSqlPatterns($sSql)

     {
     {

-        if ($sTablespace) {
-            $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
-        } else {
-            $sSql = str_replace($sTemplate, '', $sSql);
+        $sSql = str_replace('{www-user}', CONST_Database_Web_User, $sSql);
+
+        $aPatterns = array(
+                      '{ts:address-data}' => CONST_Tablespace_Address_Data,
+                      '{ts:address-index}' => CONST_Tablespace_Address_Index,
+                      '{ts:search-data}' => CONST_Tablespace_Search_Data,
+                      '{ts:search-index}' =>  CONST_Tablespace_Search_Index,
+                      '{ts:aux-data}' =>  CONST_Tablespace_Aux_Data,
+                      '{ts:aux-index}' =>  CONST_Tablespace_Aux_Index,
+        );
+
+        foreach ($aPatterns as $sPattern => $sTablespace) {
+            if ($sTablespace) {
+                $sSql = str_replace($sPattern, 'TABLESPACE "'.$sTablespace.'"', $sSql);
+            } else {
+                $sSql = str_replace($sPattern, '', $sSql);
+            }

         }
         }

+

         return $sSql;
     }
 
         return $sSql;
     }
 


@@ -868,17 +820,18 @@ class SetupFunctions
     }
 
     /**
     }
 
     /**

-     * Execute the SQL command on the open database.
+     * Drop table with the given name if it exists.

      *
      *

-     * @param string $sSQL SQL command to execute.
+     * @param string $sName Name of table to remove.

      *
      * @return null
      *
      * @pre connect() must have been called.
      */
      *
      * @return null
      *
      * @pre connect() must have been called.
      */

-    private function pgExec($sSQL)
+    private function dropTable($sName)

     {
     {

-        $this->oDB->exec($sSQL);
+        if ($this->bVerbose) echo "Dropping table $sName\n";
+        $this->oDB->exec('DROP TABLE IF EXISTS '.$sName.' CASCADE');

     }
 
     /**
     }
 
     /**