travis: reduce the size of diff download

[nominatim.git] / lib / setup / SetupClass.php

diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php
index 96eb7f8a8618a5888faf892d5f824276b8ba9cd7..385eff70eb6cda7160cb84f7a6b5c5ff0c96b08e 100755 (executable)
--- a/lib/setup/SetupClass.php
+++ b/lib/setup/SetupClass.php
@@ -3,6 +3,7 @@
 namespace Nominatim\Setup;
 
 require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php');
 namespace Nominatim\Setup;
 
 require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php');

+require_once(CONST_BasePath.'/lib/Shell.php');

 
 class SetupFunctions
 {
 
 class SetupFunctions
 {


@@ -10,11 +11,13 @@ class SetupFunctions
     protected $iInstances;
     protected $sModulePath;
     protected $aDSNInfo;
     protected $iInstances;
     protected $sModulePath;
     protected $aDSNInfo;

+    protected $bQuiet;

     protected $bVerbose;
     protected $sIgnoreErrors;
     protected $bEnableDiffUpdates;
     protected $bEnableDebugStatements;
     protected $bNoPartitions;
     protected $bVerbose;
     protected $sIgnoreErrors;
     protected $bEnableDiffUpdates;
     protected $bEnableDebugStatements;
     protected $bNoPartitions;

+    protected $bDrop;

     protected $oDB = null;
 
     public function __construct(array $aCMDResult)
     protected $oDB = null;
 
     public function __construct(array $aCMDResult)


@@ -49,6 +52,7 @@ class SetupFunctions
         }
 
         // setting member variables based on command line options stored in $aCMDResult
         }
 
         // setting member variables based on command line options stored in $aCMDResult

+        $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet'];

         $this->bVerbose = $aCMDResult['verbose'];
 
         //setting default values which are not set by the update.php array
         $this->bVerbose = $aCMDResult['verbose'];
 
         //setting default values which are not set by the update.php array


@@ -72,6 +76,8 @@ class SetupFunctions
         } else {
             $this->bEnableDiffUpdates = false;
         }
         } else {
             $this->bEnableDiffUpdates = false;
         }

+
+        $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop'];

     }
 
     public function createDB()
     }
 
     public function createDB()


@@ -79,23 +85,27 @@ class SetupFunctions
         info('Create DB');
         $oDB = new \Nominatim\DB;
 
         info('Create DB');
         $oDB = new \Nominatim\DB;
 

-        if ($oDB->databaseExists()) {
+        if ($oDB->checkConnection()) {

             fail('database already exists ('.CONST_Database_DSN.')');
         }
 
             fail('database already exists ('.CONST_Database_DSN.')');
         }
 

-        $sCreateDBCmd = 'createdb -E UTF-8'
-            .' -p '.escapeshellarg($this->aDSNInfo['port'])
-            .' '.escapeshellarg($this->aDSNInfo['database']);
+        $oCmd = (new \Nominatim\Shell('createdb'))
+                ->addParams('-E', 'UTF-8')
+                ->addParams('-p', $this->aDSNInfo['port']);
+

         if (isset($this->aDSNInfo['username'])) {
         if (isset($this->aDSNInfo['username'])) {

-            $sCreateDBCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
+            $oCmd->addParams('-U', $this->aDSNInfo['username']);
+        }
+        if (isset($this->aDSNInfo['password'])) {
+            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);

         }
         }

-

         if (isset($this->aDSNInfo['hostspec'])) {
         if (isset($this->aDSNInfo['hostspec'])) {

-            $sCreateDBCmd .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']);
+            $oCmd->addParams('-h', $this->aDSNInfo['hostspec']);

         }
         }

+        $oCmd->addParams($this->aDSNInfo['database']);

 
 

-        $result = $this->runWithPgEnv($sCreateDBCmd);
-        if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
+        $result = $oCmd->run();
+        if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());

     }
 
     public function connect()
     }
 
     public function connect()


@@ -169,43 +179,58 @@ class SetupFunctions
     {
         info('Import data');
 
     {
         info('Import data');
 

-        $osm2pgsql = CONST_Osm2pgsql_Binary;
-        if (!file_exists($osm2pgsql)) {
+        if (!file_exists(CONST_Osm2pgsql_Binary)) {

             echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
             echo "Normally you should not need to set this manually.\n";
             echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
             echo "Normally you should not need to set this manually.\n";

-            fail("osm2pgsql not found in '$osm2pgsql'");
+            fail("osm2pgsql not found in '".CONST_Osm2pgsql_Binary."'");

         }
 
         }
 

-        $osm2pgsql .= ' -S '.escapeshellarg(CONST_Import_Style);
+        $oCmd = new \Nominatim\Shell(CONST_Osm2pgsql_Binary);
+        $oCmd->addParams('--style', CONST_Import_Style);

 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {

-            $osm2pgsql .= ' --flat-nodes '.escapeshellarg(CONST_Osm2pgsql_Flatnode_File);
-        }
-
-        if (CONST_Tablespace_Osm2pgsql_Data)
-            $osm2pgsql .= ' --tablespace-slim-data '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Data);
-        if (CONST_Tablespace_Osm2pgsql_Index)
-            $osm2pgsql .= ' --tablespace-slim-index '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Index);
-        if (CONST_Tablespace_Place_Data)
-            $osm2pgsql .= ' --tablespace-main-data '.escapeshellarg(CONST_Tablespace_Place_Data);
-        if (CONST_Tablespace_Place_Index)
-            $osm2pgsql .= ' --tablespace-main-index '.escapeshellarg(CONST_Tablespace_Place_Index);
-        $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
-        $osm2pgsql .= ' -C '.escapeshellarg($this->iCacheMemory);
-        $osm2pgsql .= ' -P '.escapeshellarg($this->aDSNInfo['port']);
+            $oCmd->addParams('--flat-nodes', CONST_Osm2pgsql_Flatnode_File);
+        }
+        if (CONST_Tablespace_Osm2pgsql_Data) {
+            $oCmd->addParams('--tablespace-slim-data', CONST_Tablespace_Osm2pgsql_Data);
+        }
+        if (CONST_Tablespace_Osm2pgsql_Index) {
+            $oCmd->addParams('--tablespace-slim-index', CONST_Tablespace_Osm2pgsql_Index);
+        }
+        if (CONST_Tablespace_Place_Data) {
+            $oCmd->addParams('--tablespace-main-data', CONST_Tablespace_Place_Data);
+        }
+        if (CONST_Tablespace_Place_Index) {
+            $oCmd->addParams('--tablespace-main-index', CONST_Tablespace_Place_Index);
+        }
+        $oCmd->addParams('--latlong', '--slim', '--create');
+        $oCmd->addParams('--output', 'gazetteer');
+        $oCmd->addParams('--hstore');
+        $oCmd->addParams('--number-processes', 1);
+        $oCmd->addParams('--cache', $this->iCacheMemory);
+        $oCmd->addParams('--port', $this->aDSNInfo['port']);
+

         if (isset($this->aDSNInfo['username'])) {
         if (isset($this->aDSNInfo['username'])) {

-            $osm2pgsql .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
+            $oCmd->addParams('--username', $this->aDSNInfo['username']);
+        }
+        if (isset($this->aDSNInfo['password'])) {
+            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);

         }
         if (isset($this->aDSNInfo['hostspec'])) {
         }
         if (isset($this->aDSNInfo['hostspec'])) {

-            $osm2pgsql .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
+            $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);

         }
         }

-        $osm2pgsql .= ' -d '.escapeshellarg($this->aDSNInfo['database']).' '.escapeshellarg($sOSMFile);
-
-        $this->runWithPgEnv($osm2pgsql);
+        $oCmd->addParams('--database', $this->aDSNInfo['database']);
+        $oCmd->addParams($sOSMFile);
+        $oCmd->run();

 
         if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) {
             fail('No Data');
         }
 
         if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) {
             fail('No Data');
         }

+
+        if ($this->bDrop) {
+            $this->dropTable('planet_osm_nodes');
+            $this->removeFlatnodeFile();
+        }

     }
 
     public function createFunctions()
     }
 
     public function createFunctions()


@@ -235,6 +260,16 @@ class SetupFunctions
         $oAlParser->createTable($this->oDB, 'address_levels');
     }
 
         $oAlParser->createTable($this->oDB, 'address_levels');
     }
 

+    public function createTableTriggers()
+    {
+        info('Create Tables');
+
+        $sTemplate = file_get_contents(CONST_BasePath.'/sql/table-triggers.sql');
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
+
+        $this->pgsqlRunScript($sTemplate, false);
+    }
+

     public function createPartitionTables()
     {
         info('Create Partition Tables');
     public function createPartitionTables()
     {
         info('Create Partition Tables');


@@ -397,8 +432,10 @@ class SetupFunctions
 
         $aFilenames = glob(CONST_Tiger_Data_Path.'/*.sql');
         info('Found '.count($aFilenames).' SQL files in path '.CONST_Tiger_Data_Path);
 
         $aFilenames = glob(CONST_Tiger_Data_Path.'/*.sql');
         info('Found '.count($aFilenames).' SQL files in path '.CONST_Tiger_Data_Path);

-        if (empty($aFilenames)) return;
-
+        if (empty($aFilenames)) {
+            warn('Tiger data import selected but no files found in path '.CONST_Tiger_Data_Path);
+            return;
+        }

         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 


@@ -507,34 +544,48 @@ class SetupFunctions
 
     public function index($bIndexNoanalyse)
     {
 
     public function index($bIndexNoanalyse)
     {

-        $sOutputFile = '';
-        $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i'
-            .' -d '.escapeshellarg($this->aDSNInfo['database'])
-            .' -P '.escapeshellarg($this->aDSNInfo['port'])
-            .' -t '.escapeshellarg($this->iInstances.$sOutputFile);
+        $oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py'))
+                    ->addParams('--database', $this->aDSNInfo['database'])
+                    ->addParams('--port', $this->aDSNInfo['port'])
+                    ->addParams('--threads', $this->iInstances);
+
+        if (!$this->bQuiet) {
+            $oBaseCmd->addParams('-v');
+        }
+        if ($this->bVerbose) {
+            $oBaseCmd->addParams('-v');
+        }

         if (isset($this->aDSNInfo['hostspec'])) {
         if (isset($this->aDSNInfo['hostspec'])) {

-            $sBaseCmd .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
+            $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']);

         }
         if (isset($this->aDSNInfo['username'])) {
         }
         if (isset($this->aDSNInfo['username'])) {

-            $sBaseCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
+            $oBaseCmd->addParams('--user', $this->aDSNInfo['username']);
+        }
+        if (isset($this->aDSNInfo['password'])) {
+            $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);

         }
 
         info('Index ranks 0 - 4');
         }
 
         info('Index ranks 0 - 4');

-        $iStatus = $this->runWithPgEnv($sBaseCmd.' -R 4');
+        $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
+        echo $oCmd->escapedCmd();
+        
+        $iStatus = $oCmd->run();

         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 5 - 25');
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 5 - 25');

-        $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 5 -R 25');
+        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
+        $iStatus = $oCmd->run();

         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 26 - 30');
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 26 - 30');

-        $iStatus = $this->runWithPgEnv($sBaseCmd.' -r 26');
+        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26);
+        $iStatus = $oCmd->run();

         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }


@@ -548,7 +599,19 @@ class SetupFunctions
     {
         info('Create Search indices');
 
     {
         info('Create Search indices');
 

+        $sSQL = 'SELECT relname FROM pg_class, pg_index ';
+        $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid';
+        $aInvalidIndices = $this->oDB->getCol($sSQL);
+
+        foreach ($aInvalidIndices as $sIndexName) {
+            info("Cleaning up invalid index $sIndexName");
+            $this->oDB->exec("DROP INDEX $sIndexName;");
+        }
+

         $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');

+        if (!$this->bDrop) {
+            $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_updates.src.sql');
+        }

         if (!$this->dbReverseOnly()) {
             $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql');
         }
         if (!$this->dbReverseOnly()) {
             $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql');
         }


@@ -612,7 +675,7 @@ class SetupFunctions
                        );
 
         $aDropTables = array();
                        );
 
         $aDropTables = array();

-        $aHaveTables = $this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");
+        $aHaveTables = $this->oDB->getListOfTables();

 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;
 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;


@@ -628,6 +691,11 @@ class SetupFunctions
             $this->dropTable($sDrop);
         }
 
             $this->dropTable($sDrop);
         }
 

+        $this->removeFlatnodeFile();
+    }
+
+    private function removeFlatnodeFile()
+    {

         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
             if (file_exists(CONST_Osm2pgsql_Flatnode_File)) {
                 if ($this->bVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
             if (file_exists(CONST_Osm2pgsql_Flatnode_File)) {
                 if ($this->bVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n";


@@ -649,10 +717,21 @@ class SetupFunctions
     private function createSqlFunctions()
     {
         $sBasePath = CONST_BasePath.'/sql/functions/';
     private function createSqlFunctions()
     {
         $sBasePath = CONST_BasePath.'/sql/functions/';

-        $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
+        $sTemplate = file_get_contents($sBasePath.'utils.sql');

         $sTemplate .= file_get_contents($sBasePath.'normalization.sql');
         $sTemplate .= file_get_contents($sBasePath.'normalization.sql');

+        $sTemplate .= file_get_contents($sBasePath.'ranking.sql');

         $sTemplate .= file_get_contents($sBasePath.'importance.sql');
         $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql');
         $sTemplate .= file_get_contents($sBasePath.'importance.sql');
         $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql');

+        $sTemplate .= file_get_contents($sBasePath.'interpolation.sql');
+        if ($this->oDB->tableExists('place')) {
+            $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql');
+        }
+        if ($this->oDB->tableExists('placex')) {
+            $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql');
+        }
+        if ($this->oDB->tableExists('location_postcode')) {
+            $sTemplate .= file_get_contents($sBasePath.'postcode_triggers.sql');
+        }

         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
         if ($this->bEnableDiffUpdates) {
             $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
         if ($this->bEnableDiffUpdates) {
             $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);


@@ -698,21 +777,21 @@ class SetupFunctions
     {
         if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 
     {
         if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 

-        $sCMD = 'psql'
-            .' -p '.escapeshellarg($this->aDSNInfo['port'])
-            .' -d '.escapeshellarg($this->aDSNInfo['database']);
+        $oCmd = (new \Nominatim\Shell('psql'))
+                ->addParams('--port', $this->aDSNInfo['port'])
+                ->addParams('--dbname', $this->aDSNInfo['database']);
+

         if (!$this->bVerbose) {
         if (!$this->bVerbose) {

-            $sCMD .= ' -q';
+            $oCmd->addParams('--quiet');

         }
         if (isset($this->aDSNInfo['hostspec'])) {
         }
         if (isset($this->aDSNInfo['hostspec'])) {

-            $sCMD .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']);
+            $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);

         }
         if (isset($this->aDSNInfo['username'])) {
         }
         if (isset($this->aDSNInfo['username'])) {

-            $sCMD .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
+            $oCmd->addParams('--username', $this->aDSNInfo['username']);

         }
         }

-        $aProcEnv = null;

         if (isset($this->aDSNInfo['password'])) {
         if (isset($this->aDSNInfo['password'])) {

-            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
+            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);

         }
         $ahGzipPipes = null;
         if (preg_match('/\\.gz$/', $sFilename)) {
         }
         $ahGzipPipes = null;
         if (preg_match('/\\.gz$/', $sFilename)) {


@@ -721,12 +800,14 @@ class SetupFunctions
                              1 => array('pipe', 'w'),
                              2 => array('file', '/dev/null', 'a')
                             );
                              1 => array('pipe', 'w'),
                              2 => array('file', '/dev/null', 'a')
                             );

-            $hGzipProcess = proc_open('zcat '.escapeshellarg($sFilename), $aDescriptors, $ahGzipPipes);
+            $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename);
+
+            $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes);

             if (!is_resource($hGzipProcess)) fail('unable to start zcat');
             $aReadPipe = $ahGzipPipes[1];
             fclose($ahGzipPipes[0]);
         } else {
             if (!is_resource($hGzipProcess)) fail('unable to start zcat');
             $aReadPipe = $ahGzipPipes[1];
             fclose($ahGzipPipes[0]);
         } else {

-            $sCMD .= ' -f '.escapeshellarg($sFilename);
+            $oCmd->addParams('--file', $sFilename);

             $aReadPipe = array('pipe', 'r');
         }
         $aDescriptors = array(
             $aReadPipe = array('pipe', 'r');
         }
         $aDescriptors = array(


@@ -735,7 +816,8 @@ class SetupFunctions
                          2 => array('file', '/dev/null', 'a')
                         );
         $ahPipes = null;
                          2 => array('file', '/dev/null', 'a')
                         );
         $ahPipes = null;

-        $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
+
+        $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);

         if (!is_resource($hProcess)) fail('unable to start pgsql');
         // TODO: error checking
         while (!feof($ahPipes[1])) {
         if (!is_resource($hProcess)) fail('unable to start pgsql');
         // TODO: error checking
         while (!feof($ahPipes[1])) {


@@ -776,21 +858,6 @@ class SetupFunctions
         return $sSql;
     }
 
         return $sSql;
     }
 

-    private function runWithPgEnv($sCmd)
-    {
-        if ($this->bVerbose) {
-            echo "Execute: $sCmd\n";
-        }
-
-        $aProcEnv = null;
-
-        if (isset($this->aDSNInfo['password'])) {
-            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
-        }
-
-        return runWithEnv($sCmd, $aProcEnv);
-    }
-

     /**
      * Drop table with the given name if it exists.
      *
     /**
      * Drop table with the given name if it exists.
      *


@@ -803,7 +870,7 @@ class SetupFunctions
     private function dropTable($sName)
     {
         if ($this->bVerbose) echo "Dropping table $sName\n";
     private function dropTable($sName)
     {
         if ($this->bVerbose) echo "Dropping table $sName\n";

-        $this->oDB->exec('DROP TABLE IF EXISTS '.$sName.' CASCADE');
+        $this->oDB->deleteTable($sName);

     }
 
     /**
     }
 
     /**