Merge pull request #1348 from mtmail/checkmodulepresence-to-raise-exception

[nominatim.git] / lib / setup / SetupClass.php

diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php
index 013668fa5fd7a2b40190ec20b285588e7adab7cf..c14190c3a1be66020216defca9686262ac44f3f0 100755 (executable)
--- a/lib/setup/SetupClass.php
+++ b/lib/setup/SetupClass.php
@@ -10,7 +10,7 @@ class SetupFunctions
     protected $iInstances;
     protected $sModulePath;
     protected $aDSNInfo;
     protected $iInstances;
     protected $sModulePath;
     protected $aDSNInfo;

-    protected $sVerbose;
+    protected $bVerbose;

     protected $sIgnoreErrors;
     protected $bEnableDiffUpdates;
     protected $bEnableDebugStatements;
     protected $sIgnoreErrors;
     protected $bEnableDiffUpdates;
     protected $bEnableDebugStatements;


@@ -40,13 +40,13 @@ class SetupFunctions
         info('module path: ' . $this->sModulePath);
 
         // parse database string
         info('module path: ' . $this->sModulePath);
 
         // parse database string

-        $this->aDSNInfo = array_filter(\DB::parseDSN(CONST_Database_DSN));
+        $this->aDSNInfo = \Nominatim\DB::parseDSN(CONST_Database_DSN);

         if (!isset($this->aDSNInfo['port'])) {
             $this->aDSNInfo['port'] = 5432;
         }
 
         // setting member variables based on command line options stored in $aCMDResult
         if (!isset($this->aDSNInfo['port'])) {
             $this->aDSNInfo['port'] = 5432;
         }
 
         // setting member variables based on command line options stored in $aCMDResult

-        $this->sVerbose = $aCMDResult['verbose'];
+        $this->bVerbose = $aCMDResult['verbose'];

 
         //setting default values which are not set by the update.php array
         if (isset($aCMDResult['ignore-errors'])) {
 
         //setting default values which are not set by the update.php array
         if (isset($aCMDResult['ignore-errors'])) {


@@ -74,8 +74,9 @@ class SetupFunctions
     public function createDB()
     {
         info('Create DB');
     public function createDB()
     {
         info('Create DB');

-        $sDB = \DB::connect(CONST_Database_DSN, false);
-        if (!\PEAR::isError($sDB)) {
+        $oDB = new \Nominatim\DB;
+
+        if ($oDB->databaseExists()) {

             fail('database already exists ('.CONST_Database_DSN.')');
         }
 
             fail('database already exists ('.CONST_Database_DSN.')');
         }
 


@@ -94,14 +95,15 @@ class SetupFunctions
 
     public function connect()
     {
 
     public function connect()
     {

-        $this->oDB =& getDB();
+        $this->oDB = new \Nominatim\DB();
+        $this->oDB->connect();

     }
 
     public function setupDB()
     {
         info('Setup DB');
 
     }
 
     public function setupDB()
     {
         info('Setup DB');
 

-        $fPostgresVersion = getPostgresVersion($this->oDB);
+        $fPostgresVersion = $this->oDB->getPostgresVersion();

         echo 'Postgres version found: '.$fPostgresVersion."\n";
 
         if ($fPostgresVersion < 9.01) {
         echo 'Postgres version found: '.$fPostgresVersion."\n";
 
         if ($fPostgresVersion < 9.01) {


@@ -114,7 +116,7 @@ class SetupFunctions
         // For extratags and namedetails the hstore_to_json converter is
         // needed which is only available from Postgresql 9.3+. For older
         // versions add a dummy function that returns nothing.
         // For extratags and namedetails the hstore_to_json converter is
         // needed which is only available from Postgresql 9.3+. For older
         // versions add a dummy function that returns nothing.

-        $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'"));
+        $iNumFunc = $this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'");

 
         if ($iNumFunc == 0) {
             $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
 
         if ($iNumFunc == 0) {
             $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");


@@ -122,7 +124,7 @@ class SetupFunctions
         }
 
 
         }
 
 

-        $fPostgisVersion = getPostgisVersion($this->oDB);
+        $fPostgisVersion = $this->oDB->getPostgisVersion();

         echo 'Postgis version found: '.$fPostgisVersion."\n";
 
         if ($fPostgisVersion < 2.1) {
         echo 'Postgis version found: '.$fPostgisVersion."\n";
 
         if ($fPostgisVersion < 2.1) {


@@ -134,7 +136,7 @@ class SetupFunctions
             $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
         }
 
             $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
         }
 

-        $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
+        $i = $this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'");

         if ($i == 0) {
             echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
             echo "\n          createuser ".CONST_Database_Web_User."\n\n";
         if ($i == 0) {
             echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
             echo "\n          createuser ".CONST_Database_Web_User."\n\n";


@@ -142,9 +144,7 @@ class SetupFunctions
         }
 
         // Try accessing the C module, so we know early if something is wrong
         }
 
         // Try accessing the C module, so we know early if something is wrong

-        if (!checkModulePresence()) {
-            fail('error loading nominatim.so module');
-        }
+        checkModulePresence(); // raises exception on failure

 
         if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
             echo 'Error: you need to download the country_osm_grid first:';
 
         if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
             echo 'Error: you need to download the country_osm_grid first:';


@@ -189,6 +189,8 @@ class SetupFunctions
             fail("osm2pgsql not found in '$osm2pgsql'");
         }
 
             fail("osm2pgsql not found in '$osm2pgsql'");
         }
 

+        $osm2pgsql .= ' -S '.CONST_Import_Style;
+

         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
             $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
         }
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
             $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
         }


@@ -214,7 +216,7 @@ class SetupFunctions
 
         $this->runWithPgEnv($osm2pgsql);
 
 
         $this->runWithPgEnv($osm2pgsql);
 

-        if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) {
+        if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) {

             fail('No Data');
         }
     }
             fail('No Data');
         }
     }


@@ -223,11 +225,9 @@ class SetupFunctions
     {
         info('Create Functions');
 
     {
         info('Create Functions');
 

-        // Try accessing the C module, so we know eif something is wrong
-        // update.php calls this function
-        if (!checkModulePresence()) {
-            fail('error loading nominatim.so module');
-        }
+        // Try accessing the C module, so we know early if something is wrong
+        checkModulePresence(); // raises exception on failure
+

         $this->createSqlFunctions();
     }
 
         $this->createSqlFunctions();
     }
 


@@ -376,7 +376,8 @@ class SetupFunctions
         echo '.';
 
         $sSQL = 'select distinct partition from country_name';
         echo '.';
 
         $sSQL = 'select distinct partition from country_name';

-        $aPartitions = chksql($this->oDB->getCol($sSQL));
+        $aPartitions = $this->oDB->getCol($sSQL);
+

         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {
             $this->pgExec('TRUNCATE location_road_'.$sPartition);
         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {
             $this->pgExec('TRUNCATE location_road_'.$sPartition);


@@ -397,34 +398,48 @@ class SetupFunctions
 
         info('Load Data');
         $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
 
         info('Load Data');
         $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';

+

         $aDBInstances = array();
         $iLoadThreads = max(1, $this->iInstances - 1);
         for ($i = 0; $i < $iLoadThreads; $i++) {
         $aDBInstances = array();
         $iLoadThreads = max(1, $this->iInstances - 1);
         for ($i = 0; $i < $iLoadThreads; $i++) {

-            $aDBInstances[$i] =& getDB(true);
+            // https://secure.php.net/manual/en/function.pg-connect.php
+            $DSN = CONST_Database_DSN;
+            $DSN = preg_replace('/^pgsql:/', '', $DSN);
+            $DSN = preg_replace('/;/', ' ', $DSN);
+            $aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW);
+            pg_ping($aDBInstances[$i]);
+        }
+
+        for ($i = 0; $i < $iLoadThreads; $i++) {

             $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
             $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
             $sSQL .= "          and ST_GeometryType(geometry) = 'ST_LineString')";
             $sSQL .= ' and ST_IsValid(geometry)';
             $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
             $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
             $sSQL .= "          and ST_GeometryType(geometry) = 'ST_LineString')";
             $sSQL .= ' and ST_IsValid(geometry)';

-            if ($this->sVerbose) echo "$sSQL\n";
-            if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
-                fail(pg_last_error($aDBInstances[$i]->connection));
+            if ($this->bVerbose) echo "$sSQL\n";
+            if (!pg_send_query($aDBInstances[$i], $sSQL)) {
+                fail(pg_last_error($aDBInstances[$i]));

             }
         }
 
         // last thread for interpolation lines
             }
         }
 
         // last thread for interpolation lines

-        $aDBInstances[$iLoadThreads] =& getDB(true);
+        // https://secure.php.net/manual/en/function.pg-connect.php
+        $DSN = CONST_Database_DSN;
+        $DSN = preg_replace('/^pgsql:/', '', $DSN);
+        $DSN = preg_replace('/;/', ' ', $DSN);
+        $aDBInstances[$iLoadThreads] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW);
+        pg_ping($aDBInstances[$iLoadThreads]);

         $sSQL = 'insert into location_property_osmline';
         $sSQL .= ' (osm_id, address, linegeo)';
         $sSQL .= ' SELECT osm_id, address, geometry from place where ';
         $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
         $sSQL = 'insert into location_property_osmline';
         $sSQL .= ' (osm_id, address, linegeo)';
         $sSQL .= ' SELECT osm_id, address, geometry from place where ';
         $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";

-        if ($this->sVerbose) echo "$sSQL\n";
-        if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
-            fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
+        if ($this->bVerbose) echo "$sSQL\n";
+        if (!pg_send_query($aDBInstances[$iLoadThreads], $sSQL)) {
+            fail(pg_last_error($aDBInstances[$iLoadThreads]));

         }
 
         $bFailed = false;
         for ($i = 0; $i <= $iLoadThreads; $i++) {
         }
 
         $bFailed = false;
         for ($i = 0; $i <= $iLoadThreads; $i++) {

-            while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) {
+            while (($hPGresult = pg_get_result($aDBInstances[$i])) !== false) {

                 $resultStatus = pg_result_status($hPGresult);
                 // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
                 // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,
                 $resultStatus = pg_result_status($hPGresult);
                 // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
                 // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,


@@ -440,17 +455,22 @@ class SetupFunctions
         if ($bFailed) {
             fail('SQL errors loading placex and/or location_property_osmline tables');
         }
         if ($bFailed) {
             fail('SQL errors loading placex and/or location_property_osmline tables');
         }

+
+        for ($i = 0; $i < $this->iInstances; $i++) {
+            pg_close($aDBInstances[$i]);
+        }
+

         echo "\n";
         info('Reanalysing database');
         $this->pgsqlRunScript('ANALYSE');
 
         $sDatabaseDate = getDatabaseDate($this->oDB);
         echo "\n";
         info('Reanalysing database');
         $this->pgsqlRunScript('ANALYSE');
 
         $sDatabaseDate = getDatabaseDate($this->oDB);

-        pg_query($this->oDB->connection, 'TRUNCATE import_status');
-        if ($sDatabaseDate === false) {
+        $this->oDB->exec('TRUNCATE import_status');
+        if (!$sDatabaseDate) {

             warn('could not determine database date.');
         } else {
             $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
             warn('could not determine database date.');
         } else {
             $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";

-            pg_query($this->oDB->connection, $sSQL);
+            $this->oDB->exec($sSQL);

             echo "Latest data imported from $sDatabaseDate.\n";
         }
     }
             echo "Latest data imported from $sDatabaseDate.\n";
         }
     }


@@ -475,7 +495,12 @@ class SetupFunctions
 
         $aDBInstances = array();
         for ($i = 0; $i < $this->iInstances; $i++) {
 
         $aDBInstances = array();
         for ($i = 0; $i < $this->iInstances; $i++) {

-            $aDBInstances[$i] =& getDB(true);
+            // https://secure.php.net/manual/en/function.pg-connect.php
+            $DSN = CONST_Database_DSN;
+            $DSN = preg_replace('/^pgsql:/', '', $DSN);
+            $DSN = preg_replace('/;/', ' ', $DSN);
+            $aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW | PGSQL_CONNECT_ASYNC);
+            pg_ping($aDBInstances[$i]);

         }
 
         foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {
         }
 
         foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {


@@ -485,11 +510,11 @@ class SetupFunctions
             $iLines = 0;
             while (true) {
                 for ($i = 0; $i < $this->iInstances; $i++) {
             $iLines = 0;
             while (true) {
                 for ($i = 0; $i < $this->iInstances; $i++) {

-                    if (!pg_connection_busy($aDBInstances[$i]->connection)) {
-                        while (pg_get_result($aDBInstances[$i]->connection));
+                    if (!pg_connection_busy($aDBInstances[$i])) {
+                        while (pg_get_result($aDBInstances[$i]));

                         $sSQL = fgets($hFile, 100000);
                         if (!$sSQL) break 2;
                         $sSQL = fgets($hFile, 100000);
                         if (!$sSQL) break 2;

-                        if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
+                        if (!pg_send_query($aDBInstances[$i], $sSQL)) fail(pg_last_error($aDBInstances[$i]));

                         $iLines++;
                         if ($iLines == 1000) {
                             echo '.';
                         $iLines++;
                         if ($iLines == 1000) {
                             echo '.';


@@ -505,13 +530,17 @@ class SetupFunctions
             while ($bAnyBusy) {
                 $bAnyBusy = false;
                 for ($i = 0; $i < $this->iInstances; $i++) {
             while ($bAnyBusy) {
                 $bAnyBusy = false;
                 for ($i = 0; $i < $this->iInstances; $i++) {

-                    if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
+                    if (pg_connection_busy($aDBInstances[$i])) $bAnyBusy = true;

                 }
                 usleep(10);
             }
             echo "\n";
         }
 
                 }
                 usleep(10);
             }
             echo "\n";
         }
 

+        for ($i = 0; $i < $this->iInstances; $i++) {
+            pg_close($aDBInstances[$i]);
+        }
+

         info('Creating indexes on Tiger data');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
         $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
         info('Creating indexes on Tiger data');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
         $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);


@@ -690,11 +719,12 @@ class SetupFunctions
                         'new_query_log',
                         'spatial_ref_sys',
                         'country_name',
                         'new_query_log',
                         'spatial_ref_sys',
                         'country_name',

-                        'place_classtype_*'
+                        'place_classtype_*',
+                        'country_osm_grid'

                        );
 
         $aDropTables = array();
                        );
 
         $aDropTables = array();

-        $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"));
+        $aHaveTables = $this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");

 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;
 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;


@@ -707,15 +737,15 @@ class SetupFunctions
             if (!$bFound) array_push($aDropTables, $sTable);
         }
         foreach ($aDropTables as $sDrop) {
             if (!$bFound) array_push($aDropTables, $sTable);
         }
         foreach ($aDropTables as $sDrop) {

-            if ($this->sVerbose) echo "Dropping table $sDrop\n";
-            @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE");
+            if ($this->bVerbose) echo "Dropping table $sDrop\n";
+            $this->oDB->exec("DROP TABLE $sDrop CASCADE");

             // ignore warnings/errors as they might be caused by a table having
             // been deleted already by CASCADE
         }
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
             if (file_exists(CONST_Osm2pgsql_Flatnode_File)) {
             // ignore warnings/errors as they might be caused by a table having
             // been deleted already by CASCADE
         }
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
             if (file_exists(CONST_Osm2pgsql_Flatnode_File)) {

-                if ($this->sVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
+                if ($this->bVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n";

                 unlink(CONST_Osm2pgsql_Flatnode_File);
             }
         }
                 unlink(CONST_Osm2pgsql_Flatnode_File);
             }
         }


@@ -739,7 +769,7 @@ class SetupFunctions
         runSQLScript(
             $sScript,
             $bfatal,
         runSQLScript(
             $sScript,
             $bfatal,

-            $this->sVerbose,
+            $this->bVerbose,

             $this->sIgnoreErrors
         );
     }
             $this->sIgnoreErrors
         );
     }


@@ -773,7 +803,7 @@ class SetupFunctions
     private function pgsqlRunPartitionScript($sTemplate)
     {
         $sSQL = 'select distinct partition from country_name';
     private function pgsqlRunPartitionScript($sTemplate)
     {
         $sSQL = 'select distinct partition from country_name';

-        $aPartitions = chksql($this->oDB->getCol($sSQL));
+        $aPartitions = $this->oDB->getCol($sSQL);

         if (!$this->bNoPartitions) $aPartitions[] = 0;
 
         preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
         if (!$this->bNoPartitions) $aPartitions[] = 0;
 
         preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);


@@ -793,7 +823,7 @@ class SetupFunctions
         if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 
         $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'];
         if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 
         $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'];

-        if (!$this->sVerbose) {
+        if (!$this->bVerbose) {

             $sCMD .= ' -q';
         }
         if (isset($this->aDSNInfo['hostspec'])) {
             $sCMD .= ' -q';
         }
         if (isset($this->aDSNInfo['hostspec'])) {


@@ -856,6 +886,10 @@ class SetupFunctions
 
     private function runWithPgEnv($sCmd)
     {
 
     private function runWithPgEnv($sCmd)
     {

+        if ($this->bVerbose) {
+            echo "Execute: $sCmd\n";
+        }
+

         $aProcEnv = null;
 
         if (isset($this->aDSNInfo['password'])) {
         $aProcEnv = null;
 
         if (isset($this->aDSNInfo['password'])) {


@@ -876,9 +910,7 @@ class SetupFunctions
      */
     private function pgExec($sSQL)
     {
      */
     private function pgExec($sSQL)
     {

-        if (!pg_query($this->oDB->connection, $sSQL)) {
-            fail(pg_last_error($this->oDB->connection));
-        }
+        $this->oDB->exec($sSQL);

     }
 
     /**
     }
 
     /**


@@ -888,7 +920,6 @@ class SetupFunctions
      */
     private function dbReverseOnly()
     {
      */
     private function dbReverseOnly()
     {

-        $sSQL = "SELECT count(*) FROM pg_tables WHERE tablename = 'search_name'";
-        return !(chksql($this->oDB->getOne($sSQL)));
+        return !($this->oDB->tableExists('search_name'));

     }
 }
     }
 }