]> git.openstreetmap.org Git - nominatim.git/blobdiff - lib/setup/SetupClass.php
projects / nominatim.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
restrict size of features that get a full address search
[nominatim.git] / lib / setup / SetupClass.php
diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php
index fbcf9b5c7bb67aa146d4f00c67450be4102d5f42..fd31f0fe4143b834b2fbf63d693ad2f08b17eefc 100755 (executable)
--- a/lib/setup/SetupClass.php
+++ b/lib/setup/SetupClass.php
@@ -2,49 +2,58 @@
 
 namespace Nominatim\Setup;
 
 
 namespace Nominatim\Setup;
 
+require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php');
+require_once(CONST_BasePath.'/lib/Shell.php');
+
 class SetupFunctions
 {
 class SetupFunctions
 {
-    protected $iCacheMemory;            // set in constructor
-    protected $iInstances;              // set in constructor
-    protected $sModulePath;             // set in constructor
-    protected $aDSNInfo;                // set in constructor = DB::parseDSN(CONST_Database_DSN);
-    protected $sVerbose;                // set in constructor
-    protected $sIgnoreErrors;           // set in constructor
-    protected $bEnableDiffUpdates;      // set in constructor
-    protected $bEnableDebugStatements;  // set in constructor
-    protected $bNoPartitions;           // set in constructor
-    protected $oDB = null;              // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
-                                                // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
-
-    public function __construct($callingFunction, array $aCMDResult = array())
+    protected $iCacheMemory;
+    protected $iInstances;
+    protected $sModulePath;
+    protected $aDSNInfo;
+    protected $bQuiet;
+    protected $bVerbose;
+    protected $sIgnoreErrors;
+    protected $bEnableDiffUpdates;
+    protected $bEnableDebugStatements;
+    protected $bNoPartitions;
+    protected $bDrop;
+    protected $oDB = null;
+
+    public function __construct(array $aCMDResult)
     {
         // by default, use all but one processor, but never more than 15.
         $this->iInstances = isset($aCMDResult['threads'])
             ? $aCMDResult['threads']
             : (min(16, getProcessorCount()) - 1);
 
     {
         // by default, use all but one processor, but never more than 15.
         $this->iInstances = isset($aCMDResult['threads'])
             ? $aCMDResult['threads']
             : (min(16, getProcessorCount()) - 1);
 
-
         if ($this->iInstances < 1) {
             $this->iInstances = 1;
             warn('resetting threads to '.$this->iInstances);
         }
 
         if ($this->iInstances < 1) {
             $this->iInstances = 1;
             warn('resetting threads to '.$this->iInstances);
         }
 
-        // Assume we can steal all the cache memory in the box (unless told otherwise)
         if (isset($aCMDResult['osm2pgsql-cache'])) {
             $this->iCacheMemory = $aCMDResult['osm2pgsql-cache'];
         if (isset($aCMDResult['osm2pgsql-cache'])) {
             $this->iCacheMemory = $aCMDResult['osm2pgsql-cache'];
+        } elseif (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
+            // When flatnode files are enabled then disable cache per default.
+            $this->iCacheMemory = 0;
         } else {
         } else {
+            // Otherwise: Assume we can steal all the cache memory in the box.
             $this->iCacheMemory = getCacheMemoryMB();
         }
 
         $this->sModulePath = CONST_Database_Module_Path;
         info('module path: ' . $this->sModulePath);
             $this->iCacheMemory = getCacheMemoryMB();
         }
 
         $this->sModulePath = CONST_Database_Module_Path;
         info('module path: ' . $this->sModulePath);
-       
-        // prepares DB for import or update, sets the Data Source Name
-        $this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN);
-        if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
- 
+
+        // parse database string
+        $this->aDSNInfo = \Nominatim\DB::parseDSN(CONST_Database_DSN);
+        if (!isset($this->aDSNInfo['port'])) {
+            $this->aDSNInfo['port'] = 5432;
+        }
+
         // setting member variables based on command line options stored in $aCMDResult
         // setting member variables based on command line options stored in $aCMDResult
-        $this->sVerbose = $aCMDResult['verbose'];
+        $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet'];
+        $this->bVerbose = $aCMDResult['verbose'];
 
         //setting default values which are not set by the update.php array
         if (isset($aCMDResult['ignore-errors'])) {
 
         //setting default values which are not set by the update.php array
         if (isset($aCMDResult['ignore-errors'])) {
@@ -62,81 +71,66 @@ class SetupFunctions
         } else {
             $this->bNoPartitions = false;
         }
         } else {
             $this->bNoPartitions = false;
         }
-
-        // if class is instantiated by update.php, we have to set EnableDiffUpdates to true
-        // otherwise set to value provided by setup.php's command line arg array
-        if ($callingFunction == 'update') {
-            $this->bEnableDiffUpdates = true;
-        } elseif ($callingFunction == 'setup') {
+        if (isset($aCMDResult['enable-diff-updates'])) {
             $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
             $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
+        } else {
+            $this->bEnableDiffUpdates = false;
         }
         }
+
+        $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop'];
     }
 
     public function createDB()
     {
         info('Create DB');
     }
 
     public function createDB()
     {
         info('Create DB');
-        $sDB = \DB::connect(CONST_Database_DSN, false);
-        if (!\PEAR::isError($sDB)) {
+        $oDB = new \Nominatim\DB;
+
+        if ($oDB->checkConnection()) {
             fail('database already exists ('.CONST_Database_DSN.')');
         }
 
             fail('database already exists ('.CONST_Database_DSN.')');
         }
 
-        $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database'];
-        if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
-            $sCreateDBCmd .= ' -U '.$this->aDSNInfo['username'];
-        }
+        $oCmd = (new \Nominatim\Shell('createdb'))
+                ->addParams('-E', 'UTF-8')
+                ->addParams('-p', $this->aDSNInfo['port']);
 
 
-        if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
-            $sCreateDBCmd .= ' -h '.$this->aDSNInfo['hostspec'];
+        if (isset($this->aDSNInfo['username'])) {
+            $oCmd->addParams('-U', $this->aDSNInfo['username']);
         }
         }
-
-        $aProcEnv = null;
-        if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
-            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
+        if (isset($this->aDSNInfo['password'])) {
+            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
+        }
+        if (isset($this->aDSNInfo['hostspec'])) {
+            $oCmd->addParams('-h', $this->aDSNInfo['hostspec']);
         }
         }
+        $oCmd->addParams($this->aDSNInfo['database']);
 
 
-        $result = runWithEnv($sCreateDBCmd, $aProcEnv);
-        if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
+        $result = $oCmd->run();
+        if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
     }
 
     public function setupDB()
     {
         info('Setup DB');
     }
 
     public function setupDB()
     {
         info('Setup DB');
-        $this->oDB =& getDB();
 
 
-        $fPostgresVersion = getPostgresVersion($this->oDB);
+        $fPostgresVersion = $this->db()->getPostgresVersion();
         echo 'Postgres version found: '.$fPostgresVersion."\n";
 
         echo 'Postgres version found: '.$fPostgresVersion."\n";
 
-        if ($fPostgresVersion < 9.1) {
-            fail('Minimum supported version of Postgresql is 9.1.');
+        if ($fPostgresVersion < 9.03) {
+            fail('Minimum supported version of Postgresql is 9.3.');
         }
 
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
 
         }
 
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
 
-        // For extratags and namedetails the hstore_to_json converter is
-        // needed which is only available from Postgresql 9.3+. For older
-        // versions add a dummy function that returns nothing.
-        $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'"));
-
-        if ($iNumFunc == 0) {
-            $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
-            warn('Postgresql is too old. extratags and namedetails API not available.');
-        }
-
-
-        $fPostgisVersion = getPostgisVersion($this->oDB);
+        $fPostgisVersion = $this->db()->getPostgisVersion();
         echo 'Postgis version found: '.$fPostgisVersion."\n";
 
         echo 'Postgis version found: '.$fPostgisVersion."\n";
 
-        if ($fPostgisVersion < 2.1) {
-            // Functions were renamed in 2.1 and throw an annoying deprecation warning
-            $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
-            $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint');
-        }
         if ($fPostgisVersion < 2.2) {
         if ($fPostgisVersion < 2.2) {
-            $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
+            echo "Minimum required Postgis version 2.2\n";
+            exit(1);
         }
 
         }
 
-        $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
+        $i = $this->db()->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'");
         if ($i == 0) {
             echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
             echo "\n          createuser ".CONST_Database_Web_User."\n\n";
         if ($i == 0) {
             echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
             echo "\n          createuser ".CONST_Database_Web_User."\n\n";
@@ -144,9 +138,7 @@ class SetupFunctions
         }
 
         // Try accessing the C module, so we know early if something is wrong
         }
 
         // Try accessing the C module, so we know early if something is wrong
-        if (!checkModulePresence()) {
-            fail('error loading nominatim.so module');
-        }
+        checkModulePresence(); // raises exception on failure
 
         if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
             echo 'Error: you need to download the country_osm_grid first:';
 
         if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
             echo 'Error: you need to download the country_osm_grid first:';
@@ -154,171 +146,130 @@ class SetupFunctions
             exit(1);
         }
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
             exit(1);
         }
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
-        $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
-        $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz');
+        $this->pgsqlRunScriptFile(CONST_ExtraDataPath.'/country_osm_grid.sql.gz');
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
+        $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode_table.sql');
 
 
-
-        if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
-            $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
+        $sPostcodeFilename = CONST_BasePath.'/data/gb_postcode_data.sql.gz';
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
         } else {
         } else {
-            warn('external UK postcode table not found.');
+            warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
         }
 
         }
 
-        if (CONST_Use_Extra_US_Postcodes) {
-            $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
+        $sPostcodeFilename = CONST_BasePath.'/data/us_postcode_data.sql.gz';
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
+        } else {
+            warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
         }
 
         if ($this->bNoPartitions) {
             $this->pgsqlRunScript('update country_name set partition = 0');
         }
         }
 
         if ($this->bNoPartitions) {
             $this->pgsqlRunScript('update country_name set partition = 0');
         }
-
-        // the following will be needed by createFunctions later but
-        // is only defined in the subsequently called createTables
-        // Create dummies here that will be overwritten by the proper
-        // versions in create-tables.
-        $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()');
-        $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false);
     }
 
     public function importData($sOSMFile)
     {
         info('Import data');
 
     }
 
     public function importData($sOSMFile)
     {
         info('Import data');
 
-        $osm2pgsql = CONST_Osm2pgsql_Binary;
-        if (!file_exists($osm2pgsql)) {
+        if (!file_exists(CONST_Osm2pgsql_Binary)) {
             echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
             echo "Normally you should not need to set this manually.\n";
             echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
             echo "Normally you should not need to set this manually.\n";
-            fail("osm2pgsql not found in '$osm2pgsql'");
+            fail("osm2pgsql not found in '".CONST_Osm2pgsql_Binary."'");
         }
 
         }
 
-
+        $oCmd = new \Nominatim\Shell(CONST_Osm2pgsql_Binary);
+        $oCmd->addParams('--style', CONST_Import_Style);
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
-            $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
-        }
-
-        if (CONST_Tablespace_Osm2pgsql_Data)
-            $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data;
-        if (CONST_Tablespace_Osm2pgsql_Index)
-            $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index;
-        if (CONST_Tablespace_Place_Data)
-            $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data;
-        if (CONST_Tablespace_Place_Index)
-            $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index;
-        $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
-        $osm2pgsql .= ' -C '.$this->iCacheMemory;
-        $osm2pgsql .= ' -P '.$this->aDSNInfo['port'];
-        if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
-            $osm2pgsql .= ' -U '.$this->aDSNInfo['username'];
-        }
-        if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
-            $osm2pgsql .= ' -H '.$this->aDSNInfo['hostspec'];
-        }
-        $aProcEnv = null;
-        if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
-            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
-        }
-        $osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile;
-        runWithEnv($osm2pgsql, $aProcEnv);
-        if ($this->oDB == null) $this->oDB =& getDB();
-        if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) {
+            $oCmd->addParams('--flat-nodes', CONST_Osm2pgsql_Flatnode_File);
+        }
+        if (CONST_Tablespace_Osm2pgsql_Data) {
+            $oCmd->addParams('--tablespace-slim-data', CONST_Tablespace_Osm2pgsql_Data);
+        }
+        if (CONST_Tablespace_Osm2pgsql_Index) {
+            $oCmd->addParams('--tablespace-slim-index', CONST_Tablespace_Osm2pgsql_Index);
+        }
+        if (CONST_Tablespace_Place_Data) {
+            $oCmd->addParams('--tablespace-main-data', CONST_Tablespace_Place_Data);
+        }
+        if (CONST_Tablespace_Place_Index) {
+            $oCmd->addParams('--tablespace-main-index', CONST_Tablespace_Place_Index);
+        }
+        $oCmd->addParams('--latlong', '--slim', '--create');
+        $oCmd->addParams('--output', 'gazetteer');
+        $oCmd->addParams('--hstore');
+        $oCmd->addParams('--number-processes', 1);
+        $oCmd->addParams('--cache', $this->iCacheMemory);
+        $oCmd->addParams('--port', $this->aDSNInfo['port']);
+
+        if (isset($this->aDSNInfo['username'])) {
+            $oCmd->addParams('--username', $this->aDSNInfo['username']);
+        }
+        if (isset($this->aDSNInfo['password'])) {
+            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
+        }
+        if (isset($this->aDSNInfo['hostspec'])) {
+            $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
+        }
+        $oCmd->addParams('--database', $this->aDSNInfo['database']);
+        $oCmd->addParams($sOSMFile);
+        $oCmd->run();
+
+        if (!$this->sIgnoreErrors && !$this->db()->getRow('select * from place limit 1')) {
             fail('No Data');
         }
             fail('No Data');
         }
+
+        if ($this->bDrop) {
+            $this->dropTable('planet_osm_nodes');
+            $this->removeFlatnodeFile();
+        }
     }
 
     public function createFunctions()
     {
         info('Create Functions');
 
     }
 
     public function createFunctions()
     {
         info('Create Functions');
 
-        // Try accessing the C module, so we know eif something is wrong
-        // update.php calls this function
-        if (!checkModulePresence()) {
-            fail('error loading nominatim.so module');
-        }
+        // Try accessing the C module, so we know early if something is wrong
+        checkModulePresence(); // raises exception on failure
+
         $this->createSqlFunctions();
     }
 
         $this->createSqlFunctions();
     }
 
-    public function createTables()
+    public function createTables($bReverseOnly = false)
     {
         info('Create Tables');
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
     {
         info('Create Tables');
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-data}',
-            CONST_Tablespace_Address_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-index}',
-            CONST_Tablespace_Address_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-data}',
-            CONST_Tablespace_Search_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-index}',
-            CONST_Tablespace_Search_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $this->pgsqlRunScript($sTemplate, false);
 
         $this->pgsqlRunScript($sTemplate, false);
-    }
 
 
-    public function createPartitionTables()
-    {
-        info('Create Partition Tables');
+        if ($bReverseOnly) {
+            $this->dropTable('search_name');
+        }
 
 
-        $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-data}',
-            CONST_Tablespace_Address_Data,
-            $sTemplate
-        );
+        $oAlParser = new AddressLevelParser(CONST_Address_Level_Config);
+        $oAlParser->createTable($this->db(), 'address_levels');
+    }
 
 
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-index}',
-            CONST_Tablespace_Address_Index,
-            $sTemplate
-        );
+    public function createTableTriggers()
+    {
+        info('Create Tables');
 
 
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-data}',
-            CONST_Tablespace_Search_Data,
-            $sTemplate
-        );
+        $sTemplate = file_get_contents(CONST_BasePath.'/sql/table-triggers.sql');
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
 
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-index}',
-            CONST_Tablespace_Search_Index,
-            $sTemplate
-        );
+        $this->pgsqlRunScript($sTemplate, false);
+    }
 
 
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
+    public function createPartitionTables()
+    {
+        info('Create Partition Tables');
 
 
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $this->pgsqlRunPartitionScript($sTemplate);
     }
 
         $this->pgsqlRunPartitionScript($sTemplate);
     }
@@ -333,19 +284,14 @@ class SetupFunctions
 
     public function importWikipediaArticles()
     {
 
     public function importWikipediaArticles()
     {
-        $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
-        $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
+        $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikimedia-importance.sql.gz';
         if (file_exists($sWikiArticlesFile)) {
         if (file_exists($sWikiArticlesFile)) {
-            info('Importing wikipedia articles');
-            $this->pgsqlRunDropAndRestore($sWikiArticlesFile);
-        } else {
-            warn('wikipedia article dump file not found - places will have default importance');
-        }
-        if (file_exists($sWikiRedirectsFile)) {
-            info('Importing wikipedia redirects');
-            $this->pgsqlRunDropAndRestore($sWikiRedirectsFile);
+            info('Importing wikipedia articles and redirects');
+            $this->dropTable('wikipedia_article');
+            $this->dropTable('wikipedia_redirect');
+            $this->pgsqlRunScriptFile($sWikiArticlesFile);
         } else {
         } else {
-            warn('wikipedia redirect dump file not found - some place importance values may be missing');
+            warn('wikipedia importance dump file not found - places will have default importance');
         }
     }
 
         }
     }
 
@@ -353,43 +299,42 @@ class SetupFunctions
     {
         info('Drop old Data');
 
     {
         info('Drop old Data');
 
-        if ($this->oDB == null) $this->oDB =& getDB();
+        $oDB = $this->db();
 
 
-        if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection));
-        echo '.';
-        if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection));
-        echo '.';
-        if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection));
+        $oDB->exec('TRUNCATE word');
         echo '.';
         echo '.';
-        if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection));
+        $oDB->exec('TRUNCATE placex');
         echo '.';
         echo '.';
-        if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection));
+        $oDB->exec('TRUNCATE location_property_osmline');
         echo '.';
         echo '.';
-        if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection));
+        $oDB->exec('TRUNCATE place_addressline');
         echo '.';
         echo '.';
-        if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection));
+        $oDB->exec('TRUNCATE location_area');
         echo '.';
         echo '.';
-        if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection));
+        if (!$this->dbReverseOnly()) {
+            $oDB->exec('TRUNCATE search_name');
+            echo '.';
+        }
+        $oDB->exec('TRUNCATE search_name_blank');
         echo '.';
         echo '.';
-        if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection));
+        $oDB->exec('DROP SEQUENCE seq_place');
         echo '.';
         echo '.';
-        if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection));
+        $oDB->exec('CREATE SEQUENCE seq_place start 100000');
         echo '.';
 
         $sSQL = 'select distinct partition from country_name';
         echo '.';
 
         $sSQL = 'select distinct partition from country_name';
-        $aPartitions = chksql($this->oDB->getCol($sSQL));
+        $aPartitions = $oDB->getCol($sSQL);
+
         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {
         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {
-            if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection));
+            $oDB->exec('TRUNCATE location_road_'.$sPartition);
             echo '.';
         }
 
         // used by getorcreate_word_id to ignore frequent partial words
         $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
         $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
             echo '.';
         }
 
         // used by getorcreate_word_id to ignore frequent partial words
         $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
         $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
-        if (!pg_query($this->oDB->connection, $sSQL)) {
-            fail(pg_last_error($this->oDB->connection));
-        }
+        $oDB->exec($sSQL);
         echo ".\n";
 
         // pre-create the word list
         echo ".\n";
 
         // pre-create the word list
@@ -400,34 +345,48 @@ class SetupFunctions
 
         info('Load Data');
         $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
 
         info('Load Data');
         $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
+
         $aDBInstances = array();
         $iLoadThreads = max(1, $this->iInstances - 1);
         for ($i = 0; $i < $iLoadThreads; $i++) {
         $aDBInstances = array();
         $iLoadThreads = max(1, $this->iInstances - 1);
         for ($i = 0; $i < $iLoadThreads; $i++) {
-            $aDBInstances[$i] =& getDB(true);
+            // https://secure.php.net/manual/en/function.pg-connect.php
+            $DSN = CONST_Database_DSN;
+            $DSN = preg_replace('/^pgsql:/', '', $DSN);
+            $DSN = preg_replace('/;/', ' ', $DSN);
+            $aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW);
+            pg_ping($aDBInstances[$i]);
+        }
+
+        for ($i = 0; $i < $iLoadThreads; $i++) {
             $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
             $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
             $sSQL .= "          and ST_GeometryType(geometry) = 'ST_LineString')";
             $sSQL .= ' and ST_IsValid(geometry)';
             $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
             $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
             $sSQL .= "          and ST_GeometryType(geometry) = 'ST_LineString')";
             $sSQL .= ' and ST_IsValid(geometry)';
-            if ($this->sVerbose) echo "$sSQL\n";
-            if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
-                fail(pg_last_error($aDBInstances[$i]->connection));
+            if ($this->bVerbose) echo "$sSQL\n";
+            if (!pg_send_query($aDBInstances[$i], $sSQL)) {
+                fail(pg_last_error($aDBInstances[$i]));
             }
         }
 
         // last thread for interpolation lines
             }
         }
 
         // last thread for interpolation lines
-        $aDBInstances[$iLoadThreads] =& getDB(true);
+        // https://secure.php.net/manual/en/function.pg-connect.php
+        $DSN = CONST_Database_DSN;
+        $DSN = preg_replace('/^pgsql:/', '', $DSN);
+        $DSN = preg_replace('/;/', ' ', $DSN);
+        $aDBInstances[$iLoadThreads] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW);
+        pg_ping($aDBInstances[$iLoadThreads]);
         $sSQL = 'insert into location_property_osmline';
         $sSQL .= ' (osm_id, address, linegeo)';
         $sSQL .= ' SELECT osm_id, address, geometry from place where ';
         $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
         $sSQL = 'insert into location_property_osmline';
         $sSQL .= ' (osm_id, address, linegeo)';
         $sSQL .= ' SELECT osm_id, address, geometry from place where ';
         $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
-        if ($this->sVerbose) echo "$sSQL\n";
-        if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
-            fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
+        if ($this->bVerbose) echo "$sSQL\n";
+        if (!pg_send_query($aDBInstances[$iLoadThreads], $sSQL)) {
+            fail(pg_last_error($aDBInstances[$iLoadThreads]));
         }
 
         $bFailed = false;
         for ($i = 0; $i <= $iLoadThreads; $i++) {
         }
 
         $bFailed = false;
         for ($i = 0; $i <= $iLoadThreads; $i++) {
-            while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) {
+            while (($hPGresult = pg_get_result($aDBInstances[$i])) !== false) {
                 $resultStatus = pg_result_status($hPGresult);
                 // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
                 // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,
                 $resultStatus = pg_result_status($hPGresult);
                 // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
                 // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,
@@ -443,17 +402,22 @@ class SetupFunctions
         if ($bFailed) {
             fail('SQL errors loading placex and/or location_property_osmline tables');
         }
         if ($bFailed) {
             fail('SQL errors loading placex and/or location_property_osmline tables');
         }
+
+        for ($i = 0; $i < $this->iInstances; $i++) {
+            pg_close($aDBInstances[$i]);
+        }
+
         echo "\n";
         info('Reanalysing database');
         $this->pgsqlRunScript('ANALYSE');
 
         echo "\n";
         info('Reanalysing database');
         $this->pgsqlRunScript('ANALYSE');
 
-        $sDatabaseDate = getDatabaseDate($this->oDB);
-        pg_query($this->oDB->connection, 'TRUNCATE import_status');
-        if ($sDatabaseDate === false) {
+        $sDatabaseDate = getDatabaseDate($oDB);
+        $oDB->exec('TRUNCATE import_status');
+        if (!$sDatabaseDate) {
             warn('could not determine database date.');
         } else {
             $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
             warn('could not determine database date.');
         } else {
             $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
-            pg_query($this->oDB->connection, $sSQL);
+            $oDB->exec($sSQL);
             echo "Latest data imported from $sDatabaseDate.\n";
         }
     }
             echo "Latest data imported from $sDatabaseDate.\n";
         }
     }
@@ -462,37 +426,39 @@ class SetupFunctions
     {
         info('Import Tiger data');
 
     {
         info('Import Tiger data');
 
+        $aFilenames = glob(CONST_Tiger_Data_Path.'/*.sql');
+        info('Found '.count($aFilenames).' SQL files in path '.CONST_Tiger_Data_Path);
+        if (empty($aFilenames)) {
+            warn('Tiger data import selected but no files found in path '.CONST_Tiger_Data_Path);
+            return;
+        }
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
+
         $this->pgsqlRunScript($sTemplate, false);
 
         $aDBInstances = array();
         for ($i = 0; $i < $this->iInstances; $i++) {
         $this->pgsqlRunScript($sTemplate, false);
 
         $aDBInstances = array();
         for ($i = 0; $i < $this->iInstances; $i++) {
-            $aDBInstances[$i] =& getDB(true);
+            // https://secure.php.net/manual/en/function.pg-connect.php
+            $DSN = CONST_Database_DSN;
+            $DSN = preg_replace('/^pgsql:/', '', $DSN);
+            $DSN = preg_replace('/;/', ' ', $DSN);
+            $aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW | PGSQL_CONNECT_ASYNC);
+            pg_ping($aDBInstances[$i]);
         }
 
         }
 
-        foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {
+        foreach ($aFilenames as $sFile) {
             echo $sFile.': ';
             $hFile = fopen($sFile, 'r');
             $sSQL = fgets($hFile, 100000);
             $iLines = 0;
             while (true) {
                 for ($i = 0; $i < $this->iInstances; $i++) {
             echo $sFile.': ';
             $hFile = fopen($sFile, 'r');
             $sSQL = fgets($hFile, 100000);
             $iLines = 0;
             while (true) {
                 for ($i = 0; $i < $this->iInstances; $i++) {
-                    if (!pg_connection_busy($aDBInstances[$i]->connection)) {
-                        while (pg_get_result($aDBInstances[$i]->connection));
+                    if (!pg_connection_busy($aDBInstances[$i])) {
+                        while (pg_get_result($aDBInstances[$i]));
                         $sSQL = fgets($hFile, 100000);
                         if (!$sSQL) break 2;
                         $sSQL = fgets($hFile, 100000);
                         if (!$sSQL) break 2;
-                        if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
+                        if (!pg_send_query($aDBInstances[$i], $sSQL)) fail(pg_last_error($aDBInstances[$i]));
                         $iLines++;
                         if ($iLines == 1000) {
                             echo '.';
                         $iLines++;
                         if ($iLines == 1000) {
                             echo '.';
@@ -508,37 +474,28 @@ class SetupFunctions
             while ($bAnyBusy) {
                 $bAnyBusy = false;
                 for ($i = 0; $i < $this->iInstances; $i++) {
             while ($bAnyBusy) {
                 $bAnyBusy = false;
                 for ($i = 0; $i < $this->iInstances; $i++) {
-                    if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
+                    if (pg_connection_busy($aDBInstances[$i])) $bAnyBusy = true;
                 }
                 usleep(10);
             }
             echo "\n";
         }
 
                 }
                 usleep(10);
             }
             echo "\n";
         }
 
+        for ($i = 0; $i < $this->iInstances; $i++) {
+            pg_close($aDBInstances[$i]);
+        }
+
         info('Creating indexes on Tiger data');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
         info('Creating indexes on Tiger data');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-data}',
-            CONST_Tablespace_Aux_Data,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
+
         $this->pgsqlRunScript($sTemplate, false);
     }
 
     public function calculatePostcodes($bCMDResultAll)
     {
         info('Calculate Postcodes');
         $this->pgsqlRunScript($sTemplate, false);
     }
 
     public function calculatePostcodes($bCMDResultAll)
     {
         info('Calculate Postcodes');
-        if ($this->oDB == null) $this->oDB =& getDB();
-        if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) {
-            fail(pg_last_error($this->oDB->connection));
-        }
-
+        $this->db()->exec('TRUNCATE location_postcode');
 
         $sSQL  = 'INSERT INTO location_postcode';
         $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
 
         $sSQL  = 'INSERT INTO location_postcode';
         $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
@@ -549,22 +506,17 @@ class SetupFunctions
         $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
         $sSQL .= '       AND geometry IS NOT null';
         $sSQL .= ' GROUP BY country_code, pc';
         $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
         $sSQL .= '       AND geometry IS NOT null';
         $sSQL .= ' GROUP BY country_code, pc';
+        $this->db()->exec($sSQL);
 
 
-        if (!pg_query($this->oDB->connection, $sSQL)) {
-            fail(pg_last_error($this->oDB->connection));
-        }
-
-        if (CONST_Use_Extra_US_Postcodes) {
-            // only add postcodes that are not yet available in OSM
-            $sSQL  = 'INSERT INTO location_postcode';
-            $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
-            $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
-            $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
-            $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
-            $sSQL .= '        (SELECT postcode FROM location_postcode';
-            $sSQL .= "          WHERE country_code = 'us')";
-            if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
-        }
+        // only add postcodes that are not yet available in OSM
+        $sSQL  = 'INSERT INTO location_postcode';
+        $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+        $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
+        $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
+        $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
+        $sSQL .= '        (SELECT postcode FROM location_postcode';
+        $sSQL .= "          WHERE country_code = 'us')";
+        $this->db()->exec($sSQL);
 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';
 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';
@@ -573,83 +525,104 @@ class SetupFunctions
         $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
         $sSQL .= '           (SELECT postcode FROM location_postcode';
         $sSQL .= "             WHERE country_code = 'gb')";
         $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
         $sSQL .= '           (SELECT postcode FROM location_postcode';
         $sSQL .= "             WHERE country_code = 'gb')";
-        if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
+        $this->db()->exec($sSQL);
 
         if (!$bCMDResultAll) {
             $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
             $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
 
         if (!$bCMDResultAll) {
             $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
             $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
-            if (!pg_query($this->oDB->connection, $sSQL)) {
-                fail(pg_last_error($this->oDB->connection));
-            }
+            $this->db()->exec($sSQL);
         }
         }
+
         $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
         $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
         $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
         $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
-
-        if (!pg_query($this->oDB->connection, $sSQL)) {
-            fail(pg_last_error($this->oDB->connection));
-        }
+        $this->db()->exec($sSQL);
     }
 
     public function index($bIndexNoanalyse)
     {
     }
 
     public function index($bIndexNoanalyse)
     {
-        $sOutputFile = '';
-        $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '
-            .$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile;
-        if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
-            $sBaseCmd .= ' -H '.$this->aDSNInfo['hostspec'];
+        checkModulePresence(); // raises exception on failure
+
+        $oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py'))
+                    ->addParams('--database', $this->aDSNInfo['database'])
+                    ->addParams('--port', $this->aDSNInfo['port'])
+                    ->addParams('--threads', $this->iInstances);
+
+        if (!$this->bQuiet) {
+            $oBaseCmd->addParams('-v');
+        }
+        if ($this->bVerbose) {
+            $oBaseCmd->addParams('-v');
+        }
+        if (isset($this->aDSNInfo['hostspec'])) {
+            $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']);
         }
         }
-        if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
-            $sBaseCmd .= ' -U '.$this->aDSNInfo['username'];
+        if (isset($this->aDSNInfo['username'])) {
+            $oBaseCmd->addParams('--user', $this->aDSNInfo['username']);
         }
         }
-        $aProcEnv = null;
-        if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
-            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
+        if (isset($this->aDSNInfo['password'])) {
+            $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
         }
 
         info('Index ranks 0 - 4');
         }
 
         info('Index ranks 0 - 4');
-        $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv);
+        $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
+        echo $oCmd->escapedCmd();
+
+        $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
+
+        info('Index administrative boundaries');
+        $oCmd = (clone $oBaseCmd)->addParams('-b');
+        $iStatus = $oCmd->run();
+        if ($iStatus != 0) {
+            fail('error status ' . $iStatus . ' running nominatim!');
+        }
+
         info('Index ranks 5 - 25');
         info('Index ranks 5 - 25');
-        $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv);
+        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
+        $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
+
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
+
         info('Index ranks 26 - 30');
         info('Index ranks 26 - 30');
-        $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv);
+        $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26);
+        $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
+
         info('Index postcodes');
         info('Index postcodes');
-        if ($this->oDB == null) $this->oDB =& getDB();
         $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
         $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
-        if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
+        $this->db()->exec($sSQL);
     }
 
     public function createSearchIndices()
     {
         info('Create Search indices');
 
     }
 
     public function createSearchIndices()
     {
         info('Create Search indices');
 
+        $sSQL = 'SELECT relname FROM pg_class, pg_index ';
+        $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid';
+        $aInvalidIndices = $this->db()->getCol($sSQL);
+
+        foreach ($aInvalidIndices as $sIndexName) {
+            info("Cleaning up invalid index $sIndexName");
+            $this->db()->exec("DROP INDEX $sIndexName;");
+        }
+
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
-        $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
-        $sTemplate = $this->replaceTablespace(
-            '{ts:address-index}',
-            CONST_Tablespace_Address_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:search-index}',
-            CONST_Tablespace_Search_Index,
-            $sTemplate
-        );
-        $sTemplate = $this->replaceTablespace(
-            '{ts:aux-index}',
-            CONST_Tablespace_Aux_Index,
-            $sTemplate
-        );
+        if (!$this->bDrop) {
+            $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_updates.src.sql');
+        }
+        if (!$this->dbReverseOnly()) {
+            $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql');
+        }
+        $sTemplate = $this->replaceSqlPatterns($sTemplate);
+
         $this->pgsqlRunScript($sTemplate);
     }
 
         $this->pgsqlRunScript($sTemplate);
     }
 
@@ -703,12 +676,12 @@ class SetupFunctions
                         'new_query_log',
                         'spatial_ref_sys',
                         'country_name',
                         'new_query_log',
                         'spatial_ref_sys',
                         'country_name',
-                        'place_classtype_*'
+                        'place_classtype_*',
+                        'country_osm_grid'
                        );
 
                        );
 
-        if ($this->oDB = null) $this->oDB =& getDB();
         $aDropTables = array();
         $aDropTables = array();
-        $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"));
+        $aHaveTables = $this->db()->getListOfTables();
 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;
 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;
@@ -721,48 +694,101 @@ class SetupFunctions
             if (!$bFound) array_push($aDropTables, $sTable);
         }
         foreach ($aDropTables as $sDrop) {
             if (!$bFound) array_push($aDropTables, $sTable);
         }
         foreach ($aDropTables as $sDrop) {
-            if ($this->sVerbose) echo "dropping table $sDrop\n";
-            @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE");
-            // ignore warnings/errors as they might be caused by a table having
-            // been deleted already by CASCADE
+            $this->dropTable($sDrop);
         }
 
         }
 
-        if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
-            if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
-            unlink(CONST_Osm2pgsql_Flatnode_File);
-        }
+        $this->removeFlatnodeFile();
     }
 
     }
 
-    private function pgsqlRunDropAndRestore($sDumpFile)
+    /**
+     * Setup settings-frontend.php in the build/website directory
+     *
+     * @return null
+     */
+    public function setupWebsite()
     {
     {
-        if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
-        $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
-        if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
-            $sCMD .= ' -h '.$this->aDSNInfo['hostspec'];
-        }
-        if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
-            $sCMD .= ' -U '.$this->aDSNInfo['username'];
+        $rOutputFile = fopen(CONST_InstallPath.'/settings/settings-frontend.php', 'w');
+
+        fwrite($rOutputFile, "<?php
+@define('CONST_BasePath', '".CONST_BasePath."');
+if (file_exists(getenv('NOMINATIM_SETTINGS'))) require_once(getenv('NOMINATIM_SETTINGS'));
+
+@define('CONST_Database_DSN', '".CONST_Database_DSN."');
+@define('CONST_Default_Language', ".(CONST_Default_Language ? ("'".CONST_Default_Language."'") : 'false').");
+@define('CONST_Log_DB', ".(CONST_Log_DB ? 'true' : 'false').");
+@define('CONST_Log_File', ".(CONST_Log_File ? ("'".CONST_Log_File."'")  : 'false').");
+@define('CONST_Max_Word_Frequency', '".CONST_Max_Word_Frequency."');
+@define('CONST_NoAccessControl', ".CONST_NoAccessControl.");
+@define('CONST_Places_Max_ID_count', ".CONST_Places_Max_ID_count.");
+@define('CONST_PolygonOutput_MaximumTypes', ".CONST_PolygonOutput_MaximumTypes.");
+@define('CONST_Search_AreaPolygons', ".CONST_Search_AreaPolygons.");
+@define('CONST_Search_BatchMode', ".(CONST_Search_BatchMode ? 'true' : 'false').");
+@define('CONST_Search_NameOnlySearchFrequencyThreshold', ".CONST_Search_NameOnlySearchFrequencyThreshold.");
+@define('CONST_Search_ReversePlanForAll', ".CONST_Search_ReversePlanForAll.");
+@define('CONST_Term_Normalization_Rules', \"".CONST_Term_Normalization_Rules."\");
+@define('CONST_Use_Aux_Location_data', ".(CONST_Use_Aux_Location_data ? 'true' : 'false').");
+@define('CONST_Use_US_Tiger_Data', ".(CONST_Use_US_Tiger_Data ? 'true' : 'false').");
+@define('CONST_MapIcon_URL', ".(CONST_MapIcon_URL ? ("'".CONST_MapIcon_URL."'") : 'false').');
+');
+        info(CONST_InstallPath.'/settings/settings-frontend.php has been set up successfully');
+    }
+
+    /**
+     * Return the connection to the database.
+     *
+     * @return Database object.
+     *
+     * Creates a new connection if none exists yet. Otherwise reuses the
+     * already established connection.
+     */
+    private function db()
+    {
+        if (is_null($this->oDB)) {
+            $this->oDB = new \Nominatim\DB();
+            $this->oDB->connect();
         }
         }
-        $aProcEnv = null;
-        if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
-            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
+
+        return $this->oDB;
+    }
+
+    private function removeFlatnodeFile()
+    {
+        if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
+            if (file_exists(CONST_Osm2pgsql_Flatnode_File)) {
+                if ($this->bVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
+                unlink(CONST_Osm2pgsql_Flatnode_File);
+            }
         }
         }
-        $iReturn = runWithEnv($sCMD, $aProcEnv);    // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)"
     }
     }
-                     
+
     private function pgsqlRunScript($sScript, $bfatal = true)
     {
         runSQLScript(
             $sScript,
             $bfatal,
     private function pgsqlRunScript($sScript, $bfatal = true)
     {
         runSQLScript(
             $sScript,
             $bfatal,
-            $this->sVerbose,
+            $this->bVerbose,
             $this->sIgnoreErrors
         );
     }
 
     private function createSqlFunctions()
     {
             $this->sIgnoreErrors
         );
     }
 
     private function createSqlFunctions()
     {
-        $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
+        $sBasePath = CONST_BasePath.'/sql/functions/';
+        $sTemplate = file_get_contents($sBasePath.'utils.sql');
+        $sTemplate .= file_get_contents($sBasePath.'normalization.sql');
+        $sTemplate .= file_get_contents($sBasePath.'ranking.sql');
+        $sTemplate .= file_get_contents($sBasePath.'importance.sql');
+        $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql');
+        $sTemplate .= file_get_contents($sBasePath.'interpolation.sql');
+        if ($this->db()->tableExists('place')) {
+            $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql');
+        }
+        if ($this->db()->tableExists('placex')) {
+            $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql');
+        }
+        if ($this->db()->tableExists('location_postcode')) {
+            $sTemplate .= file_get_contents($sBasePath.'postcode_triggers.sql');
+        }
         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
         if ($this->bEnableDiffUpdates) {
             $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
         if ($this->bEnableDiffUpdates) {
             $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
@@ -779,15 +805,17 @@ class SetupFunctions
         if (!CONST_Use_Aux_Location_data) {
             $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
         }
         if (!CONST_Use_Aux_Location_data) {
             $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
         }
+
+        $sReverseOnly = $this->dbReverseOnly() ? 'true' : 'false';
+        $sTemplate = str_replace('%REVERSE-ONLY%', $sReverseOnly, $sTemplate);
+
         $this->pgsqlRunScript($sTemplate);
     }
 
     private function pgsqlRunPartitionScript($sTemplate)
     {
         $this->pgsqlRunScript($sTemplate);
     }
 
     private function pgsqlRunPartitionScript($sTemplate)
     {
-        if ($this->oDB == null) $this->oDB =& getDB();
-
         $sSQL = 'select distinct partition from country_name';
         $sSQL = 'select distinct partition from country_name';
-        $aPartitions = chksql($this->oDB->getCol($sSQL));
+        $aPartitions = $this->db()->getCol($sSQL);
         if (!$this->bNoPartitions) $aPartitions[] = 0;
 
         preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
         if (!$this->bNoPartitions) $aPartitions[] = 0;
 
         preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
@@ -806,19 +834,21 @@ class SetupFunctions
     {
         if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 
     {
         if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 
-        $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'];
-        if (!$this->sVerbose) {
-            $sCMD .= ' -q';
+        $oCmd = (new \Nominatim\Shell('psql'))
+                ->addParams('--port', $this->aDSNInfo['port'])
+                ->addParams('--dbname', $this->aDSNInfo['database']);
+
+        if (!$this->bVerbose) {
+            $oCmd->addParams('--quiet');
         }
         }
-        if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
-            $sCMD .= ' -h '.$this->aDSNInfo['hostspec'];
+        if (isset($this->aDSNInfo['hostspec'])) {
+            $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
         }
         }
-        if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
-            $sCMD .= ' -U '.$this->aDSNInfo['username'];
+        if (isset($this->aDSNInfo['username'])) {
+            $oCmd->addParams('--username', $this->aDSNInfo['username']);
         }
         }
-        $aProcEnv = null;
-        if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
-            $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
+        if (isset($this->aDSNInfo['password'])) {
+            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
         }
         $ahGzipPipes = null;
         if (preg_match('/\\.gz$/', $sFilename)) {
         }
         $ahGzipPipes = null;
         if (preg_match('/\\.gz$/', $sFilename)) {
@@ -827,12 +857,14 @@ class SetupFunctions
                              1 => array('pipe', 'w'),
                              2 => array('file', '/dev/null', 'a')
                             );
                              1 => array('pipe', 'w'),
                              2 => array('file', '/dev/null', 'a')
                             );
-            $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
+            $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename);
+
+            $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes);
             if (!is_resource($hGzipProcess)) fail('unable to start zcat');
             $aReadPipe = $ahGzipPipes[1];
             fclose($ahGzipPipes[0]);
         } else {
             if (!is_resource($hGzipProcess)) fail('unable to start zcat');
             $aReadPipe = $ahGzipPipes[1];
             fclose($ahGzipPipes[0]);
         } else {
-            $sCMD .= ' -f '.$sFilename;
+            $oCmd->addParams('--file', $sFilename);
             $aReadPipe = array('pipe', 'r');
         }
         $aDescriptors = array(
             $aReadPipe = array('pipe', 'r');
         }
         $aDescriptors = array(
@@ -841,7 +873,8 @@ class SetupFunctions
                          2 => array('file', '/dev/null', 'a')
                         );
         $ahPipes = null;
                          2 => array('file', '/dev/null', 'a')
                         );
         $ahPipes = null;
-        $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
+
+        $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
         if (!is_resource($hProcess)) fail('unable to start pgsql');
         // TODO: error checking
         while (!feof($ahPipes[1])) {
         if (!is_resource($hProcess)) fail('unable to start pgsql');
         // TODO: error checking
         while (!feof($ahPipes[1])) {
@@ -858,13 +891,50 @@ class SetupFunctions
         }
     }
 
         }
     }
 
-    private function replaceTablespace($sTemplate, $sTablespace, $sSql)
+    private function replaceSqlPatterns($sSql)
     {
     {
-        if ($sTablespace) {
-            $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
-        } else {
-            $sSql = str_replace($sTemplate, '', $sSql);
+        $sSql = str_replace('{www-user}', CONST_Database_Web_User, $sSql);
+
+        $aPatterns = array(
+                      '{ts:address-data}' => CONST_Tablespace_Address_Data,
+                      '{ts:address-index}' => CONST_Tablespace_Address_Index,
+                      '{ts:search-data}' => CONST_Tablespace_Search_Data,
+                      '{ts:search-index}' =>  CONST_Tablespace_Search_Index,
+                      '{ts:aux-data}' =>  CONST_Tablespace_Aux_Data,
+                      '{ts:aux-index}' =>  CONST_Tablespace_Aux_Index,
+        );
+
+        foreach ($aPatterns as $sPattern => $sTablespace) {
+            if ($sTablespace) {
+                $sSql = str_replace($sPattern, 'TABLESPACE "'.$sTablespace.'"', $sSql);
+            } else {
+                $sSql = str_replace($sPattern, '', $sSql);
+            }
         }
         }
+
         return $sSql;
     }
         return $sSql;
     }
+
+    /**
+     * Drop table with the given name if it exists.
+     *
+     * @param string $sName Name of table to remove.
+     *
+     * @return null
+     */
+    private function dropTable($sName)
+    {
+        if ($this->bVerbose) echo "Dropping table $sName\n";
+        $this->db()->deleteTable($sName);
+    }
+
+    /**
+     * Check if the database is in reverse-only mode.
+     *
+     * @return True if there is no search_name table and infrastructure.
+     */
+    private function dbReverseOnly()
+    {
+        return !($this->db()->tableExists('search_name'));
+    }
 }
 }
Open Source search based on OpenStreetMap data