Merge pull request #1923 from lonvia/split-indexing-for-boundries

[nominatim.git] / lib / setup / SetupClass.php

diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php
index 385eff70eb6cda7160cb84f7a6b5c5ff0c96b08e..2815f8c4b11a120bad4746776a7b6006ffc392db 100755 (executable)
--- a/lib/setup/SetupClass.php
+++ b/lib/setup/SetupClass.php
@@ -108,17 +108,11 @@ class SetupFunctions
         if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
     }
 
         if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
     }
 

-    public function connect()
-    {
-        $this->oDB = new \Nominatim\DB();
-        $this->oDB->connect();
-    }
-

     public function setupDB()
     {
         info('Setup DB');
 
     public function setupDB()
     {
         info('Setup DB');
 

-        $fPostgresVersion = $this->oDB->getPostgresVersion();
+        $fPostgresVersion = $this->db()->getPostgresVersion();

         echo 'Postgres version found: '.$fPostgresVersion."\n";
 
         if ($fPostgresVersion < 9.03) {
         echo 'Postgres version found: '.$fPostgresVersion."\n";
 
         if ($fPostgresVersion < 9.03) {


@@ -128,7 +122,7 @@ class SetupFunctions
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
 
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
 

-        $fPostgisVersion = $this->oDB->getPostgisVersion();
+        $fPostgisVersion = $this->db()->getPostgisVersion();

         echo 'Postgis version found: '.$fPostgisVersion."\n";
 
         if ($fPostgisVersion < 2.2) {
         echo 'Postgis version found: '.$fPostgisVersion."\n";
 
         if ($fPostgisVersion < 2.2) {


@@ -136,7 +130,7 @@ class SetupFunctions
             exit(1);
         }
 
             exit(1);
         }
 

-        $i = $this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'");
+        $i = $this->db()->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'");

         if ($i == 0) {
             echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
             echo "\n          createuser ".CONST_Database_Web_User."\n\n";
         if ($i == 0) {
             echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
             echo "\n          createuser ".CONST_Database_Web_User."\n\n";


@@ -223,7 +217,7 @@ class SetupFunctions
         $oCmd->addParams($sOSMFile);
         $oCmd->run();
 
         $oCmd->addParams($sOSMFile);
         $oCmd->run();
 

-        if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) {
+        if (!$this->sIgnoreErrors && !$this->db()->getRow('select * from place limit 1')) {

             fail('No Data');
         }
 
             fail('No Data');
         }
 


@@ -257,7 +251,7 @@ class SetupFunctions
         }
 
         $oAlParser = new AddressLevelParser(CONST_Address_Level_Config);
         }
 
         $oAlParser = new AddressLevelParser(CONST_Address_Level_Config);

-        $oAlParser->createTable($this->oDB, 'address_levels');
+        $oAlParser->createTable($this->db(), 'address_levels');

     }
 
     public function createTableTriggers()
     }
 
     public function createTableTriggers()


@@ -305,40 +299,42 @@ class SetupFunctions
     {
         info('Drop old Data');
 
     {
         info('Drop old Data');
 

-        $this->oDB->exec('TRUNCATE word');
+        $oDB = $this->db();
+
+        $oDB->exec('TRUNCATE word');

         echo '.';
         echo '.';

-        $this->oDB->exec('TRUNCATE placex');
+        $oDB->exec('TRUNCATE placex');

         echo '.';
         echo '.';

-        $this->oDB->exec('TRUNCATE location_property_osmline');
+        $oDB->exec('TRUNCATE location_property_osmline');

         echo '.';
         echo '.';

-        $this->oDB->exec('TRUNCATE place_addressline');
+        $oDB->exec('TRUNCATE place_addressline');

         echo '.';
         echo '.';

-        $this->oDB->exec('TRUNCATE location_area');
+        $oDB->exec('TRUNCATE location_area');

         echo '.';
         if (!$this->dbReverseOnly()) {
         echo '.';
         if (!$this->dbReverseOnly()) {

-            $this->oDB->exec('TRUNCATE search_name');
+            $oDB->exec('TRUNCATE search_name');

             echo '.';
         }
             echo '.';
         }

-        $this->oDB->exec('TRUNCATE search_name_blank');
+        $oDB->exec('TRUNCATE search_name_blank');

         echo '.';
         echo '.';

-        $this->oDB->exec('DROP SEQUENCE seq_place');
+        $oDB->exec('DROP SEQUENCE seq_place');

         echo '.';
         echo '.';

-        $this->oDB->exec('CREATE SEQUENCE seq_place start 100000');
+        $oDB->exec('CREATE SEQUENCE seq_place start 100000');

         echo '.';
 
         $sSQL = 'select distinct partition from country_name';
         echo '.';
 
         $sSQL = 'select distinct partition from country_name';

-        $aPartitions = $this->oDB->getCol($sSQL);
+        $aPartitions = $oDB->getCol($sSQL);

 
         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {
 
         if (!$this->bNoPartitions) $aPartitions[] = 0;
         foreach ($aPartitions as $sPartition) {

-            $this->oDB->exec('TRUNCATE location_road_'.$sPartition);
+            $oDB->exec('TRUNCATE location_road_'.$sPartition);

             echo '.';
         }
 
         // used by getorcreate_word_id to ignore frequent partial words
         $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
         $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
             echo '.';
         }
 
         // used by getorcreate_word_id to ignore frequent partial words
         $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
         $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';

-        $this->oDB->exec($sSQL);
+        $oDB->exec($sSQL);

         echo ".\n";
 
         // pre-create the word list
         echo ".\n";
 
         // pre-create the word list


@@ -415,13 +411,13 @@ class SetupFunctions
         info('Reanalysing database');
         $this->pgsqlRunScript('ANALYSE');
 
         info('Reanalysing database');
         $this->pgsqlRunScript('ANALYSE');
 

-        $sDatabaseDate = getDatabaseDate($this->oDB);
-        $this->oDB->exec('TRUNCATE import_status');
+        $sDatabaseDate = getDatabaseDate($oDB);
+        $oDB->exec('TRUNCATE import_status');

         if (!$sDatabaseDate) {
             warn('could not determine database date.');
         } else {
             $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
         if (!$sDatabaseDate) {
             warn('could not determine database date.');
         } else {
             $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";

-            $this->oDB->exec($sSQL);
+            $oDB->exec($sSQL);

             echo "Latest data imported from $sDatabaseDate.\n";
         }
     }
             echo "Latest data imported from $sDatabaseDate.\n";
         }
     }


@@ -499,7 +495,7 @@ class SetupFunctions
     public function calculatePostcodes($bCMDResultAll)
     {
         info('Calculate Postcodes');
     public function calculatePostcodes($bCMDResultAll)
     {
         info('Calculate Postcodes');

-        $this->oDB->exec('TRUNCATE location_postcode');
+        $this->db()->exec('TRUNCATE location_postcode');

 
         $sSQL  = 'INSERT INTO location_postcode';
         $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
 
         $sSQL  = 'INSERT INTO location_postcode';
         $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';


@@ -510,7 +506,7 @@ class SetupFunctions
         $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
         $sSQL .= '       AND geometry IS NOT null';
         $sSQL .= ' GROUP BY country_code, pc';
         $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
         $sSQL .= '       AND geometry IS NOT null';
         $sSQL .= ' GROUP BY country_code, pc';

-        $this->oDB->exec($sSQL);
+        $this->db()->exec($sSQL);

 
         // only add postcodes that are not yet available in OSM
         $sSQL  = 'INSERT INTO location_postcode';
 
         // only add postcodes that are not yet available in OSM
         $sSQL  = 'INSERT INTO location_postcode';


@@ -520,7 +516,7 @@ class SetupFunctions
         $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
         $sSQL .= '        (SELECT postcode FROM location_postcode';
         $sSQL .= "          WHERE country_code = 'us')";
         $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
         $sSQL .= '        (SELECT postcode FROM location_postcode';
         $sSQL .= "          WHERE country_code = 'us')";

-        $this->oDB->exec($sSQL);
+        $this->db()->exec($sSQL);

 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';
 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';


@@ -529,21 +525,23 @@ class SetupFunctions
         $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
         $sSQL .= '           (SELECT postcode FROM location_postcode';
         $sSQL .= "             WHERE country_code = 'gb')";
         $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
         $sSQL .= '           (SELECT postcode FROM location_postcode';
         $sSQL .= "             WHERE country_code = 'gb')";

-        $this->oDB->exec($sSQL);
+        $this->db()->exec($sSQL);

 
         if (!$bCMDResultAll) {
             $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
             $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
 
         if (!$bCMDResultAll) {
             $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
             $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';

-            $this->oDB->exec($sSQL);
+            $this->db()->exec($sSQL);

         }
 
         $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
         $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
         }
 
         $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
         $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';

-        $this->oDB->exec($sSQL);
+        $this->db()->exec($sSQL);

     }
 
     public function index($bIndexNoanalyse)
     {
     }
 
     public function index($bIndexNoanalyse)
     {

+        checkModulePresence(); // raises exception on failure
+

         $oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py'))
                     ->addParams('--database', $this->aDSNInfo['database'])
                     ->addParams('--port', $this->aDSNInfo['port'])
         $oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py'))
                     ->addParams('--database', $this->aDSNInfo['database'])
                     ->addParams('--port', $this->aDSNInfo['port'])


@@ -568,19 +566,27 @@ class SetupFunctions
         info('Index ranks 0 - 4');
         $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
         echo $oCmd->escapedCmd();
         info('Index ranks 0 - 4');
         $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
         echo $oCmd->escapedCmd();

-        
+

         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 

+        info('Index administrative boundaries');
+        $oCmd = (clone $oBaseCmd)->addParams('-b');
+        $iStatus = $oCmd->run();
+        if ($iStatus != 0) {
+            fail('error status ' . $iStatus . ' running nominatim!');
+        }
+

         info('Index ranks 5 - 25');
         $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }
         info('Index ranks 5 - 25');
         $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
         $iStatus = $oCmd->run();
         if ($iStatus != 0) {
             fail('error status ' . $iStatus . ' running nominatim!');
         }

+

         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 26 - 30');
         if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
 
         info('Index ranks 26 - 30');


@@ -592,7 +598,7 @@ class SetupFunctions
 
         info('Index postcodes');
         $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
 
         info('Index postcodes');
         $sSQL = 'UPDATE location_postcode SET indexed_status = 0';

-        $this->oDB->exec($sSQL);
+        $this->db()->exec($sSQL);

     }
 
     public function createSearchIndices()
     }
 
     public function createSearchIndices()


@@ -601,11 +607,11 @@ class SetupFunctions
 
         $sSQL = 'SELECT relname FROM pg_class, pg_index ';
         $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid';
 
         $sSQL = 'SELECT relname FROM pg_class, pg_index ';
         $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid';

-        $aInvalidIndices = $this->oDB->getCol($sSQL);
+        $aInvalidIndices = $this->db()->getCol($sSQL);

 
         foreach ($aInvalidIndices as $sIndexName) {
             info("Cleaning up invalid index $sIndexName");
 
         foreach ($aInvalidIndices as $sIndexName) {
             info("Cleaning up invalid index $sIndexName");

-            $this->oDB->exec("DROP INDEX $sIndexName;");
+            $this->db()->exec("DROP INDEX $sIndexName;");

         }
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
         }
 
         $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');


@@ -675,7 +681,7 @@ class SetupFunctions
                        );
 
         $aDropTables = array();
                        );
 
         $aDropTables = array();

-        $aHaveTables = $this->oDB->getListOfTables();
+        $aHaveTables = $this->db()->getListOfTables();

 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;
 
         foreach ($aHaveTables as $sTable) {
             $bFound = false;


@@ -694,6 +700,62 @@ class SetupFunctions
         $this->removeFlatnodeFile();
     }
 
         $this->removeFlatnodeFile();
     }
 

+    /**
+     * Setup settings-frontend.php in the build/website directory
+     *
+     * @return null
+     */
+    public function setupWebsite()
+    {
+        $rOutputFile = fopen(CONST_InstallPath.'/settings/settings-frontend.php', 'w');
+
+        fwrite($rOutputFile, "<?php
+@define('CONST_BasePath', '".CONST_BasePath."');
+if (file_exists(getenv('NOMINATIM_SETTINGS'))) require_once(getenv('NOMINATIM_SETTINGS'));
+
+@define('CONST_Database_DSN', '".CONST_Database_DSN."'); // or add ;host=...;port=...;user=...;password=...
+@define('CONST_Default_Language', ".(CONST_Default_Language ? ("'".CONST_Default_Language."'") : 'false').");
+@define('CONST_Default_Lat', ".CONST_Default_Lat.");
+@define('CONST_Default_Lon', ".CONST_Default_Lon.");
+@define('CONST_Default_Zoom', ".CONST_Default_Zoom.");
+@define('CONST_Map_Tile_URL', '".CONST_Map_Tile_URL."');
+@define('CONST_Map_Tile_Attribution', '".CONST_Map_Tile_Attribution."'); // Set if tile source isn't osm.org
+@define('CONST_Log_DB', ".(CONST_Log_DB ? 'true' : 'false').");
+@define('CONST_Log_File', ".(CONST_Log_File ? ("'".CONST_Log_File."'")  : 'false').");
+@define('CONST_Max_Word_Frequency', '".CONST_Max_Word_Frequency."');
+@define('CONST_NoAccessControl', ".CONST_NoAccessControl.");
+@define('CONST_Places_Max_ID_count', ".CONST_Places_Max_ID_count.");
+@define('CONST_PolygonOutput_MaximumTypes', ".CONST_PolygonOutput_MaximumTypes.");
+@define('CONST_Search_AreaPolygons', ".CONST_Search_AreaPolygons.");
+@define('CONST_Search_BatchMode', ".(CONST_Search_BatchMode ? 'true' : 'false').");
+@define('CONST_Search_NameOnlySearchFrequencyThreshold', ".CONST_Search_NameOnlySearchFrequencyThreshold.");
+@define('CONST_Search_ReversePlanForAll', ".CONST_Search_ReversePlanForAll.");
+@define('CONST_Term_Normalization_Rules', \"".CONST_Term_Normalization_Rules."\");
+@define('CONST_Use_Aux_Location_data', ".(CONST_Use_Aux_Location_data ? 'true' : 'false').");
+@define('CONST_Use_US_Tiger_Data', ".(CONST_Use_US_Tiger_Data ? 'true' : 'false').");
+@define('CONST_Website_BaseURL', '".CONST_Website_BaseURL."');
+");
+        info(CONST_InstallPath.'/settings/settings-frontend.php has been set up successfully');
+    }
+
+    /**
+     * Return the connection to the database.
+     *
+     * @return Database object.
+     *
+     * Creates a new connection if none exists yet. Otherwise reuses the
+     * already established connection.
+     */
+    private function db()
+    {
+        if (is_null($this->oDB)) {
+            $this->oDB = new \Nominatim\DB();
+            $this->oDB->connect();
+        }
+
+        return $this->oDB;
+    }
+

     private function removeFlatnodeFile()
     {
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
     private function removeFlatnodeFile()
     {
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {


@@ -723,13 +785,13 @@ class SetupFunctions
         $sTemplate .= file_get_contents($sBasePath.'importance.sql');
         $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql');
         $sTemplate .= file_get_contents($sBasePath.'interpolation.sql');
         $sTemplate .= file_get_contents($sBasePath.'importance.sql');
         $sTemplate .= file_get_contents($sBasePath.'address_lookup.sql');
         $sTemplate .= file_get_contents($sBasePath.'interpolation.sql');

-        if ($this->oDB->tableExists('place')) {
+        if ($this->db()->tableExists('place')) {

             $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql');
         }
             $sTemplate .= file_get_contents($sBasePath.'place_triggers.sql');
         }

-        if ($this->oDB->tableExists('placex')) {
+        if ($this->db()->tableExists('placex')) {

             $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql');
         }
             $sTemplate .= file_get_contents($sBasePath.'placex_triggers.sql');
         }

-        if ($this->oDB->tableExists('location_postcode')) {
+        if ($this->db()->tableExists('location_postcode')) {

             $sTemplate .= file_get_contents($sBasePath.'postcode_triggers.sql');
         }
         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
             $sTemplate .= file_get_contents($sBasePath.'postcode_triggers.sql');
         }
         $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);


@@ -758,7 +820,7 @@ class SetupFunctions
     private function pgsqlRunPartitionScript($sTemplate)
     {
         $sSQL = 'select distinct partition from country_name';
     private function pgsqlRunPartitionScript($sTemplate)
     {
         $sSQL = 'select distinct partition from country_name';

-        $aPartitions = $this->oDB->getCol($sSQL);
+        $aPartitions = $this->db()->getCol($sSQL);

         if (!$this->bNoPartitions) $aPartitions[] = 0;
 
         preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
         if (!$this->bNoPartitions) $aPartitions[] = 0;
 
         preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);


@@ -864,13 +926,11 @@ class SetupFunctions
      * @param string $sName Name of table to remove.
      *
      * @return null
      * @param string $sName Name of table to remove.
      *
      * @return null

-     *
-     * @pre connect() must have been called.

      */
     private function dropTable($sName)
     {
         if ($this->bVerbose) echo "Dropping table $sName\n";
      */
     private function dropTable($sName)
     {
         if ($this->bVerbose) echo "Dropping table $sName\n";

-        $this->oDB->deleteTable($sName);
+        $this->db()->deleteTable($sName);

     }
 
     /**
     }
 
     /**


@@ -880,6 +940,6 @@ class SetupFunctions
      */
     private function dbReverseOnly()
     {
      */
     private function dbReverseOnly()
     {

-        return !($this->oDB->tableExists('search_name'));
+        return !($this->db()->tableExists('search_name'));

     }
 }
     }
 }