Merge remote-tracking branch 'upstream/master'

[nominatim.git] / utils / setup.php

diff --git a/utils/setup.php b/utils/setup.php
index d9c761b276920d2f75f4dbb660b2fb59546d8f23..edf0b14d460f27156569b65f872f2bb8aa934e6d 100755 (executable)
--- a/utils/setup.php
+++ b/utils/setup.php
@@ -147,7 +147,15 @@
                pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');

-               pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
+               pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
+               if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz'))
+               {
+                       pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
+               }
+               else
+               {
+                       echo "WARNING: external UK postcode table not found.\n";
+               }

                pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
                pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');


@@ -156,6 +164,13 @@
                {
                        pgsqlRunScript('update country_name set partition = 0');
                }
                {
                        pgsqlRunScript('update country_name set partition = 0');
                }

+
+               // the following will be needed by create_functions later but
+               // is only defined in the subsequently called create_tables.
+               // Create dummies here that will be overwritten by the proper
+               // versions in create-tables.
+               pgsqlRunScript('CREATE TABLE place_boundingbox ()');
+               pgsqlRunScript('create type wikipedia_article_match as ()');

        }
 
        if ($aCMDResult['import-data'] || $aCMDResult['all'])
        }
 
        if ($aCMDResult['import-data'] || $aCMDResult['all'])


@@ -174,8 +189,9 @@
                {
                        $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
                }
                {
                        $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
                }

+               $osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';

                $osm2pgsql .= ' -lsc -O gazetteer --hstore';
                $osm2pgsql .= ' -lsc -O gazetteer --hstore';

-               $osm2pgsql .= ' -C '.$iCacheMemory;
+               $osm2pgsql .= ' -C 18000';

                $osm2pgsql .= ' -P '.$aDSNInfo['port'];
                $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
                passthruCheckReturn($osm2pgsql);
                $osm2pgsql .= ' -P '.$aDSNInfo['port'];
                $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
                passthruCheckReturn($osm2pgsql);


@@ -501,7 +517,7 @@
                $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
                $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
                $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
                $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
                $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
                $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";

-               $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
+               $sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";

                if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 
                $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
                if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 
                $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";


@@ -677,10 +693,29 @@
                // Convert database DSN to psql parameters
                $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
                if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
                // Convert database DSN to psql parameters
                $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
                if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;

-               $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
+               $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+
+               $ahGzipPipes = null;
+               if (preg_match('/\\.gz$/', $sFilename))
+               {
+                       $aDescriptors = array(
+                               0 => array('pipe', 'r'),
+                               1 => array('pipe', 'w'),
+                               2 => array('file', '/dev/null', 'a')
+                       );
+                       $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
+                       if (!is_resource($hGzipProcess)) fail('unable to start zcat');
+                       $aReadPipe = $ahGzipPipes[1];
+                       fclose($ahGzipPipes[0]);
+               }
+               else
+               {
+                       $sCMD .= ' -f '.$sFilename;
+                       $aReadPipe = array('pipe', 'r');
+               }

 
                $aDescriptors = array(
 
                $aDescriptors = array(

-                       0 => array('pipe', 'r'),
+                       0 => $aReadPipe,

                        1 => array('pipe', 'w'),
                        2 => array('file', '/dev/null', 'a')
                );
                        1 => array('pipe', 'w'),
                        2 => array('file', '/dev/null', 'a')
                );


@@ -688,7 +723,6 @@
                $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
                if (!is_resource($hProcess)) fail('unable to start pgsql');
 
                $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
                if (!is_resource($hProcess)) fail('unable to start pgsql');
 

-               fclose($ahPipes[0]);

 
                // TODO: error checking
                while(!feof($ahPipes[1]))
 
                // TODO: error checking
                while(!feof($ahPipes[1]))


@@ -697,7 +731,17 @@
                }
                fclose($ahPipes[1]);
 
                }
                fclose($ahPipes[1]);
 

-               proc_close($hProcess);
+               $iReturn = proc_close($hProcess);
+               if ($iReturn > 0)
+               {
+                       fail("pgsql returned with error code ($iReturn)");
+               }
+               if ($ahGzipPipes)
+               {
+                       fclose($ahGzipPipes[1]);
+                       proc_close($hGzipProcess);
+               }
+

        }
 
        function pgsqlRunScript($sScript)
        }
 
        function pgsqlRunScript($sScript)


@@ -706,6 +750,7 @@
                $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
                if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
                $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
                $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
                if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
                $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];

+               $sCMD .= ' -v ON_ERROR_STOP=1';

                $aDescriptors = array(
                        0 => array('pipe', 'r'),
                        1 => STDOUT, 
                $aDescriptors = array(
                        0 => array('pipe', 'r'),
                        1 => STDOUT, 


@@ -721,7 +766,11 @@
                        $sScript = substr($sScript, $written);
                }
                fclose($ahPipes[0]);
                        $sScript = substr($sScript, $written);
                }
                fclose($ahPipes[0]);

-               proc_close($hProcess);
+               $iReturn = proc_close($hProcess);
+               if ($iReturn > 0)
+               {
+                       fail("pgsql returned with error code ($iReturn)");
+               }

        }
 
        function pgsqlRunRestoreData($sDumpFile)
        }
 
        function pgsqlRunRestoreData($sDumpFile)


@@ -749,7 +798,11 @@
                }
                fclose($ahPipes[1]);
 
                }
                fclose($ahPipes[1]);
 

-               proc_close($hProcess);
+               $iReturn = proc_close($hProcess);
+               if ($iReturn > 0)
+               {
+                       fail("pgsql returned with error code ($iReturn)");
+               }

        }
 
        function pgsqlRunDropAndRestore($sDumpFile)
        }
 
        function pgsqlRunDropAndRestore($sDumpFile)


@@ -777,7 +830,11 @@
                }
                fclose($ahPipes[1]);
 
                }
                fclose($ahPipes[1]);
 

-               proc_close($hProcess);
+               $iReturn = proc_close($hProcess);
+               if ($iReturn > 0)
+               {
+                       fail("pgsql returned with error code ($iReturn)");
+               }

        }
 
        function passthruCheckReturn($cmd)
        }
 
        function passthruCheckReturn($cmd)