]> git.openstreetmap.org Git - nominatim.git/blob - utils/setup.php
use the new country data files
[nominatim.git] / utils / setup.php
1 #!/usr/bin/php -Cq
2 <?php
3
4         require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
5         ini_set('memory_limit', '800M');
6
7         $aCMDOptions = array(
8                 "Create and setup nominatim search system",
9                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
10                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
11                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
12
13                 array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
14                 array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
15
16                 array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
17
18                 array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
19                 array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
20                 array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
21                 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
22                 array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
23                 array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
24                 array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
25                 array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
26                 array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
27                 array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
28                 array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
29                 array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
30                 array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
31                 array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
32                 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
33                 array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
34                 array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
35                 array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
36                 array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
37                 array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse opertions during index (EXPERT)'),
38                 array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
39                 array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
40                 array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
41         );
42         getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
43
44         $bDidSomething = false;
45
46         // This is a pretty hard core defult - the number of processors in the box - 1
47         $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
48         if ($iInstances < 1)
49         {
50                 $iInstances = 1;
51                 echo "WARNING: resetting threads to $iInstances\n";
52         }
53         if ($iInstances > getProcessorCount())
54         {
55                 $iInstances = getProcessorCount();
56                 echo "WARNING: resetting threads to $iInstances\n";
57         }
58
59         // Assume we can steal all the cache memory in the box (unless told otherwise)
60         $iCacheMemory = (isset($aCMDResult['osm2pgsql-cache'])?$aCMDResult['osm2pgsql-cache']:getCacheMemoryMB());
61         if ($iCacheMemory > getTotalMemoryMB())
62         {
63                 $iCacheMemory = getCacheMemoryMB();
64                 echo "WARNING: resetting cache memory to $iCacheMemory\n";
65         }
66
67         if (isset($aCMDResult['osm-file']) && !isset($aCMDResult['osmosis-init-date']))
68         {
69                 $sBaseFile = basename($aCMDResult['osm-file']);
70                 if (preg_match('#^planet-([0-9]{2})([0-9]{2})([0-9]{2})[.]#', $sBaseFile, $aMatch))
71                 {
72                         $iTime = mktime(0, 0, 0, $aMatch[2], $aMatch[3], '20'.$aMatch[1]);
73                         $iTime -= (60*60*24);
74                         $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z';
75                 }
76         }
77         $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
78         if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
79
80         if ($aCMDResult['create-db'] || $aCMDResult['all'])
81         {
82                 echo "Create DB\n";
83                 $bDidSomething = true;
84                 $oDB =& DB::connect(CONST_Database_DSN, false);
85                 if (!PEAR::isError($oDB))
86                 {
87                         fail('database already exists ('.CONST_Database_DSN.')');
88                 }
89                 passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
90         }
91
92         if ($aCMDResult['create-db'] || $aCMDResult['all'])
93         {
94                 echo "Create DB (2)\n";
95                 $bDidSomething = true;
96                 // TODO: path detection, detection memory, etc.
97
98                 $oDB =& getDB();
99                 passthru('createlang plpgsql '.$aDSNInfo['database']);
100                 $pgver = (float) CONST_Postgresql_Version;
101                 if ($pgver < 9.1) {
102                         pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
103                 } else {
104                         pgsqlRunScript('CREATE EXTENSION hstore');
105                 }
106                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
107                 pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
108                 pgsqlRunDropAndRestore(CONST_BasePath.'/data/country_name.sql.bin');
109                 pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
110                 pgsqlRunDropAndRestore(CONST_BasePath.'/data/country_osm_grid.sql.bin');
111                 pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
112                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
113                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
114                 pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
115                 pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
116         }
117
118         if ($aCMDResult['import-data'] || $aCMDResult['all'])
119         {
120                 echo "Import\n";
121                 $bDidSomething = true;
122
123                 $osm2pgsql = CONST_Osm2pgsql_Binary;
124                 if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql");
125                 $osm2pgsql .= ' -lsc -O gazetteer --hstore';
126                 $osm2pgsql .= ' -C '.$iCacheMemory;
127                 $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
128                 passthru($osm2pgsql);
129
130                 $oDB =& getDB();
131                 $x = $oDB->getRow('select * from place limit 1');
132                 if (PEAR::isError($x)) {
133                         fail($x->getMessage());
134                 }
135                 if (!$x) fail('No Data');
136         }
137
138         if ($aCMDResult['create-functions'] || $aCMDResult['all'])
139         {
140                 echo "Functions\n";
141                 $bDidSomething = true;
142                 if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
143                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
144                 $sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
145                 if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
146                 if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
147                 pgsqlRunScript($sTemplate);
148         }
149
150         if ($aCMDResult['create-minimal-tables'])
151         {
152                 echo "Minimal Tables\n";
153                 $bDidSomething = true;
154                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
155
156                 $sScript = '';
157
158                 // Backstop the import process - easliest possible import id
159                 $sScript .= "insert into import_npi_log values (18022);\n";
160
161                 $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
162                 if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
163
164                 while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
165                 {
166                         list($sClass, $sType) = explode(' ', trim($sLine));
167                         $sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
168                         $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
169
170                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
171                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
172
173                         $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
174                         $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
175                 }
176                 fclose($hFile);
177                 pgsqlRunScript($sScript);
178         }
179
180         if ($aCMDResult['create-tables'] || $aCMDResult['all'])
181         {
182                 echo "Tables\n";
183                 $bDidSomething = true;
184                 pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
185
186                 // re-run the functions
187                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
188                 $sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
189                 pgsqlRunScript($sTemplate);
190         }
191
192         if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
193         {
194                 echo "Partitions\n";
195                 $bDidSomething = true;
196                 $oDB =& getDB();
197                 $sSQL = 'select partition from country_name order by country_code';
198                 $aPartitions = $oDB->getCol($sSQL);
199                 if (PEAR::isError($aPartitions))
200                 {
201                         fail($aPartitions->getMessage());
202                 }
203                 $aPartitions[] = 0;
204
205                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
206                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
207                 foreach($aMatches as $aMatch)
208                 {
209                         $sResult = '';
210                         foreach($aPartitions as $sPartitionName)
211                         {
212                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
213                         }
214                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
215                 }
216
217                 pgsqlRunScript($sTemplate);
218         }
219
220         if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all'])
221         {
222                 $bDidSomething = true;
223                 $sWikiArticlesFile = CONST_BasePath.'/data/wikipedia_article.sql.bin';
224                 $sWikiRedirectsFile = CONST_BasePath.'/data/wikipedia_redirect.sql.bin';
225                 if (file_exists($sWikiArticlesFile))
226                 {
227                         echo "Importing wikipedia articles...";
228                         pgsqlRunDropAndRestore($sWikiArticlesFile);
229                         echo "...done\n";
230                 }
231                 else
232                 {
233                         echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
234                 }
235                 if (file_exists($sWikiRedirectsFile))
236                 {
237                         echo "Importing wikipedia redirects...";
238                         pgsqlRunDropAndRestore($sWikiRedirectsFile);
239                         echo "...done\n";
240                 }
241                 else
242                 {
243                         echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
244                 }
245         }
246
247
248         if ($aCMDResult['load-data'] || $aCMDResult['all'])
249         {
250                 echo "Load Data\n";
251                 $bDidSomething = true;
252
253                 $oDB =& getDB();
254                 if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection));
255                 echo '.';
256                 if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
257                 echo '.';
258                 if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
259                 echo '.';
260                 if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
261                 echo '.';
262                 if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection));
263                 echo '.';
264                 if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection));
265                 echo '.';
266                 if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection));
267                 echo '.';
268                 if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection));
269                 echo '.';
270                 if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
271                 echo '.';
272
273                 // pre-create the word list
274                 if (!$aCMDResult['disable-token-precalc'])
275                 {
276                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
277                         echo '.';
278                         if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
279                         echo '.';
280                         if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
281                         echo '.';
282                 }
283
284                 $aDBInstances = array();
285                 for($i = 0; $i < $iInstances; $i++)
286                 {
287                         $aDBInstances[$i] =& getDB(true);
288                         $sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
289                         $sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
290                         $sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
291                         if ($aCMDResult['verbose']) echo "$sSQL\n";
292                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
293                 }
294                 $bAnyBusy = true;
295                 while($bAnyBusy)
296                 {
297                         $bAnyBusy = false;
298                         for($i = 0; $i < $iInstances; $i++)
299                         {
300                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
301                         }
302                         sleep(1);
303                         echo '.';
304                 }
305                 echo "\n";
306                 echo "Reanalysing database...\n";
307                 pgsqlRunScript('ANALYSE');
308         }
309
310         if ($aCMDResult['create-roads'])
311         {
312                 $bDidSomething = true;
313
314                 $oDB =& getDB();
315                 $aDBInstances = array();
316                 for($i = 0; $i < $iInstances; $i++)
317                 {
318                         $aDBInstances[$i] =& getDB(true);
319                         if (!pg_query($aDBInstances[$i]->connection, 'set enable_bitmapscan = off')) fail(pg_last_error($oDB->connection));
320                         $sSQL = 'select count(*) from (select insertLocationRoad(partition, place_id, country_code, geometry) from ';
321                         $sSQL .= 'placex where osm_id % '.$iInstances.' = '.$i.' and rank_search between 26 and 27 and class = \'highway\') as x ';
322                         if ($aCMDResult['verbose']) echo "$sSQL\n";
323                         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
324                 }
325                 $bAnyBusy = true;
326                 while($bAnyBusy)
327                 {
328                         $bAnyBusy = false;
329                         for($i = 0; $i < $iInstances; $i++)
330                         {
331                                 if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
332                         }
333                         sleep(1);
334                         echo '.';
335                 }
336                 echo "\n";
337         }
338
339         if ($aCMDResult['import-tiger-data'])
340         {
341                 $bDidSomething = true;
342
343                 $aDBInstances = array();
344                 for($i = 0; $i < $iInstances; $i++)
345                 {
346                         $aDBInstances[$i] =& getDB(true);
347                 }
348
349                 foreach(glob(CONST_BasePath.'/data/tiger2011/*.sql') as $sFile)
350                 {
351                         echo $sFile.': ';
352                         $hFile = fopen($sFile, "r");
353                         $sSQL = fgets($hFile, 100000);
354                         $iLines = 0;
355
356                         while(true)
357                         {
358                                 for($i = 0; $i < $iInstances; $i++)
359                                 {
360                                         if (!pg_connection_busy($aDBInstances[$i]->connection))
361                                         {
362                                                 while(pg_get_result($aDBInstances[$i]->connection));
363                                                 $sSQL = fgets($hFile, 100000);
364                                                 if (!$sSQL) break 2;
365                                                 if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
366                                                 $iLines++;
367                                                 if ($iLines == 1000)
368                                                 {
369                                                         echo ".";
370                                                         $iLines = 0;
371                                                 }
372                                         }
373                                 }
374                                 usleep(10);
375                         }
376
377                         fclose($hFile);
378
379                         $bAnyBusy = true;
380                         while($bAnyBusy)
381                         {
382                                 $bAnyBusy = false;
383                                 for($i = 0; $i < $iInstances; $i++)
384                                 {
385                                         if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
386                                 }
387                                 usleep(10);
388                         }
389                         echo "\n";
390                 }
391         }
392
393         if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all'])
394         {
395                 $bDidSomething = true;
396                 $oDB =& getDB();
397                 if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
398                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
399                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,country_code,";
400                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,postcode,";
401                 $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
402                 $sSQL .= "from placex where postcode is not null group by country_code,postcode) as x";
403                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
404
405                 $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,country_code,geometry) ";
406                 $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
407                 $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
408                 if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
409         }
410
411         if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date']))
412         {
413                 $bDidSomething = true;
414                 $oDB =& getDB();
415
416                 if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
417                 if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
418                 else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
419
420                 $sDate = $aCMDResult['osmosis-init-date'];
421                 $aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
422                 $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?';
423                 $sURL .= 'Y='.$aDate['year'].'&m='.$aDate['month'].'&d='.$aDate['day'];
424                 $sURL .= '&H='.$aDate['hour'].'&i='.$aDate['minute'].'&s=0';
425                 $sURL .= '&stream=minute';
426                 echo "Getting state file: $sURL\n";
427                 $sStateFile = file_get_contents($sURL);
428                 if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
429                 file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
430                 echo "Updating DB status\n";
431                 pg_query($oDB->connection, 'TRUNCATE import_status');
432                 $sSQL = "INSERT INTO import_status VALUES('".$sDate."')";
433                 pg_query($oDB->connection, $sSQL);
434
435         }
436
437         if ($aCMDResult['index'] || $aCMDResult['all'])
438         {
439                 $bDidSomething = true;
440                 $sOutputFile = '';
441                 if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
442                 $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
443                 passthru($sBaseCmd.' -R 4');
444 var_dump($sBaseCmd.' -r 15 -R 15');
445                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
446                 passthru($sBaseCmd.' -r 5 -R 25');
447                 if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
448                 passthru($sBaseCmd.' -r 26');
449         }
450
451         if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
452         {
453                 echo "Search indices\n";
454                 $bDidSomething = true;
455                 $oDB =& getDB();
456                 $sSQL = 'select partition from country_name order by country_code';
457                 $aPartitions = $oDB->getCol($sSQL);
458                 if (PEAR::isError($aPartitions))
459                 {
460                         fail($aPartitions->getMessage());
461                 }
462                 $aPartitions[] = 0;
463
464                 $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
465                 preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
466                 foreach($aMatches as $aMatch)
467                 {
468                         $sResult = '';
469                         foreach($aPartitions as $sPartitionName)
470                         {
471                                 $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
472                         }
473                         $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
474                 }
475
476                 pgsqlRunScript($sTemplate);
477         }
478
479         if (isset($aCMDResult['create-website']))
480         {
481                 $bDidSomething = true;
482                 $sTargetDir = $aCMDResult['create-website'];
483                 if (!is_dir($sTargetDir)) fail('please specify a directory to setup');
484                 @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php');
485                 @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
486                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
487                 @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
488                 @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
489                 @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
490                 echo "Symlinks created\n";
491         }
492
493         if (!$bDidSomething)
494         {
495                 showUsage($aCMDOptions, true);
496         }
497
498         function pgsqlRunScriptFile($sFilename)
499         {
500                 if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
501
502                 // Convert database DSN to psql paramaters
503                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
504                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
505                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
506
507                 $aDescriptors = array(
508                         0 => array('pipe', 'r'),
509                         1 => array('pipe', 'w'),
510                         2 => array('file', '/dev/null', 'a')
511                 );
512                 $ahPipes = null;
513                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
514                 if (!is_resource($hProcess)) fail('unable to start pgsql');
515
516                 fclose($ahPipes[0]);
517
518                 // TODO: error checking
519                 while(!feof($ahPipes[1]))
520                 {
521                         echo fread($ahPipes[1], 4096);
522                 }
523                 fclose($ahPipes[1]);
524
525                 proc_close($hProcess);
526         }
527
528         function pgsqlRunScript($sScript)
529         {
530                 // Convert database DSN to psql paramaters
531                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
532                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
533                 $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
534                 $aDescriptors = array(
535                         0 => array('pipe', 'r'),
536                         1 => STDOUT, 
537                         2 => STDERR
538                 );
539                 $ahPipes = null;
540                 $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
541                 if (!is_resource($hProcess)) fail('unable to start pgsql');
542
543                 while(strlen($sScript))
544                 {
545                         $written = fwrite($ahPipes[0], $sScript);
546                         $sScript = substr($sScript, $written);
547                 }
548                 fclose($ahPipes[0]);
549                 proc_close($hProcess);
550         }
551
552         function pgsqlRunRestoreData($sDumpFile)
553         {
554                 // Convert database DSN to psql paramaters
555                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
556                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
557                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile;
558
559                 $aDescriptors = array(
560                         0 => array('pipe', 'r'),
561                         1 => array('pipe', 'w'),
562                         2 => array('file', '/dev/null', 'a')
563                 );
564                 $ahPipes = null;
565                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
566                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
567
568                 fclose($ahPipes[0]);
569
570                 // TODO: error checking
571                 while(!feof($ahPipes[1]))
572                 {
573                         echo fread($ahPipes[1], 4096);
574                 }
575                 fclose($ahPipes[1]);
576
577                 proc_close($hProcess);
578         }
579
580         function pgsqlRunDropAndRestore($sDumpFile)
581         {
582                 // Convert database DSN to psql paramaters
583                 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
584                 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
585                 $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
586
587                 $aDescriptors = array(
588                         0 => array('pipe', 'r'),
589                         1 => array('pipe', 'w'),
590                         2 => array('file', '/dev/null', 'a')
591                 );
592                 $ahPipes = null;
593                 $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
594                 if (!is_resource($hProcess)) fail('unable to start pg_restore');
595
596                 fclose($ahPipes[0]);
597
598                 // TODO: error checking
599                 while(!feof($ahPipes[1]))
600                 {
601                         echo fread($ahPipes[1], 4096);
602                 }
603                 fclose($ahPipes[1]);
604
605                 proc_close($hProcess);
606         }