4 require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
5 require_once(CONST_BasePath.'/lib/init-cmd.php');
6 require_once(CONST_BasePath.'/lib/setup_functions.php');
7 require_once(CONST_BasePath.'/lib/setup/SetupClass.php');
9 ini_set('memory_limit', '800M');
11 use Nominatim\Setup\SetupFunctions as SetupFunctions;
13 // (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
16 'Import / update / index osm data',
17 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
18 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
19 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
21 array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
22 array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
23 array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
24 array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
25 array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
26 array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
28 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
30 array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
31 array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
32 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
34 array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
35 array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
36 array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
37 array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
39 array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
40 array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
41 array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
43 array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
44 array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
45 array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
48 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
50 if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
52 if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
54 date_default_timezone_set('Etc/UTC');
58 $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
59 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
61 // cache memory to be used by osm2pgsql, should not be more than the available memory
62 $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
63 if ($iCacheMemory + 500 > getTotalMemoryMB()) {
64 $iCacheMemory = getCacheMemoryMB();
65 echo "WARNING: resetting cache memory to $iCacheMemory\n";
67 $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
68 if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
69 $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username'];
71 if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
72 $sOsm2pgsqlCmd .= ' -H ' . $aDSNInfo['hostspec'];
75 if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
76 $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
79 if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
80 $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
83 if ($aResult['init-updates']) {
84 // sanity check that the replication URL is correct
85 $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
86 if ($sBaseState === false) {
87 echo "\nCannot find state.txt file at the configured replication URL.\n";
88 echo "Does the URL point to a directory containing OSM update data?\n\n";
89 fail('replication URL not reachable.');
91 // sanity check for pyosmium-get-changes
92 if (!CONST_Pyosmium_Binary) {
93 echo "\nCONST_Pyosmium_Binary not configured.\n";
94 echo "You need to install pyosmium and set up the path to pyosmium-get-changes\n";
95 echo "in your local settings file.\n\n";
96 fail('CONST_Pyosmium_Binary not configured');
99 $sCmd = CONST_Pyosmium_Binary.' --help';
100 exec($sCmd, $aOutput, $iRet);
102 echo "Cannot execute pyosmium-get-changes.\n";
103 echo "Make sure you have pyosmium installed correctly\n";
104 echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n";
105 fail('pyosmium-get-changes not found or not usable');
108 if (!$aResult['no-update-functions']) {
109 // instantiate setupClass to use the function therein
110 $cSetup = new SetupFunctions(array(
111 'enable-diff-updates' => true,
112 'verbose' => $aResult['verbose']
114 $cSetup->createFunctions();
117 $sDatabaseDate = getDatabaseDate($oDB);
118 if ($sDatabaseDate === false) {
119 fail('Cannot determine date of database.');
121 $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
123 // get the appropriate state id
125 $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
126 exec($sCmd, $aOutput, $iRet);
127 if ($iRet != 0 || $aOutput[0] == 'None') {
128 fail('Error running pyosmium tools');
131 pg_query($oDB->connection, 'TRUNCATE import_status');
132 $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
133 $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';
134 if (!pg_query($oDB->connection, $sSQL)) {
135 fail('Could not enter sequence into database.');
138 echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";
141 if ($aResult['check-for-updates']) {
142 $aLastState = chksql($oDB->getRow('SELECT sequence_id FROM import_status'));
144 if (!$aLastState['sequence_id']) {
145 fail('Updates not set up. Please run ./utils/update.php --init-updates.');
148 system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet);
152 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
153 // import diffs and files directly (e.g. from osmosis --rri)
154 $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
156 if (!file_exists($sNextFile)) {
157 fail("Cannot open $sNextFile\n");
161 $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
163 $iErrorLevel = runWithEnv($sCMD, $aProcEnv);
166 fail("Error from osm2pgsql, $iErrorLevel\n");
169 // Don't update the import status - we don't know what this file contains
172 if ($aResult['calculate-postcodes']) {
173 info('Update postcodes centroids');
174 $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
175 runSQLScript($sTemplate, true, true);
178 $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
180 $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
182 if (isset($aResult['import-node']) && $aResult['import-node']) {
184 $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
186 $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
190 if (isset($aResult['import-way']) && $aResult['import-way']) {
192 $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
194 $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
198 if (isset($aResult['import-relation']) && $aResult['import-relation']) {
200 $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
202 $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
207 file_put_contents($sTemporaryFile, file_get_contents($sContentURL));
212 // import generated change file
213 $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
215 $iErrorLevel = runWithEnv($sCMD, $aProcEnv);
217 fail("osm2pgsql exited with error level $iErrorLevel\n");
221 if ($aResult['deduplicate']) {
224 if (getPostgresVersion($oDB) < 9.3) {
225 fail('ERROR: deduplicate is only currently supported in postgresql 9.3');
228 $sSQL = 'select partition from country_name order by country_code';
229 $aPartitions = chksql($oDB->getCol($sSQL));
232 // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
233 foreach ($aPartitions as $i => $sPartition) {
234 $sSQL = 'select count(*) from search_name_'.$sPartition;
235 $nEntries = chksql($oDB->getOne($sSQL));
236 if ($nEntries == 0) {
237 unset($aPartitions[$i]);
241 $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '";
242 $sSQL .= ' and class is null and type is null and country_code is null';
243 $sSQL .= ' group by word_token having count(*) > 1 order by word_token';
244 $aDuplicateTokens = chksql($oDB->getAll($sSQL));
245 foreach ($aDuplicateTokens as $aToken) {
246 if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
247 echo 'Deduping '.$aToken['word_token']."\n";
248 $sSQL = 'select word_id,';
249 $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
250 $sSQL .= " from word where word_token = '".$aToken['word_token'];
251 $sSQL .= "' and class is null and type is null and country_code is null order by num desc";
252 $aTokenSet = chksql($oDB->getAll($sSQL));
254 $aKeep = array_shift($aTokenSet);
255 $iKeepID = $aKeep['word_id'];
257 foreach ($aTokenSet as $aRemove) {
258 $sSQL = 'update search_name set';
259 $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),';
260 $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
261 $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
262 chksql($oDB->query($sSQL));
264 $sSQL = 'update search_name set';
265 $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
266 $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']';
267 chksql($oDB->query($sSQL));
269 $sSQL = 'update location_area_country set';
270 $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
271 $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
272 chksql($oDB->query($sSQL));
274 foreach ($aPartitions as $sPartition) {
275 $sSQL = 'update search_name_'.$sPartition.' set';
276 $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')';
277 $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
278 chksql($oDB->query($sSQL));
280 $sSQL = 'update location_area_country set';
281 $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
282 $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
283 chksql($oDB->query($sSQL));
286 $sSQL = 'delete from word where word_id = '.$aRemove['word_id'];
287 chksql($oDB->query($sSQL));
292 if ($aResult['recompute-word-counts']) {
293 info('Recompute frequency of full-word search terms');
294 $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
295 runSQLScript($sTemplate, true, true);
298 if ($aResult['index']) {
299 $sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank'];
300 if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
301 $sCmd .= ' -H ' . $aDSNInfo['hostspec'];
303 if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
304 $sCmd .= ' -U ' . $aDSNInfo['username'];
307 runWithEnv($sCmd, $aProcEnv);
310 if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
312 if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
313 fail('Error: Update interval too low for download.geofabrik.de. ' .
314 "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
317 $sImportFile = CONST_InstallPath.'/osmosischange.osc';
318 $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
319 $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
320 $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
321 if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
322 $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec'];
324 if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
325 $sCMDIndex .= ' -U ' . $aDSNInfo['username'];
329 $fStartTime = time();
330 $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));
332 if (!$aLastState['sequence_id']) {
333 echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
337 echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n";
339 $sBatchEnd = $aLastState['lastimportdate'];
340 $iEndSequence = $aLastState['sequence_id'];
342 if ($aLastState['indexed'] == 't') {
343 // Sleep if the update interval has not yet been reached.
344 $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;
345 if ($fNextUpdate > $fStartTime) {
346 $iSleepTime = $fNextUpdate - $fStartTime;
347 echo "Waiting for next update for $iSleepTime sec.";
351 // Download the next batch of changes.
353 $fCMDStartTime = time();
354 $iNextSeq = (int) $aLastState['sequence_id'];
356 echo "$sCMDDownload -I $iNextSeq\n";
357 if (file_exists($sImportFile)) {
358 unlink($sImportFile);
360 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
363 echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
364 sleep(CONST_Replication_Recheck_Interval);
365 } elseif ($iResult != 0) {
366 echo 'ERROR: updates failed.';
369 $iEndSequence = (int)$aOutput[0];
373 // get the newest object from the diff file
376 exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
378 echo "Diff file is empty. skipping import.\n";
379 if (!$aResult['import-osmosis-all']) {
386 fail('Error getting date from diff file.');
388 $sBatchEnd = $sBatchEnd[0];
391 $fCMDStartTime = time();
392 echo $sCMDImport."\n";
394 $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv);
396 echo "Error executing osm2pgsql: $iErrorLevel\n";
400 // write the update logs
401 $iFileSize = filesize($sImportFile);
402 $sSQL = 'INSERT INTO import_osmosis_log';
403 $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
404 $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
405 $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
406 $sSQL .= date('Y-m-d H:i:s')."','import')";
408 chksql($oDB->query($sSQL));
411 $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence";
413 chksql($oDB->query($sSQL));
414 echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
418 if (!$aResult['no-index']) {
419 $sThisIndexCmd = $sCMDIndex;
420 $fCMDStartTime = time();
422 echo "$sThisIndexCmd\n";
423 $iErrorLevel = runWithEnv($sThisIndexCmd, $aProcEnv);
425 echo "Error: $iErrorLevel\n";
429 $sSQL = 'INSERT INTO import_osmosis_log';
430 $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
431 $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
432 $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
433 $sSQL .= date('Y-m-d H:i:s')."','index')";
436 echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
438 $sSQL = 'update import_status set indexed = true';
442 $fDuration = time() - $fStartTime;
443 echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";
444 if (!$aResult['import-osmosis-all']) exit(0);