3 require_once(CONST_BasePath.'/lib/init-cmd.php');
4 require_once(CONST_BasePath.'/lib/setup_functions.php');
5 require_once(CONST_BasePath.'/lib/setup/SetupClass.php');
6 require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php');
8 ini_set('memory_limit', '800M');
10 use Nominatim\Setup\SetupFunctions as SetupFunctions;
12 // (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
15 'Import / update / index osm data',
16 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
17 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
18 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
20 array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
21 array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
22 array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
23 array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
24 array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
25 array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
27 array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
29 array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
30 array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
31 array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
33 array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
34 array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
35 array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
36 array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
38 array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
39 array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
40 array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
42 array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
43 array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
44 array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'),
45 array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
48 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
50 if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
52 if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
54 date_default_timezone_set('Etc/UTC');
56 $oDB = new Nominatim\DB();
59 $aDSNInfo = Nominatim\DB::parseDSN(CONST_Database_DSN);
60 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
62 // cache memory to be used by osm2pgsql, should not be more than the available memory
63 $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
64 if ($iCacheMemory + 500 > getTotalMemoryMB()) {
65 $iCacheMemory = getCacheMemoryMB();
66 echo "WARNING: resetting cache memory to $iCacheMemory\n";
68 $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -S '.CONST_Import_Style.' -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
69 if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
70 $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username'];
72 if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
73 $sOsm2pgsqlCmd .= ' -H ' . $aDSNInfo['hostspec'];
76 if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
77 $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
80 if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
81 $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
84 if ($aResult['init-updates']) {
85 // sanity check that the replication URL is correct
86 $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
87 if ($sBaseState === false) {
88 echo "\nCannot find state.txt file at the configured replication URL.\n";
89 echo "Does the URL point to a directory containing OSM update data?\n\n";
90 fail('replication URL not reachable.');
92 // sanity check for pyosmium-get-changes
93 if (!CONST_Pyosmium_Binary) {
94 echo "\nCONST_Pyosmium_Binary not configured.\n";
95 echo "You need to install pyosmium and set up the path to pyosmium-get-changes\n";
96 echo "in your local settings file.\n\n";
97 fail('CONST_Pyosmium_Binary not configured');
100 $sCmd = CONST_Pyosmium_Binary.' --help';
101 exec($sCmd, $aOutput, $iRet);
103 echo "Cannot execute pyosmium-get-changes.\n";
104 echo "Make sure you have pyosmium installed correctly\n";
105 echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n";
106 fail('pyosmium-get-changes not found or not usable');
109 if (!$aResult['no-update-functions']) {
110 // instantiate setupClass to use the function therein
111 $cSetup = new SetupFunctions(array(
112 'enable-diff-updates' => true,
113 'verbose' => $aResult['verbose']
116 $cSetup->createFunctions();
119 $sDatabaseDate = getDatabaseDate($oDB);
120 if (!$sDatabaseDate) {
121 fail('Cannot determine date of database.');
123 $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
125 // get the appropriate state id
127 $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
128 exec($sCmd, $aOutput, $iRet);
129 if ($iRet != 0 || $aOutput[0] == 'None') {
130 fail('Error running pyosmium tools');
133 $oDB->exec('TRUNCATE import_status');
134 $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
135 $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';
139 } catch (\Nominatim\DatabaseError $e) {
140 fail('Could not enter sequence into database.');
143 echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";
146 if ($aResult['check-for-updates']) {
147 $aLastState = chksql($oDB->getRow('SELECT sequence_id FROM import_status'));
149 if (!$aLastState['sequence_id']) {
150 fail('Updates not set up. Please run ./utils/update.php --init-updates.');
153 system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet);
157 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
158 // import diffs and files directly (e.g. from osmosis --rri)
159 $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
161 if (!file_exists($sNextFile)) {
162 fail("Cannot open $sNextFile\n");
166 $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
168 $iErrorLevel = runWithEnv($sCMD, $aProcEnv);
171 fail("Error from osm2pgsql, $iErrorLevel\n");
174 // Don't update the import status - we don't know what this file contains
177 if ($aResult['calculate-postcodes']) {
178 info('Update postcodes centroids');
179 $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
180 runSQLScript($sTemplate, true, true);
183 $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
185 $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
187 if (isset($aResult['import-node']) && $aResult['import-node']) {
189 $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
191 $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
195 if (isset($aResult['import-way']) && $aResult['import-way']) {
197 $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
199 $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
203 if (isset($aResult['import-relation']) && $aResult['import-relation']) {
205 $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
207 $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
212 file_put_contents($sTemporaryFile, file_get_contents($sContentURL));
217 // import generated change file
218 $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
220 $iErrorLevel = runWithEnv($sCMD, $aProcEnv);
222 fail("osm2pgsql exited with error level $iErrorLevel\n");
226 if ($aResult['deduplicate']) {
227 $oDB = new Nominatim\DB();
230 if ($oDB->getPostgresVersion() < 9.3) {
231 fail('ERROR: deduplicate is only currently supported in postgresql 9.3');
234 $sSQL = 'select partition from country_name order by country_code';
235 $aPartitions = chksql($oDB->getCol($sSQL));
238 // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
239 foreach ($aPartitions as $i => $sPartition) {
240 $sSQL = 'select count(*) from search_name_'.$sPartition;
241 $nEntries = chksql($oDB->getOne($sSQL));
242 if ($nEntries == 0) {
243 unset($aPartitions[$i]);
247 $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '";
248 $sSQL .= ' and class is null and type is null and country_code is null';
249 $sSQL .= ' group by word_token having count(*) > 1 order by word_token';
250 $aDuplicateTokens = chksql($oDB->getAll($sSQL));
251 foreach ($aDuplicateTokens as $aToken) {
252 if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
253 echo 'Deduping '.$aToken['word_token']."\n";
254 $sSQL = 'select word_id,';
255 $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
256 $sSQL .= " from word where word_token = '".$aToken['word_token'];
257 $sSQL .= "' and class is null and type is null and country_code is null order by num desc";
258 $aTokenSet = chksql($oDB->getAll($sSQL));
260 $aKeep = array_shift($aTokenSet);
261 $iKeepID = $aKeep['word_id'];
263 foreach ($aTokenSet as $aRemove) {
264 $sSQL = 'update search_name set';
265 $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),';
266 $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
267 $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
268 chksql($oDB->exec($sSQL));
270 $sSQL = 'update search_name set';
271 $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
272 $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']';
273 chksql($oDB->exec($sSQL));
275 $sSQL = 'update location_area_country set';
276 $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
277 $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
278 chksql($oDB->exec($sSQL));
280 foreach ($aPartitions as $sPartition) {
281 $sSQL = 'update search_name_'.$sPartition.' set';
282 $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')';
283 $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
284 chksql($oDB->exec($sSQL));
286 $sSQL = 'update location_area_country set';
287 $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
288 $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
289 chksql($oDB->exec($sSQL));
292 $sSQL = 'delete from word where word_id = '.$aRemove['word_id'];
293 chksql($oDB->exec($sSQL));
298 if ($aResult['recompute-word-counts']) {
299 info('Recompute frequency of full-word search terms');
300 $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
301 runSQLScript($sTemplate, true, true);
304 if ($aResult['index']) {
305 $sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank'];
306 if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
307 $sCmd .= ' -H ' . $aDSNInfo['hostspec'];
309 if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
310 $sCmd .= ' -U ' . $aDSNInfo['username'];
313 runWithEnv($sCmd, $aProcEnv);
315 $oDB->exec('update import_status set indexed = true');
318 if ($aResult['update-address-levels']) {
319 echo 'Updating address levels from '.CONST_Address_Level_Config.".\n";
320 $oAlParser = new \Nominatim\Setup\AddressLevelParser(CONST_Address_Level_Config);
321 $oAlParser->createTable($oDB, 'address_levels');
324 if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
326 if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
327 fail('Error: Update interval too low for download.geofabrik.de. ' .
328 "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
331 $sImportFile = CONST_InstallPath.'/osmosischange.osc';
332 $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
333 $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
334 $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
335 if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
336 $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec'];
338 if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
339 $sCMDIndex .= ' -U ' . $aDSNInfo['username'];
343 $fStartTime = time();
344 $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));
346 if (!$aLastState['sequence_id']) {
347 echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
351 echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n";
353 $sBatchEnd = $aLastState['lastimportdate'];
354 $iEndSequence = $aLastState['sequence_id'];
356 if ($aLastState['indexed'] == 't') {
357 // Sleep if the update interval has not yet been reached.
358 $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;
359 if ($fNextUpdate > $fStartTime) {
360 $iSleepTime = $fNextUpdate - $fStartTime;
361 echo "Waiting for next update for $iSleepTime sec.";
365 // Download the next batch of changes.
367 $fCMDStartTime = time();
368 $iNextSeq = (int) $aLastState['sequence_id'];
370 echo "$sCMDDownload -I $iNextSeq\n";
371 if (file_exists($sImportFile)) {
372 unlink($sImportFile);
374 exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
377 echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
378 sleep(CONST_Replication_Recheck_Interval);
379 } elseif ($iResult != 0) {
380 echo 'ERROR: updates failed.';
383 $iEndSequence = (int)$aOutput[0];
387 // get the newest object from the diff file
390 exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
392 echo "Diff file is empty. skipping import.\n";
393 if (!$aResult['import-osmosis-all']) {
400 fail('Error getting date from diff file.');
402 $sBatchEnd = $sBatchEnd[0];
405 $fCMDStartTime = time();
406 echo $sCMDImport."\n";
408 $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv);
410 echo "Error executing osm2pgsql: $iErrorLevel\n";
414 // write the update logs
415 $iFileSize = filesize($sImportFile);
416 $sSQL = 'INSERT INTO import_osmosis_log';
417 $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
418 $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
419 $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
420 $sSQL .= date('Y-m-d H:i:s')."','import')";
422 chksql($oDB->exec($sSQL));
425 $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence";
427 chksql($oDB->exec($sSQL));
428 echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
432 if (!$aResult['no-index']) {
433 $sThisIndexCmd = $sCMDIndex;
434 $fCMDStartTime = time();
436 echo "$sThisIndexCmd\n";
437 $iErrorLevel = runWithEnv($sThisIndexCmd, $aProcEnv);
439 echo "Error: $iErrorLevel\n";
443 $sSQL = 'INSERT INTO import_osmosis_log';
444 $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
445 $sSQL .= " values ('$sBatchEnd',$iEndSequence,NULL,'";
446 $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
447 $sSQL .= date('Y-m-d H:i:s')."','index')";
450 echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
452 $sSQL = 'update import_status set indexed = true';
455 if ($aResult['import-osmosis-all']) {
456 echo "Error: --no-index cannot be used with continuous imports (--import-osmosis-all).\n";
461 $fDuration = time() - $fStartTime;
462 echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";
463 if (!$aResult['import-osmosis-all']) exit(0);