From: Sarah Hoffmann <lonvia@denofr.de>
Date: Wed, 3 Apr 2013 20:40:33 +0000 (+0200)
Subject: Merge remote-tracking branch 'upstream/master'
X-Git-Tag: deploy~628
X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/ca11fc90e6e646335045e087f03d824236c798f2?ds=inline;hp=-c

Merge remote-tracking branch 'upstream/master'
---

ca11fc90e6e646335045e087f03d824236c798f2
diff --combined settings/settings.php
index 91595272,771cf80f..159c24ac
--- a/settings/settings.php
+++ b/settings/settings.php
@@@ -4,7 -4,7 +4,7 @@@
  
  	// General settings
  	@define('CONST_Debug', false);
- 	@define('CONST_Database_DSN', 'pgsql://@/nominatim');
+ 	@define('CONST_Database_DSN', 'pgsql://@/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database>
  	@define('CONST_Max_Word_Frequency', '50000');
  
  	// Software versions
@@@ -16,6 -16,12 +16,12 @@@
  	@define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-'.CONST_Postgis_Version);
  	@define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
  	@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
+ 
+ 	// Replication settings
+ 	@define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
+ 	@define('CONST_Replication_MaxInterval', '3600');
+ 	@define('CONST_Replication_Update_Interval', '60');  // How often upstream publishes diffs
+ 	@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
  
  	// Connection buckets to rate limit people being nasty
  	@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
@@@ -42,14 -48,12 +48,14 @@@
  
  	// Website settings
  	@define('CONST_NoAccessControl', true);
 -	@define('CONST_ClosedForIndexing', false);
 -	@define('CONST_ClosedForIndexingExceptionIPs', '');
  	@define('CONST_BlockedIPs', '');
 +	@define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks');
 +	@define('CONST_WhitelistedIPs', '');
 +	@define('CONST_BlockedUserAgents', '');
 +	@define('CONST_BlockReverseMaxLoad', 15);
  	@define('CONST_BulkUserIPs', '');
  
 -	@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
 +	@define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/');
  	@define('CONST_Tile_Default', 'Mapnik');
  
  	@define('CONST_Default_Language', 'xx');
diff --combined utils/setup.php
index 68e7a8b3,3f2e6d05..f8da1970
--- a/utils/setup.php
+++ b/utils/setup.php
@@@ -155,9 -155,8 +155,9 @@@
  			echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
  			fail("osm2pgsql not found in '$osm2pgsql'");
  		}
 +		$osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
  		$osm2pgsql .= ' -lsc -O gazetteer --hstore';
 -		$osm2pgsql .= ' -C '.$iCacheMemory;
 +		$osm2pgsql .= ' -C 16000';
  		$osm2pgsql .= ' -P '.$aDSNInfo['port'];
  		$osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
  		passthruCheckReturn($osm2pgsql);
@@@ -482,7 -481,7 +482,7 @@@
  		$sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,calculated_country_code,";
  		$sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select calculated_country_code,postcode,";
  		$sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
 -		$sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
 +		$sSQL .= "from placex where postcode is not null and calculated_country_code not in ('ie') group by calculated_country_code,postcode) as x";
  		if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
  
  		$sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
@@@ -496,7 -495,11 +496,11 @@@
  		$bDidSomething = true;
  		$oDB =& getDB();
  
- 		if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
+ 		if (!file_exists(CONST_Osmosis_Binary))
+ 		{
+ 			echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
+ 			fail("osmosis not found in '".CONST_Osmosis_Binary."'");
+ 		}
  		if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
  		{
  			echo "settings/configuration.txt already exists\n";
@@@ -504,8 -507,9 +508,9 @@@
  		else
  		{
  			passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
- 			// server layout changed afer license change, fix path to minutely diffs
- 			passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
+ 			// update osmosis configuration.txt with our settings
+ 			passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
+ 			passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
  		}
  
  		// Find the last node in the DB
@@@ -518,10 -522,12 +523,12 @@@
  		$iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
  
  
- 		// Search for the correct state file - uses file timestamps
- 		$sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
- 		$sRep = file_get_contents($sRepURL);
- 		preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
+ 		// Search for the correct state file - uses file timestamps so need to sort by date descending
+ 		$sRepURL = CONST_Replication_Url."/";
+ 		$sRep = file_get_contents($sRepURL."?C=M;O=D");
+ 		// download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
+ 		// planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
+ 		preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
  		$aPrevRepMatch = false;
  		foreach($aRepMatches as $aRepMatch)
  		{
@@@ -531,8 -537,8 +538,8 @@@
  		if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
  
  		$sRepURL .= $aRepMatch[1];
- 		$sRep = file_get_contents($sRepURL);
- 		preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
+ 		$sRep = file_get_contents($sRepURL."?C=M;O=D");
+ 		preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
  		$aPrevRepMatch = false;
  		foreach($aRepMatches as $aRepMatch)
  		{
@@@ -542,8 -548,8 +549,8 @@@
  		if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
  
  		$sRepURL .= $aRepMatch[1];
- 		$sRep = file_get_contents($sRepURL);
- 		preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
+ 		$sRep = file_get_contents($sRepURL."?C=M;O=D");
+ 		preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
  		$aPrevRepMatch = false;
  		foreach($aRepMatches as $aRepMatch)
  		{
diff --combined utils/update.php
index 0918730d,76d6f858..e8972f1d
--- a/utils/update.php
+++ b/utils/update.php
@@@ -46,6 -46,7 +46,6 @@@
  		showUsage($aCMDOptions, true, 'Select either import of hourly or daily');
  	}
  
 -	if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
  	if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
  /*
  	// Lock to prevent multiple copies running
@@@ -343,7 -344,6 +343,7 @@@
  
  	if ($aResult['index'])
  	{
 +		if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
  		passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
  	}
  
@@@ -353,8 -353,9 +353,9 @@@
  		$sOsmosisCMD = CONST_Osmosis_Binary;
  		$sOsmosisConfigDirectory = CONST_BasePath.'/settings';
  		$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
+ 		$sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
  		$sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sImportFile;
 -		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'];
 +		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'];
  		if (!$aResult['no-npi']) {
  			$sCMDIndex .= '-F ';
  		}
@@@ -372,7 -373,30 +373,30 @@@
  //			{
  				if (!file_exists($sImportFile))
  				{
- 					// Use osmosis to download the file
+ 					// First check if there are new updates published (except for minutelies - there's always new diffs to process)
+ 					if ( CONST_Replication_Update_Interval > 60 )
+ 					{
+ 
+ 						unset($aReplicationLag);
+ 						exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+ 						while ($iErrorLevel == 1 || $aReplicationLag[0] < 1)
+ 						{
+ 							if ($iErrorLevel)
+ 							{
+ 								echo "Error: $iErrorLevel. ";
+ 								echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
+ 							}
+ 							else
+ 							{
+ 								echo ".";
+ 							}
+ 							sleep(CONST_Replication_Recheck_Interval);
+ 							unset($aReplicationLag);
+ 							exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+ 						}
+ 						// There are new replication files - use osmosis to download the file
+ 						echo "\nReplication Delay is ".$aReplicationLag[0]."\n";
+ 					}
  					$fCMDStartTime = time();
  					echo $sCMDDownload."\n";
  					exec($sCMDDownload, $sJunk, $iErrorLevel);
@@@ -414,16 -438,7 +438,16 @@@
  			$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
  
  			// Index file
 -			$sThisIndexCmd = $sCMDIndex;
 +			if (!isset($aResult['index-instances']))
 +			{
 +				if (getLoadAverage() < 15)
 +					$iIndexInstances = 2;
 +				else
 +					$iIndexInstances = 1;
 +			} else
 +				$iIndexInstances = $aResult['index-instances'];
 +
 +			$sThisIndexCmd = $sCMDIndex.' -t '.$iIndexInstances;
  
  			if (!$aResult['no-npi'])
  			{
@@@ -492,8 -507,16 +516,16 @@@
  			echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
  			if (!$aResult['import-osmosis-all']) exit;
  
- 			echo "Sleeping ".max(0,60-$fDuration)." seconds\n";
- 			sleep(max(0,60-$fDuration));
+ 			if ( CONST_Replication_Update_Interval > 60 )
+ 			{
+ 				$iSleep = round(CONST_Replication_Update_Interval*0.8);
+ 			}
+ 			else
+ 			{
+ 				$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
+ 			}
+ 			echo "Sleeping $iSleep seconds\n";
+ 			sleep($iSleep);
  		}
  
  	}
diff --combined website/search.php
index ed67b419,5667181b..d0c45c70
--- a/website/search.php
+++ b/website/search.php
@@@ -71,7 -71,6 +71,7 @@@
  	if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true;
  	if (isset($aLangPrefOrder['name:ru'])) $bReverseInPlan = true;
  	if (isset($aLangPrefOrder['name:ja'])) $bReverseInPlan = true;
 +	if (isset($aLangPrefOrder['name:pl'])) $bReverseInPlan = true;
  
  	$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
  
@@@ -154,7 -153,7 +154,7 @@@
  				array('county', 9, 13),
  				array('state', 8, 8),
  				array('country', 4, 4),
- 				array('postalcode', 16, 25),
+ 				array('postalcode', 5, 11),
  				);
  	$aStructuredQuery = array();
  	$sAllowedTypesSQLList = '';
@@@ -978,17 -977,12 +978,17 @@@
  								else
  									$sSQL .= " limit ".$iLimit;
  
 -								if (CONST_Debug) { var_dump($sSQL); }
 +								if (CONST_Debug) var_dump($sSQL);
 +								$iStartTime = time();
  								$aViewBoxPlaceIDs = $oDB->getAll($sSQL);
  								if (PEAR::IsError($aViewBoxPlaceIDs))
  								{
  									failInternalError("Could not get places for search terms.", $sSQL, $aViewBoxPlaceIDs);
  								}
 +								if (time() - $iStartTime > 60) {
 +									file_put_contents(CONST_BasePath.'/log/long_queries.log', date('Y-m-d H:i:s', $iStartTime).' '.$sSQL."\n", FILE_APPEND);
 +								}
 +
  //var_dump($aViewBoxPlaceIDs);
  								// Did we have an viewbox matches?
  								$aPlaceIDs = array();