From: Sarah Hoffmann <lonvia@denofr.de>
Date: Mon, 8 Oct 2012 05:43:37 +0000 (+0200)
Subject: Merge branch 'master' of http://github.com/twain47/Nominatim
X-Git-Tag: deploy~673
X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/03d6c48ef1bbc4ccd057125db6df015c0f8e53f0?ds=inline;hp=-c

Merge branch 'master' of http://github.com/twain47/Nominatim

Conflicts:
	settings/settings.php
	utils/specialphrases.php
	website/search.php
---

03d6c48ef1bbc4ccd057125db6df015c0f8e53f0
diff --combined settings/settings.php
index f6aa5730,f441521f..ce4bec5a
--- a/settings/settings.php
+++ b/settings/settings.php
@@@ -5,26 -5,25 +5,27 @@@
  	// General settings
  	@define('CONST_Debug', false);
  	@define('CONST_Database_DSN', 'pgsql://@/nominatim');
+ 	@define('CONST_Max_Word_Frequency', '50000');
  
  	// Paths
  	@define('CONST_Postgresql_Version', '9.1');
  	@define('CONST_Path_Postgresql_Contrib', '/usr/share/postgresql/'.CONST_Postgresql_Version.'/contrib');
  	@define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-1.5');
  	@define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
- 	@define('CONST_Osmosis_Binary', CONST_BasePath.'/../osmosis-0.40.1/bin/osmosis');
+ 	@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
  
  	// Website settings
 -	@define('CONST_ClosedForIndexing', false);
 -	@define('CONST_ClosedForIndexingExceptionIPs', '');
  	@define('CONST_BlockedIPs', '');
 +	@define('CONST_IPBanFile', CONST_BasePath.'/settings/ip_blocks');
 +	@define('CONST_WhitelistedIPs', '');
 +	@define('CONST_BlockedUserAgents', '');
 +	@define('CONST_BlockReverseMaxLoad', 15);
  	@define('CONST_BulkUserIPs', '');
  
 -	@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
 +	@define('CONST_Website_BaseURL', 'http://nominatim.openstreetmap.org/');
  	@define('CONST_Tile_Default', 'Mapnik');
  
 -	@define('CONST_Default_Language', 'xx');
 +	@define('CONST_Default_Language', 'en');
  	@define('CONST_Default_Lat', 20.0);
  	@define('CONST_Default_Lon', 0.0);
  	@define('CONST_Default_Zoom', 2);
diff --combined utils/setup.php
index 8e3d1cb5,bfc7c720..4cd6a5f4
--- a/utils/setup.php
+++ b/utils/setup.php
@@@ -146,9 -146,8 +146,9 @@@
  			echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
  			fail("osm2pgsql not found in '$osm2pgsql'");
  		}
 +		$osm2pgsql .= ' --tablespace-slim-index ssd --tablespace-main-index ssd --tablespace-main-data ssd --tablespace-slim-data data';
  		$osm2pgsql .= ' -lsc -O gazetteer --hstore';
 -		$osm2pgsql .= ' -C '.$iCacheMemory;
 +		$osm2pgsql .= ' -C 16000';
  		$osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
  		passthruCheckReturn($osm2pgsql);
  
@@@ -272,7 -271,7 +272,7 @@@
  
  	if ($aCMDResult['load-data'] || $aCMDResult['all'])
  	{
- 		echo "Load Data\n";
+ 		echo "Drop old Data\n";
  		$bDidSomething = true;
  
  		$oDB =& getDB();
@@@ -308,17 -307,18 +308,18 @@@
  			echo '.';
  		}
  
+ 		// used by getorcreate_word_id to ignore frequent partial words
+ 		if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection));
+ 		echo ".\n";
+ 
  		// pre-create the word list
  		if (!$aCMDResult['disable-token-precalc'])
  		{
- 			if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
- 			echo '.';
- 			if (!pg_query($oDB->connection, 'select count(make_keywords(v)) from (select distinct postcode as v from place) as w where v is not null;')) fail(pg_last_error($oDB->connection));
- 			echo '.';
- 			if (!pg_query($oDB->connection, 'select count(getorcreate_housenumber_id(v)) from (select distinct housenumber as v from place where housenumber is not null) as w;')) fail(pg_last_error($oDB->connection));
- 			echo '.';
+ 			echo "Loading word list\n";
+ 			pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
  		}
  
+ 		echo "Load Data\n";
  		$aDBInstances = array();
  		for($i = 0; $i < $iInstances; $i++)
  		{
@@@ -341,7 -341,6 +342,7 @@@
  			echo '.';
  		}
  		echo "\n";
 +		pgsqlRunScript('ALTER TABLE place SET TABLESPACE "data"');
  		echo "Reanalysing database...\n";
  		pgsqlRunScript('ANALYSE');
  	}
@@@ -459,7 -458,12 +460,12 @@@
  
  		if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
  		if (file_exists(CONST_BasePath.'/settings/configuration.txt')) echo "settings/configuration.txt already exists\n";
- 		else passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
+ 		else
+ 		{
+ 			passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
+ 			// server layout changed afer license change, fix path to minutely diffs
+ 			passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
+ 		}
  
  		$sDate = $aCMDResult['osmosis-init-date'];
  		$aDate = date_parse_from_format("Y-m-d\TH-i", $sDate);
diff --combined utils/specialphrases.php
index 94aa74d8,f1a0d0d0..02aaaa0f
--- a/utils/specialphrases.php
+++ b/utils/specialphrases.php
@@@ -86,23 -86,17 +86,17 @@@
  
  		foreach($aPairs as $aPair)
  		{
- 			if ($aPair[0] == 'yes') continue;
- 			if ($aPair[1] == 'yes') continue;
- 			if ($aPair[0] == 'highway') continue;
- 			if ($aPair[1] == 'highway') continue;
- 
- 			echo "drop table if exists place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]).";\n";
  			echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." as ";
  			echo "select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
  			echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."';\n";
  
  			echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid ";
 -			echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid);\n";
 +			echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid) tablespace ssd;\n";
  
  			echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id ";
 -			echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id);\n";
 +			echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id) tablespace ssd;\n";
  
 -            echo "GRANT SELECT ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." TO \"www-data\";";
 +            echo "GRANT SELECT ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." TO \"www-data\";\n";
  
  		}
  
diff --combined website/search.php
index 461e4530,f0325f78..a2d8729f
--- a/website/search.php
+++ b/website/search.php
@@@ -41,7 -41,6 +41,7 @@@
  	if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true;
  	if (isset($aLangPrefOrder['name:ru'])) $bReverseInPlan = true;
  	if (isset($aLangPrefOrder['name:ja'])) $bReverseInPlan = true;
 +	if (isset($aLangPrefOrder['name:pl'])) $bReverseInPlan = true;
  
  	$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
  
@@@ -246,7 -245,7 +246,7 @@@
  				$sQuery = str_replace($aSpecialTerm[0], ' ', $sQuery);
  				$sToken = $oDB->getOne("select make_standard_name('".$aSpecialTerm[1]."') as string");
  				$sSQL = 'select * from (select word_id,word_token, word, class, type, location, country_code, operator';
- 				$sSQL .= ' from word where word_token in (\' '.$sToken.'\')) as x where (class is not null and class not in (\'place\',\'highway\')) or country_code is not null';
+ 				$sSQL .= ' from word where word_token in (\' '.$sToken.'\')) as x where (class is not null and class not in (\'place\')) or country_code is not null';
  				if (CONST_Debug) var_Dump($sSQL);
  				$aSearchWords = $oDB->getAll($sSQL);
  				$aNewSearches = array();
@@@ -313,12 -312,7 +313,7 @@@
  			// Check which tokens we have, get the ID numbers			
  			$sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator';
  			$sSQL .= ' from word where word_token in ('.join(',',array_map("getDBQuoted",$aTokens)).')';
- 			// HACK WARNING
- 			// (mis)using search_name_count to exclude words that return too many
- 			// search results. saerch_name_count is currently set to 1 by hand
- 			// because there is no fast way to extract this count from a live database. 
- 			$sSQL .= ' and search_name_count = 0';
- //			$sSQL .= ' and (class is null or class not in (\'highway\'))';
+ 			$sSQL .= ' and search_name_count < '.CONST_Max_Word_Frequency;
  //			$sSQL .= ' group by word_token, word, class, type, location, country_code';
  
  			if (CONST_Debug) var_Dump($sSQL);
@@@ -797,16 -791,11 +792,16 @@@
  									$sSQL .= " limit ".$iLimit;
  
  								if (CONST_Debug) var_dump($sSQL);
 +								$iStartTime = time();
  								$aViewBoxPlaceIDs = $oDB->getAll($sSQL);
  								if (PEAR::IsError($aViewBoxPlaceIDs))
  								{
  									failInternalError("Could not get places for search terms.", $sSQL, $aViewBoxPlaceIDs);
  								}
 +								if (time() - $iStartTime > 60) {
 +									file_put_contents(CONST_BasePath.'/log/long_queries.log', date('Y-m-d H:i:s', $iStartTime).' '.$sSQL."\n", FILE_APPEND);
 +								}
 +
  //var_dump($aViewBoxPlaceIDs);
  								// Did we have an viewbox matches?
  								$aPlaceIDs = array();