From: Sarah Hoffmann Date: Fri, 22 Jan 2016 21:55:34 +0000 (+0100) Subject: Merge remote-tracking branch 'mtmail/mobile-responsive' X-Git-Tag: v.2.5.0~13 X-Git-Url: https://git.openstreetmap.org./nominatim.git/commitdiff_plain/620100a5ac943e86e54f81f40dc52f1dd0d61330?hp=cd37227a3c08c2078037278c6e2ad9c4dd550df8 Merge remote-tracking branch 'mtmail/mobile-responsive' --- diff --git a/tests-php/Nominatim/NominatimTest.php b/tests-php/Nominatim/NominatimTest.php index 7ab68934..4c5638b1 100644 --- a/tests-php/Nominatim/NominatimTest.php +++ b/tests-php/Nominatim/NominatimTest.php @@ -74,4 +74,78 @@ class NominatimTest extends \PHPUnit_Framework_TestCase } + + + public function test_getWordSets() + { + + // given an array of arrays like + // array( array('a','b'), array('c','d') ) + // returns a summary as string: '(a|b),(c|d)' + function serialize_sets($aSets) + { + $aParts = array(); + foreach($aSets as $aSet){ + $aParts[] = '(' . join('|', $aSet) . ')'; + } + return join(',', $aParts); + } + + $this->assertEquals( + array(array('')), + getWordSets(array(),0) + ); + + $this->assertEquals( + '(a)', + serialize_sets( getWordSets(array("a"),0) ) + ); + + $this->assertEquals( + '(a b),(a|b)', + serialize_sets( getWordSets(array('a','b'),0) ) + ); + + $this->assertEquals( + '(a b c),(a|b c),(a|b|c),(a b|c)', + serialize_sets( getWordSets(array('a','b','c'),0) ) + ); + + $this->assertEquals( + '(a b c d),(a|b c d),(a|b|c d),(a|b|c|d),(a|b c|d),(a b|c d),(a b|c|d),(a b c|d)', + serialize_sets( getWordSets(array('a','b','c','d'),0) ) + ); + + + // Inverse + $this->assertEquals( + '(a b c),(c|a b),(c|b|a),(b c|a)', + serialize_sets( getInverseWordSets(array('a','b','c'),0) ) + ); + + + // make sure we don't create too many sets + // 4 words => 8 sets + // 10 words => 511 sets + // 15 words => 12911 sets + // 18 words => 65536 sets + // 20 words => 169766 sets + // 22 words => 401930 sets + // 28 words => 3505699 sets (needs more than 4GB via 'phpunit -d memory_limit=' to run) + $this->assertEquals( + 8, + count( getWordSets(array_fill( 0, 4, 'a'),0) ) + ); + + + $this->assertEquals( + 65536, + count( getWordSets(array_fill( 0, 18, 'a'),0) ) + ); + + + + } + + } diff --git a/tests-php/README.txt b/tests-php/README.txt index 8aee5d8e..d551c1da 100644 --- a/tests-php/README.txt +++ b/tests-php/README.txt @@ -7,7 +7,7 @@ installed. To execute the test suite run $ cd tests-php -$ phpunit +$ phpunit ./ It will read phpunit.xml which points to the library, test path, bootstrap strip and set other parameters. diff --git a/utils/tigerAddressImport.py b/utils/tigerAddressImport.py index b05b7394..1eb10d76 100755 --- a/utils/tigerAddressImport.py +++ b/utils/tigerAddressImport.py @@ -197,32 +197,33 @@ county_fips = { '02013' : 'Aleutians East, AK' , '02016' : 'Aleutians West, AK' , '02020' : 'Anchorage, AK' , -'02230' : 'Skagway Municipality' , '02050' : 'Bethel, AK' , '02060' : 'Bristol Bay, AK' , '02068' : 'Denali, AK' , '02070' : 'Dillingham, AK' , '02090' : 'Fairbanks North Star, AK' , '02100' : 'Haines, AK' , -'02105' : 'Hoonah-Angoon Census Area' , +'02105' : 'Hoonah-Angoon Census Area, AK' , '02110' : 'Juneau, AK' , '02122' : 'Kenai Peninsula, AK' , '02130' : 'Ketchikan Gateway, AK' , '02150' : 'Kodiak Island, AK' , +'02158' : 'Kusilvak Census Area, AK' , '02164' : 'Lake and Peninsula, AK' , '02170' : 'Matanuska-Susitna, AK' , '02180' : 'Nome, AK' , '02185' : 'North Slope, AK' , '02188' : 'Northwest Arctic, AK' , -'02195' : 'Petersburg Census Area' , -'02198' : 'Prince of Wales-Hyder Census Area' , +'02195' : 'Petersburg, AK' , +'02198' : 'Prince of Wales-Hyder Census Area, AK' , '02201' : 'Prince of Wales-Outer Ketchikan, AK' , '02220' : 'Sitka, AK' , +'02230' : 'Skagway, AK' , '02232' : 'Skagway-Hoonah-Angoon, AK' , '02240' : 'Southeast Fairbanks, AK' , '02261' : 'Valdez-Cordova, AK' , '02270' : 'Wade Hampton, AK' , -'02275' : 'Wrangell City and Borough' , +'02275' : 'Wrangell City and Borough, AK' , '02280' : 'Wrangell-Petersburg, AK' , '02282' : 'Yakutat, AK' , '02290' : 'Yukon-Koyukuk, AK' , @@ -786,7 +787,7 @@ county_fips = { '17093' : 'Kendall, IL' , '17095' : 'Knox, IL' , '17097' : 'Lake, IL' , -'17099' : 'La Salle, IL' , +'17099' : 'LaSalle, IL' , '17101' : 'Lawrence, IL' , '17103' : 'Lee, IL' , '17105' : 'Livingston, IL' , @@ -1289,7 +1290,7 @@ county_fips = { '22053' : 'Jefferson Davis, LA' , '22055' : 'Lafayette, LA' , '22057' : 'Lafourche, LA' , -'22059' : 'La Salle, LA' , +'22059' : 'LaSalle, LA' , '22061' : 'Lincoln, LA' , '22063' : 'Livingston, LA' , '22065' : 'Madison, LA' , @@ -1365,7 +1366,7 @@ county_fips = { '24043' : 'Washington, MD' , '24045' : 'Wicomico, MD' , '24047' : 'Worcester, MD' , -'24510' : 'Baltimore city, MD' , +'24510' : 'Baltimore City, MD' , '25000' : 'MASSACHUSETTS' , '25001' : 'Barnstable, MA' , '25003' : 'Berkshire, MA' , @@ -1751,7 +1752,7 @@ county_fips = { '29225' : 'Webster, MO' , '29227' : 'Worth, MO' , '29229' : 'Wright, MO' , -'29510' : 'St. Louis city, MO' , +'29510' : 'St. Louis City, MO' , '30000' : 'MONTANA' , '30001' : 'Beaverhead, MT' , '30003' : 'Big Horn, MT' , @@ -1921,7 +1922,7 @@ county_fips = { '32029' : 'Storey, NV' , '32031' : 'Washoe, NV' , '32033' : 'White Pine, NV' , -'32510' : 'Carson City city, NV' , +'32510' : 'Carson City, NV' , '33000' : 'NEW HAMPSHIRE' , '33001' : 'Belknap, NH' , '33003' : 'Carroll, NH' , @@ -2583,6 +2584,7 @@ county_fips = { '46097' : 'Miner, SD' , '46099' : 'Minnehaha, SD' , '46101' : 'Moody, SD' , +'46102' : 'Oglala Lakota, SD' , '46103' : 'Pennington, SD' , '46105' : 'Perkins, SD' , '46107' : 'Potter, SD' , @@ -3326,97 +3328,97 @@ county_fips = { '56043' : 'Washakie, WY' , '56045' : 'Weston, WY' , '60000' : 'AMERICAN SAMOA', -'60010' : 'Eastern district, AS' , -'60020' : 'Manua district, AS' , -'60030' : 'Rose island, AS' , -'60040' : 'Swains island, AS' , -'60050' : 'Western district, AS' , +'60010' : 'Eastern District, AS' , +'60020' : 'Manu\'a District, AS' , +'60030' : 'Rose Island, AS' , +'60040' : 'Swains Island, AS' , +'60050' : 'Western District, AS' , '66000' : 'GUAM' , '66010' : 'Guam, GU' , '69000' : 'COMMONWEALTH OF THE NORTHERN MARIANA ISLANDS' , -'69085' : 'Commonwealth of the Northern Mariana Islands, MP' , -'69100' : 'Commonwealth of the Northern Mariana Islands, MP' , -'69110' : 'Commonwealth of the Northern Mariana Islands, MP' , -'69120' : 'Commonwealth of the Northern Mariana Islands, MP' , -'72000' : 'PUERTO RICO', -'72001' : 'Adjuntas, PR', -'72003' : 'Aguada, PR', -'72005' : 'Aguadilla, PR', -'72007' : 'Aguas Buenas, PR', -'72009' : 'Aibonito, PR', -'72011' : 'Anasco, PR', -'72013' : 'Arecibo, PR', -'72015' : 'Arroyo, PR', -'72017' : 'Barceloneta, PR', -'72019' : 'Barranquitas, PR', -'72021' : 'Bayamon, PR', -'72023' : 'Cabo Rojo, PR', -'72025' : 'Caguas, PR', -'72027' : 'Camuy, PR', -'72029' : 'Canovanas, PR', -'72031' : 'Carolina, PR', -'72033' : 'Catano, PR', -'72035' : 'Cayey, PR', -'72037' : 'Ceiba, PR', -'72039' : 'Ciales, PR', -'72041' : 'Cidra, PR', -'72043' : 'Coamo, PR', -'72045' : 'Comerio, PR', -'72047' : 'Corozal, PR', -'72049' : 'Culebra, PR', -'72051' : 'Dorado, PR', -'72053' : 'Fajardo, PR', -'72054' : 'Florida, PR', -'72055' : 'Guanica, PR', -'72057' : 'Guayama, PR', -'72059' : 'Guayanilla, PR', -'72061' : 'Guaynabo, PR', -'72063' : 'Gurabo, PR', -'72065' : 'Hatillo, PR', -'72067' : 'Hormigueros, PR', -'72069' : 'Humacao, PR', -'72071' : 'Isabela, PR', -'72073' : 'Jayuya, PR', -'72075' : 'Juana Diaz, PR', -'72077' : 'Juncos, PR', -'72079' : 'Lajas, PR', -'72081' : 'Lares, PR', -'72083' : 'Las Marias, PR', -'72085' : 'Las Piedras, PR', -'72087' : 'Loiza, PR', -'72089' : 'Luquillo, PR', -'72091' : 'Manati, PR', -'72093' : 'Maricao, PR', -'72095' : 'Maunabo, PR', -'72097' : 'Mayaguez, PR', -'72099' : 'Moca, PR', -'72101' : 'Morovis, PR', -'72103' : 'Naguabo, PR', -'72105' : 'Naranjito, PR', -'72107' : 'Orocovis, PR', -'72109' : 'Patillas, PR', -'72111' : 'Penuelas, PR', -'72113' : 'Ponce, PR', -'72115' : 'Quebradillas, PR', -'72117' : 'Rincon, PR', -'72119' : 'Rio Grande, PR', -'72121' : 'Sabana Grande, PR', -'72123' : 'Salinas, PR', -'72125' : 'San German, PR', -'72127' : 'San Juan, PR', -'72129' : 'San Lorenzo, PR', -'72131' : 'San Sebastian, PR', -'72133' : 'Santa Isabel, PR', -'72135' : 'Toa Alta, PR', -'72137' : 'Toa Baja, PR', -'72139' : 'Trujillo Alto, PR', -'72141' : 'Utuado, PR', -'72143' : 'Vega Alta, PR', -'72145' : 'Vega Baja, PR', -'72147' : 'Vieques, PR', -'72149' : 'Villalba, PR', -'72151' : 'Yabucoa, PR', -'72153' : 'Yauco, PR', +'69085' : 'Northern Islands, MP' , +'69100' : 'Rota, MP' , +'69110' : 'Saipan, MP' , +'69120' : 'Tinian, MP' , +'72000' : 'PUERTO RICO' , +'72001' : 'Adjuntas, PR' , +'72003' : 'Aguada, PR' , +'72005' : 'Aguadilla, PR' , +'72007' : 'Aguas Buenas, PR' , +'72009' : 'Aibonito, PR' , +'72011' : 'Anasco, PR' , +'72013' : 'Arecibo, PR' , +'72015' : 'Arroyo, PR' , +'72017' : 'Barceloneta, PR' , +'72019' : 'Barranquitas, PR' , +'72021' : 'Bayamon, PR' , +'72023' : 'Cabo Rojo, PR' , +'72025' : 'Caguas, PR' , +'72027' : 'Camuy, PR' , +'72029' : 'Canovanas, PR' , +'72031' : 'Carolina, PR' , +'72033' : 'Catano, PR' , +'72035' : 'Cayey, PR' , +'72037' : 'Ceiba, PR' , +'72039' : 'Ciales, PR' , +'72041' : 'Cidra, PR' , +'72043' : 'Coamo, PR' , +'72045' : 'Comerio, PR' , +'72047' : 'Corozal, PR' , +'72049' : 'Culebra, PR' , +'72051' : 'Dorado, PR' , +'72053' : 'Fajardo, PR' , +'72054' : 'Florida, PR' , +'72055' : 'Guanica, PR' , +'72057' : 'Guayama, PR' , +'72059' : 'Guayanilla, PR' , +'72061' : 'Guaynabo, PR' , +'72063' : 'Gurabo, PR' , +'72065' : 'Hatillo, PR' , +'72067' : 'Hormigueros, PR' , +'72069' : 'Humacao, PR' , +'72071' : 'Isabela, PR' , +'72073' : 'Jayuya, PR' , +'72075' : 'Juana Diaz, PR' , +'72077' : 'Juncos, PR' , +'72079' : 'Lajas, PR' , +'72081' : 'Lares, PR' , +'72083' : 'Las Marias, PR' , +'72085' : 'Las Piedras, PR' , +'72087' : 'Loiza, PR' , +'72089' : 'Luquillo, PR' , +'72091' : 'Manati, PR' , +'72093' : 'Maricao, PR' , +'72095' : 'Maunabo, PR' , +'72097' : 'Mayaguez, PR' , +'72099' : 'Moca, PR' , +'72101' : 'Morovis, PR' , +'72103' : 'Naguabo, PR' , +'72105' : 'Naranjito, PR' , +'72107' : 'Orocovis, PR' , +'72109' : 'Patillas, PR' , +'72111' : 'Penuelas, PR' , +'72113' : 'Ponce, PR' , +'72115' : 'Quebradillas, PR' , +'72117' : 'Rincon, PR' , +'72119' : 'Rio Grande, PR' , +'72121' : 'Sabana Grande, PR' , +'72123' : 'Salinas, PR' , +'72125' : 'San German, PR' , +'72127' : 'San Juan, PR' , +'72129' : 'San Lorenzo, PR' , +'72131' : 'San Sebastian, PR' , +'72133' : 'Santa Isabel, PR' , +'72135' : 'Toa Alta, PR' , +'72137' : 'Toa Baja, PR' , +'72139' : 'Trujillo Alto, PR' , +'72141' : 'Utuado, PR' , +'72143' : 'Vega Alta, PR' , +'72145' : 'Vega Baja, PR' , +'72147' : 'Vieques, PR' , +'72149' : 'Villalba, PR' , +'72151' : 'Yabucoa, PR' , +'72153' : 'Yauco, PR' , '78000' : 'VIRGIN ISLANDS' , '78010' : 'St. Croix, VI' , '78020' : 'St. John, VI' , diff --git a/utils/update.php b/utils/update.php index 13790450..cc275463 100755 --- a/utils/update.php +++ b/utils/update.php @@ -54,15 +54,13 @@ // Lock to prevent multiple copies running if (exec('/bin/ps uww | grep '.basename(__FILE__).' | grep -v /dev/null | grep -v grep -c', $aOutput2, $iResult) > 1) { - echo "Copy already running\n"; - exit; + fail("Copy already running\n"); } if (!isset($aResult['max-load'])) $aResult['max-load'] = 1.9; if (!isset($aResult['max-blocking'])) $aResult['max-blocking'] = 3; if (getBlockingProcesses() > $aResult['max-blocking']) { - echo "Too many blocking processes for import\n"; - exit; + fail("Too many blocking processes for import\n"); } */ @@ -120,8 +118,7 @@ $sNextFile = $aResult['import-diff']; if (!file_exists($sNextFile)) { - echo "Cannot open $sNextFile\n"; - exit; + fail("Cannot open $sNextFile\n"); } // Don't update the import status - we don't know what this file contains $sUpdateSQL = 'update import_status set lastimportdate = now() where false'; @@ -137,8 +134,7 @@ if ($iErrorLevel) { - echo "Error from osm2pgsql, $iErrorLevel\n"; - exit; + fail("Error from osm2pgsql, $iErrorLevel\n"); } // Move the date onwards @@ -205,8 +201,7 @@ exec($sCMD, $sJunk, $iErrorLevel); if ($iErrorLevel) { - echo "Error converting osm to osc, osmosis returned: $iErrorLevel\n"; - exit; + fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n"); } } else @@ -221,8 +216,7 @@ $hProc = proc_open($sCMD, $aSpec, $aPipes); if (!is_resource($hProc)) { - echo "Error converting osm to osc, osmosis failed\n"; - exit; + fail("Error converting osm to osc, osmosis failed\n"); } fwrite($aPipes[0], $sModifyXMLstr); fclose($aPipes[0]); @@ -237,7 +231,7 @@ echo "Error converting osm to osc, osmosis returned: $iError\n"; echo $sOut; echo $sErrors; - exit; + exit(-1); } } @@ -247,8 +241,7 @@ exec($sCMD, $sJunk, $iErrorLevel); if ($iErrorLevel) { - echo "osm2pgsql exited with error level $iErrorLevel\n"; - exit; + fail("osm2pgsql exited with error level $iErrorLevel\n"); } } @@ -257,8 +250,7 @@ $pgver = (float) CONST_Postgresql_Version; if ($pgver < 9.3) { - echo "ERROR: deduplicate is only currently supported in postgresql 9.3"; - exit; + fail("ERROR: deduplicate is only currently supported in postgresql 9.3"); } $oDB =& getDB(); @@ -281,7 +273,7 @@ if (PEAR::isError($aTokenSet)) { var_dump($aTokenSet, $sSQL); - exit; + exit(1); } $aKeep = array_shift($aTokenSet); @@ -297,7 +289,7 @@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } $sSQL = "update search_name set"; @@ -307,7 +299,7 @@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } $sSQL = "update location_area_country set"; @@ -317,7 +309,7 @@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } foreach ($aPartitions as $sPartition) @@ -329,7 +321,7 @@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } $sSQL = "update location_area_country set"; @@ -339,7 +331,7 @@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } } @@ -348,7 +340,7 @@ if (PEAR::isError($x)) { var_dump($x); - exit; + exit(1); } } @@ -364,8 +356,7 @@ { if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { - echo "Error: Update interval too low for download.geofabrik.de. Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n"; - exit; + fail("Error: Update interval too low for download.geofabrik.de. Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n"); } $sImportFile = CONST_BasePath.'/data/osmosischange.osc'; @@ -527,7 +518,7 @@ $fDuration = time() - $fStartTime; echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n"; - if (!$aResult['import-osmosis-all']) exit; + if (!$aResult['import-osmosis-all']) exit(0); if ( CONST_Replication_Update_Interval > 60 ) { @@ -549,7 +540,7 @@ if (PEAR::isError($iNPIID)) { var_dump($iNPIID); - exit; + exit(1); } $sConfigDirectory = CONST_BasePath.'/settings'; $sCMDImportTemplate = $sBasePath.'/nominatim/nominatim -d gazetteer -P 5433 -I -T '.$sBasePath.'/nominatim/partitionedtags.def -F '; @@ -580,8 +571,7 @@ exec($sCMDImport, $sJunk, $iErrorLevel); if ($iErrorLevel) { - echo "Error: $iErrorLevel\n"; - exit; + fail("Error: $iErrorLevel\n"); } $sBatchEnd = $iNPIID; echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; diff --git a/vagrant-provision.sh b/vagrant-provision.sh index 68e279c5..3fae5b68 100755 --- a/vagrant-provision.sh +++ b/vagrant-provision.sh @@ -29,10 +29,9 @@ USERNAME=vagrant sudo apt-get update -qq sudo apt-get upgrade -y -# sudo apt-get install -y git-core screen sudo apt-get install -y build-essential libxml2-dev libgeos-dev libpq-dev libbz2-dev \ libtool automake libproj-dev libboost-dev libboost-system-dev \ - libboost-filesystem-dev libboost-thread-dev + libboost-filesystem-dev libboost-thread-dev libexpat-dev sudo apt-get autoremove -y # get arrow-keys working in terminal (e.g. editing in vi) @@ -62,8 +61,7 @@ sudo -u postgres createuser -s $USERNAME ### ### PHP for frontend ### -sudo apt-get install -y php5 php5-pgsql php-pear -sudo pear install DB +sudo apt-get install -y php5 php5-pgsql php-pear php-db # get rid of some warning @@ -156,15 +154,12 @@ sudo chown $USERNAME /var/www/nominatim ## Test suite (Python) ## https://github.com/twain47/Nominatim/tree/master/tests ## -sudo apt-get install -y python-dev python-pip python-Levenshtein tidy -sudo pip install lettuce nose pytidylib haversine psycopg2 shapely +sudo apt-get install -y python-dev python-pip python-Levenshtein python-shapely \ + python-psycopg2 tidy python-nose python-tidylib +sudo pip install lettuce==0.2.18 six==1.7 haversine -## ## Test suite (PHP) ## https://github.com/twain47/Nominatim/tree/master/tests-php -## -wget --no-clobber -q https://phar.phpunit.de/phpunit.phar -chmod +x phpunit.phar -sudo mv phpunit.phar /usr/local/bin/phpunit +sudo apt-get install -y phpunit