utils/importWikipedia.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4 require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
   5 require_once(CONST_BasePath.'/lib/init-cmd.php');
   6 ini_set('memory_limit', '800M');
   7
   8 $aCMDOptions = array(
   9     "Create and setup nominatim search system",
  10     array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
  11     array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
  12     array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
  13
  14     array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create wikipedia tables'),
  15     array('parse-articles', '', 0, 1, 0, 0, 'bool', 'Parse wikipedia articles'),
  16     array('link', '', 0, 1, 0, 0, 'bool', 'Try to link to existing OSM ids'),
  17 );
  18 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
  19
  20 /*
  21 $sTestPageText = <<<EOD
  22 {{Coord|47|N|2|E|type:country_region:FR|display=title}}
  23 {{ Infobox Amusement park
  24 | name = Six Flags Great Adventure
  25 | image = [[File:SixFlagsGreatAdventure logo.png]]
  26 | caption = Six Flags Great Adventure logo
  27 | location = [[Jackson, New Jersey|Jackson]]
  28 | location2 = New Jersey
  29 | location3 = United States
  30 | address = 1 Six Flags Boulevard<ref name="drivedir"/>
  31 | season = March/April through October/November
  32 | opening_date = July 1, 1974
  33 | previous_names = Great Adventure
  34 | area_acre = 2200
  35 | rides = 45 park admission rides
  36 | coasters = 12
  37 | water_rides = 2
  38 | owner = [[Six Flags]]
  39 | general_manager =
  40 | homepage = [http://www.sixflags.com/parks/greatadventure/ Six Flags Great Adventure]
  41 }}
  42 EOD;
  43 var_dump(_templatesToProperties(_parseWikipediaContent($sTestPageText)));
  44 exit;
  45 //| coordinates = {{Coord|40|08|16.65|N|74|26|26.69|W|region:US-NJ_type:landmark|display=inline,title}}
  46 */
  47 /*
  48
  49     $a = array();
  50     $a[] = 'test';
  51
  52     $oDB &= getDB();
  53
  54     if ($aCMDResult['drop-tables'])
  55     {
  56         $oDB->query('DROP TABLE wikipedia_article');
  57         $oDB->query('DROP TABLE wikipedia_link');
  58     }
  59 */
  60
  61 if ($aCMDResult['create-tables'])
  62 {
  63     $sSQL = <<<'EOD'
  64 CREATE TABLE wikipedia_article (
  65     language text NOT NULL,
  66     title text NOT NULL,
  67     langcount integer,
  68     othercount integer,
  69     totalcount integer,
  70     lat double precision,
  71     lon double precision,
  72     importance double precision,
  73     title_en text,
  74     osm_type character(1),
  75     osm_id bigint,
  76     infobox_type text,
  77     population bigint,
  78     website text
  79 );
  80         $oDB->query($sSQL);
  81
  82         $oDB->query("SELECT AddGeometryColumn('wikipedia_article', 'location', 4326, 'GEOMETRY', 2)");
  83
  84         $sSQL = <<<'EOD'
  85 CREATE TABLE wikipedia_link (
  86   from_id INTEGER,
  87   to_name text
  88   );
  89 EOD;
  90     $oDB->query($sSQL);
  91 }
  92
  93 function degreesAndMinutesToDecimal($iDegrees, $iMinutes=0, $fSeconds=0, $sNSEW='N')
  94 {
  95     $sNSEW = strtoupper($sNSEW);
  96     return ($sNSEW == 'S' || $sNSEW == 'W'?-1:1) * ((float)$iDegrees + (float)$iMinutes/60 + (float)$fSeconds/3600);
  97 }
  98
  99 function _parseWikipediaContent($sPageText)
 100 {
 101     $sPageText = str_replace("\n", ' ', $sPageText);
 102     $sPageText = preg_replace('#<!--.*?-->#m', '', $sPageText);
 103     $sPageText = preg_replace('#<math>.*?<\\/math>#m', '', $sPageText);
 104
 105     $aPageText = preg_split('#({{|}}|\\[\\[|\\]\\]|[|])#', $sPageText, -1, PREG_SPLIT_DELIM_CAPTURE);
 106
 107     $aPageProperties = array();
 108     $sPageBody = '';
 109     $aTemplates = array();
 110     $aLinks = array();
 111
 112     $aTemplateStack = array();
 113     $aState = array('body');
 114     foreach($aPageText as $i => $sPart)
 115     {
 116         switch($sPart)
 117         {
 118         case '{{':
 119             array_unshift($aTemplateStack, array('', array()));
 120             array_unshift($aState, 'template');
 121             break;
 122         case '}}':
 123             if ($aState[0] == 'template' || $aState[0] == 'templateparam')
 124             {
 125                 $aTemplate = array_shift($aTemplateStack);
 126                 array_shift($aState);
 127
 128                 $aTemplates[] = $aTemplate;
 129
 130             }
 131             break;
 132         case '[[':
 133             $sLinkPage = '';
 134             $sLinkSyn = '';
 135             array_unshift($aState, 'link');
 136             break;
 137         case ']]':
 138             if ($aState[0] == 'link' || $aState[0] == 'linksynonim')
 139             {
 140                 if (!$sLinkSyn) $sLinkSyn = $sLinkPage;
 141                 if (substr($sLinkPage, 0, 6) == 'Image:') $sLinkSyn = substr($sLinkPage, 6);
 142
 143                 $aLinks[] = array($sLinkPage, $sLinkSyn);
 144
 145                 array_shift($aState);
 146                 switch($aState[0])
 147                 {
 148                 case 'template':
 149                     $aTemplateStack[0][0] .= trim($sPart);
 150                     break;
 151                 case 'templateparam':
 152                     $aTemplateStack[0][1][0] .= $sLinkSyn;
 153                     break;
 154                 case 'link':
 155                     $sLinkPage .= trim($sPart);
 156                     break;
 157                 case 'linksynonim':
 158                     $sLinkSyn .= $sPart;
 159                     break;
 160                 case 'body':
 161                     $sPageBody .= $sLinkSyn;
 162                     break;
 163                 default:
 164                     var_dump($aState, $sPageName, $aTemplateStack, $sPart, $aPageText);
 165                     fail('unknown state');
 166                 }
 167             }
 168             break;
 169         case '|':
 170             if ($aState[0] == 'template' || $aState[0] == 'templateparam')
 171             {
 172                 // Create a new template paramater
 173                 $aState[0] = 'templateparam';
 174                 array_unshift($aTemplateStack[0][1], '');
 175             }
 176             if ($aState[0] == 'link') $aState[0] = 'linksynonim';
 177             break;
 178         default:
 179             switch($aState[0])
 180             {
 181             case 'template':
 182                 $aTemplateStack[0][0] .= trim($sPart);
 183                 break;
 184             case 'templateparam':
 185                 $aTemplateStack[0][1][0] .= $sPart;
 186                 break;
 187             case 'link':
 188                 $sLinkPage .= trim($sPart);
 189                 break;
 190             case 'linksynonim':
 191                 $sLinkSyn .= $sPart;
 192                 break;
 193             case 'body':
 194                 $sPageBody .= $sPart;
 195                 break;
 196             default:
 197                 var_dump($aState, $aPageText);
 198                 fail('unknown state');
 199             }
 200             break;
 201         }
 202     }
 203     return $aTemplates;
 204 }
 205
 206 function _templatesToProperties($aTemplates)
 207 {
 208     $aPageProperties = array();
 209     foreach($aTemplates as $iTemplate => $aTemplate)
 210     {
 211         $aParams = array();
 212         foreach(array_reverse($aTemplate[1]) as $iParam => $sParam)
 213         {
 214             if (($iPos = strpos($sParam, '=')) === FALSE)
 215             {
 216                 $aParams[] = trim($sParam);
 217             }
 218             else
 219             {
 220                 $aParams[trim(substr($sParam, 0, $iPos))] = trim(substr($sParam, $iPos+1));
 221             }
 222         }
 223         $aTemplates[$iTemplate][1] = $aParams;
 224         if (!isset($aPageProperties['sOfficialName']) && isset($aParams['official_name']) && $aParams['official_name']) $aPageProperties['sOfficialName'] = $aParams['official_name'];
 225         if (!isset($aPageProperties['iPopulation']) && isset($aParams['population']) && $aParams['population'] && preg_match('#^[0-9.,]+#', $aParams['population']))
 226         {
 227             $aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population']);
 228         }
 229         if (!isset($aPageProperties['iPopulation']) && isset($aParams['population_total']) && $aParams['population_total'] && preg_match('#^[0-9.,]+#', $aParams['population_total']))
 230         {
 231             $aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population_total']);
 232         }
 233         if (!isset($aPageProperties['iPopulation']) && isset($aParams['population_urban']) && $aParams['population_urban'] && preg_match('#^[0-9.,]+#', $aParams['population_urban']))
 234         {
 235             $aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population_urban']);
 236         }
 237         if (!isset($aPageProperties['iPopulation']) && isset($aParams['population_estimate']) && $aParams['population_estimate'] && preg_match('#^[0-9.,]+#', $aParams['population_estimate']))
 238         {
 239             $aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population_estimate']);
 240         }
 241         if (!isset($aPageProperties['sWebsite']) && isset($aParams['website']) && $aParams['website'])
 242         {
 243             if (preg_match('#^\\[?([^ \\]]+)[^\\]]*\\]?$#', $aParams['website'], $aMatch))
 244             {
 245                 $aPageProperties['sWebsite'] = $aMatch[1];
 246                 if (strpos($aPageProperties['sWebsite'],':/'.'/') === FALSE)
 247                 {
 248                     $aPageProperties['sWebsite'] = 'http:/'.'/'.$aPageProperties['sWebsite'];
 249                 }
 250             }
 251         }
 252         if (!isset($aPageProperties['sTopLevelDomain']) && isset($aParams['cctld']) && $aParams['cctld'])
 253         {
 254             $aPageProperties['sTopLevelDomain'] = str_replace(array('[',']','.'),'', $aParams['cctld']);
 255         }
 256
 257         if (!isset($aPageProperties['sInfoboxType']) && strtolower(substr($aTemplate[0],0,7)) == 'infobox')
 258         {
 259             $aPageProperties['sInfoboxType'] = trim(substr($aTemplate[0],8));
 260             // $aPageProperties['aInfoboxParams'] = $aParams;
 261         }
 262
 263         // Assume the first template with lots of params is the type (fallback for infobox)
 264         if (!isset($aPageProperties['sPossibleInfoboxType']) && sizeof($aParams) > 10)
 265         {
 266             $aPageProperties['sPossibleInfoboxType'] = trim($aTemplate[0]);
 267             // $aPageProperties['aInfoboxParams'] = $aParams;
 268         }
 269
 270         // do we have a lat/lon
 271         if (!isset($aPageProperties['fLat']))
 272         {
 273             if (isset($aParams['latd']) && isset($aParams['longd']))
 274             {
 275                 $aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams['latd'], @$aParams['latm'], @$aParams['lats'], @$aParams['latNS']);
 276                 $aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams['longd'], @$aParams['longm'], @$aParams['longs'], @$aParams['longEW']);
 277             }
 278             if (isset($aParams['lat_degrees']) && isset($aParams['lat_degrees']))
 279             {
 280                 $aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams['lat_degrees'], @$aParams['lat_minutes'], @$aParams['lat_seconds'], @$aParams['lat_direction']);
 281                 $aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams['long_degrees'], @$aParams['long_minutes'], @$aParams['long_seconds'], @$aParams['long_direction']);
 282             }
 283             if (isset($aParams['latitude']) && isset($aParams['longitude']))
 284             {
 285                 if (preg_match('#[0-9.]+#', $aParams['latitude']) && preg_match('#[0-9.]+#', $aParams['longitude']))
 286                 {
 287                     $aPageProperties['fLat'] = (float)$aParams['latitude'];
 288                     $aPageProperties['fLon'] = (float)$aParams['longitude'];
 289                 }
 290             }
 291             if (strtolower($aTemplate[0]) == 'coord')
 292             {
 293                 if (isset($aParams[3]) && (strtoupper($aParams[3]) == 'N' || strtoupper($aParams[3]) == 'S'))
 294                 {
 295                     $aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams[0], $aParams[1], $aParams[2], $aParams[3]);
 296                     $aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams[4], $aParams[5], $aParams[6], $aParams[7]);
 297                 }
 298                 elseif (isset($aParams[0]) && isset($aParams[1]) && isset($aParams[2]) && (strtoupper($aParams[2]) == 'N' || strtoupper($aParams[2]) == 'S'))
 299                 {
 300                     $aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams[0], $aParams[1], 0, $aParams[2]);
 301                     $aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams[3], $aParams[4], 0, $aParams[5]);
 302                 }
 303                 else if (isset($aParams[0]) && isset($aParams[1]) && (strtoupper($aParams[1]) == 'N' || strtoupper($aParams[1]) == 'S'))
 304                 {
 305                     $aPageProperties['fLat'] = (strtoupper($aParams[1]) == 'N'?1:-1) * (float)$aParams[0];
 306                     $aPageProperties['fLon'] = (strtoupper($aParams[3]) == 'E'?1:-1) * (float)$aParams[2];
 307                 }
 308                 else if (isset($aParams[0]) && is_numeric($aParams[0]) && isset($aParams[1]) && is_numeric($aParams[1]))
 309                 {
 310                     $aPageProperties['fLat'] = (float)$aParams[0];
 311                     $aPageProperties['fLon'] = (float)$aParams[1];
 312                 }
 313             }
 314             if (isset($aParams['Latitude']) && isset($aParams['Longitude']))
 315             {
 316                 $aParams['Latitude'] = str_replace('&nbsp;',' ',$aParams['Latitude']);
 317                 $aParams['Longitude'] = str_replace('&nbsp;',' ',$aParams['Longitude']);
 318                 if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([NS]) to ([0-9]+)°( ([0-9]+)′)? ([NS])#', $aParams['Latitude'], $aMatch))
 319                 {
 320                     $aPageProperties['fLat'] =
 321                         (degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4])
 322                         +degreesAndMinutesToDecimal($aMatch[5], $aMatch[7], 0, $aMatch[8])) / 2;
 323                 }
 324                 else if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([NS])#', $aParams['Latitude'], $aMatch))
 325                 {
 326                     $aPageProperties['fLat'] = degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4]);
 327                 }
 328
 329                 if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([EW]) to ([0-9]+)°( ([0-9]+)′)? ([EW])#', $aParams['Longitude'], $aMatch))
 330                 {
 331                     $aPageProperties['fLon'] =
 332                         (degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4])
 333                         +degreesAndMinutesToDecimal($aMatch[5], $aMatch[7], 0, $aMatch[8])) / 2;
 334                 }
 335                 else if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([EW])#', $aParams['Longitude'], $aMatch))
 336                 {
 337                     $aPageProperties['fLon'] = degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4]);
 338                 }
 339             }
 340         }
 341     }
 342     if (isset($aPageProperties['sPossibleInfoboxType']))
 343     {
 344         if (!isset($aPageProperties['sInfoboxType'])) $aPageProperties['sInfoboxType'] = '#'.$aPageProperties['sPossibleInfoboxType'];
 345         unset($aPageProperties['sPossibleInfoboxType']);
 346     }
 347     return $aPageProperties;
 348 }
 349
 350 if (isset($aCMDResult['parse-wikipedia']))
 351 {
 352     $oDB =& getDB();
 353     $aArticleNames = $oDB->getCol('select page_title from content where page_namespace = 0 and page_id %10 = '.$aCMDResult['parse-wikipedia'].' and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' and page_content ilike \'%lon%\'))');
 354 //      $aArticleNames = $oDB->getCol($sSQL = 'select page_title from content where page_namespace = 0 and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' and page_content ilike \'%lon%\')) and page_title in (\'Virginia\')');
 355     foreach($aArticleNames as $sArticleName)
 356     {
 357         $sPageText = $oDB->getOne('select page_content from content where page_namespace = 0 and page_title = \''.pg_escape_string($sArticleName).'\'');
 358         $aP = _templatesToProperties(_parseWikipediaContent($sPageText));
 359
 360         if (isset($aP['sInfoboxType']))
 361         {
 362             $aP['sInfoboxType'] = preg_replace('#\\s+#',' ',$aP['sInfoboxType']);
 363             $sSQL = 'update wikipedia_article set ';
 364             $sSQL .= 'infobox_type = \''.pg_escape_string($aP['sInfoboxType']).'\'';
 365             $sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
 366             $oDB->query($sSQL);
 367         }
 368         if (isset($aP['iPopulation']))
 369         {
 370             $sSQL = 'update wikipedia_article set ';
 371             $sSQL .= 'population = \''.pg_escape_string($aP['iPopulation']).'\'';
 372             $sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
 373             $oDB->query($sSQL);
 374         }
 375         if (isset($aP['sWebsite']))
 376         {
 377             $sSQL = 'update wikipedia_article set ';
 378             $sSQL .= 'website = \''.pg_escape_string($aP['sWebsite']).'\'';
 379             $sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
 380             $oDB->query($sSQL);
 381         }
 382         if (isset($aP['fLat']) && ($aP['fLat']!='-0' || $aP['fLon']!='-0'))
 383         {
 384             if (!isset($aP['sInfoboxType'])) $aP['sInfoboxType'] = '';
 385             echo $sArticleName.'|'.$aP['sInfoboxType'].'|'.$aP['fLat'].'|'.$aP['fLon'] ."\n";
 386             $sSQL = 'update wikipedia_article set ';
 387             $sSQL .= 'lat = \''.pg_escape_string($aP['fLat']).'\',';
 388             $sSQL .= 'lon = \''.pg_escape_string($aP['fLon']).'\'';
 389             $sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
 390             $oDB->query($sSQL);
 391         }
 392     }
 393 }
 394
 395 function nominatimXMLStart($hParser, $sName, $aAttr)
 396 {
 397         global $aNominatRecords;
 398         switch($sName)
 399         {
 400         case 'PLACE':
 401                 $aNominatRecords[] = $aAttr;
 402                 break;
 403         }
 404 }
 405
 406 function nominatimXMLEnd($hParser, $sName)
 407 {
 408 }
 409
 410
 411 if (isset($aCMDResult['link']))
 412 {
 413     $oDB =& getDB();
 414     $aWikiArticles = $oDB->getAll("select * from wikipedia_article where language = 'en' and lat is not null and osm_type is null and totalcount < 31 order by importance desc limit 200000");
 415
 416     // If you point this script at production OSM you will be blocked
 417     $sNominatimBaseURL = 'http://SEVERNAME/search.php';
 418
 419     foreach($aWikiArticles as $aRecord)
 420     {
 421         $aRecord['name'] = str_replace('_',' ',$aRecord['title']);
 422
 423         $sURL = $sNominatimBaseURL.'?format=xml&accept-language=en';
 424
 425         echo "\n-- ".$aRecord['name'].", ".$aRecord['infobox_type']."\n";
 426         $fMaxDist = 0.0000001;
 427         $bUnknown = false;
 428         switch(strtolower($aRecord['infobox_type']))
 429         {
 430         case 'former country':
 431             continue 2;
 432         case 'sea':
 433             $fMaxDist = 60; // effectively turn it off
 434             $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
 435             break;
 436         case 'country':
 437         case 'island':
 438         case 'islands':
 439         case 'continent':
 440             $fMaxDist = 60; // effectively turn it off
 441             $sURL .= "&featuretype=country";
 442             $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
 443             break;
 444         case 'prefecture japan':
 445             $aRecord['name'] = trim(str_replace(' Prefecture',' ', $aRecord['name']));
 446         case 'state':
 447         case '#us state':
 448         case 'county':
 449         case 'u.s. state':
 450         case 'u.s. state symbols':
 451         case 'german state':
 452         case 'province or territory of canada';
 453         case 'indian jurisdiction';
 454         case 'province';
 455         case 'french region':
 456         case 'region of italy':
 457         case 'kommune':
 458         case '#australia state or territory':
 459         case 'russian federal subject':
 460             $fMaxDist = 4;
 461             $sURL .= "&featuretype=state";
 462             $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
 463             break;
 464         case 'protected area':
 465             $fMaxDist = 1;
 466             $sURL .= "&nearlat=".$aRecord['lat'];
 467             $sURL .= "&nearlon=".$aRecord['lon'];
 468             $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
 469             break;
 470         case 'settlement':
 471             $bUnknown = true;
 472         case 'french commune':
 473         case 'italian comune':
 474         case 'uk place':
 475         case 'italian comune':
 476         case 'australian place':
 477         case 'german place':
 478         case '#geobox':
 479         case 'u.s. county':
 480         case 'municipality':
 481         case 'city japan':
 482         case 'russian inhabited locality':
 483         case 'finnish municipality/land area':
 484         case 'england county':
 485         case 'israel municipality':
 486         case 'russian city':
 487         case 'city':
 488             $fMaxDist = 0.2;
 489             $sURL .= "&featuretype=settlement";
 490             $sURL .= "&viewbox=".($aRecord['lon']-0.5).",".($aRecord['lat']+0.5).",".($aRecord['lon']+0.5).",".($aRecord['lat']-0.5);
 491             break;
 492         case 'mountain':
 493         case 'mountain pass':
 494         case 'river':
 495         case 'lake':
 496         case 'airport':
 497             $fMaxDist = 0.2;
 498             $sURL .= "&viewbox=".($aRecord['lon']-0.5).",".($aRecord['lat']+0.5).",".($aRecord['lon']+0.5).",".($aRecord['lat']-0.5);
 499
 500         case 'ship begin':
 501             $fMaxDist = 0.1;
 502             $aTypes = array('wreck');
 503             $sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
 504             $sURL .= "&nearlat=".$aRecord['lat'];
 505             $sURL .= "&nearlon=".$aRecord['lon'];
 506             break;
 507         case 'road':
 508         case 'university':
 509         case 'company':
 510         case 'department':
 511             $fMaxDist = 0.005;
 512             $sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
 513             $sURL .= "&bounded=1";
 514             $sURL .= "&nearlat=".$aRecord['lat'];
 515             $sURL .= "&nearlon=".$aRecord['lon'];
 516             break;
 517         default:
 518             $bUnknown = true;
 519             $fMaxDist = 0.005;
 520             $sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
 521 //              $sURL .= "&bounded=1";
 522             $sURL .= "&nearlat=".$aRecord['lat'];
 523             $sURL .= "&nearlon=".$aRecord['lon'];
 524             echo "-- Unknown: ".$aRecord['infobox_type']."\n";
 525             break;
 526         }
 527         $sNameURL = $sURL.'&q='.urlencode($aRecord['name']);
 528
 529         var_Dump($sNameURL);
 530         $sXML = file_get_contents($sNameURL);
 531
 532         $aNominatRecords = array();
 533         $hXMLParser = xml_parser_create();
 534         xml_set_element_handler($hXMLParser, 'nominatimXMLStart', 'nominatimXMLEnd');
 535         xml_parse($hXMLParser, $sXML, true);
 536         xml_parser_free($hXMLParser);
 537
 538         if (!isset($aNominatRecords[0]))
 539         {
 540             $aNameParts = preg_split('#[(,]#',$aRecord['name']);
 541             if (sizeof($aNameParts) > 1)
 542             {
 543                 $sNameURL = $sURL.'&q='.urlencode(trim($aNameParts[0]));
 544                 var_Dump($sNameURL);
 545                 $sXML = file_get_contents($sNameURL);
 546
 547                 $aNominatRecords = array();
 548                 $hXMLParser = xml_parser_create();
 549                 xml_set_element_handler($hXMLParser, 'nominatimXMLStart', 'nominatimXMLEnd');
 550                 xml_parse($hXMLParser, $sXML, true);
 551                 xml_parser_free($hXMLParser);#
 552             }
 553         }
 554
 555         // assume first is best/right
 556         for($i = 0; $i < sizeof($aNominatRecords); $i++)
 557         {
 558             $fDiff = ($aRecord['lat']-$aNominatRecords[$i]['LAT']) * ($aRecord['lat']-$aNominatRecords[$i]['LAT']);
 559             $fDiff += ($aRecord['lon']-$aNominatRecords[$i]['LON']) * ($aRecord['lon']-$aNominatRecords[$i]['LON']);
 560             $fDiff = sqrt($fDiff);
 561             if ($bUnknown) {
 562                 // If it was an unknown type base it on the rank of the found result
 563                 $iRank = (int)$aNominatRecords[$i]['PLACE_RANK'];
 564                 if ($iRank <= 4) $fMaxDist = 2;
 565                 elseif ($iRank <= 8) $fMaxDist = 1;
 566                 elseif ($iRank <= 10) $fMaxDist = 0.8;
 567                 elseif ($iRank <= 12) $fMaxDist = 0.6;
 568                 elseif ($iRank <= 17) $fMaxDist = 0.2;
 569                 elseif ($iRank <= 18) $fMaxDist = 0.1;
 570                 elseif ($iRank <= 22) $fMaxDist = 0.02;
 571                 elseif ($iRank <= 26) $fMaxDist = 0.001;
 572                 else $fMaxDist = 0.001;
 573             }
 574             echo "-- FOUND \"".substr($aNominatRecords[$i]['DISPLAY_NAME'],0,50)."\", ".$aNominatRecords[$i]['CLASS'].", ".$aNominatRecords[$i]['TYPE'].", ".$aNominatRecords[$i]['PLACE_RANK'].", ".$aNominatRecords[$i]['OSM_TYPE']." (dist:$fDiff, max:$fMaxDist)\n";
 575             if ($fDiff > $fMaxDist)
 576             {
 577                 echo "-- Diff too big $fDiff (max: $fMaxDist)".$aRecord['lat'].','.$aNominatRecords[$i]['LAT'].' & '.$aRecord['lon'].','.$aNominatRecords[$i]['LON']." \n";
 578             }
 579             else
 580             {
 581                 $sSQL = "update wikipedia_article set osm_type=";
 582                 switch($aNominatRecords[$i]['OSM_TYPE'])
 583                 {
 584                 case 'relation': $sSQL .= "'R'"; break;
 585                 case 'way': $sSQL .= "'W'"; break;
 586                 case 'node': $sSQL .= "'N'"; break;
 587                 }
 588                 $sSQL .= ", osm_id=".$aNominatRecords[$i]['OSM_ID']." where language = '".pg_escape_string($aRecord['language'])."' and title = '".pg_escape_string($aRecord['title'])."'";
 589                 $oDB->query($sSQL);
 590                 break;
 591             }
 592         }
 593     }
 594 }