wikidata/parse.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4 $hFile = @fopen("wikidatawiki-20130623-pages-articles.xml", "r");
   5
   6 $hFileEntity = fopen("entity.csv", "w");
   7 $hFileEntityLabel = fopen("entity_label.csv", "w");
   8 $hFileEntityDescription = fopen("entity_description.csv", "w");
   9 $hFileEntityAlias = fopen("entity_alias.csv", "w");
  10 $hFileEntityLink = fopen("entity_link.csv", "w");
  11 $hFileEntityProperty = fopen("entity_property.csv", "w");
  12
  13 $iCount = 0;
  14
  15 $sTitle = '';
  16 $iNS = false;
  17 $iID = false;
  18
  19 if ($hFile) {
  20         while (($sLine = fgets($hFile, 4000000)) !== false) {
  21                 if (substr($sLine, 0, 11) == '    <title>') {
  22                         $sTitle = substr($sLine, 11, -9);
  23                 }
  24                 else if (substr($sLine, 0, 8) == '    <ns>') {
  25                         $iNS = (int)substr($sLine, 8, -6);
  26                 }
  27                 else if (substr($sLine, 0, 8) == '    <id>') {
  28                         $iID = (int)substr($sLine, 8, -6);
  29                 }
  30                 else if (substr($sLine, 0, 33) == '      <text xml:space="preserve">') {
  31                         if ($iNS == -2) continue;
  32                         if ($iNS == -1) continue;
  33                         if ($iNS == 1) continue;
  34                         if ($iNS == 2) continue;
  35                         if ($iNS == 3) continue;
  36                         if ($iNS == 4) continue;
  37                         if ($iNS == 5) continue;
  38                         if ($iNS == 6) continue;
  39                         if ($iNS == 7) continue;
  40                         if ($iNS == 8) continue;
  41                         if ($iNS == 9) continue;
  42                         if ($iNS == 10) continue;
  43                         if ($iNS == 11) continue;
  44                         if ($iNS == 12) continue;
  45                         if ($iNS == 13) continue;
  46                         if ($iNS == 14) continue;
  47                         if ($iNS == 15) continue;
  48                         if ($iNS == 121) continue;
  49                         if ($iNS == 123) continue;
  50                         if ($iNS == 829) continue;
  51                         if ($iNS == 1198) continue;
  52                         if ($iNS == 1199) continue;
  53                         $sText = html_entity_decode(substr($sLine, 33, -8), ENT_COMPAT, 'UTF-8');
  54                         $aArticle = json_decode($sText, true);
  55
  56                         if (array_diff(array_keys($aArticle), array("label", "description", "aliases", "links", "entity", "claims", "datatype")) != array()) {
  57                                 // DEBUG
  58                                 var_dump($sTitle);
  59                                 var_dump(array_keys($aArticle));
  60                                 var_dump($aArticle);
  61                                 exit;
  62                         }
  63
  64                         $iPID = $iQID = null;
  65                         if ($aArticle['entity'][0] == 'p') {
  66                                 $iPID = (int)substr($aArticle['entity'], 1);
  67                         } else if ($aArticle['entity'][0] == 'q') {
  68                                 $iQID = (int)substr($aArticle['entity'], 1);
  69                         } else {
  70                                 continue;
  71                         }
  72
  73                         echo ".";
  74
  75                         fputcsv($hFileEntity, array($iID,$sTitle,$iPID,$iQID,@$aArticle['datatype']));
  76
  77                         foreach($aArticle['label'] as $sLang => $sLabel) {
  78                                 fputcsv($hFileEntityLabel, array($iID,$sLang,$sLabel));
  79                                 // echo "insert into entity_label values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
  80                         }
  81
  82                         foreach($aArticle['description'] as $sLang => $sLabel) {
  83                                 fputcsv($hFileEntityDescription, array($iID,$sLang,$sLabel));
  84                                 // echo "insert into entity_description values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
  85                         }
  86
  87                         foreach($aArticle['aliases'] as $sLang => $aLabels) {
  88                                 $aUniqueAlias = array();
  89                                 foreach($aLabels as $sLabel) {
  90                                         if (!isset($aUniqueAlias[$sLabel]) && $sLabel) {
  91                                         fputcsv($hFileEntityAlias, array($iID,$sLang,$sLabel));
  92                                         // echo "insert into entity_alias values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
  93                                         $aUniqueAlias[$sLabel] = true;
  94                                 }
  95                         }
  96                 }
  97
  98                 foreach($aArticle['links'] as $sLang => $sLabel) {
  99                         fputcsv($hFileEntityLink, array($iID,$sLang,$sLabel));
 100                         // echo "insert into entity_link values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
 101                 }
 102
 103
 104                 if (isset($aArticle['claims'])) {
 105
 106                         foreach($aArticle['claims'] as $iClaim => $aClaim) {
 107
 108                                 $bFail = false;
 109                         if ($aClaim['m'][0] == 'novalue') continue;
 110                         if ($aClaim['m'][0] == 'somevalue') continue;
 111                         $iPID = (int)$aClaim['m'][1];
 112                         if ($aClaim['m'][0] != 'value') $bFail = true;
 113                         if ($aClaim['m'][2]== 'wikibase-entityid') {
 114
 115                                 if ($aClaim['m'][3]['entity-type'] != 'item') $bFail = true;
 116                                 fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,$aClaim['m'][3]['numeric-id'],null,null));
 117                                 // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,".$aClaim['m'][3]['numeric-id'].",null);\n";
 118
 119                         } elseif ($aClaim['m'][2] == 'globecoordinate') {
 120
 121                                 if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true;
 122                                 fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,"SRID=4326;POINT(".((float)$aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")",null));
 123                                 // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']).", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n";
 124
 125                         } elseif ($aClaim['m'][2] == 'time') {
 126                                 // TODO!
 127 /*
 128                 if ($aClaim['m'][3]['calendarmodel'] == 'http://www.wikidata.org/entity/Q1985727') {
 129                         // Gregorian
 130                         if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
 131                                 if ((int)$aMatch[2] < 4700 && ) {
 132                                         $sDateString = $aMatch[2].'-'.$aMatch[3].($aClaim['m'][3]['timezone']>=0?'+':'').$aClaim['m'][3]['timezone'].($aMatch[1]=='-'?' bc':'');
 133                                         fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,null,$sDateString));
 134                                 }
 135                         } else {
 136 //                              $bFail = true;
 137                         }
 138                 } elseif ( $aClaim['m'][3]['calendarmodel'] != 'http://www.wikidata.org/entity/Q1985786') {
 139 / *
 140                         // Julian
 141                         if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
 142 var_dump($aMatch);
 143 exit;
 144 $iDayCount = juliantojd(2, 11, 1732);
 145 var_dump($iDayCount, jdtogregorian($iDayCount));
 146                         } else {
 147                                 $bFail = true;
 148 exit;
 149                         }
 150 exit;
 151 * /
 152                 } else {
 153 //                      $bFail = true;
 154                 }
 155 */
 156                         } elseif ($aClaim['m'][2] == 'string') {
 157
 158                                 // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",'".pg_escape_string($aClaim['m'][3])."',null,null);\n";
 159                                 fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,$aClaim['m'][3],null,null,null));
 160
 161                         } else {
 162
 163                                 $bFail = true;
 164
 165                         }
 166
 167                         // Don't care about sources:    if ($aClaim['refs'] != array()) $bFail = true;
 168
 169                         if ($bFail) {
 170                                 var_dump($sTitle);
 171                                 var_dump($aClaim);
 172                         } else {
 173                                 // process
 174                         }
 175
 176                 }
 177
 178                         }
 179                 }
 180         }
 181         fclose($hFile);
 182         fclose($hFileEntity);
 183         fclose($hFileEntityLabel);
 184         fclose($hFileEntityDescription);
 185         fclose($hFileEntityAlias);
 186         fclose($hFileEntityLink);
 187         fclose($hFileEntityProperty);
 188 }