wikidata/parse.php

   1 #!/usr/bin/php -Cq
   2 <?php
   3
   4 $hFile = @fopen('wikidatawiki-20130623-pages-articles.xml', 'r');
   5
   6 $hFileEntity = fopen('entity.csv', 'w');
   7 $hFileEntityLabel = fopen('entity_label.csv', 'w');
   8 $hFileEntityDescription = fopen('entity_description.csv', 'w');
   9 $hFileEntityAlias = fopen('entity_alias.csv', 'w');
  10 $hFileEntityLink = fopen('entity_link.csv', 'w');
  11 $hFileEntityProperty = fopen('entity_property.csv', 'w');
  12
  13 $iCount = 0;
  14
  15 $sTitle = '';
  16 $iNS = false;
  17 $iID = false;
  18
  19 if ($hFile) {
  20     while (($sLine = fgets($hFile, 4000000)) !== false) {
  21         if (substr($sLine, 0, 11) == '    <title>') {
  22             $sTitle = substr($sLine, 11, -9);
  23         } elseif (substr($sLine, 0, 8) == '    <ns>') {
  24             $iNS = (int)substr($sLine, 8, -6);
  25         } elseif (substr($sLine, 0, 8) == '    <id>') {
  26             $iID = (int)substr($sLine, 8, -6);
  27         } elseif (substr($sLine, 0, 33) == '      <text xml:space="preserve">') {
  28             if ($iNS == -2) continue;
  29             if ($iNS == -1) continue;
  30             if ($iNS == 1) continue;
  31             if ($iNS == 2) continue;
  32             if ($iNS == 3) continue;
  33             if ($iNS == 4) continue;
  34             if ($iNS == 5) continue;
  35             if ($iNS == 6) continue;
  36             if ($iNS == 7) continue;
  37             if ($iNS == 8) continue;
  38             if ($iNS == 9) continue;
  39             if ($iNS == 10) continue;
  40             if ($iNS == 11) continue;
  41             if ($iNS == 12) continue;
  42             if ($iNS == 13) continue;
  43             if ($iNS == 14) continue;
  44             if ($iNS == 15) continue;
  45             if ($iNS == 121) continue;
  46             if ($iNS == 123) continue;
  47             if ($iNS == 829) continue;
  48             if ($iNS == 1198) continue;
  49             if ($iNS == 1199) continue;
  50             $sText = html_entity_decode(substr($sLine, 33, -8), ENT_COMPAT, 'UTF-8');
  51             $aArticle = json_decode($sText, true);
  52
  53             if (array_diff(array_keys($aArticle), array('label', 'description', 'aliases', 'links', 'entity', 'claims', 'datatype')) != array()) {
  54                 // DEBUG
  55                 var_dump($sTitle);
  56                 var_dump(array_keys($aArticle));
  57                 var_dump($aArticle);
  58                 exit;
  59             }
  60
  61             $iPID = $iQID = null;
  62             if ($aArticle['entity'][0] == 'p') {
  63                 $iPID = (int) substr($aArticle['entity'], 1);
  64             } elseif ($aArticle['entity'][0] == 'q') {
  65                 $iQID = (int) substr($aArticle['entity'], 1);
  66             } else {
  67                 continue;
  68             }
  69
  70             echo '.';
  71
  72             fputcsv($hFileEntity, array($iID, $sTitle, $iPID, $iQID, @$aArticle['datatype']));
  73
  74             foreach ($aArticle['label'] as $sLang => $sLabel) {
  75                 fputcsv($hFileEntityLabel, array($iID, $sLang, $sLabel));
  76                 // echo "insert into entity_label values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
  77             }
  78
  79             foreach ($aArticle['description'] as $sLang => $sLabel) {
  80                 fputcsv($hFileEntityDescription, array($iID, $sLang, $sLabel));
  81                 // echo "insert into entity_description values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
  82             }
  83
  84             foreach ($aArticle['aliases'] as $sLang => $aLabels) {
  85                 $aUniqueAlias = array();
  86                 foreach ($aLabels as $sLabel) {
  87                     if (!isset($aUniqueAlias[$sLabel]) && $sLabel) {
  88                         fputcsv($hFileEntityAlias, array($iID, $sLang, $sLabel));
  89                         // echo "insert into entity_alias values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
  90                         $aUniqueAlias[$sLabel] = true;
  91                     }
  92                 }
  93             }
  94
  95             foreach ($aArticle['links'] as $sLang => $sLabel) {
  96                 fputcsv($hFileEntityLink, array($iID, $sLang, $sLabel));
  97                 // echo "insert into entity_link values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
  98             }
  99
 100
 101             if (isset($aArticle['claims'])) {
 102                 //
 103                 foreach ($aArticle['claims'] as $iClaim => $aClaim) {
 104                     //
 105                     $bFail = false;
 106                     if ($aClaim['m'][0] == 'novalue') continue;
 107                     if ($aClaim['m'][0] == 'somevalue') continue;
 108                     $iPID = (int)$aClaim['m'][1];
 109                     if ($aClaim['m'][0] != 'value') $bFail = true;
 110                     if ($aClaim['m'][2]== 'wikibase-entityid') {
 111                         //
 112                         if ($aClaim['m'][3]['entity-type'] != 'item') $bFail = true;
 113                         fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, null, $aClaim['m'][3]['numeric-id'], null, null));
 114                         // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,".$aClaim['m'][3]['numeric-id'].",null);\n";
 115                     } elseif ($aClaim['m'][2] == 'globecoordinate') {
 116                         //
 117                         if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true;
 118                         fputcsv(
 119                             $hFileEntityProperty,
 120                             array(
 121                              $iID,
 122                              $iClaim,
 123                              $iPID,
 124                              null,
 125                              null,
 126                              'SRID=4326;POINT('.((float) $aClaim['m'][3]['longitude']).' '.((float)$aClaim['m'][3]['latitude']).')', null
 127                             )
 128                         );
 129                         /* echo "insert into entity_property values (nextval('seq_entity_property'),";
 130                          * echo $iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']);
 131                          * echo ", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n";
 132                          */
 133                     } elseif ($aClaim['m'][2] == 'time') {
 134                         // TODO!
 135                         /*
 136                         if ($aClaim['m'][3]['calendarmodel'] == 'http://www.wikidata.org/entity/Q1985727') {
 137                             // Gregorian
 138                             if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
 139                                 if ((int)$aMatch[2] < 4700 && ) {
 140                                     $sDateString = $aMatch[2].'-'.$aMatch[3].($aClaim['m'][3]['timezone']>=0?'+':'').$aClaim['m'][3]['timezone'].($aMatch[1]=='-'?' bc':'');
 141                                     fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,null,$sDateString));
 142                                 }
 143                             } else {
 144                                 // $bFail = true;
 145                             }
 146                         } elseif ( $aClaim['m'][3]['calendarmodel'] != 'http://www.wikidata.org/entity/Q1985786') {
 147                             // Julian
 148                             if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
 149                                 var_dump($aMatch);
 150                                 exit;
 151                                 $iDayCount = juliantojd(2, 11, 1732);
 152                                 var_dump($iDayCount, jdtogregorian($iDayCount));
 153                         } else {
 154                             $bFail = true;
 155                             exit;
 156                         }
 157                         exit;
 158                     } else {
 159                         // $bFail = true;
 160                     }
 161                     */
 162                     } elseif ($aClaim['m'][2] == 'string') {
 163                         // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",'".pg_escape_string($aClaim['m'][3])."',null,null);\n";
 164                         fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, $aClaim['m'][3], null, null, null));
 165                     } else {
 166                         $bFail = true;
 167                     }
 168
 169                     // Don't care about sources:    if ($aClaim['refs'] != array()) $bFail = true;
 170
 171                     if ($bFail) {
 172                         var_dump($sTitle);
 173                         var_dump($aClaim);
 174                     } else {
 175                         // process
 176                     }
 177                 }
 178             }
 179         }
 180     }
 181     fclose($hFile);
 182     fclose($hFileEntity);
 183     fclose($hFileEntityLabel);
 184     fclose($hFileEntityDescription);
 185     fclose($hFileEntityAlias);
 186     fclose($hFileEntityLink);
 187     fclose($hFileEntityProperty);
 188 }