wikidata/pull_wikipedia_logs.php

   1 <?php
   2
   3 for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60)
   4 {
   5     $sYear = date("Y", $iTimestamp);
   6     $sMonth = date("Y-m", $iTimestamp);
   7     $sDay = date("Ymd", $iTimestamp);
   8
   9     for($iHour = 0; $iHour < 24; $iHour++)
  10     {
  11         $sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
  12         echo $sFilename."\n";
  13         if (!file_exists($sFilename.'.gz'))
  14         {
  15             exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
  16         }
  17
  18         exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
  19
  20         $hPrevTotals = @fopen("totals.txt", "r");
  21         $hDayTotals = @fopen("hour.txt", "r");
  22         $hNewTotals = @fopen("newtotals.txt", "w");
  23
  24         $sPrevKey = $sDayKey = true;
  25         $sPrevLine = true;
  26         $sDayLine = true;
  27
  28         do
  29         {
  30             if ($sPrevKey === $sDayKey)
  31             {
  32                 if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n");
  33                 $sPrevLine = true;
  34                 $sDayLine = true;
  35             }
  36             else if ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false))
  37             {
  38                 fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n");
  39                 $sDayLine = true;
  40             }
  41             else if ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false))
  42             {
  43                 fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n");
  44                 $sPrevLine = true;
  45             }
  46
  47             if ($sPrevLine === true)
  48             {
  49                 $sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false;
  50                 if ($sPrevLine !== false)
  51                 {
  52                     $aPrevLine = explode(' ', $sPrevLine);
  53                     $sPrevKey = $aPrevLine[0].' '.$aPrevLine[1];
  54                     $iPrevValue = (int)$aPrevLine[2];
  55                 }
  56                 else
  57                 {
  58                     $sPrevKey = false;
  59                     $iPrevValue =  0;
  60                 }
  61             }
  62
  63             if ($sDayLine === true)
  64             {
  65                 $sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false;
  66                 if ($sDayLine !== false)
  67                 {
  68                     preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch);
  69                     $sDayKey = $aMatch[1].' '.$aMatch[2];
  70                     $iDayValue = (int)$aMatch[3];
  71                 }
  72                 else
  73                 {
  74                     $sDayKey = false;
  75                     $iDayValue = 0;
  76                 }
  77             }
  78
  79         } while ($sPrevLine !== false || $sDayLine !== false);
  80
  81         @fclose($hPrevTotals);
  82         @fclose($hDayTotals);
  83         @fclose($hNewTotals);
  84
  85         @unlink("totals.txt");
  86         rename("newtotals.txt", "totals.txt");
  87     }
  88 }
  89
  90 // Notes:
  91 /*
  92  gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" | sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1' and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$
  93  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = s,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
  94  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = coalesce(hits,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
  95 */