wikidata/pull_wikipedia_logs.php

   1 <?php
   2
   3         for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60)
   4         {
   5                 $sYear = date("Y", $iTimestamp);
   6                 $sMonth = date("Y-m", $iTimestamp);
   7                 $sDay = date("Ymd", $iTimestamp);
   8
   9                 for($iHour = 0; $iHour < 24; $iHour++)
  10                 {
  11                         $sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
  12                         echo $sFilename."\n";
  13                         if (!file_exists($sFilename.'.gz'))
  14                         {
  15                                 exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
  16                         }
  17
  18                         exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
  19
  20                         $hPrevTotals = @fopen("totals.txt", "r");
  21                         $hDayTotals = @fopen("hour.txt", "r");
  22                         $hNewTotals = @fopen("newtotals.txt", "w");
  23
  24                         $sPrevKey = $sDayKey = true;
  25                         $sPrevLine = true;
  26                         $sDayLine = true;
  27
  28                         do
  29                         {
  30                                 if ($sPrevKey === $sDayKey)
  31                                 {
  32                                         if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n");
  33                                         $sPrevLine = true;
  34                                         $sDayLine = true;
  35                                 }
  36                                 else if ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false))
  37                                 {
  38                                         fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n");
  39                                         $sDayLine = true;
  40                                 }
  41                                 else if ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false))
  42                                 {
  43                                         fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n");
  44                                         $sPrevLine = true;
  45                                 }
  46
  47                                 if ($sPrevLine === true)
  48                                 {
  49                                         $sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false;
  50                                         if ($sPrevLine !== false)
  51                                         {
  52                                                 $aPrevLine = explode(' ', $sPrevLine);
  53                                                 $sPrevKey = $aPrevLine[0].' '.$aPrevLine[1];
  54                                                 $iPrevValue = (int)$aPrevLine[2];
  55                                         }
  56                                         else
  57                                         {
  58                                                 $sPrevKey = false;
  59                                                 $iPrevValue =  0;
  60                                         }
  61                                 }
  62
  63                                 if ($sDayLine === true)
  64                                 {
  65                                         $sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false;
  66                                         if ($sDayLine !== false)
  67                                         {
  68                                                 preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch);
  69                                                 $sDayKey = $aMatch[1].' '.$aMatch[2];
  70                                                 $iDayValue = (int)$aMatch[3];
  71                                         }
  72                                         else
  73                                         {
  74                                                 $sDayKey = false;
  75                                                 $iDayValue = 0;
  76                                         }
  77                                 }
  78
  79                         } while ($sPrevLine !== false || $sDayLine !== false);
  80
  81                         @fclose($hPrevTotals);
  82                         @fclose($hDayTotals);
  83                         @fclose($hNewTotals);
  84
  85                         @unlink("totals.txt");
  86                         rename("newtotals.txt", "totals.txt");
  87                 }
  88         }
  89
  90 // Notes:
  91 /*
  92  gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" | sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1' and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$
  93  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = s,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
  94  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = coalesce(hits,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
  95 */