]> git.openstreetmap.org Git - nominatim.git/blob - wikidata/pull_wikipedia_logs.php
Removed unnecessary test for tiger.
[nominatim.git] / wikidata / pull_wikipedia_logs.php
1 <?php
2
3         for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60)
4         {
5                 $sYear = date("Y", $iTimestamp);
6                 $sMonth = date("Y-m", $iTimestamp);
7                 $sDay = date("Ymd", $iTimestamp);
8
9                 for($iHour = 0; $iHour < 24; $iHour++)
10                 {
11                         $sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
12                         echo $sFilename."\n";
13                         if (!file_exists($sFilename.'.gz'))
14                         {
15                                 exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
16                         }
17
18                         exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
19
20                         $hPrevTotals = @fopen("totals.txt", "r");
21                         $hDayTotals = @fopen("hour.txt", "r");
22                         $hNewTotals = @fopen("newtotals.txt", "w");
23
24                         $sPrevKey = $sDayKey = true;
25                         $sPrevLine = true;
26                         $sDayLine = true;
27
28                         do
29                         {
30                                 if ($sPrevKey === $sDayKey)
31                                 {
32                                         if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n");
33                                         $sPrevLine = true;
34                                         $sDayLine = true;
35                                 }
36                                 else if ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false))
37                                 {
38                                         fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n");
39                                         $sDayLine = true;
40                                 }
41                                 else if ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false))
42                                 {
43                                         fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n");
44                                         $sPrevLine = true;
45                                 }
46
47                                 if ($sPrevLine === true)
48                                 {
49                                         $sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false;
50                                         if ($sPrevLine !== false)
51                                         {
52                                                 $aPrevLine = explode(' ', $sPrevLine);
53                                                 $sPrevKey = $aPrevLine[0].' '.$aPrevLine[1];
54                                                 $iPrevValue = (int)$aPrevLine[2];
55                                         }
56                                         else
57                                         {
58                                                 $sPrevKey = false;
59                                                 $iPrevValue =  0;
60                                         }
61                                 }
62
63                                 if ($sDayLine === true)
64                                 {
65                                         $sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false;
66                                         if ($sDayLine !== false)
67                                         {
68                                                 preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch);
69                                                 $sDayKey = $aMatch[1].' '.$aMatch[2];
70                                                 $iDayValue = (int)$aMatch[3];
71                                         }
72                                         else
73                                         {
74                                                 $sDayKey = false;
75                                                 $iDayValue = 0;
76                                         }
77                                 }
78
79                         } while ($sPrevLine !== false || $sDayLine !== false);
80
81                         @fclose($hPrevTotals);
82                         @fclose($hDayTotals);
83                         @fclose($hNewTotals);
84
85                         @unlink("totals.txt");
86                         rename("newtotals.txt", "totals.txt");
87                 }
88         }
89
90 // Notes:
91 /*
92  gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" | sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1' and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$
93  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = s,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
94  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = coalesce(hits,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
95 */