US TIGER 2019 data got released

[nominatim.git] / wikidata / pull_wikipedia_logs.php

diff --git a/wikidata/pull_wikipedia_logs.php b/wikidata/pull_wikipedia_logs.php
index a7f6a9a3e21c3cb8b8a70ecd16b06fe97fdfb959..3e172069d7113f1a171c3d49e55b429133445f5c 100644 (file)
--- a/wikidata/pull_wikipedia_logs.php
+++ b/wikidata/pull_wikipedia_logs.php
@@ -1,12 +1,12 @@
 <?php
 
 for ($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60) {
 <?php
 
 for ($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60) {

-    $sYear = date("Y", $iTimestamp);
-    $sMonth = date("Y-m", $iTimestamp);
-    $sDay = date("Ymd", $iTimestamp);
+    $sYear = date('Y', $iTimestamp);
+    $sMonth = date('Y-m', $iTimestamp);
+    $sDay = date('Ymd', $iTimestamp);

 
     for ($iHour = 0; $iHour < 24; $iHour++) {
 
     for ($iHour = 0; $iHour < 24; $iHour++) {

-        $sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
+        $sFilename = sprintf('pagecounts-'.$sDay.'-%02d0000', $iHour);

         echo $sFilename."\n";
         if (!file_exists($sFilename.'.gz')) {
             exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
         echo $sFilename."\n";
         if (!file_exists($sFilename.'.gz')) {
             exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');


@@ -14,9 +14,9 @@ for ($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6,
 
         exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
 
 
         exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
 

-        $hPrevTotals = @fopen("totals.txt", "r");
-        $hDayTotals = @fopen("hour.txt", "r");
-        $hNewTotals = @fopen("newtotals.txt", "w");
+        $hPrevTotals = @fopen('totals.txt', 'r');
+        $hDayTotals = @fopen('hour.txt', 'r');
+        $hNewTotals = @fopen('newtotals.txt', 'w');

 
         $sPrevKey = $sDayKey = true;
         $sPrevLine = true;
 
         $sPrevKey = $sDayKey = true;
         $sPrevLine = true;


@@ -64,14 +64,16 @@ for ($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6,
         @fclose($hDayTotals);
         @fclose($hNewTotals);
 
         @fclose($hDayTotals);
         @fclose($hNewTotals);
 

-        @unlink("totals.txt");
-        rename("newtotals.txt", "totals.txt");
+        @unlink('totals.txt');
+        rename('newtotals.txt', 'totals.txt');

     }
 }
 
 // Notes:
 /*
     }
 }
 
 // Notes:
 /*

- gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" | sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1' and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$
+    gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" |
+    sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1'
+      and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$

  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = s,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = coalesce(hits,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
 */
  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = s,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
  cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = coalesce(hits,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
 */