Planet redirect: add pbf+full history

Fixes https://github.com/openstreetmap/operations/issues/355

diff --git a/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb b/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb
index 81c9f2ae3d260a2c5a819a213ae11d006d944339..33abc665149e5988dd1ddd92a1c2ea0f21a48c89 100644 (file)
--- a/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb
+++ b/cookbooks/planet/templates/default/planet-mirror-redirect-update.erb
@@ -8,9 +8,34 @@
        Modifies .htaccess
 */
 $_YEAR = date('Y');
        Modifies .htaccess
 */
 $_YEAR = date('Y');

-$_PLANET_FOLDER = '<%= node[:planet][:dump][:xml_directory] %>/'.$_YEAR.'/';
-$_PLANET_REGEX = "/^(planet|changesets)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/";
-$_MIRROR = 'https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/planet/'.$_YEAR.'/';
+
+$_MIRROR = 'https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/';
+
+$_FOLDERS = [
+              "planet_bz2" => [
+                          'FOLDER'     => '<%= node[:planet][:dump][:xml_directory] %>/'.$_YEAR.'/',
+                          'REGEX'      => '/^(planet|changesets|discussions)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/',
+                          'DIR_PREFIX' => 'planet/'.$_YEAR.'/'
+                          ],
+
+              "history_bz2" => [
+                          'FOLDER' => '<%= node[:planet][:dump][:xml_history_directory] %>/'.$_YEAR.'/',
+                          'REGEX'  => '/^(history)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/',
+                          'DIR_PREFIX' => 'planet/full-history/'.$_YEAR.'/'
+                          ],
+              "planet_pbf" => [
+                          'FOLDER'     => '<%= node[:planet][:dump][:pbf_directory] %>/',
+                          'REGEX'      => '/^(planet)\-\d{6}(\-nolt)?\.osm\.pbf$/',
+                          'DIR_PREFIX' => 'pbf/'
+                          ],
+
+              "history_pbf" => [
+                          'FOLDER' => '<%= node[:planet][:dump][:pbf_history_directory] %>/',
+                          'REGEX'  => '/^(history)\-\d{6}(\-nolt)?\.osm\.pbf$/',
+                          'DIR_PREFIX' => 'pbf/full-history/'
+                          ]
+            ];
+

 $_PLANET_HTACCESS = realpath('<%= node[:planet][:dump][:xml_directory] %>/..').'/.htaccess';
 
 function _MIRROR_FILE_SIZE($url) {
 $_PLANET_HTACCESS = realpath('<%= node[:planet][:dump][:xml_directory] %>/..').'/.htaccess';
 
 function _MIRROR_FILE_SIZE($url) {


@@ -26,28 +51,36 @@ function _MIRROR_FILE_SIZE($url) {
 
 if (!is_writable($_PLANET_HTACCESS)) die('File '.$_PLANET_HTACCESS.' is not writable by current user.'."\n");
 
 
 if (!is_writable($_PLANET_HTACCESS)) die('File '.$_PLANET_HTACCESS.' is not writable by current user.'."\n");
 

-if (is_dir($_PLANET_FOLDER)) {
-       $htaccess_contents = file_get_contents($_PLANET_HTACCESS);
-       $htaccess_handle = fopen($_PLANET_HTACCESS, 'a');
-       if ($dh = opendir($_PLANET_FOLDER)) {
-               while (($file = readdir($dh)) !== false ) {
-                       if (preg_match($_PLANET_REGEX,$file)) {
-                               $file_slashed = 'planet/'.$_YEAR.'/'.str_replace(array('.','-'), array('\.','\-'), $file);
-                               if (strpos($htaccess_contents,$file_slashed) === false) {
-                                       $file_size = filesize($_PLANET_FOLDER.$file);
-                                       sleep(rand(2,5));
-                                       $file_mirror_size = _MIRROR_FILE_SIZE($_MIRROR.$file);
-                                       if ($file_size==$file_mirror_size) {
-                                                       echo 'Adding: '.$file."\n";
-                                                       fwrite($htaccess_handle,        'RewriteRule'."\t".
-                                                                                       '^('.$file_slashed.')$'."\t".
-                                                                                       'https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/$1'."\t".
-                                                                                       '[R,L]'."\n");
-                                       }
-                               }
-                       }
-               }
-               closedir($dh);
-               fclose($htaccess_handle);
-       }
+$htaccess_contents = file_get_contents($_PLANET_HTACCESS);
+$htaccess_handle = fopen($_PLANET_HTACCESS, 'a');
+
+foreach ($_FOLDERS as $FOLDER) {
+
+  $_PLANET_FOLDER = $FOLDER["FOLDER"];
+  $_PLANET_REGEX  = $FOLDER["REGEX"];
+
+  if (!(is_dir($_PLANET_FOLDER))) {
+    continue;
+  }
+
+  if ($dh = opendir($_PLANET_FOLDER)) {
+    while (($file = readdir($dh)) !== false ) {
+      if (preg_match($_PLANET_REGEX, $file)) {
+        $file_slashed = $FOLDER['DIR_PREFIX'].str_replace(array('.','-'), array('\.','\-'), $file);
+        if (strpos($htaccess_contents, $file_slashed) === false) {
+          $file_size = filesize($_PLANET_FOLDER.$file);
+          sleep(rand(2,5));
+          $file_mirror_size = _MIRROR_FILE_SIZE($_MIRROR.$FOLDER['DIR_PREFIX'].$file);
+          if ($file_size==$file_mirror_size) {
+            echo 'Adding: '.$file."\n";
+            fwrite($htaccess_handle, 'RewriteRule'."\t". '^('.$file_slashed.')$'."\t".$_MIRROR.'$1'."\t".'[R,L]'."\n");
+          }
+        }
+      }
+    }
+    closedir($dh);
+  }

 }
 }

+
+fclose($htaccess_handle);
+