Bring some more bits of the planet server under chef control

diff --git a/cookbooks/planet/files/default/bin/apache-latest-planet-filename b/cookbooks/planet/files/default/bin/apache-latest-planet-filename
new file mode 100755 (executable)
index 0000000..e5e3581
--- /dev/null
+++ b/cookbooks/planet/files/default/bin/apache-latest-planet-filename
@@ -0,0 +1,10 @@
+#!/usr/bin/perl
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+$| = 1;
+while (<STDIN>) {
+       my $where = readlink("/store/planet/planet/planet-latest.osm.bz2");
+       s/planet\/planet\-latest\.osm\.bz2/planet\/$where/g;
+       print $_;
+}

diff --git a/cookbooks/planet/files/default/bin/planet-mirror-redirect-update b/cookbooks/planet/files/default/bin/planet-mirror-redirect-update
new file mode 100755 (executable)
index 0000000..9731376
--- /dev/null
+++ b/cookbooks/planet/files/default/bin/planet-mirror-redirect-update
@@ -0,0 +1,56 @@
+#!/usr/bin/php
+<?
+/*
+       DO NOT EDIT - This file is being maintained by Chef
+
+       planet-mirror-redirect
+       Check if planet file exists on mirror and link if not yet linked
+       Modifies .htaccess
+*/
+$_YEAR = date('Y');
+$_PLANET_FOLDER = '/store/planet/planet/'.$_YEAR.'/';
+$_PLANET_REGEX = "/^(planet|changesets)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/";
+$_MIRROR = 'http://ftp.heanet.ie/mirrors/openstreetmap.org/planet/'.$_YEAR.'/';
+$_PLANET_HTACCESS = $_PLANET_FOLDER.'../../.htaccess';
+
+function _MIRROR_FILE_SIZE($url) {
+       $ch = @curl_init();
+       curl_setopt($ch, CURLOPT_URL, $url);
+       curl_setopt($ch, CURLOPT_NOBODY, TRUE);
+       curl_setopt($ch, CURLOPT_VERBOSE, FALSE);
+       $curl_response = @curl_exec($ch);
+       $curl_result = curl_getinfo($ch);
+       if ($curl_result['http_code']!='200') return FALSE;
+       return ($curl_result['download_content_length']);
+}                      
+
+if (posix_getuid() !== fileowner($_PLANET_HTACCESS)) die('User ID of process does not match .htaccess owner'."\n");
+
+if (is_dir($_PLANET_FOLDER)) {
+       $htaccess_contents = file_get_contents($_PLANET_HTACCESS);
+       $htaccess_handle = fopen($_PLANET_HTACCESS, 'a');
+       if ($dh = opendir($_PLANET_FOLDER)) {
+               while (($file = readdir($dh)) !== false ) {
+                       if (preg_match($_PLANET_REGEX,$file)) {
+                               $file_slashed = 'planet/'.$_YEAR.'/'.str_replace(array('.','-'), array('\.','\-'), $file);
+                               if (strpos($htaccess_contents,$file_slashed) === false) {
+                                       //PHP IS BRAINDEAD filesize borked >4GB
+                                       //$file_size = $file_stats['size'];
+                                       $file_size = trim(`stat -c%s $_PLANET_FOLDER$file`);
+                                       sleep(rand(2,5));
+                                       $file_mirror_size = _MIRROR_FILE_SIZE($_MIRROR.$file);
+                                       if ($file_size==$file_mirror_size) {
+                                                       echo 'Adding: '.$file."\n";
+                                                       fwrite($htaccess_handle,        'RewriteRule'."\t".
+                                                                                       '^('.$file_slashed.')$'."\t".
+                                                                                       'http://ftp.heanet.ie/mirrors/openstreetmap.org/$1'."\t".
+                                                                                       '[R,L]'."\n");
+                                       }
+                               }
+                       }
+               }
+               closedir($dh);
+               fclose($htaccess_handle);
+       }
+}
+?>

diff --git a/cookbooks/planet/files/default/bin/planet2pbf b/cookbooks/planet/files/default/bin/planet2pbf
new file mode 100755 (executable)
index 0000000..3c2c807
--- /dev/null
+++ b/cookbooks/planet/files/default/bin/planet2pbf
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+set -e
+user=`whoami`
+if [ "$user" != "www-data" ];
+       then
+               echo 'Error: run as www-data'
+               exit 1
+       fi
+
+if [ -f /tmp/planet2pbf.lock ]; then
+       if [ "$(ps -p `cat /tmp/planet2pbf.lock` | wc -l)" -gt 1 ]; then
+               echo planet2pbf process is still running
+               exit 1
+       else
+               # process not running, but lock file not deleted?
+               rm /tmp/planet2pbf.lock
+       fi
+fi
+echo $$ >/tmp/planet2pbf.lock
+
+YEARFULL=`TZ=GMT+24 date +%Y`
+
+cd /store/planet/planet/
+if [ -z $1 ]
+then
+       YEAR=`TZ=GMT+24 date +%y`
+       MONTH=`TZ=GMT+24 date +%m`
+       DAY=`TZ=GMT+24 date +%d`
+       if [ -f ${YEARFULL}/planet-${YEAR}${MONTH}${DAY}.osm.bz2 -o -f ${YEARFULL}/.planet-${YEAR}${MONTH}${DAY}.osm.bz2.new ];
+       then
+               planet=planet-${YEAR}${MONTH}${DAY}.osm
+       else
+               echo Error /store/planet/planet/${YEARFULL}/planet-${YEAR}${MONTH}${DAY}.osm.bz2 not available
+               exit 1
+       fi
+else
+       planet=$1
+fi
+while [ -f ${YEARFULL}/.${planet}.bz2.new ]; 
+do 
+       echo sleeping - ${planet}.bz2
+       sleep 300
+done
+if [ -f /store/planet/planet/${YEARFULL}/${planet}.bz2 ]
+then
+       echo $planet
+       if [ -f /store/planet/pbf/${planet}.pbf ];
+       then
+               echo Error: file /store/planet/pbf/${planet}.pbf exists
+               exit 1
+       fi
+       JAVACMD_OPTIONS=-Xmx2048M
+       export JAVACMD_OPTIONS
+       pbzip2 -dc /store/planet/planet/${YEARFULL}/${planet}.bz2 | /home/grant/code/osmosis-0.43/bin/osmosis --read-xml - --buffer bufferCapacity=18000 --write-pbf /store/planet/pbf/${planet}.pbf
+       cd /store/planet/pbf/
+       md5sum ${planet}.pbf >${planet}.pbf.md5
+       ln -fs ${planet}.pbf planet-latest.osm.pbf
+       ln -fs ${planet}.pbf.md5 planet-latest.osm.pbf.md5
+else
+       echo Error no file - ${planet}.bz2
+fi
+
+rm /tmp/planet2pbf.lock
+

diff --git a/cookbooks/planet/recipes/default.rb b/cookbooks/planet/recipes/default.rb
index d9db270dbcbeac72eb9fd7cf93d7c8b2d2486840..5cfdf541691f1654e34b45ef0757c9108c7dce62 100644 (file)
--- a/cookbooks/planet/recipes/default.rb
+++ b/cookbooks/planet/recipes/default.rb
@@ -19,6 +19,26 @@
 
 include_recipe "apache::ssl"
 
+package "perl"
+package "php5-cli"
+
+remote_directory "/usr/local/bin" do
+  source "bin"
+  owner "root"
+  group "root"
+  mode 0755
+  files_owner "root"
+  files_group "root"
+  files_mode 0755
+end
+
+template "/etc/cron.d/planet" do
+  source "cron.erb"
+  owner "root"
+  group "root"
+  mode 0644
+end
+
 apache_module "rewrite" do
   conf "rewrite.conf.erb"
 end

diff --git a/cookbooks/planet/templates/default/cron.erb b/cookbooks/planet/templates/default/cron.erb
new file mode 100644 (file)
index 0000000..70ec98c
--- /dev/null
+++ b/cookbooks/planet/templates/default/cron.erb
@@ -0,0 +1,4 @@
+# DO NOT EDIT - This file is being maintained by Chef
+MAILTO=horntail-www-data-cron@firefishy.com
+20 */2 * * * www-data /usr/local/bin/planet-mirror-redirect-update
+39 10  * * 4 www-data /usr/local/bin/planet2pbf