From: Tom Hughes <tom@compton.nu>
Date: Tue, 18 Jun 2013 20:10:19 +0000 (+0100)
Subject: Bring some more bits of the planet server under chef control
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/fc9f955d9561602e9c5e55146fd28e7f71d9d532

Bring some more bits of the planet server under chef control
---

diff --git a/cookbooks/planet/files/default/bin/apache-latest-planet-filename b/cookbooks/planet/files/default/bin/apache-latest-planet-filename
new file mode 100755
index 000000000..e5e35819a
--- /dev/null
+++ b/cookbooks/planet/files/default/bin/apache-latest-planet-filename
@@ -0,0 +1,10 @@
+#!/usr/bin/perl
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+$| = 1;
+while (<STDIN>) {
+	my $where = readlink("/store/planet/planet/planet-latest.osm.bz2");
+	s/planet\/planet\-latest\.osm\.bz2/planet\/$where/g;
+	print $_;
+}
diff --git a/cookbooks/planet/files/default/bin/planet-mirror-redirect-update b/cookbooks/planet/files/default/bin/planet-mirror-redirect-update
new file mode 100755
index 000000000..973137671
--- /dev/null
+++ b/cookbooks/planet/files/default/bin/planet-mirror-redirect-update
@@ -0,0 +1,56 @@
+#!/usr/bin/php
+<?
+/*
+	DO NOT EDIT - This file is being maintained by Chef
+
+	planet-mirror-redirect
+	Check if planet file exists on mirror and link if not yet linked
+	Modifies .htaccess
+*/
+$_YEAR = date('Y');
+$_PLANET_FOLDER = '/store/planet/planet/'.$_YEAR.'/';
+$_PLANET_REGEX = "/^(planet|changesets)\-\d{6}(\-nolt)?\.osm\.(bz2|gz)$/";
+$_MIRROR = 'http://ftp.heanet.ie/mirrors/openstreetmap.org/planet/'.$_YEAR.'/';
+$_PLANET_HTACCESS = $_PLANET_FOLDER.'../../.htaccess';
+
+function _MIRROR_FILE_SIZE($url) {
+	$ch = @curl_init();
+	curl_setopt($ch, CURLOPT_URL, $url);
+	curl_setopt($ch, CURLOPT_NOBODY, TRUE);
+	curl_setopt($ch, CURLOPT_VERBOSE, FALSE);
+	$curl_response = @curl_exec($ch);
+	$curl_result = curl_getinfo($ch);
+	if ($curl_result['http_code']!='200') return FALSE;
+	return ($curl_result['download_content_length']);
+}			
+
+if (posix_getuid() !== fileowner($_PLANET_HTACCESS)) die('User ID of process does not match .htaccess owner'."\n");
+
+if (is_dir($_PLANET_FOLDER)) {
+	$htaccess_contents = file_get_contents($_PLANET_HTACCESS);
+	$htaccess_handle = fopen($_PLANET_HTACCESS, 'a');
+	if ($dh = opendir($_PLANET_FOLDER)) {
+		while (($file = readdir($dh)) !== false ) {
+			if (preg_match($_PLANET_REGEX,$file)) {
+				$file_slashed = 'planet/'.$_YEAR.'/'.str_replace(array('.','-'), array('\.','\-'), $file);
+				if (strpos($htaccess_contents,$file_slashed) === false) {
+					//PHP IS BRAINDEAD filesize borked >4GB
+					//$file_size = $file_stats['size'];
+					$file_size = trim(`stat -c%s $_PLANET_FOLDER$file`);
+					sleep(rand(2,5));
+					$file_mirror_size = _MIRROR_FILE_SIZE($_MIRROR.$file);
+					if ($file_size==$file_mirror_size) {
+							echo 'Adding: '.$file."\n";
+							fwrite($htaccess_handle,	'RewriteRule'."\t".
+											'^('.$file_slashed.')$'."\t".
+											'http://ftp.heanet.ie/mirrors/openstreetmap.org/$1'."\t".
+											'[R,L]'."\n");
+					}
+				}
+			}
+		}
+		closedir($dh);
+		fclose($htaccess_handle);
+	}
+}
+?>
diff --git a/cookbooks/planet/files/default/bin/planet2pbf b/cookbooks/planet/files/default/bin/planet2pbf
new file mode 100755
index 000000000..3c2c807f2
--- /dev/null
+++ b/cookbooks/planet/files/default/bin/planet2pbf
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# DO NOT EDIT - This file is being maintained by Chef
+
+set -e
+user=`whoami`
+if [ "$user" != "www-data" ];
+	then
+		echo 'Error: run as www-data'
+		exit 1
+	fi
+
+if [ -f /tmp/planet2pbf.lock ]; then
+	if [ "$(ps -p `cat /tmp/planet2pbf.lock` | wc -l)" -gt 1 ]; then
+		echo planet2pbf process is still running
+		exit 1
+	else
+		# process not running, but lock file not deleted?
+		rm /tmp/planet2pbf.lock
+	fi
+fi
+echo $$ >/tmp/planet2pbf.lock
+
+YEARFULL=`TZ=GMT+24 date +%Y`
+
+cd /store/planet/planet/
+if [ -z $1 ]
+then
+	YEAR=`TZ=GMT+24 date +%y`
+	MONTH=`TZ=GMT+24 date +%m`
+	DAY=`TZ=GMT+24 date +%d`
+	if [ -f ${YEARFULL}/planet-${YEAR}${MONTH}${DAY}.osm.bz2 -o -f ${YEARFULL}/.planet-${YEAR}${MONTH}${DAY}.osm.bz2.new ];
+	then
+		planet=planet-${YEAR}${MONTH}${DAY}.osm
+	else
+		echo Error /store/planet/planet/${YEARFULL}/planet-${YEAR}${MONTH}${DAY}.osm.bz2 not available
+		exit 1
+	fi
+else
+	planet=$1
+fi
+while [ -f ${YEARFULL}/.${planet}.bz2.new ]; 
+do 
+	echo sleeping - ${planet}.bz2
+	sleep 300
+done
+if [ -f /store/planet/planet/${YEARFULL}/${planet}.bz2 ]
+then
+	echo $planet
+	if [ -f /store/planet/pbf/${planet}.pbf ];
+	then
+		echo Error: file /store/planet/pbf/${planet}.pbf exists
+		exit 1
+	fi
+	JAVACMD_OPTIONS=-Xmx2048M
+	export JAVACMD_OPTIONS
+	pbzip2 -dc /store/planet/planet/${YEARFULL}/${planet}.bz2 | /home/grant/code/osmosis-0.43/bin/osmosis --read-xml - --buffer bufferCapacity=18000 --write-pbf /store/planet/pbf/${planet}.pbf
+	cd /store/planet/pbf/
+	md5sum ${planet}.pbf >${planet}.pbf.md5
+	ln -fs ${planet}.pbf planet-latest.osm.pbf
+	ln -fs ${planet}.pbf.md5 planet-latest.osm.pbf.md5
+else
+	echo Error no file - ${planet}.bz2
+fi
+
+rm /tmp/planet2pbf.lock
+
diff --git a/cookbooks/planet/recipes/default.rb b/cookbooks/planet/recipes/default.rb
index d9db270db..5cfdf5416 100644
--- a/cookbooks/planet/recipes/default.rb
+++ b/cookbooks/planet/recipes/default.rb
@@ -19,6 +19,26 @@
 
 include_recipe "apache::ssl"
 
+package "perl"
+package "php5-cli"
+
+remote_directory "/usr/local/bin" do
+  source "bin"
+  owner "root"
+  group "root"
+  mode 0755
+  files_owner "root"
+  files_group "root"
+  files_mode 0755
+end
+
+template "/etc/cron.d/planet" do
+  source "cron.erb"
+  owner "root"
+  group "root"
+  mode 0644
+end
+
 apache_module "rewrite" do
   conf "rewrite.conf.erb"
 end
diff --git a/cookbooks/planet/templates/default/cron.erb b/cookbooks/planet/templates/default/cron.erb
new file mode 100644
index 000000000..70ec98c04
--- /dev/null
+++ b/cookbooks/planet/templates/default/cron.erb
@@ -0,0 +1,4 @@
+# DO NOT EDIT - This file is being maintained by Chef
+MAILTO=horntail-www-data-cron@firefishy.com
+20 */2 * * * www-data /usr/local/bin/planet-mirror-redirect-update
+39 10  * * 4 www-data /usr/local/bin/planet2pbf