From 243ec909a52ccbdb5bfdd5becdfb80ab6a76419e Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Thu, 20 Jun 2013 21:12:06 +0100 Subject: [PATCH] Add a cookbook to configure the backup server --- cookbooks/backup/README.rdoc | 8 ++ cookbooks/backup/attributes/default.rb | 1 + cookbooks/backup/files/default/expire-backups | 86 +++++++++++++++++++ cookbooks/backup/metadata.rb | 6 ++ cookbooks/backup/recipes/default.rb | 37 ++++++++ .../backup/templates/default/expire.cron.erb | 10 +++ roles/backup.rb | 37 ++++++++ roles/horntail.rb | 20 +---- 8 files changed, 186 insertions(+), 19 deletions(-) create mode 100644 cookbooks/backup/README.rdoc create mode 100644 cookbooks/backup/attributes/default.rb create mode 100644 cookbooks/backup/files/default/expire-backups create mode 100644 cookbooks/backup/metadata.rb create mode 100644 cookbooks/backup/recipes/default.rb create mode 100644 cookbooks/backup/templates/default/expire.cron.erb create mode 100644 roles/backup.rb diff --git a/cookbooks/backup/README.rdoc b/cookbooks/backup/README.rdoc new file mode 100644 index 000000000..3de2ec7a3 --- /dev/null +++ b/cookbooks/backup/README.rdoc @@ -0,0 +1,8 @@ += DESCRIPTION: + += REQUIREMENTS: + += ATTRIBUTES: + += USAGE: + diff --git a/cookbooks/backup/attributes/default.rb b/cookbooks/backup/attributes/default.rb new file mode 100644 index 000000000..c0b8de57f --- /dev/null +++ b/cookbooks/backup/attributes/default.rb @@ -0,0 +1 @@ +default[:stats][:sites] = [] diff --git a/cookbooks/backup/files/default/expire-backups b/cookbooks/backup/files/default/expire-backups new file mode 100644 index 000000000..4e26e3711 --- /dev/null +++ b/cookbooks/backup/files/default/expire-backups @@ -0,0 +1,86 @@ +#!/usr/bin/perl + +# DO NOT EDIT - This file is being maintained by Chef + +use strict; +use warnings; + +use Date::Calc qw(Today Add_Delta_Days Add_Delta_YMD Day_of_Week); +use Getopt::Long; + +my $days = 0; +my $weeks = 0; +my $months = 0; + +GetOptions("days=i" => \$days, "weeks=i" => \$weeks, "months=i" => \$months); + +my $dir = shift; +my $prefix = shift; +my @dates; + +for my $day (0 .. $days - 1) +{ + push(@dates, sprintf("%04d-%02d-%02d", Subtract_Delta_Days(Today(), $day))); +} + +for my $week (0 .. $weeks - 1) +{ + push(@dates, sprintf("%04d-%02d-%02d", Subtract_Delta_Days(Monday(), $week * 7))); +} + +for my $month (0 .. $months - 1) +{ + push(@dates, sprintf("%04d-%02d-%02d", Subtract_Delta_YMD(First_of_Month(), 0, $month, 0))); +} + +my $dates = join("|", @dates); +my $match = qr/^${prefix}-\d{4}-\d{2}-\d{2}\./; +my $keep = qr/^${prefix}-(?:${dates})\./; + +opendir(DIR, "$dir") || die "Can't open ${dir}: $!"; + +while (my $file = readdir(DIR)) +{ +# print "Expiring $file\n" if $file =~ $match && $file !~ $keep; + unlink("${dir}/${file}") if $file =~ $match && $file !~ $keep; +} + +closedir(DIR); + +exit 0; + +sub Monday +{ + my @today = Today(); + + return Subtract_Delta_Days(@today, Day_of_Week(@today) - 1); +} + +sub First_of_Month +{ + my($year,$month,$day) = Today(); + + return($year,$month,1); +} + +sub Subtract_Delta_Days +{ + my $year = shift; + my $month = shift; + my $day = shift; + my $days = shift; + + return Add_Delta_Days($year, $month, $day, -$days); +} + +sub Subtract_Delta_YMD +{ + my $year = shift; + my $month = shift; + my $day = shift; + my $years = shift; + my $months = shift; + my $days = shift; + + return Add_Delta_YMD($year, $month, $day, -$years, -$months, -$days); +} diff --git a/cookbooks/backup/metadata.rb b/cookbooks/backup/metadata.rb new file mode 100644 index 000000000..a6ab006c5 --- /dev/null +++ b/cookbooks/backup/metadata.rb @@ -0,0 +1,6 @@ +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache 2.0" +description "Installs and configures backup.openstreetmap.org" +long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc')) +version "1.0.0" diff --git a/cookbooks/backup/recipes/default.rb b/cookbooks/backup/recipes/default.rb new file mode 100644 index 000000000..a9b70f168 --- /dev/null +++ b/cookbooks/backup/recipes/default.rb @@ -0,0 +1,37 @@ +# +# Cookbook Name:: backup +# Recipe:: default +# +# Copyright 2013, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +directory "/store/backup" do + owner "osmbackup" + group "osmbackup" + mode 02755 +end + +cookbook_file "/usr/local/bin/expire-backups" do + owner "root" + group "root" + mode 0755 +end + +template "/etc/cron.daily/expire-backups" do + source "expire.cron.erb" + owner "root" + group "root" + mode 0644 +end diff --git a/cookbooks/backup/templates/default/expire.cron.erb b/cookbooks/backup/templates/default/expire.cron.erb new file mode 100644 index 000000000..03abaec05 --- /dev/null +++ b/cookbooks/backup/templates/default/expire.cron.erb @@ -0,0 +1,10 @@ +#!/bin/sh + +# DO NOT EDIT - This file is being maintained by Chef + +for prefix in chef git lists osmf-blog osmf-crm osmf-ledgersmb osmf-wiki osqa otrs sotm svn trac wiki +do + /usr/local/bin/expire-backups --days=3 --weeks=3 --months=3 /store/backup $prefix +done + +/usr/local/bin/expire-backups --weeks=5 /store/backup osm diff --git a/roles/backup.rb b/roles/backup.rb new file mode 100644 index 000000000..852a46a8a --- /dev/null +++ b/roles/backup.rb @@ -0,0 +1,37 @@ +name "backup" +description "Role applied to backup.openstreetmap.org" + +default_attributes( + :accounts => { + :users => { + :osmbackup => { :status => :role } + } + }, + :rsyncd => { + :modules => { + :backup => { + :comment => "Backups", + :path => "/store/backup", + :read_only => false, + :write_only => true, + :list => false, + :uid => "osmbackup", + :gid => "osmbackup", + :transfer_logging => false, + :hosts_allow => [ + "128.40.168.0/24", # ucl external + "146.179.159.160/27", # ic internal + "193.63.75.96/27", # ic external + "2001:630:12:500::/64", # ic external + "127.0.0.0/8", # localhost + "::1" # localhost + ] + } + } + } +) + +run_list( + "recipe[rsyncd]", + "recipe[backup]" +) diff --git a/roles/horntail.rb b/roles/horntail.rb index 0512ddb5e..e8a8192de 100644 --- a/roles/horntail.rb +++ b/roles/horntail.rb @@ -4,7 +4,6 @@ description "Master role applied to horntail" default_attributes( :accounts => { :users => { - :osmbackup => { :status => :role }, :gravitystorm => { :status => :user } } }, @@ -112,24 +111,6 @@ default_attributes( "127.0.0.0/8", # localhost "::1" # localhost ] - }, - :backup => { - :comment => "Backups", - :path => "/store/backup", - :read_only => false, - :write_only => true, - :list => false, - :uid => "osmbackup", - :gid => "osmbackup", - :transfer_logging => false, - :hosts_allow => [ - "128.40.168.0/24", # ucl external - "146.179.159.160/27", # ic internal - "193.63.75.96/27", # ic external - "2001:630:12:500::/64", # ic external - "127.0.0.0/8", # localhost - "::1" # localhost - ] } } } @@ -143,6 +124,7 @@ run_list( "role[planet]", "role[stats]", "role[web-storage]", + "role[backup]", "recipe[rsyncd]", "recipe[openvpn]" ) -- 2.39.5