cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "osmosis"
  24
  25 db_passwords = data_bag_item("db", "passwords")
  26
  27 package %w[
  28   postgresql-client
  29   ruby
  30   ruby-dev
  31   ruby-libxml
  32   make
  33   gcc
  34   libpq-dev
  35   osmdbt
  36 ]
  37
  38 gem_package "pg"
  39
  40 remote_directory "/opt/flush" do
  41   source "flush"
  42   owner "root"
  43   group "root"
  44   mode "755"
  45   files_owner "root"
  46   files_group "root"
  47   files_mode "755"
  48 end
  49
  50 execute "/opt/flush/Makefile" do
  51   action :nothing
  52   command "make"
  53   cwd "/opt/flush"
  54   user "root"
  55   group "root"
  56   subscribes :run, "remote_directory[/opt/flush]"
  57 end
  58
  59 remote_directory "/usr/local/bin" do
  60   source "replication-bin"
  61   owner "root"
  62   group "root"
  63   mode "755"
  64   files_owner "root"
  65   files_group "root"
  66   files_mode "755"
  67 end
  68
  69 template "/usr/local/bin/replicate-minute" do
  70   source "replicate-minute.erb"
  71   owner "root"
  72   group "root"
  73   mode "755"
  74 end
  75
  76 template "/usr/local/bin/users-agreed" do
  77   source "users-agreed.erb"
  78   owner "root"
  79   group "root"
  80   mode "755"
  81 end
  82
  83 template "/usr/local/bin/users-deleted" do
  84   source "users-deleted.erb"
  85   owner "root"
  86   group "root"
  87   mode "755"
  88 end
  89
  90 remote_directory "/store/planet/users_deleted" do
  91   source "users_deleted"
  92   owner "planet"
  93   group "planet"
  94   mode "755"
  95   files_owner "root"
  96   files_group "root"
  97   files_mode "644"
  98 end
  99
 100 remote_directory "/store/planet/replication" do
 101   source "replication-cgi"
 102   owner "root"
 103   group "root"
 104   mode "755"
 105   files_owner "root"
 106   files_group "root"
 107   files_mode "755"
 108 end
 109
 110 directory "/store/planet/replication/changesets" do
 111   owner "planet"
 112   group "planet"
 113   mode "755"
 114 end
 115
 116 directory "/store/planet/replication/day" do
 117   owner "planet"
 118   group "planet"
 119   mode "755"
 120 end
 121
 122 directory "/store/planet/replication/hour" do
 123   owner "planet"
 124   group "planet"
 125   mode "755"
 126 end
 127
 128 directory "/store/planet/replication/minute" do
 129   owner "planet"
 130   group "planet"
 131   mode "755"
 132 end
 133
 134 directory "/store/planet/replication/test" do
 135   owner "planet"
 136   group "planet"
 137   mode "755"
 138 end
 139
 140 directory "/store/planet/replication/test/minute" do
 141   owner "planet"
 142   group "planet"
 143   mode "755"
 144 end
 145
 146 systemd_tmpfile "/run/replication" do
 147   type "d"
 148   owner "planet"
 149   group "planet"
 150   mode "755"
 151 end
 152
 153 directory "/etc/replication" do
 154   owner "root"
 155   group "root"
 156   mode "755"
 157 end
 158
 159 directory "/var/run/lock/changeset-replication/" do
 160   owner "planet"
 161   group "planet"
 162   mode "750"
 163 end
 164
 165 template "/etc/replication/auth.conf" do
 166   source "replication.auth.erb"
 167   user "root"
 168   group "planet"
 169   mode "640"
 170   variables :password => db_passwords["planetdiff"]
 171 end
 172
 173 osmdbt_config = {
 174   "database" => {
 175     "host" => node[:web][:database_host],
 176     "dbname" => "openstreetmap",
 177     "user" => "planetdiff",
 178     "password" => db_passwords["planetdiff"],
 179     "replication_slot" => "osmdbt"
 180   },
 181   "log_dir" => "/var/lib/replication/minute",
 182   "changes_dir" => "/store/planet/replication/test/minute",
 183   "tmp_dir" => "/tmp",
 184   "run_dir" => "/run/replication"
 185 }
 186
 187 file "/etc/replication/osmdbt-config.yaml" do
 188   user "root"
 189   group "planet"
 190   mode "640"
 191   content YAML.dump(osmdbt_config)
 192 end
 193
 194 systemd_service "replication-minutely" do
 195   description "Minutely replication"
 196   user "planet"
 197   working_directory "/etc/replication"
 198   exec_start "/usr/local/bin/replicate-minute"
 199   private_tmp true
 200   private_devices true
 201   protect_system "full"
 202   protect_home true
 203   restrict_address_families %w[AF_INET AF_INET6]
 204   no_new_privileges true
 205 end
 206
 207 template "/etc/replication/changesets.conf" do
 208   source "changesets.conf.erb"
 209   user "root"
 210   group "planet"
 211   mode "640"
 212   variables :password => db_passwords["planetdiff"]
 213 end
 214
 215 template "/etc/replication/users-agreed.conf" do
 216   source "users-agreed.conf.erb"
 217   user "planet"
 218   group "planet"
 219   mode "600"
 220   variables :password => db_passwords["planetdiff"]
 221 end
 222
 223 directory "/var/lib/replication" do
 224   owner "planet"
 225   group "planet"
 226   mode "755"
 227 end
 228
 229 directory "/var/lib/replication/minute" do
 230   owner "planet"
 231   group "planet"
 232   mode "755"
 233 end
 234
 235 directory "/var/lib/replication/hour" do
 236   owner "planet"
 237   group "planet"
 238   mode "755"
 239 end
 240
 241 template "/var/lib/replication/hour/configuration.txt" do
 242   source "replication.config.erb"
 243   owner "planet"
 244   group "planet"
 245   mode "644"
 246   variables :base => "minute", :interval => 3600
 247 end
 248
 249 link "/var/lib/replication/hour/data" do
 250   to "/store/planet/replication/hour"
 251 end
 252
 253 directory "/var/lib/replication/day" do
 254   owner "planet"
 255   group "planet"
 256   mode "755"
 257 end
 258
 259 template "/var/lib/replication/day/configuration.txt" do
 260   source "replication.config.erb"
 261   owner "planet"
 262   group "planet"
 263   mode "644"
 264   variables :base => "hour", :interval => 86400
 265 end
 266
 267 link "/var/lib/replication/day/data" do
 268   to "/store/planet/replication/day"
 269 end
 270
 271 if node[:planet][:replication] == "enabled"
 272   cron_d "users-agreed" do
 273     minute "0"
 274     hour "7"
 275     user "planet"
 276     command "/usr/local/bin/users-agreed"
 277     mailto "zerebubuth@gmail.com"
 278   end
 279
 280   cron_d "users-deleted" do
 281     minute "0"
 282     hour "17"
 283     user "planet"
 284     command "/usr/local/bin/users-deleted"
 285     mailto "zerebubuth@gmail.com"
 286   end
 287
 288   cron_d "replication-changesets" do
 289     user "planet"
 290     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 291     mailto "zerebubuth@gmail.com"
 292   end
 293
 294   systemd_timer "replication-minutely" do
 295     description "Minutely replication"
 296     on_boot_sec 60
 297     on_unit_active_sec 60
 298   end
 299
 300   cron_d "replication-minutely" do
 301     user "planet"
 302     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 303     mailto "brett@bretth.com"
 304     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 305   end
 306
 307   cron_d "replication-hourly" do
 308     minute "2,7,12,17"
 309     user "planet"
 310     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 311     mailto "brett@bretth.com"
 312     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 313   end
 314
 315   cron_d "replication-daily" do
 316     minute "5,10,15,20"
 317     user "planet"
 318     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 319     mailto "brett@bretth.com"
 320     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 321   end
 322 else
 323   cron_d "users-agreed" do
 324     action :delete
 325   end
 326
 327   cron_d "users-deleted" do
 328     action :delete
 329   end
 330
 331   cron_d "replication-changesets" do
 332     action :delete
 333   end
 334
 335   cron_d "replication-minutely" do
 336     action :delete
 337   end
 338
 339   cron_d "replication-hourly" do
 340     action :delete
 341   end
 342
 343   cron_d "replication-daily" do
 344     action :delete
 345   end
 346 end