cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "osmosis"
  24
  25 db_passwords = data_bag_item("db", "passwords")
  26
  27 package %w[
  28   postgresql-client
  29   ruby
  30   ruby-dev
  31   ruby-libxml
  32   make
  33   gcc
  34   libpq-dev
  35   osmdbt
  36 ]
  37
  38 gem_package "pg"
  39
  40 remote_directory "/opt/flush" do
  41   source "flush"
  42   owner "root"
  43   group "root"
  44   mode "755"
  45   files_owner "root"
  46   files_group "root"
  47   files_mode "755"
  48 end
  49
  50 execute "/opt/flush/Makefile" do
  51   action :nothing
  52   command "make"
  53   cwd "/opt/flush"
  54   user "root"
  55   group "root"
  56   subscribes :run, "remote_directory[/opt/flush]"
  57 end
  58
  59 remote_directory "/usr/local/bin" do
  60   source "replication-bin"
  61   owner "root"
  62   group "root"
  63   mode "755"
  64   files_owner "root"
  65   files_group "root"
  66   files_mode "755"
  67 end
  68
  69 template "/usr/local/bin/replicate-minute" do
  70   source "replicate-minute.erb"
  71   owner "root"
  72   group "root"
  73   mode "755"
  74 end
  75
  76 template "/usr/local/bin/users-agreed" do
  77   source "users-agreed.erb"
  78   owner "root"
  79   group "root"
  80   mode "755"
  81 end
  82
  83 template "/usr/local/bin/users-deleted" do
  84   source "users-deleted.erb"
  85   owner "root"
  86   group "root"
  87   mode "755"
  88 end
  89
  90 remote_directory "/store/planet/users_deleted" do
  91   source "users_deleted"
  92   owner "planet"
  93   group "planet"
  94   mode "755"
  95   files_owner "root"
  96   files_group "root"
  97   files_mode "644"
  98 end
  99
 100 remote_directory "/store/planet/replication" do
 101   source "replication-cgi"
 102   owner "root"
 103   group "root"
 104   mode "755"
 105   files_owner "root"
 106   files_group "root"
 107   files_mode "755"
 108 end
 109
 110 directory "/store/planet/replication/changesets" do
 111   owner "planet"
 112   group "planet"
 113   mode "755"
 114 end
 115
 116 directory "/store/planet/replication/day" do
 117   owner "planet"
 118   group "planet"
 119   mode "755"
 120 end
 121
 122 directory "/store/planet/replication/hour" do
 123   owner "planet"
 124   group "planet"
 125   mode "755"
 126 end
 127
 128 directory "/store/planet/replication/minute" do
 129   owner "planet"
 130   group "planet"
 131   mode "755"
 132 end
 133
 134 directory "/store/planet/replication/test" do
 135   owner "planet"
 136   group "planet"
 137   mode "755"
 138 end
 139
 140 directory "/store/planet/replication/test/minute" do
 141   owner "planet"
 142   group "planet"
 143   mode "755"
 144 end
 145
 146 directory "/store/replication" do
 147   owner "planet"
 148   group "planet"
 149   mode "755"
 150 end
 151
 152 directory "/store/replication/minute" do
 153   owner "planet"
 154   group "planet"
 155   mode "755"
 156 end
 157
 158 systemd_tmpfile "/run/replication" do
 159   type "d"
 160   owner "planet"
 161   group "planet"
 162   mode "755"
 163 end
 164
 165 directory "/etc/replication" do
 166   owner "root"
 167   group "root"
 168   mode "755"
 169 end
 170
 171 directory "/var/run/lock/changeset-replication/" do
 172   owner "planet"
 173   group "planet"
 174   mode "750"
 175 end
 176
 177 template "/etc/replication/auth.conf" do
 178   source "replication.auth.erb"
 179   user "root"
 180   group "planet"
 181   mode "640"
 182   variables :password => db_passwords["planetdiff"]
 183 end
 184
 185 osmdbt_config = {
 186   "database" => {
 187     "host" => node[:web][:database_host],
 188     "dbname" => "openstreetmap",
 189     "user" => "planetdiff",
 190     "password" => db_passwords["planetdiff"],
 191     "replication_slot" => "osmdbt"
 192   },
 193   "log_dir" => "/var/lib/replication/minute",
 194   "changes_dir" => "/store/planet/replication/test/minute",
 195   "tmp_dir" => "/store/replication/minute",
 196   "run_dir" => "/run/replication"
 197 }
 198
 199 file "/etc/replication/osmdbt-config.yaml" do
 200   user "root"
 201   group "planet"
 202   mode "640"
 203   content YAML.dump(osmdbt_config)
 204 end
 205
 206 systemd_service "replication-minutely" do
 207   description "Minutely replication"
 208   user "planet"
 209   working_directory "/etc/replication"
 210   exec_start "/usr/local/bin/replicate-minute"
 211   private_tmp true
 212   private_devices true
 213   protect_system "full"
 214   protect_home true
 215   restrict_address_families %w[AF_INET AF_INET6]
 216   no_new_privileges true
 217 end
 218
 219 systemd_timer "replication-minutely" do
 220   description "Minutely replication"
 221   on_boot_sec 60
 222   on_unit_active_sec 60
 223 end
 224
 225 template "/etc/replication/changesets.conf" do
 226   source "changesets.conf.erb"
 227   user "root"
 228   group "planet"
 229   mode "640"
 230   variables :password => db_passwords["planetdiff"]
 231 end
 232
 233 template "/etc/replication/users-agreed.conf" do
 234   source "users-agreed.conf.erb"
 235   user "planet"
 236   group "planet"
 237   mode "600"
 238   variables :password => db_passwords["planetdiff"]
 239 end
 240
 241 directory "/var/lib/replication" do
 242   owner "planet"
 243   group "planet"
 244   mode "755"
 245 end
 246
 247 directory "/var/lib/replication/minute" do
 248   owner "planet"
 249   group "planet"
 250   mode "755"
 251 end
 252
 253 directory "/var/lib/replication/hour" do
 254   owner "planet"
 255   group "planet"
 256   mode "755"
 257 end
 258
 259 template "/var/lib/replication/hour/configuration.txt" do
 260   source "replication.config.erb"
 261   owner "planet"
 262   group "planet"
 263   mode "644"
 264   variables :base => "minute", :interval => 3600
 265 end
 266
 267 link "/var/lib/replication/hour/data" do
 268   to "/store/planet/replication/hour"
 269 end
 270
 271 directory "/var/lib/replication/day" do
 272   owner "planet"
 273   group "planet"
 274   mode "755"
 275 end
 276
 277 template "/var/lib/replication/day/configuration.txt" do
 278   source "replication.config.erb"
 279   owner "planet"
 280   group "planet"
 281   mode "644"
 282   variables :base => "hour", :interval => 86400
 283 end
 284
 285 link "/var/lib/replication/day/data" do
 286   to "/store/planet/replication/day"
 287 end
 288
 289 if node[:planet][:replication] == "enabled"
 290   cron_d "users-agreed" do
 291     minute "0"
 292     hour "7"
 293     user "planet"
 294     command "/usr/local/bin/users-agreed"
 295     mailto "zerebubuth@gmail.com"
 296   end
 297
 298   cron_d "users-deleted" do
 299     minute "0"
 300     hour "17"
 301     user "planet"
 302     command "/usr/local/bin/users-deleted"
 303     mailto "zerebubuth@gmail.com"
 304   end
 305
 306   cron_d "replication-changesets" do
 307     user "planet"
 308     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 309     mailto "zerebubuth@gmail.com"
 310   end
 311
 312   service "replication-minutely.timer" do
 313     action [:enable, :start]
 314   end
 315
 316   cron_d "replication-minutely" do
 317     user "planet"
 318     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 319     mailto "brett@bretth.com"
 320     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 321   end
 322
 323   cron_d "replication-hourly" do
 324     minute "2,7,12,17"
 325     user "planet"
 326     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 327     mailto "brett@bretth.com"
 328     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 329   end
 330
 331   cron_d "replication-daily" do
 332     minute "5,10,15,20"
 333     user "planet"
 334     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 335     mailto "brett@bretth.com"
 336     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 337   end
 338 else
 339   cron_d "users-agreed" do
 340     action :delete
 341   end
 342
 343   cron_d "users-deleted" do
 344     action :delete
 345   end
 346
 347   cron_d "replication-changesets" do
 348     action :delete
 349   end
 350
 351   service "replication-minutely.timer" do
 352     action [:stop, :disable]
 353   end
 354
 355   cron_d "replication-minutely" do
 356     action :delete
 357   end
 358
 359   cron_d "replication-hourly" do
 360     action :delete
 361   end
 362
 363   cron_d "replication-daily" do
 364     action :delete
 365   end
 366 end