cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "osmosis"
  24
  25 db_passwords = data_bag_item("db", "passwords")
  26
  27 package %w[
  28   postgresql-client
  29   ruby
  30   ruby-dev
  31   ruby-libxml
  32   make
  33   gcc
  34   libpq-dev
  35   osmdbt
  36 ]
  37
  38 gem_package "pg"
  39
  40 remote_directory "/opt/flush" do
  41   source "flush"
  42   owner "root"
  43   group "root"
  44   mode "755"
  45   files_owner "root"
  46   files_group "root"
  47   files_mode "755"
  48 end
  49
  50 execute "/opt/flush/Makefile" do
  51   action :nothing
  52   command "make"
  53   cwd "/opt/flush"
  54   user "root"
  55   group "root"
  56   subscribes :run, "remote_directory[/opt/flush]"
  57 end
  58
  59 remote_directory "/usr/local/bin" do
  60   source "replication-bin"
  61   owner "root"
  62   group "root"
  63   mode "755"
  64   files_owner "root"
  65   files_group "root"
  66   files_mode "755"
  67 end
  68
  69 template "/usr/local/bin/replicate-minute" do
  70   source "replicate-minute.erb"
  71   owner "root"
  72   group "root"
  73   mode "755"
  74 end
  75
  76 template "/usr/local/bin/users-agreed" do
  77   source "users-agreed.erb"
  78   owner "root"
  79   group "root"
  80   mode "755"
  81 end
  82
  83 template "/usr/local/bin/users-deleted" do
  84   source "users-deleted.erb"
  85   owner "root"
  86   group "root"
  87   mode "755"
  88 end
  89
  90 remote_directory "/store/planet/users_deleted" do
  91   source "users_deleted"
  92   owner "planet"
  93   group "planet"
  94   mode "755"
  95   files_owner "root"
  96   files_group "root"
  97   files_mode "644"
  98 end
  99
 100 remote_directory "/store/planet/replication" do
 101   source "replication-cgi"
 102   owner "root"
 103   group "root"
 104   mode "755"
 105   files_owner "root"
 106   files_group "root"
 107   files_mode "755"
 108 end
 109
 110 directory "/store/planet/replication/changesets" do
 111   owner "planet"
 112   group "planet"
 113   mode "755"
 114 end
 115
 116 directory "/store/planet/replication/day" do
 117   owner "planet"
 118   group "planet"
 119   mode "755"
 120 end
 121
 122 directory "/store/planet/replication/hour" do
 123   owner "planet"
 124   group "planet"
 125   mode "755"
 126 end
 127
 128 directory "/store/planet/replication/minute" do
 129   owner "planet"
 130   group "planet"
 131   mode "755"
 132 end
 133
 134 directory "/store/planet/replication/test" do
 135   owner "planet"
 136   group "planet"
 137   mode "755"
 138 end
 139
 140 directory "/store/planet/replication/test/day" do
 141   owner "planet"
 142   group "planet"
 143   mode "755"
 144 end
 145
 146 directory "/store/planet/replication/test/hour" do
 147   owner "planet"
 148   group "planet"
 149   mode "755"
 150 end
 151
 152 directory "/store/planet/replication/test/minute" do
 153   owner "planet"
 154   group "planet"
 155   mode "755"
 156 end
 157
 158 directory "/store/replication" do
 159   owner "planet"
 160   group "planet"
 161   mode "755"
 162 end
 163
 164 directory "/store/replication/minute" do
 165   owner "planet"
 166   group "planet"
 167   mode "755"
 168 end
 169
 170 systemd_tmpfile "/run/replication" do
 171   type "d"
 172   owner "planet"
 173   group "planet"
 174   mode "755"
 175 end
 176
 177 directory "/etc/replication" do
 178   owner "root"
 179   group "root"
 180   mode "755"
 181 end
 182
 183 directory "/var/run/lock/changeset-replication/" do
 184   owner "planet"
 185   group "planet"
 186   mode "750"
 187 end
 188
 189 directory "/var/lib/replication" do
 190   owner "planet"
 191   group "planet"
 192   mode "755"
 193 end
 194
 195 directory "/var/lib/replication/test" do
 196   owner "planet"
 197   group "planet"
 198   mode "755"
 199 end
 200
 201 template "/etc/replication/auth.conf" do
 202   source "replication.auth.erb"
 203   user "root"
 204   group "planet"
 205   mode "640"
 206   variables :password => db_passwords["planetdiff"]
 207 end
 208
 209 osmdbt_config = {
 210   "database" => {
 211     "host" => node[:web][:database_host],
 212     "dbname" => "openstreetmap",
 213     "user" => "planetdiff",
 214     "password" => db_passwords["planetdiff"],
 215     "replication_slot" => "osmdbt"
 216   },
 217   "log_dir" => "/var/lib/replication/minute",
 218   "changes_dir" => "/store/planet/replication/test/minute",
 219   "tmp_dir" => "/store/replication/minute",
 220   "run_dir" => "/run/replication"
 221 }
 222
 223 file "/etc/replication/osmdbt-config.yaml" do
 224   user "root"
 225   group "planet"
 226   mode "640"
 227   content YAML.dump(osmdbt_config)
 228 end
 229
 230 systemd_service "replication-minutely" do
 231   description "Minutely replication"
 232   user "planet"
 233   working_directory "/etc/replication"
 234   exec_start "/usr/local/bin/replicate-minute"
 235   private_tmp true
 236   private_devices true
 237   protect_system "full"
 238   protect_home true
 239   restrict_address_families %w[AF_INET AF_INET6]
 240   no_new_privileges true
 241 end
 242
 243 systemd_timer "replication-minutely" do
 244   description "Minutely replication"
 245   on_boot_sec 60
 246   on_unit_active_sec 60
 247   accuracy_sec 5
 248 end
 249
 250 directory "/var/lib/replication/test/hour" do
 251   owner "planet"
 252   group "planet"
 253   mode "755"
 254 end
 255
 256 template "/var/lib/replication/test/hour/configuration.txt" do
 257   source "replication.config.erb"
 258   owner "planet"
 259   group "planet"
 260   mode "644"
 261   variables :base => "test/minute", :interval => 3600
 262 end
 263
 264 link "/var/lib/replication/test/hour/data" do
 265   to "/store/planet/replication/test/hour"
 266 end
 267
 268 systemd_service "replication-hourly" do
 269   description "Hourly replication"
 270   user "planet"
 271   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour"
 272   private_tmp true
 273   private_devices true
 274   protect_system "full"
 275   protect_home true
 276   restrict_address_families %w[AF_INET AF_INET6]
 277   no_new_privileges true
 278 end
 279
 280 systemd_timer "replication-hourly" do
 281   description "Daily replication"
 282   on_calendar "*-*-* *:02/15:00"
 283 end
 284
 285 directory "/var/lib/replication/test/day" do
 286   owner "planet"
 287   group "planet"
 288   mode "755"
 289 end
 290
 291 template "/var/lib/replication/test/day/configuration.txt" do
 292   source "replication.config.erb"
 293   owner "planet"
 294   group "planet"
 295   mode "644"
 296   variables :base => "test/hour", :interval => 86400
 297 end
 298
 299 link "/var/lib/replication/test/day/data" do
 300   to "/store/planet/replication/test/day"
 301 end
 302
 303 systemd_service "replication-daily" do
 304   description "Daily replication"
 305   user "planet"
 306   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day"
 307   private_tmp true
 308   private_devices true
 309   protect_system "full"
 310   protect_home true
 311   restrict_address_families %w[AF_INET AF_INET6]
 312   no_new_privileges true
 313 end
 314
 315 systemd_timer "replication-daily" do
 316   description "Daily replication"
 317   on_calendar "*-*-* *:02/15:00"
 318 end
 319
 320 template "/etc/replication/changesets.conf" do
 321   source "changesets.conf.erb"
 322   user "root"
 323   group "planet"
 324   mode "640"
 325   variables :password => db_passwords["planetdiff"]
 326 end
 327
 328 template "/etc/replication/users-agreed.conf" do
 329   source "users-agreed.conf.erb"
 330   user "planet"
 331   group "planet"
 332   mode "600"
 333   variables :password => db_passwords["planetdiff"]
 334 end
 335
 336 directory "/var/lib/replication/minute" do
 337   owner "planet"
 338   group "planet"
 339   mode "755"
 340 end
 341
 342 directory "/var/lib/replication/hour" do
 343   owner "planet"
 344   group "planet"
 345   mode "755"
 346 end
 347
 348 template "/var/lib/replication/hour/configuration.txt" do
 349   source "replication.config.erb"
 350   owner "planet"
 351   group "planet"
 352   mode "644"
 353   variables :base => "minute", :interval => 3600
 354 end
 355
 356 link "/var/lib/replication/hour/data" do
 357   to "/store/planet/replication/hour"
 358 end
 359
 360 directory "/var/lib/replication/day" do
 361   owner "planet"
 362   group "planet"
 363   mode "755"
 364 end
 365
 366 template "/var/lib/replication/day/configuration.txt" do
 367   source "replication.config.erb"
 368   owner "planet"
 369   group "planet"
 370   mode "644"
 371   variables :base => "hour", :interval => 86400
 372 end
 373
 374 link "/var/lib/replication/day/data" do
 375   to "/store/planet/replication/day"
 376 end
 377
 378 if node[:planet][:replication] == "enabled"
 379   cron_d "users-agreed" do
 380     minute "0"
 381     hour "7"
 382     user "planet"
 383     command "/usr/local/bin/users-agreed"
 384     mailto "zerebubuth@gmail.com"
 385   end
 386
 387   cron_d "users-deleted" do
 388     minute "0"
 389     hour "17"
 390     user "planet"
 391     command "/usr/local/bin/users-deleted"
 392     mailto "zerebubuth@gmail.com"
 393   end
 394
 395   cron_d "replication-changesets" do
 396     user "planet"
 397     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 398     mailto "zerebubuth@gmail.com"
 399   end
 400
 401   service "replication-minutely.timer" do
 402     action [:enable, :start]
 403   end
 404
 405   service "replication-hourly.timer" do
 406     action [:enable, :start]
 407   end
 408
 409   service "replication-daily.timer" do
 410     action [:enable, :start]
 411   end
 412
 413   cron_d "replication-minutely" do
 414     user "planet"
 415     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 416     mailto "brett@bretth.com"
 417     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 418   end
 419
 420   cron_d "replication-hourly" do
 421     minute "2,7,12,17"
 422     user "planet"
 423     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 424     mailto "brett@bretth.com"
 425     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 426   end
 427
 428   cron_d "replication-daily" do
 429     minute "5,10,15,20"
 430     user "planet"
 431     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 432     mailto "brett@bretth.com"
 433     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 434   end
 435 else
 436   cron_d "users-agreed" do
 437     action :delete
 438   end
 439
 440   cron_d "users-deleted" do
 441     action :delete
 442   end
 443
 444   cron_d "replication-changesets" do
 445     action :delete
 446   end
 447
 448   service "replication-minutely.timer" do
 449     action [:stop, :disable]
 450   end
 451
 452   service "replication-hourly.timer" do
 453     action [:stop, :disable]
 454   end
 455
 456   service "replication-daily.timer" do
 457     action [:stop, :disable]
 458   end
 459
 460   cron_d "replication-minutely" do
 461     action :delete
 462   end
 463
 464   cron_d "replication-hourly" do
 465     action :delete
 466   end
 467
 468   cron_d "replication-daily" do
 469     action :delete
 470   end
 471 end