cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "apt"
  24 include_recipe "osmosis"
  25
  26 db_passwords = data_bag_item("db", "passwords")
  27
  28 package %w[
  29   postgresql-client
  30   ruby
  31   ruby-dev
  32   ruby-libxml
  33   make
  34   gcc
  35   libc6-dev
  36   libpq-dev
  37   osmdbt
  38 ]
  39
  40 gem_package "pg"
  41
  42 remote_directory "/opt/flush" do
  43   source "flush"
  44   owner "root"
  45   group "root"
  46   mode "755"
  47   files_owner "root"
  48   files_group "root"
  49   files_mode "755"
  50 end
  51
  52 execute "/opt/flush/Makefile" do
  53   action :nothing
  54   command "make"
  55   cwd "/opt/flush"
  56   user "root"
  57   group "root"
  58   subscribes :run, "remote_directory[/opt/flush]"
  59 end
  60
  61 remote_directory "/usr/local/bin" do
  62   source "replication-bin"
  63   owner "root"
  64   group "root"
  65   mode "755"
  66   files_owner "root"
  67   files_group "root"
  68   files_mode "755"
  69 end
  70
  71 template "/usr/local/bin/replicate-minute" do
  72   source "replicate-minute.erb"
  73   owner "root"
  74   group "root"
  75   mode "755"
  76 end
  77
  78 template "/usr/local/bin/users-agreed" do
  79   source "users-agreed.erb"
  80   owner "root"
  81   group "root"
  82   mode "755"
  83 end
  84
  85 template "/usr/local/bin/users-deleted" do
  86   source "users-deleted.erb"
  87   owner "root"
  88   group "root"
  89   mode "755"
  90 end
  91
  92 remote_directory "/store/planet/users_deleted" do
  93   source "users_deleted"
  94   owner "planet"
  95   group "planet"
  96   mode "755"
  97   files_owner "root"
  98   files_group "root"
  99   files_mode "644"
 100 end
 101
 102 remote_directory "/store/planet/replication" do
 103   source "replication-cgi"
 104   owner "root"
 105   group "root"
 106   mode "755"
 107   files_owner "root"
 108   files_group "root"
 109   files_mode "755"
 110 end
 111
 112 directory "/store/planet/replication/changesets" do
 113   owner "planet"
 114   group "planet"
 115   mode "755"
 116 end
 117
 118 directory "/store/planet/replication/day" do
 119   owner "planet"
 120   group "planet"
 121   mode "755"
 122 end
 123
 124 directory "/store/planet/replication/hour" do
 125   owner "planet"
 126   group "planet"
 127   mode "755"
 128 end
 129
 130 directory "/store/planet/replication/minute" do
 131   owner "planet"
 132   group "planet"
 133   mode "755"
 134 end
 135
 136 directory "/store/planet/replication/test" do
 137   owner "planet"
 138   group "planet"
 139   mode "755"
 140 end
 141
 142 directory "/store/planet/replication/test/day" do
 143   owner "planet"
 144   group "planet"
 145   mode "755"
 146 end
 147
 148 directory "/store/planet/replication/test/hour" do
 149   owner "planet"
 150   group "planet"
 151   mode "755"
 152 end
 153
 154 directory "/store/planet/replication/test/minute" do
 155   owner "planet"
 156   group "planet"
 157   mode "755"
 158 end
 159
 160 directory "/store/replication" do
 161   owner "planet"
 162   group "planet"
 163   mode "755"
 164 end
 165
 166 directory "/store/replication/minute" do
 167   owner "planet"
 168   group "planet"
 169   mode "755"
 170 end
 171
 172 systemd_tmpfile "/run/replication" do
 173   type "d"
 174   owner "planet"
 175   group "planet"
 176   mode "755"
 177 end
 178
 179 directory "/etc/replication" do
 180   owner "root"
 181   group "root"
 182   mode "755"
 183 end
 184
 185 directory "/var/run/lock/changeset-replication/" do
 186   owner "planet"
 187   group "planet"
 188   mode "750"
 189 end
 190
 191 directory "/var/lib/replication" do
 192   owner "planet"
 193   group "planet"
 194   mode "755"
 195 end
 196
 197 directory "/var/lib/replication/test" do
 198   owner "planet"
 199   group "planet"
 200   mode "755"
 201 end
 202
 203 template "/etc/replication/auth.conf" do
 204   source "replication.auth.erb"
 205   user "root"
 206   group "planet"
 207   mode "640"
 208   variables :password => db_passwords["planetdiff"]
 209 end
 210
 211 osmdbt_config = {
 212   "database" => {
 213     "host" => node[:web][:database_host],
 214     "dbname" => "openstreetmap",
 215     "user" => "planetdiff",
 216     "password" => db_passwords["planetdiff"],
 217     "replication_slot" => "osmdbt"
 218   },
 219   "log_dir" => "/var/lib/replication/minute",
 220   "changes_dir" => "/store/planet/replication/test/minute",
 221   "tmp_dir" => "/store/replication/minute",
 222   "run_dir" => "/run/replication"
 223 }
 224
 225 file "/etc/replication/osmdbt-config.yaml" do
 226   user "root"
 227   group "planet"
 228   mode "640"
 229   content YAML.dump(osmdbt_config)
 230 end
 231
 232 systemd_service "replication-minutely" do
 233   description "Minutely replication"
 234   user "planet"
 235   working_directory "/etc/replication"
 236   exec_start "/usr/local/bin/replicate-minute"
 237   private_tmp true
 238   private_devices true
 239   protect_system "full"
 240   protect_home true
 241   restrict_address_families %w[AF_INET AF_INET6]
 242   no_new_privileges true
 243 end
 244
 245 systemd_timer "replication-minutely" do
 246   description "Minutely replication"
 247   on_boot_sec 60
 248   on_unit_active_sec 60
 249   accuracy_sec 5
 250 end
 251
 252 directory "/var/lib/replication/test/hour" do
 253   owner "planet"
 254   group "planet"
 255   mode "755"
 256 end
 257
 258 template "/var/lib/replication/test/hour/configuration.txt" do
 259   source "replication.config.erb"
 260   owner "planet"
 261   group "planet"
 262   mode "644"
 263   variables :base => "test/minute", :interval => 3600
 264 end
 265
 266 link "/var/lib/replication/test/hour/data" do
 267   to "/store/planet/replication/test/hour"
 268 end
 269
 270 systemd_service "replication-hourly" do
 271   description "Hourly replication"
 272   user "planet"
 273   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour"
 274   private_tmp true
 275   private_devices true
 276   protect_system "full"
 277   protect_home true
 278   restrict_address_families %w[AF_INET AF_INET6]
 279   no_new_privileges true
 280 end
 281
 282 systemd_timer "replication-hourly" do
 283   description "Daily replication"
 284   on_calendar "*-*-* *:02/15:00"
 285 end
 286
 287 directory "/var/lib/replication/test/day" do
 288   owner "planet"
 289   group "planet"
 290   mode "755"
 291 end
 292
 293 template "/var/lib/replication/test/day/configuration.txt" do
 294   source "replication.config.erb"
 295   owner "planet"
 296   group "planet"
 297   mode "644"
 298   variables :base => "test/hour", :interval => 86400
 299 end
 300
 301 link "/var/lib/replication/test/day/data" do
 302   to "/store/planet/replication/test/day"
 303 end
 304
 305 systemd_service "replication-daily" do
 306   description "Daily replication"
 307   user "planet"
 308   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day"
 309   private_tmp true
 310   private_devices true
 311   protect_system "full"
 312   protect_home true
 313   restrict_address_families %w[AF_INET AF_INET6]
 314   no_new_privileges true
 315 end
 316
 317 systemd_timer "replication-daily" do
 318   description "Daily replication"
 319   on_calendar "*-*-* *:02/15:00"
 320 end
 321
 322 template "/etc/replication/changesets.conf" do
 323   source "changesets.conf.erb"
 324   user "root"
 325   group "planet"
 326   mode "640"
 327   variables :password => db_passwords["planetdiff"]
 328 end
 329
 330 template "/etc/replication/users-agreed.conf" do
 331   source "users-agreed.conf.erb"
 332   user "planet"
 333   group "planet"
 334   mode "600"
 335   variables :password => db_passwords["planetdiff"]
 336 end
 337
 338 directory "/var/lib/replication/minute" do
 339   owner "planet"
 340   group "planet"
 341   mode "755"
 342 end
 343
 344 directory "/var/lib/replication/hour" do
 345   owner "planet"
 346   group "planet"
 347   mode "755"
 348 end
 349
 350 template "/var/lib/replication/hour/configuration.txt" do
 351   source "replication.config.erb"
 352   owner "planet"
 353   group "planet"
 354   mode "644"
 355   variables :base => "minute", :interval => 3600
 356 end
 357
 358 link "/var/lib/replication/hour/data" do
 359   to "/store/planet/replication/hour"
 360 end
 361
 362 directory "/var/lib/replication/day" do
 363   owner "planet"
 364   group "planet"
 365   mode "755"
 366 end
 367
 368 template "/var/lib/replication/day/configuration.txt" do
 369   source "replication.config.erb"
 370   owner "planet"
 371   group "planet"
 372   mode "644"
 373   variables :base => "hour", :interval => 86400
 374 end
 375
 376 link "/var/lib/replication/day/data" do
 377   to "/store/planet/replication/day"
 378 end
 379
 380 if node[:planet][:replication] == "enabled"
 381   cron_d "users-agreed" do
 382     minute "0"
 383     hour "7"
 384     user "planet"
 385     command "/usr/local/bin/users-agreed"
 386     mailto "zerebubuth@gmail.com"
 387   end
 388
 389   cron_d "users-deleted" do
 390     minute "0"
 391     hour "17"
 392     user "planet"
 393     command "/usr/local/bin/users-deleted"
 394     mailto "zerebubuth@gmail.com"
 395   end
 396
 397   cron_d "replication-changesets" do
 398     user "planet"
 399     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 400     mailto "zerebubuth@gmail.com"
 401   end
 402
 403   service "replication-minutely.timer" do
 404     action [:enable, :start]
 405   end
 406
 407   service "replication-hourly.timer" do
 408     action [:enable, :start]
 409   end
 410
 411   service "replication-daily.timer" do
 412     action [:enable, :start]
 413   end
 414
 415   cron_d "replication-minutely" do
 416     user "planet"
 417     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 418     mailto "brett@bretth.com"
 419     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 420   end
 421
 422   cron_d "replication-hourly" do
 423     minute "2,7,12,17"
 424     user "planet"
 425     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 426     mailto "brett@bretth.com"
 427     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 428   end
 429
 430   cron_d "replication-daily" do
 431     minute "5,10,15,20"
 432     user "planet"
 433     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 434     mailto "brett@bretth.com"
 435     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 436   end
 437 else
 438   cron_d "users-agreed" do
 439     action :delete
 440   end
 441
 442   cron_d "users-deleted" do
 443     action :delete
 444   end
 445
 446   cron_d "replication-changesets" do
 447     action :delete
 448   end
 449
 450   service "replication-minutely.timer" do
 451     action [:stop, :disable]
 452   end
 453
 454   service "replication-hourly.timer" do
 455     action [:stop, :disable]
 456   end
 457
 458   service "replication-daily.timer" do
 459     action [:stop, :disable]
 460   end
 461
 462   cron_d "replication-minutely" do
 463     action :delete
 464   end
 465
 466   cron_d "replication-hourly" do
 467     action :delete
 468   end
 469
 470   cron_d "replication-daily" do
 471     action :delete
 472   end
 473 end