cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "apt"
  24 include_recipe "osmosis"
  25
  26 db_passwords = data_bag_item("db", "passwords")
  27
  28 ## Install required packages
  29
  30 package %w[
  31   postgresql-client
  32   ruby
  33   ruby-dev
  34   ruby-libxml
  35   make
  36   gcc
  37   libc6-dev
  38   libpq-dev
  39   osmdbt
  40 ]
  41
  42 gem_package "pg"
  43
  44 ## Build preload library to flush files
  45
  46 remote_directory "/opt/flush" do
  47   source "flush"
  48   owner "root"
  49   group "root"
  50   mode "755"
  51   files_owner "root"
  52   files_group "root"
  53   files_mode "755"
  54 end
  55
  56 execute "/opt/flush/Makefile" do
  57   action :nothing
  58   command "make"
  59   cwd "/opt/flush"
  60   user "root"
  61   group "root"
  62   subscribes :run, "remote_directory[/opt/flush]"
  63 end
  64
  65 ## Install scripts
  66
  67 remote_directory "/usr/local/bin" do
  68   source "replication-bin"
  69   owner "root"
  70   group "root"
  71   mode "755"
  72   files_owner "root"
  73   files_group "root"
  74   files_mode "755"
  75 end
  76
  77 template "/usr/local/bin/replicate-minute" do
  78   source "replicate-minute.erb"
  79   owner "root"
  80   group "root"
  81   mode "755"
  82 end
  83
  84 template "/usr/local/bin/users-agreed" do
  85   source "users-agreed.erb"
  86   owner "root"
  87   group "root"
  88   mode "755"
  89 end
  90
  91 template "/usr/local/bin/users-deleted" do
  92   source "users-deleted.erb"
  93   owner "root"
  94   group "root"
  95   mode "755"
  96 end
  97
  98 ## Published deleted users directory
  99
 100 remote_directory "/store/planet/users_deleted" do
 101   source "users_deleted"
 102   owner "planet"
 103   group "planet"
 104   mode "755"
 105   files_owner "root"
 106   files_group "root"
 107   files_mode "644"
 108 end
 109
 110 ## Published replication directory
 111
 112 remote_directory "/store/planet/replication" do
 113   source "replication-cgi"
 114   owner "root"
 115   group "root"
 116   mode "755"
 117   files_owner "root"
 118   files_group "root"
 119   files_mode "755"
 120 end
 121
 122 directory "/store/planet/replication/test" do
 123   owner "planet"
 124   group "planet"
 125   mode "755"
 126 end
 127
 128 ## Configuration directory
 129
 130 directory "/etc/replication" do
 131   owner "root"
 132   group "root"
 133   mode "755"
 134 end
 135
 136 ## Transient state directory
 137
 138 systemd_tmpfile "/run/replication" do
 139   type "d"
 140   owner "planet"
 141   group "planet"
 142   mode "755"
 143 end
 144
 145 ## Persistent state directory
 146
 147 directory "/var/lib/replication" do
 148   owner "planet"
 149   group "planet"
 150   mode "755"
 151 end
 152
 153 directory "/var/lib/replication/test" do
 154   owner "planet"
 155   group "planet"
 156   mode "755"
 157 end
 158
 159 ## Users replication
 160
 161 template "/etc/replication/users-agreed.conf" do
 162   source "users-agreed.conf.erb"
 163   user "planet"
 164   group "planet"
 165   mode "600"
 166   variables :password => db_passwords["planetdiff"]
 167 end
 168
 169 ## Changeset replication
 170
 171 directory "/store/planet/replication/changesets" do
 172   owner "planet"
 173   group "planet"
 174   mode "755"
 175 end
 176
 177 template "/etc/replication/changesets.conf" do
 178   source "changesets.conf.erb"
 179   user "root"
 180   group "planet"
 181   mode "640"
 182   variables :password => db_passwords["planetdiff"]
 183 end
 184
 185 ## Minutely replication
 186
 187 directory "/store/planet/replication/minute" do
 188   owner "planet"
 189   group "planet"
 190   mode "755"
 191 end
 192
 193 directory "/var/lib/replication/minute" do
 194   owner "planet"
 195   group "planet"
 196   mode "755"
 197 end
 198
 199 template "/etc/replication/auth.conf" do
 200   source "replication.auth.erb"
 201   user "root"
 202   group "planet"
 203   mode "640"
 204   variables :password => db_passwords["planetdiff"]
 205 end
 206
 207 ## Hourly replication
 208
 209 directory "/store/planet/replication/hour" do
 210   owner "planet"
 211   group "planet"
 212   mode "755"
 213 end
 214
 215 directory "/var/lib/replication/hour" do
 216   owner "planet"
 217   group "planet"
 218   mode "755"
 219 end
 220
 221 link "/var/lib/replication/hour/data" do
 222   to "/store/planet/replication/hour"
 223 end
 224
 225 template "/var/lib/replication/hour/configuration.txt" do
 226   source "replication.config.erb"
 227   owner "planet"
 228   group "planet"
 229   mode "644"
 230   variables :base => "minute", :interval => 3600
 231 end
 232
 233 ## Daily replication
 234
 235 directory "/store/planet/replication/day" do
 236   owner "planet"
 237   group "planet"
 238   mode "755"
 239 end
 240
 241 directory "/var/lib/replication/day" do
 242   owner "planet"
 243   group "planet"
 244   mode "755"
 245 end
 246
 247 link "/var/lib/replication/day/data" do
 248   to "/store/planet/replication/day"
 249 end
 250
 251 template "/var/lib/replication/day/configuration.txt" do
 252   source "replication.config.erb"
 253   owner "planet"
 254   group "planet"
 255   mode "644"
 256   variables :base => "hour", :interval => 86400
 257 end
 258
 259 ## Minutely replication (test feed)
 260
 261 directory "/store/planet/replication/test/minute" do
 262   owner "planet"
 263   group "planet"
 264   mode "755"
 265 end
 266
 267 directory "/store/replication" do
 268   owner "planet"
 269   group "planet"
 270   mode "755"
 271 end
 272
 273 directory "/store/replication/minute" do
 274   owner "planet"
 275   group "planet"
 276   mode "755"
 277 end
 278
 279 osmdbt_config = {
 280   "database" => {
 281     "host" => node[:web][:database_host],
 282     "dbname" => "openstreetmap",
 283     "user" => "planetdiff",
 284     "password" => db_passwords["planetdiff"],
 285     "replication_slot" => "osmdbt"
 286   },
 287   "log_dir" => "/var/lib/replication/minute",
 288   "changes_dir" => "/store/planet/replication/test/minute",
 289   "tmp_dir" => "/store/replication/minute",
 290   "run_dir" => "/run/replication"
 291 }
 292
 293 file "/etc/replication/osmdbt-config.yaml" do
 294   user "root"
 295   group "planet"
 296   mode "640"
 297   content YAML.dump(osmdbt_config)
 298 end
 299
 300 systemd_service "replication-minutely" do
 301   description "Minutely replication"
 302   user "planet"
 303   working_directory "/etc/replication"
 304   exec_start "/usr/local/bin/replicate-minute"
 305   private_tmp true
 306   private_devices true
 307   protect_system "full"
 308   protect_home true
 309   restrict_address_families %w[AF_INET AF_INET6]
 310   no_new_privileges true
 311 end
 312
 313 systemd_timer "replication-minutely" do
 314   description "Minutely replication"
 315   on_boot_sec 60
 316   on_unit_active_sec 60
 317   accuracy_sec 5
 318 end
 319
 320 ### Hourly replication (test feed)
 321
 322 directory "/store/planet/replication/test/hour" do
 323   owner "planet"
 324   group "planet"
 325   mode "755"
 326 end
 327
 328 directory "/var/lib/replication/test/hour" do
 329   owner "planet"
 330   group "planet"
 331   mode "755"
 332 end
 333
 334 link "/var/lib/replication/test/hour/data" do
 335   to "/store/planet/replication/test/hour"
 336 end
 337
 338 template "/var/lib/replication/test/hour/configuration.txt" do
 339   source "replication.config.erb"
 340   owner "planet"
 341   group "planet"
 342   mode "644"
 343   variables :base => "test/minute", :interval => 3600
 344 end
 345
 346 systemd_service "replication-hourly" do
 347   description "Hourly replication"
 348   user "planet"
 349   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/hour"
 350   private_tmp true
 351   private_devices true
 352   protect_system "full"
 353   protect_home true
 354   restrict_address_families %w[AF_INET AF_INET6]
 355   no_new_privileges true
 356 end
 357
 358 systemd_timer "replication-hourly" do
 359   description "Daily replication"
 360   on_calendar "*-*-* *:02/15:00"
 361 end
 362
 363 ## Daily replication (test feed)
 364
 365 directory "/store/planet/replication/test/day" do
 366   owner "planet"
 367   group "planet"
 368   mode "755"
 369 end
 370
 371 directory "/var/lib/replication/test/day" do
 372   owner "planet"
 373   group "planet"
 374   mode "755"
 375 end
 376
 377 link "/var/lib/replication/test/day/data" do
 378   to "/store/planet/replication/test/day"
 379 end
 380
 381 template "/var/lib/replication/test/day/configuration.txt" do
 382   source "replication.config.erb"
 383   owner "planet"
 384   group "planet"
 385   mode "644"
 386   variables :base => "test/hour", :interval => 86400
 387 end
 388
 389 systemd_service "replication-daily" do
 390   description "Daily replication"
 391   user "planet"
 392   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/test/day"
 393   private_tmp true
 394   private_devices true
 395   protect_system "full"
 396   protect_home true
 397   restrict_address_families %w[AF_INET AF_INET6]
 398   no_new_privileges true
 399 end
 400
 401 systemd_timer "replication-daily" do
 402   description "Daily replication"
 403   on_calendar "*-*-* *:02/15:00"
 404 end
 405
 406 ## Enable/disable feeds
 407
 408 if node[:planet][:replication] == "enabled"
 409   cron_d "users-agreed" do
 410     minute "0"
 411     hour "7"
 412     user "planet"
 413     command "/usr/local/bin/users-agreed"
 414     mailto "zerebubuth@gmail.com"
 415   end
 416
 417   cron_d "users-deleted" do
 418     minute "0"
 419     hour "17"
 420     user "planet"
 421     command "/usr/local/bin/users-deleted"
 422     mailto "zerebubuth@gmail.com"
 423   end
 424
 425   cron_d "replication-changesets" do
 426     user "planet"
 427     command "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 428     mailto "zerebubuth@gmail.com"
 429   end
 430
 431   cron_d "replication-minutely" do
 432     user "planet"
 433     command "/usr/local/bin/osmosis -q --replicate-apidb authFile=/etc/replication/auth.conf validateSchemaVersion=false --write-replication workingDirectory=/store/planet/replication/minute"
 434     mailto "brett@bretth.com"
 435     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 436   end
 437
 438   cron_d "replication-hourly" do
 439     minute "2,7,12,17"
 440     user "planet"
 441     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 442     mailto "brett@bretth.com"
 443     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 444   end
 445
 446   cron_d "replication-daily" do
 447     minute "5,10,15,20"
 448     user "planet"
 449     command "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 450     mailto "brett@bretth.com"
 451     environment "LD_PRELOAD" => "/opt/flush/flush.so"
 452   end
 453
 454   service "replication-minutely.timer" do
 455     action [:enable, :start]
 456   end
 457
 458   service "replication-hourly.timer" do
 459     action [:enable, :start]
 460   end
 461
 462   service "replication-daily.timer" do
 463     action [:enable, :start]
 464   end
 465 else
 466   cron_d "users-agreed" do
 467     action :delete
 468   end
 469
 470   cron_d "users-deleted" do
 471     action :delete
 472   end
 473
 474   cron_d "replication-changesets" do
 475     action :delete
 476   end
 477
 478   cron_d "replication-minutely" do
 479     action :delete
 480   end
 481
 482   cron_d "replication-hourly" do
 483     action :delete
 484   end
 485
 486   cron_d "replication-daily" do
 487     action :delete
 488   end
 489
 490   service "replication-minutely.timer" do
 491     action [:stop, :disable]
 492   end
 493
 494   service "replication-hourly.timer" do
 495     action [:stop, :disable]
 496   end
 497
 498   service "replication-daily.timer" do
 499     action [:stop, :disable]
 500   end
 501 end