cookbooks/planet/recipes/replication.rb

   1 #
   2 # Cookbook:: planet
   3 # Recipe:: dump
   4 #
   5 # Copyright:: 2013, OpenStreetMap Foundation
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     https://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 require "yaml"
  21
  22 include_recipe "accounts"
  23 include_recipe "apt"
  24 include_recipe "osmosis"
  25 include_recipe "ruby"
  26 include_recipe "tools"
  27
  28 db_passwords = data_bag_item("db", "passwords")
  29
  30 ## Install required packages
  31
  32 package %w[
  33   postgresql-client
  34   ruby-libxml
  35   make
  36   gcc
  37   libc6-dev
  38   libpq-dev
  39   osmdbt
  40 ]
  41
  42 gem_package "pg" do
  43   gem_binary node[:ruby][:gem]
  44 end
  45
  46 ## Build preload library to flush files
  47
  48 remote_directory "/opt/flush" do
  49   source "flush"
  50   owner "root"
  51   group "root"
  52   mode "755"
  53   files_owner "root"
  54   files_group "root"
  55   files_mode "755"
  56 end
  57
  58 execute "/opt/flush/Makefile" do
  59   action :nothing
  60   command "make"
  61   cwd "/opt/flush"
  62   user "root"
  63   group "root"
  64   subscribes :run, "remote_directory[/opt/flush]"
  65 end
  66
  67 ## Install scripts
  68
  69 remote_directory "/usr/local/bin" do
  70   source "replication-bin"
  71   owner "root"
  72   group "root"
  73   mode "755"
  74   files_owner "root"
  75   files_group "root"
  76   files_mode "755"
  77 end
  78
  79 template "/usr/local/bin/users-agreed" do
  80   source "users-agreed.erb"
  81   owner "root"
  82   group "root"
  83   mode "755"
  84 end
  85
  86 template "/usr/local/bin/users-deleted" do
  87   source "users-deleted.erb"
  88   owner "root"
  89   group "root"
  90   mode "755"
  91 end
  92
  93 ## Published deleted users directory
  94
  95 remote_directory "/store/planet/users_deleted" do
  96   source "users_deleted"
  97   owner "planet"
  98   group "planet"
  99   mode "755"
 100   files_owner "root"
 101   files_group "root"
 102   files_mode "644"
 103 end
 104
 105 ## Published replication directory
 106
 107 remote_directory "/store/planet/replication" do
 108   source "replication-cgi"
 109   owner "root"
 110   group "root"
 111   mode "755"
 112   files_owner "root"
 113   files_group "root"
 114   files_mode "755"
 115 end
 116
 117 ## Configuration directory
 118
 119 directory "/etc/replication" do
 120   owner "root"
 121   group "root"
 122   mode "755"
 123 end
 124
 125 ## Transient state directory
 126
 127 systemd_tmpfile "/run/replication" do
 128   type "d"
 129   owner "planet"
 130   group "planet"
 131   mode "755"
 132 end
 133
 134 ## Persistent state directory
 135
 136 directory "/var/lib/replication" do
 137   owner "planet"
 138   group "planet"
 139   mode "755"
 140 end
 141
 142 ## Temporary directory
 143
 144 directory "/store/replication" do
 145   owner "planet"
 146   group "planet"
 147   mode "755"
 148 end
 149
 150 ## Users replication
 151
 152 template "/etc/replication/users-agreed.conf" do
 153   source "users-agreed.conf.erb"
 154   user "planet"
 155   group "planet"
 156   mode "600"
 157   variables :password => db_passwords["planetdiff"]
 158 end
 159
 160 systemd_service "users-agreed" do
 161   description "Update list of users accepting CTs"
 162   user "planet"
 163   exec_start "/usr/local/bin/users-agreed"
 164   nice 10
 165   sandbox :enable_network => true
 166   read_write_paths "/store/planet/users_agreed"
 167 end
 168
 169 systemd_timer "users-agreed" do
 170   description "Update list of users accepting CTs"
 171   on_calendar "7:00"
 172 end
 173
 174 systemd_service "users-deleted" do
 175   description "Update list of deleted users"
 176   user "planet"
 177   exec_start "/usr/local/bin/users-deleted"
 178   nice 10
 179   sandbox :enable_network => true
 180   read_write_paths "/store/planet/users_deleted"
 181 end
 182
 183 systemd_timer "users-deleted" do
 184   description "Update list of deleted users"
 185   on_calendar "17:00"
 186 end
 187
 188 ## Changeset replication
 189
 190 directory "/store/planet/replication/changesets" do
 191   owner "planet"
 192   group "planet"
 193   mode "755"
 194 end
 195
 196 template "/etc/replication/changesets.conf" do
 197   source "changesets.conf.erb"
 198   user "root"
 199   group "planet"
 200   mode "640"
 201   variables :password => db_passwords["planetdiff"]
 202 end
 203
 204 systemd_service "replication-changesets" do
 205   description "Changesets replication"
 206   user "planet"
 207   exec_start "/usr/local/bin/replicate-changesets /etc/replication/changesets.conf"
 208   sandbox :enable_network => true
 209   read_write_paths [
 210     "/run/replication",
 211     "/store/planet/replication/changesets"
 212   ]
 213 end
 214
 215 systemd_timer "replication-changesets" do
 216   description "Changesets replication"
 217   on_boot_sec 60
 218   on_unit_active_sec 60
 219   accuracy_sec 5
 220 end
 221
 222 ## Minutely replication
 223
 224 directory "/store/planet/replication/minute" do
 225   owner "planet"
 226   group "planet"
 227   mode "755"
 228 end
 229
 230 directory "/var/lib/replication/minute" do
 231   owner "planet"
 232   group "planet"
 233   mode "755"
 234 end
 235
 236 directory "/store/replication/minute" do
 237   owner "planet"
 238   group "planet"
 239   mode "755"
 240 end
 241
 242 osmdbt_config = {
 243   "database" => {
 244     "host" => node[:web][:database_host],
 245     "dbname" => "openstreetmap",
 246     "user" => "planetdiff",
 247     "password" => db_passwords["planetdiff"],
 248     "replication_slot" => "osmdbt"
 249   },
 250   "log_dir" => "/var/lib/replication/minute",
 251   "changes_dir" => "/store/planet/replication/minute",
 252   "tmp_dir" => "/store/replication/minute",
 253   "run_dir" => "/run/replication"
 254 }
 255
 256 file "/etc/replication/osmdbt-config.yaml" do
 257   user "root"
 258   group "planet"
 259   mode "640"
 260   content YAML.dump(osmdbt_config)
 261 end
 262
 263 systemd_service "replication-minutely" do
 264   description "Minutely replication"
 265   user "planet"
 266   working_directory "/etc/replication"
 267   exec_start "/usr/local/bin/replicate-minute"
 268   sandbox :enable_network => true
 269   read_write_paths [
 270     "/run/replication",
 271     "/store",
 272     "/var/lib/replication/minute"
 273   ]
 274 end
 275
 276 systemd_timer "replication-minutely" do
 277   description "Minutely replication"
 278   on_boot_sec 60
 279   on_unit_active_sec 60
 280   accuracy_sec 5
 281 end
 282
 283 ## Hourly replication
 284
 285 directory "/store/planet/replication/hour" do
 286   owner "planet"
 287   group "planet"
 288   mode "755"
 289 end
 290
 291 directory "/var/lib/replication/hour" do
 292   owner "planet"
 293   group "planet"
 294   mode "755"
 295 end
 296
 297 link "/var/lib/replication/hour/data" do
 298   to "/store/planet/replication/hour"
 299 end
 300
 301 template "/var/lib/replication/hour/configuration.txt" do
 302   source "replication.config.erb"
 303   owner "planet"
 304   group "planet"
 305   mode "644"
 306   variables :base => "minute", :interval => 3600
 307 end
 308
 309 systemd_service "replication-hourly" do
 310   description "Hourly replication"
 311   user "planet"
 312   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/hour"
 313   environment "LD_PRELOAD" => "/opt/flush/flush.so"
 314   sandbox :enable_network => true
 315   memory_deny_write_execute false
 316   read_write_paths [
 317     "/store/planet/replication/hour",
 318     "/var/lib/replication/hour"
 319   ]
 320 end
 321
 322 systemd_timer "replication-hourly" do
 323   description "Daily replication"
 324   on_calendar "*-*-* *:02/15:00"
 325 end
 326
 327 ## Daily replication
 328
 329 directory "/store/planet/replication/day" do
 330   owner "planet"
 331   group "planet"
 332   mode "755"
 333 end
 334
 335 directory "/var/lib/replication/day" do
 336   owner "planet"
 337   group "planet"
 338   mode "755"
 339 end
 340
 341 link "/var/lib/replication/day/data" do
 342   to "/store/planet/replication/day"
 343 end
 344
 345 template "/var/lib/replication/day/configuration.txt" do
 346   source "replication.config.erb"
 347   owner "planet"
 348   group "planet"
 349   mode "644"
 350   variables :base => "hour", :interval => 86400
 351 end
 352
 353 systemd_service "replication-daily" do
 354   description "Daily replication"
 355   user "planet"
 356   exec_start "/usr/local/bin/osmosis -q --merge-replication-files workingDirectory=/var/lib/replication/day"
 357   environment "LD_PRELOAD" => "/opt/flush/flush.so"
 358   sandbox :enable_network => true
 359   memory_deny_write_execute false
 360   read_write_paths [
 361     "/store/planet/replication/day",
 362     "/var/lib/replication/day"
 363   ]
 364 end
 365
 366 systemd_timer "replication-daily" do
 367   description "Daily replication"
 368   on_calendar "*-*-* *:02/15:00"
 369 end
 370
 371 ## Replication cleanup
 372
 373 systemd_service "replication-cleanup" do
 374   description "Cleanup replication"
 375   user "planet"
 376   exec_start "/usr/local/bin/replicate-cleanup"
 377   sandbox true
 378   read_write_paths "/var/lib/replication"
 379 end
 380
 381 systemd_timer "replication-cleanup" do
 382   description "Cleanup replication"
 383   on_boot_sec 60
 384   on_unit_active_sec 86400
 385   accuracy_sec 1800
 386 end
 387
 388 ## Enable/disable feeds
 389
 390 if node[:planet][:replication] == "enabled"
 391   service "users-agreed.timer" do
 392     action [:enable, :start]
 393   end
 394
 395   service "users-deleted.timer" do
 396     action [:enable, :start]
 397   end
 398
 399   service "replication-changesets.timer" do
 400     action [:enable, :start]
 401   end
 402
 403   service "replication-minutely.timer" do
 404     action [:enable, :start]
 405   end
 406
 407   service "replication-hourly.timer" do
 408     action [:enable, :start]
 409   end
 410
 411   service "replication-daily.timer" do
 412     action [:enable, :start]
 413   end
 414
 415   service "replication-cleanup.timer" do
 416     action [:enable, :start]
 417   end
 418 else
 419   service "users-agreed.timer" do
 420     action [:stop, :disable]
 421   end
 422
 423   service "users-deleted.timer" do
 424     action [:stop, :disable]
 425   end
 426
 427   service "replication-changesets.timer" do
 428     action [:stop, :disable]
 429   end
 430
 431   service "replication-minutely.timer" do
 432     action [:stop, :disable]
 433   end
 434
 435   service "replication-hourly.timer" do
 436     action [:stop, :disable]
 437   end
 438
 439   service "replication-daily.timer" do
 440     action [:stop, :disable]
 441   end
 442
 443   service "replication-cleanup.timer" do
 444     action [:stop, :disable]
 445   end
 446 end