X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/30d9ea905c709817bb4820380f230b5b1724b1f8..e847b156af79e857466b8ca0f51d68ef69a6dd01:/cookbooks/dev/recipes/default.rb diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index d6b27a06d..679ff6b4c 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -1,14 +1,14 @@ # -# Cookbook Name:: dev +# Cookbook:: dev # Recipe:: default # -# Copyright 2011, OpenStreetMap Foundation +# Copyright:: 2011, OpenStreetMap Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -22,46 +22,63 @@ require "securerandom" include_recipe "apache" include_recipe "passenger" +include_recipe "geoipupdate" include_recipe "git" +include_recipe "memcached" +include_recipe "munin" include_recipe "mysql" include_recipe "nodejs" +include_recipe "php::fpm" include_recipe "postgresql" include_recipe "python" +include_recipe "ruby" package %w[ - php php-cgi php-cli php-curl php-db - php-fpm php-imagick - php-mcrypt php-mysql php-pear php-pgsql php-sqlite3 -] - -package %w[ pngcrush pngquant -] - -package %w[ - python - python-argparse - python-beautifulsoup - python-cheetah - python-dateutil - python-magic - python-psycopg2 - python-gdal + python3 + python3-bs4 + python3-cheetah + python3-dateutil + python3-magic + python3-psycopg2 + python3-gdal + g++ + gcc + make + autoconf + automake + libtool + libargon2-dev + libfcgi-dev + libxml2-dev + libmemcached-dev + libboost-regex-dev + libboost-system-dev + libboost-program-options-dev + libboost-date-time-dev + libboost-filesystem-dev + libboost-locale-dev + libpqxx-dev + libcrypto++-dev + libyajl-dev + zlib1g-dev ] nodejs_package "svgo" -python_package "geojson" +python_package "geojson" do + python_version "3" +end apache_module "env" apache_module "expires" @@ -71,44 +88,35 @@ apache_module "proxy_fcgi" apache_module "rewrite" apache_module "suexec" apache_module "userdir" -apache_module "wsgi" - -package "apache2-suexec-pristine" - -gem_package "sqlite3" -gem_package "rails" do - version "3.0.9" +apache_module "wsgi" do + package "libapache2-mod-wsgi-py3" end -service "php7.0-fpm" do - action [:enable, :start] -end +package "apache2-suexec-pristine" -template "/etc/php/7.0/fpm/pool.d/default.conf" do - source "fpm-default.conf.erb" - owner "root" - group "root" - mode 0o644 - notifies :reload, "service[php7.0-fpm]" +php_fpm "default" do + pm_max_children 10 + pm_start_servers 4 + pm_min_spare_servers 2 + pm_max_spare_servers 6 end -file "/etc/php/7.0/fpm/pool.d/www.conf" do +php_fpm "www" do action :delete - notifies :reload, "service[php7.0-fpm]" end directory "/srv/dev.openstreetmap.org" do owner "root" group "root" - mode 0o755 + mode "755" end template "/srv/dev.openstreetmap.org/index.html" do source "dev.html.erb" owner "root" group "root" - mode 0o644 + mode "644" end ssl_certificate "dev.openstreetmap.org" do @@ -126,7 +134,7 @@ template "/etc/phppgadmin/config.inc.php" do source "phppgadmin.conf.erb" owner "root" group "root" - mode 0o644 + mode "644" end file "/etc/apache2/conf.d/phppgadmin" do @@ -152,15 +160,27 @@ search(:accounts, "*:*").each do |account| next unless File.directory?("#{user_home}/public_html") - port = 7000 + account["uid"].to_i - - template "/etc/php/7.0/fpm/pool.d/#{name}.conf" do - source "fpm.conf.erb" - owner "root" - group "root" - mode 0o644 - variables :user => name, :port => port - notifies :reload, "service[php7.0-fpm]" + php_fpm name do + user name + group name + pm_max_children 10 + pm_start_servers 4 + pm_min_spare_servers 2 + pm_max_spare_servers 6 + pm_max_requests 10000 + request_terminate_timeout 1800 + environment "HOSTNAME" => "$HOSTNAME", + "PATH" => "/usr/local/bin:/usr/bin:/bin", + "TMP" => "/tmp", + "TMPDIR" => "/tmp", + "TEMP" => "/tmp" + php_values "max_execution_time" => "300", + "memory_limit" => "128M", + "post_max_size" => "32M", + "upload_max_filesize" => "32M" + php_admin_values "sendmail_path" => "/usr/sbin/sendmail -t -i -f #{name}@errol.openstreetmap.org", + "open_basedir" => "/home/#{name}/:/tmp/:/usr/share/php/" + php_flags "display_errors" => "on" end ssl_certificate "#{name}.dev.openstreetmap.org" do @@ -171,21 +191,21 @@ search(:accounts, "*:*").each do |account| apache_site "#{name}.dev.openstreetmap.org" do template "apache.user.erb" directory "#{user_home}/public_html" - variables :user => name, :port => port + variables :user => name end template "/etc/sudoers.d/#{name}" do source "sudoers.user.erb" owner "root" group "root" - mode 0o440 + mode "440" variables :user => name end end -if node[:postgresql][:clusters][:"9.5/main"] +if node[:postgresql][:clusters][:"14/main"] postgresql_user "apis" do - cluster "9.5/main" + cluster "14/main" end template "/usr/local/bin/cleanup-rails-assets" do @@ -193,53 +213,187 @@ if node[:postgresql][:clusters][:"9.5/main"] source "cleanup-assets.erb" owner "root" group "root" - mode 0o755 + mode "755" + end + + systemd_service "rails-jobs@" do + description "Rails job queue runner" + type "simple" + user "apis" + working_directory "/srv/%i.apis.dev.openstreetmap.org/rails" + exec_start "#{node[:ruby][:bundle]} exec rake jobs:work" + restart "on-failure" + private_tmp true + private_devices true + protect_system "full" + protect_home true + no_new_privileges true + end + + systemd_service "cgimap@" do + description "OpenStreetMap API Server" + type "forking" + environment_file "/etc/default/cgimap-%i" + user "apis" + exec_start "/srv/%i.apis.dev.openstreetmap.org/cgimap/openstreetmap-cgimap --daemon --port $CGIMAP_PORT --instances 5" + exec_reload "/bin/kill -HUP $MAINPID" + private_tmp true + private_devices true + protect_system "full" + protect_home true + no_new_privileges true + restart "on-failure" + end + + cgimap_port = 9000 + + Dir.glob("/srv/*.apis.dev.openstreetmap.org").each do |dir| + node.default_unless[:dev][:rails][File.basename(dir).split(".").first] = {} end node[:dev][:rails].each do |name, details| database_name = details[:database] || "apis_#{name}" site_name = "#{name}.apis.dev.openstreetmap.org" - rails_directory = "/srv/#{name}.apis.dev.openstreetmap.org" + site_directory = "/srv/#{name}.apis.dev.openstreetmap.org" + log_directory = "#{site_directory}/logs" + rails_directory = "#{site_directory}/rails" + cgimap_directory = "#{site_directory}/cgimap" + gpx_directory = "#{site_directory}/gpx" if details[:repository] site_aliases = details[:aliases] || [] - secret_key_base = details[:secret_key_base] || SecureRandom.base64(96) - - node.normal[:dev][:rails][name][:secret_key_base] = secret_key_base + secret_key_base = persistent_token("dev", "rails", name, "secret_key_base") postgresql_database database_name do - cluster "9.5/main" + cluster "14/main" owner "apis" end postgresql_extension "#{database_name}_btree_gist" do - cluster "9.5/main" + cluster "14/main" database database_name extension "btree_gist" end + directory site_directory do + owner "apis" + group "apis" + mode "755" + end + + directory log_directory do + owner "apis" + group "apis" + mode "755" + end + + directory gpx_directory do + owner "apis" + group "apis" + mode "755" + end + + directory "#{gpx_directory}/traces" do + owner "apis" + group "apis" + mode "755" + end + + directory "#{gpx_directory}/images" do + owner "apis" + group "apis" + mode "755" + end + rails_port site_name do - ruby node[:passenger][:ruby_version] directory rails_directory user "apis" group "apis" repository details[:repository] revision details[:revision] - database_port node[:postgresql][:clusters][:"9.5/main"][:port] + database_port node[:postgresql][:clusters][:"14/main"][:port] database_name database_name database_username "apis" + email_from "OpenStreetMap " + gpx_dir gpx_directory + log_path "#{log_directory}/rails.log" + memcache_servers ["127.0.0.1"] + csp_enforce true run_migrations true + trace_use_job_queue true end template "#{rails_directory}/config/initializers/setup.rb" do source "rails.setup.rb.erb" owner "apis" group "apis" - mode 0o644 + mode "644" variables :site => site_name notifies :restart, "rails_port[#{site_name}]" end + service "rails-jobs@#{name}" do + action [:enable, :start] + supports :restart => true + subscribes :restart, "rails_port[#{site_name}]" + subscribes :restart, "systemd_service[#{name}]" + only_if "fgrep -q delayed_job #{rails_directory}/Gemfile.lock" + end + + if details[:cgimap_repository] + git cgimap_directory do + action :sync + repository details[:cgimap_repository] + revision details[:cgimap_revision] + user "apis" + group "apis" + end + + execute "#{cgimap_directory}/autogen.sh" do + action :nothing + command "./autogen.sh" + cwd cgimap_directory + user "apis" + group "apis" + subscribes :run, "git[#{cgimap_directory}]", :immediately + end + + execute "#{cgimap_directory}/configure" do + action :nothing + command "./configure --with-fcgi=/usr --with-boost-libdir=/usr/lib/x86_64-linux-gnu --enable-yajl" + cwd cgimap_directory + user "apis" + group "apis" + subscribes :run, "execute[#{cgimap_directory}/autogen.sh]", :immediately + end + + execute "#{cgimap_directory}/Makefile" do + action :nothing + command "make -j" + cwd cgimap_directory + user "apis" + group "apis" + subscribes :run, "execute[#{cgimap_directory}/configure]", :immediately + notifies :restart, "service[cgimap@#{name}]" + end + + template "/etc/default/cgimap-#{name}" do + source "cgimap.environment.erb" + owner "root" + group "root" + mode "640" + variables :cgimap_port => cgimap_port, + :database_port => node[:postgresql][:clusters][:"14/main"][:port], + :database_name => database_name, + :log_directory => log_directory + notifies :restart, "service[cgimap@#{name}]" + end + + service "cgimap@#{name}" do + action [:start, :enable] + end + end + ssl_certificate site_name do domains [site_name] + site_aliases notifies :reload, "service[apache2]" @@ -247,14 +401,46 @@ if node[:postgresql][:clusters][:"9.5/main"] apache_site site_name do template "apache.rails.erb" - variables :name => site_name, :aliases => site_aliases, :secret_key_base => secret_key_base + variables :application_name => name, + :aliases => site_aliases, + :secret_key_base => secret_key_base, + :cgimap_enabled => details.key?(:cgimap_repository), + :cgimap_port => cgimap_port + end + + template "/etc/logrotate.d/apis-#{name}" do + source "logrotate.apis.erb" + owner "root" + group "root" + mode "644" + variables :name => name, + :log_directory => log_directory, + :rails_directory => rails_directory end + + cgimap_port += 1 else + file "/etc/logrotate.d/apis-#{name}" do + action :delete + end + apache_site site_name do action [:delete] end - directory rails_directory do + service "cgimap@#{name}" do + action [:stop, :disable] + end + + file "/etc/default/cgimap-#{name}" do + action :delete + end + + service "rails-jobs@#{name}" do + action [:stop, :disable] + end + + directory site_directory do action :delete recursive true end @@ -265,24 +451,22 @@ if node[:postgresql][:clusters][:"9.5/main"] postgresql_database database_name do action :drop - cluster "9.5/main" + cluster "14/main" end - - node.normal[:dev][:rails].delete(name) end end directory "/srv/apis.dev.openstreetmap.org" do owner "apis" group "apis" - mode 0o755 + mode "755" end template "/srv/apis.dev.openstreetmap.org/index.html" do source "apis.html.erb" owner "apis" group "apis" - mode 0o644 + mode "644" end ssl_certificate "apis.dev.openstreetmap.org" do @@ -305,17 +489,17 @@ end directory "/srv/ooc.openstreetmap.org" do owner "root" group "root" - mode 0o755 + mode "755" end remote_directory "/srv/ooc.openstreetmap.org/html" do source "ooc" owner "root" group "root" - mode 0o755 + mode "755" files_owner "root" files_group "root" - files_mode 0o644 + files_mode "644" end ssl_certificate "ooc.openstreetmap.org" do