Replace netcat with netcat-openbsd

[chef.git] / cookbooks / dev / recipes / default.rb

diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb
index 3da970fbebf2e253af2aab2ca8b870803fd6d200..6e71b794d509ce98cf251316f6e03e5034b15ecd 100644 (file)
--- a/cookbooks/dev/recipes/default.rb
+++ b/cookbooks/dev/recipes/default.rb
@@ -25,7 +25,6 @@ include_recipe "passenger"
 include_recipe "geoipupdate"
 include_recipe "git"
 include_recipe "memcached"
 include_recipe "geoipupdate"
 include_recipe "git"
 include_recipe "memcached"

-include_recipe "munin"

 include_recipe "mysql"
 include_recipe "nodejs"
 include_recipe "php::fpm"
 include_recipe "mysql"
 include_recipe "nodejs"
 include_recipe "php::fpm"


@@ -35,10 +34,70 @@ include_recipe "ruby"
 
 package %w[
   ant
 
 package %w[
   ant

+  apache2-dev
+  aria2
+  at
+  autoconf
+  automake
+  awscli
+  cmake
+  composer

   default-jdk-headless
   default-jre-headless
   default-jdk-headless
   default-jre-headless

+  fonts-dejavu
+  fonts-dejavu-core
+  fonts-dejavu-extra
+  fonts-droid-fallback
+  fonts-liberation
+  fonts-noto-mono
+  g++
+  gcc
+  gdal-bin
+  gnuplot-nox

   golang
   golang

-  composer
+  graphviz
+  irssi
+  jq
+  libargon2-dev
+  libboost-date-time-dev
+  libboost-dev
+  libboost-filesystem-dev
+  libboost-locale-dev
+  libboost-program-options-dev
+  libboost-regex-dev
+  libboost-system-dev
+  libbrotli-dev
+  libbytes-random-secure-perl
+  libcairo2-dev
+  libcrypto++-dev
+  libcurl4-openssl-dev
+  libfcgi-dev
+  libfmt-dev
+  libglib2.0-dev
+  libiniparser-dev
+  libjson-xs-perl
+  libmapnik-dev
+  libmemcached-dev
+  libpqxx-dev
+  libtool
+  libxml-twig-perl
+  libxml2-dev
+  libyajl-dev
+  lua-any
+  luajit
+  lz4
+  lzip
+  lzop
+  mailutils
+  make
+  nano
+  netcat-openbsd
+  osm2pgsql
+  osmium-tool
+  osmosis
+  pandoc
+  pandoc
+  pbzip2

   php-apcu
   php-cgi
   php-cli
   php-apcu
   php-cgi
   php-cli


@@ -55,8 +114,12 @@ package %w[
   php-pgsql
   php-sqlite3
   php-xml
   php-pgsql
   php-sqlite3
   php-xml

+  pigz

   pngcrush
   pngquant
   pngcrush
   pngquant

+  proj-bin
+  pyosmium
+  python-is-python3

   python3
   python3-brotli
   python3-bs4
   python3
   python3-brotli
   python3-bs4


@@ -64,69 +127,22 @@ package %w[
   python3-dateutil
   python3-dev
   python3-dotenv
   python3-dateutil
   python3-dev
   python3-dotenv

+  python3-gdal

   python3-lxml
   python3-lz4
   python3-magic
   python3-pil
   python3-psycopg2
   python3-pyproj
   python3-lxml
   python3-lz4
   python3-magic
   python3-pil
   python3-psycopg2
   python3-pyproj

-  python3-gdal
-  gdal-bin
-  proj-bin
-  g++
-  gcc
-  make
-  autoconf
-  automake
-  libtool
-  libargon2-dev
-  libfcgi-dev
-  libxml2-dev
-  libmemcached-dev
-  libboost-dev
-  libboost-regex-dev
-  libboost-system-dev
-  libboost-program-options-dev
-  libboost-date-time-dev
-  libboost-filesystem-dev
-  libboost-locale-dev
-  libpqxx-dev
-  libcrypto++-dev
-  libyajl-dev
-  libfmt-dev
-  zlib1g-dev
-  nano
-  osm2pgsql
-  osmosis
-  at
-  awscli
-  mailutils
-  lua-any
-  luajit
-  fonts-dejavu
-  fonts-dejavu-core
-  fonts-dejavu-extra
-  fonts-droid-fallback
-  fonts-liberation
-  fonts-noto-mono
-  gnuplot-nox
-  graphviz
-  irssi
-  jq
-  lz4
-  lzip
-  lzop
-  pbzip2
-  pigz
+  python3-venv
+  r-base
+  redis
+  tmux

   unrar
   unzip
   unrar
   unzip

-  zip
-  netcat
-  tmux

   whois
   whois

-  redis
-  r-base
-  pandoc
+  zip
+  zlib1g-dev

 ]
 
 # Add uk_os_OSTN15_NTv2_OSGBtoETRS.tif used for reprojecting OS data
 ]
 
 # Add uk_os_OSTN15_NTv2_OSGBtoETRS.tif used for reprojecting OS data


@@ -239,7 +255,7 @@ search(:accounts, "*:*").each do |account|
                "memory_limit" => "128M",
                "post_max_size" => "32M",
                "upload_max_filesize" => "32M"
                "memory_limit" => "128M",
                "post_max_size" => "32M",
                "upload_max_filesize" => "32M"

-    php_admin_values "sendmail_path" => "/usr/sbin/sendmail -t -i -f #{name}@errol.openstreetmap.org",
+    php_admin_values "sendmail_path" => "/usr/sbin/sendmail -t -i -f #{name}@dev.openstreetmap.org",

                      "open_basedir" => "/home/#{name}/:/tmp/:/usr/share/php/"
     php_flags "display_errors" => "on"
   end
                      "open_basedir" => "/home/#{name}/:/tmp/:/usr/share/php/"
     php_flags "display_errors" => "on"
   end


@@ -268,9 +284,9 @@ node[:postgresql][:versions].each do |version|
   package "postgresql-#{version}-postgis-3"
 end
 
   package "postgresql-#{version}-postgis-3"
 end
 

-if node[:postgresql][:clusters][:"14/main"]
+if node[:postgresql][:clusters][:"15/main"]

   postgresql_user "apis" do
   postgresql_user "apis" do

-    cluster "14/main"
+    cluster "15/main"

   end
 
   template "/usr/local/bin/cleanup-rails-assets" do
   end
 
   template "/usr/local/bin/cleanup-rails-assets" do


@@ -284,14 +300,19 @@ if node[:postgresql][:clusters][:"14/main"]
   systemd_service "rails-jobs@" do
     description "Rails job queue runner"
     type "simple"
   systemd_service "rails-jobs@" do
     description "Rails job queue runner"
     type "simple"

-    environment "RAILS_ENV" => "production", "SLEEP_DELAY" => "60"
+    environment_file "/etc/default/rails-%i"

     user "apis"
     working_directory "/srv/%i.apis.dev.openstreetmap.org/rails"
     exec_start "#{node[:ruby][:bundle]} exec rails jobs:work"
     restart "on-failure"
     nice 10
     sandbox :enable_network => true
     user "apis"
     working_directory "/srv/%i.apis.dev.openstreetmap.org/rails"
     exec_start "#{node[:ruby][:bundle]} exec rails jobs:work"
     restart "on-failure"
     nice 10
     sandbox :enable_network => true

-    read_write_paths "/srv/%i.apis.dev.openstreetmap.org/logs"
+    restrict_address_families "AF_UNIX"
+    memory_deny_write_execute false
+    read_write_paths [
+      "/srv/%i.apis.dev.openstreetmap.org/logs",
+      "/srv/%i.apis.dev.openstreetmap.org/rails/storage"
+    ]

   end
 
   systemd_service "cgimap@" do
   end
 
   systemd_service "cgimap@" do


@@ -299,16 +320,17 @@ if node[:postgresql][:clusters][:"14/main"]
     type "forking"
     environment_file "/etc/default/cgimap-%i"
     user "apis"
     type "forking"
     environment_file "/etc/default/cgimap-%i"
     user "apis"

-    exec_start "/srv/%i.apis.dev.openstreetmap.org/cgimap/openstreetmap-cgimap --daemon --port $CGIMAP_PORT --instances 5"
+    group "www-data"
+    umask "0002"
+    exec_start "/srv/%i.apis.dev.openstreetmap.org/cgimap/build/openstreetmap-cgimap --daemon --instances 5"

     exec_reload "/bin/kill -HUP $MAINPID"
     exec_reload "/bin/kill -HUP $MAINPID"

+    runtime_directory "cgimap-%i"

     sandbox :enable_network => true
     restrict_address_families "AF_UNIX"
     read_write_paths ["/srv/%i.apis.dev.openstreetmap.org/logs", "/srv/%i.apis.dev.openstreetmap.org/rails/tmp"]
     restart "on-failure"
   end
 
     sandbox :enable_network => true
     restrict_address_families "AF_UNIX"
     read_write_paths ["/srv/%i.apis.dev.openstreetmap.org/logs", "/srv/%i.apis.dev.openstreetmap.org/rails/tmp"]
     restart "on-failure"
   end
 

-  cgimap_port = 9000
-

   Dir.glob("/srv/*.apis.dev.openstreetmap.org").each do |dir|
     node.default_unless[:dev][:rails][File.basename(dir).split(".").first] = {}
   end
   Dir.glob("/srv/*.apis.dev.openstreetmap.org").each do |dir|
     node.default_unless[:dev][:rails][File.basename(dir).split(".").first] = {}
   end


@@ -327,12 +349,12 @@ if node[:postgresql][:clusters][:"14/main"]
       secret_key_base = persistent_token("dev", "rails", name, "secret_key_base")
 
       postgresql_database database_name do
       secret_key_base = persistent_token("dev", "rails", name, "secret_key_base")
 
       postgresql_database database_name do

-        cluster "14/main"
+        cluster "15/main"

         owner "apis"
       end
 
       postgresql_extension "#{database_name}_btree_gist" do
         owner "apis"
       end
 
       postgresql_extension "#{database_name}_btree_gist" do

-        cluster "14/main"
+        cluster "15/main"

         database database_name
         extension "btree_gist"
       end
         database database_name
         extension "btree_gist"
       end


@@ -367,13 +389,19 @@ if node[:postgresql][:clusters][:"14/main"]
         mode "755"
       end
 
         mode "755"
       end
 

+      openssl_rsa_private_key "#{site_directory}/doorkeeper.key" do
+        owner "root"
+        group "root"
+        mode "0400"
+      end
+

       rails_port site_name do
         directory rails_directory
         user "apis"
         group "apis"
         repository details[:repository]
         revision details[:revision]
       rails_port site_name do
         directory rails_directory
         user "apis"
         group "apis"
         repository details[:repository]
         revision details[:revision]

-        database_port node[:postgresql][:clusters][:"14/main"][:port]
+        database_port node[:postgresql][:clusters][:"15/main"][:port]

         database_name database_name
         database_username "apis"
         email_from "OpenStreetMap <web@noreply.openstreetmap.org>"
         database_name database_name
         database_username "apis"
         email_from "OpenStreetMap <web@noreply.openstreetmap.org>"


@@ -383,6 +411,7 @@ if node[:postgresql][:clusters][:"14/main"]
         csp_enforce true
         run_migrations true
         trace_use_job_queue true
         csp_enforce true
         run_migrations true
         trace_use_job_queue true

+        doorkeeper_signing_key lazy { File.read("#{site_directory}/doorkeeper.key") }

       end
 
       template "#{rails_directory}/config/initializers/setup.rb" do
       end
 
       template "#{rails_directory}/config/initializers/setup.rb" do


@@ -394,11 +423,19 @@ if node[:postgresql][:clusters][:"14/main"]
         notifies :restart, "rails_port[#{site_name}]"
       end
 
         notifies :restart, "rails_port[#{site_name}]"
       end
 

+      template "/etc/default/rails-#{name}" do
+        source "rails.environment.erb"
+        owner "root"
+        group "root"
+        mode "0600"
+        variables :secret_key_base => secret_key_base
+      end
+

       service "rails-jobs@#{name}" do
         action [:enable, :start]
         supports :restart => true
         subscribes :restart, "rails_port[#{site_name}]"
       service "rails-jobs@#{name}" do
         action [:enable, :start]
         supports :restart => true
         subscribes :restart, "rails_port[#{site_name}]"

-        subscribes :restart, "systemd_service[#{name}]"
+        subscribes :restart, "systemd_service[rails-jobs@]"

         only_if "fgrep -q delayed_job #{rails_directory}/Gemfile.lock"
       end
 
         only_if "fgrep -q delayed_job #{rails_directory}/Gemfile.lock"
       end
 


@@ -411,32 +448,28 @@ if node[:postgresql][:clusters][:"14/main"]
           group "apis"
         end
 
           group "apis"
         end
 

-        execute "#{cgimap_directory}/autogen.sh" do
-          action :nothing
-          command "./autogen.sh"
-          cwd cgimap_directory
+        directory "#{cgimap_directory}/build" do

           user "apis"
           group "apis"
           user "apis"
           group "apis"

-          subscribes :run, "git[#{cgimap_directory}]", :immediately
+          mode "0755"

         end
 
         end
 

-        execute "#{cgimap_directory}/configure" do
+        execute "#{cgimap_directory}/CMakeLists.txt" do

           action :nothing
           action :nothing

-          command "./configure --with-fcgi=/usr --with-boost-libdir=/usr/lib/x86_64-linux-gnu --enable-yajl"
-          cwd cgimap_directory
+          command "cmake .."
+          cwd "#{cgimap_directory}/build"

           user "apis"
           group "apis"
           user "apis"
           group "apis"

-          subscribes :run, "execute[#{cgimap_directory}/autogen.sh]", :immediately
+          subscribes :run, "git[#{cgimap_directory}]", :immediately

         end
 
         end
 

-        execute "#{cgimap_directory}/Makefile" do
+        execute "#{cgimap_directory}/build/Makefile" do

           action :nothing
           command "make -j"
           action :nothing
           command "make -j"

-          cwd cgimap_directory
+          cwd "#{cgimap_directory}/build"

           user "apis"
           group "apis"
           user "apis"
           group "apis"

-          subscribes :run, "execute[#{cgimap_directory}/configure]", :immediately
-          notifies :restart, "service[cgimap@#{name}]"
+          subscribes :run, "execute[#{cgimap_directory}/CMakeLists.txt]", :immediately

         end
 
         template "/etc/default/cgimap-#{name}" do
         end
 
         template "/etc/default/cgimap-#{name}" do


@@ -444,15 +477,18 @@ if node[:postgresql][:clusters][:"14/main"]
           owner "root"
           group "root"
           mode "640"
           owner "root"
           group "root"
           mode "640"

-          variables :cgimap_port => cgimap_port,
-                    :database_port => node[:postgresql][:clusters][:"14/main"][:port],
+          variables :cgimap_socket => "/run/cgimap-#{name}/socket",
+                    :database_port => node[:postgresql][:clusters][:"15/main"][:port],

                     :database_name => database_name,
                     :database_name => database_name,

-                    :log_directory => log_directory
-          notifies :restart, "service[cgimap@#{name}]"
+                    :log_directory => log_directory,
+                    :options => details[:cgimap_options]

         end
 
         service "cgimap@#{name}" do
           action [:start, :enable]
         end
 
         service "cgimap@#{name}" do
           action [:start, :enable]

+          subscribes :restart, "execute[#{cgimap_directory}/build/Makefile]"
+          subscribes :restart, "template[/etc/default/cgimap-#{name}]"
+          subscribes :restart, "systemd_service[cgimap@]"

         end
       end
 
         end
       end
 


@@ -467,7 +503,7 @@ if node[:postgresql][:clusters][:"14/main"]
                   :aliases => site_aliases,
                   :secret_key_base => secret_key_base,
                   :cgimap_enabled => details.key?(:cgimap_repository),
                   :aliases => site_aliases,
                   :secret_key_base => secret_key_base,
                   :cgimap_enabled => details.key?(:cgimap_repository),

-                  :cgimap_port => cgimap_port
+                  :cgimap_socket => "/run/cgimap-#{name}/socket"

       end
 
       template "/etc/logrotate.d/apis-#{name}" do
       end
 
       template "/etc/logrotate.d/apis-#{name}" do


@@ -479,8 +515,6 @@ if node[:postgresql][:clusters][:"14/main"]
                   :log_directory => log_directory,
                   :rails_directory => rails_directory
       end
                   :log_directory => log_directory,
                   :rails_directory => rails_directory
       end

-
-      cgimap_port += 1

     else
       file "/etc/logrotate.d/apis-#{name}" do
         action :delete
     else
       file "/etc/logrotate.d/apis-#{name}" do
         action :delete


@@ -513,7 +547,7 @@ if node[:postgresql][:clusters][:"14/main"]
 
       postgresql_database database_name do
         action :drop
 
       postgresql_database database_name do
         action :drop

-        cluster "14/main"
+        cluster "15/main"

       end
     end
   end
       end
     end
   end


@@ -539,13 +573,6 @@ if node[:postgresql][:clusters][:"14/main"]
   apache_site "apis.dev.openstreetmap.org" do
     template "apache.apis.erb"
   end
   apache_site "apis.dev.openstreetmap.org" do
     template "apache.apis.erb"
   end

-
-  node[:postgresql][:clusters].each_key do |name|
-    postgresql_munin name do
-      cluster name
-      database "ALL"
-    end
-  end

 end
 
 directory "/srv/ooc.openstreetmap.org" do
 end
 
 directory "/srv/ooc.openstreetmap.org" do


@@ -575,3 +602,16 @@ end
 apache_site "ooc.openstreetmap.org" do
   template "apache.ooc.erb"
 end
 apache_site "ooc.openstreetmap.org" do
   template "apache.ooc.erb"
 end

+
+directory "/etc/systemd/system/user-.slice.d" do
+  owner "root"
+  group "root"
+  mode "0755"
+end
+
+template "/etc/systemd/system/user-.slice.d/99-chef.conf" do
+  source "user-slice.conf.erb"
+  owner "root"
+  group "root"
+  mode "0644"
+end