fix: README.md grammar punctuation

- to automate the configuration
- e.g.,

diff --git a/README.md b/README.md
index 86145db1765475d9ed7c6d07bcd1fa0c4750f76e..6b6304834a74c475269d3c9c76095855b9a30229 100644 (file)
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 
 This repository manages the configuration of all the servers run by the
 OpenStreetMap Foundation's Operations Working Group. We use
 
 This repository manages the configuration of all the servers run by the
 OpenStreetMap Foundation's Operations Working Group. We use

-[Chef](https://www.chef.io/) to automated the configuration of all of our
+[Chef](https://www.chef.io/) to automate the configuration of all of our

 servers.
 
 [OSMF Operations Working Group](https://operations.osmfoundation.org/)
 servers.
 
 [OSMF Operations Working Group](https://operations.osmfoundation.org/)


@@ -14,20 +14,20 @@ servers.
 
 We make extensive use of roles to configure the servers. In general we have:
 
 
 We make extensive use of roles to configure the servers. In general we have:
 

-## Server-specific roles (e.g. [faffy.rb](roles/faffy.rb))
+## Server-specific roles (e.g., [faffy.rb](roles/faffy.rb))

 
 These deal with particular setup or quirks of a server, such as its IP address. They also include roles representing the service they are performing, and the location they are in and any particular hardware they have that needs configuration.
 All our servers are [named after dragons](https://wiki.openstreetmap.org/wiki/Servers/Name_Ideas).
 
 
 These deal with particular setup or quirks of a server, such as its IP address. They also include roles representing the service they are performing, and the location they are in and any particular hardware they have that needs configuration.
 All our servers are [named after dragons](https://wiki.openstreetmap.org/wiki/Servers/Name_Ideas).
 

-## Hardware-specific roles (e.g. [hp-g9.rb](roles/hp-g9.rb))
+## Hardware-specific roles (e.g., [hp-g9.rb](roles/hp-g9.rb))

 
 Covers anything specific to a certain piece of hardware, like a motherboard, that could apply to multiple machines.
 
 
 Covers anything specific to a certain piece of hardware, like a motherboard, that could apply to multiple machines.
 

-## Location-specific roles (e.g. [equinix-dub.rb](roles/equinix-dub.rb))
+## Location-specific roles (e.g., [equinix-dub.rb](roles/equinix-dub.rb))

 
 These form a hierarchy of datacentres, organisations, and countries where our servers are located.
 
 
 These form a hierarchy of datacentres, organisations, and countries where our servers are located.
 

-## Service-specific roles (e.g. [web-frontend](roles/web-frontend.rb))
+## Service-specific roles (e.g., [web-frontend](roles/web-frontend.rb))

 
 These cover the services that the server is running, and will include the recipes required for that service along with any specific configurations and other cascading roles.
 
 
 These cover the services that the server is running, and will include the recipes required for that service along with any specific configurations and other cascading roles.
 

diff --git a/cookbooks/community/recipes/default.rb b/cookbooks/community/recipes/default.rb
index 000a57a7e00cac990e9ebafa1c9e4aba3ec37deb..a09762470771d1b67ec783ff61a501b8d3ed2a05 100644 (file)
--- a/cookbooks/community/recipes/default.rb
+++ b/cookbooks/community/recipes/default.rb
@@ -62,8 +62,8 @@ end
 git "/srv/community.openstreetmap.org/docker" do
   action :sync
   repository "https://github.com/discourse/discourse_docker.git"
 git "/srv/community.openstreetmap.org/docker" do
   action :sync
   repository "https://github.com/discourse/discourse_docker.git"

-  # Revision pin not possible as launch wrapper automatically updates git repo.
-  revision "main"
+  # DANGER launch wrapper automatically updates git repo if rebuild method used: https://github.com/discourse/discourse_docker/blob/107ffb40fe8b1ea40e00814468db974a4f3f8e8f/launcher#L799
+  revision "b345430a822d6275573f3d82f1ad2e2b5fa1e0b1"

   depth 1
   user "root"
   group "root"
   depth 1
   user "root"
   group "root"


@@ -122,18 +122,24 @@ notify_group "discourse_container_new_web_only" do
   notifies :run, "execute[discourse_container_data_start]", :immediately # noop if site up
   notifies :run, "execute[discourse_container_web_only_bootstrap]", :immediately # site up but runs in parallel. Slow
   notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down
   notifies :run, "execute[discourse_container_data_start]", :immediately # noop if site up
   notifies :run, "execute[discourse_container_web_only_bootstrap]", :immediately # site up but runs in parallel. Slow
   notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down

-  notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down
+  notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down
+  notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down
+  notifies :run, "execute[discourse_container_data_start]", :immediately # site down

   notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore
 end
 
 notify_group "discourse_container_new_data" do
   notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down
   notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore
 end
 
 notify_group "discourse_container_new_data" do
   notifies :run, "execute[discourse_container_web_only_destroy]", :immediately # site down

-  notifies :run, "execute[discourse_container_data_rebuild]", :immediately # site down
+  notifies :run, "execute[discourse_container_data_destroy]", :immediately # site down
+  notifies :run, "execute[discourse_container_data_bootstrap]", :immediately # site down
+  notifies :run, "execute[discourse_container_data_start]", :immediately # site down

   notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore
 end
 
 notify_group "discourse_container_new_mail_receiver" do
   notifies :run, "execute[discourse_container_web_only_start]", :immediately # site restore
 end
 
 notify_group "discourse_container_new_mail_receiver" do

-  notifies :run, "execute[discourse_container_mail_receiver_rebuild]", :immediately
+  notifies :run, "execute[discourse_container_mail_receiver_destroy]", :immediately
+  notifies :run, "execute[discourse_container_mail_receiver_bootstrap]", :immediately
+  notifies :run, "execute[discourse_container_mail_receiver_start]", :immediately

 end
 
 # Attempt at a failsafe to ensure all containers are running
 end
 
 # Attempt at a failsafe to ensure all containers are running


@@ -144,17 +150,25 @@ notify_group "discourse_container_ensure_all_running" do
   notifies :run, "execute[discourse_container_mail_receiver_start]", :delayed
 end
 
   notifies :run, "execute[discourse_container_mail_receiver_start]", :delayed
 end
 

-execute "discourse_container_data_start" do
+execute "discourse_container_data_bootstrap" do

   action :nothing
   action :nothing

-  command "./launcher start data"
+  command "./launcher bootstrap data"
+  cwd "/srv/community.openstreetmap.org/docker/"
+  user "root"
+  group "root"
+end
+
+execute "discourse_container_data_destroy" do
+  action :nothing
+  command "./launcher destroy data"

   cwd "/srv/community.openstreetmap.org/docker/"
   user "root"
   group "root"
 end
 
   cwd "/srv/community.openstreetmap.org/docker/"
   user "root"
   group "root"
 end
 

-execute "discourse_container_data_rebuild" do
+execute "discourse_container_data_start" do

   action :nothing
   action :nothing

-  command "./launcher rebuild data"
+  command "./launcher start data"

   cwd "/srv/community.openstreetmap.org/docker/"
   user "root"
   group "root"
   cwd "/srv/community.openstreetmap.org/docker/"
   user "root"
   group "root"


@@ -184,10 +198,17 @@ execute "discourse_container_web_only_start" do
   group "root"
 end
 
   group "root"
 end
 

-# Rebuild: Stop Destroy Bootstap Start
-execute "discourse_container_mail_receiver_rebuild" do
+execute "discourse_container_mail_receiver_bootstrap" do
+  action :nothing
+  command "./launcher bootstrap mail-receiver"
+  cwd "/srv/community.openstreetmap.org/docker/"
+  user "root"
+  group "root"
+end
+
+execute "discourse_container_mail_receiver_destroy" do

   action :nothing
   action :nothing

-  command "./launcher rebuild mail-receiver"
+  command "./launcher destroy mail-receiver"

   cwd "/srv/community.openstreetmap.org/docker/"
   user "root"
   group "root"
   cwd "/srv/community.openstreetmap.org/docker/"
   user "root"
   group "root"

diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb
index 073bb3e132feec3a85a09b84b07a94733d65c700..0d6790f46308a675e8383ff304ae180ed2f9816f 100644 (file)
--- a/cookbooks/dev/recipes/default.rb
+++ b/cookbooks/dev/recipes/default.rb
@@ -61,6 +61,7 @@ package %w[
   gnuplot-nox
   golang
   graphviz
   gnuplot-nox
   golang
   graphviz

+  htop

   irssi
   jq
   libargon2-dev
   irssi
   jq
   libargon2-dev


@@ -95,6 +96,7 @@ package %w[
   lzip
   lzop
   mailutils
   lzip
   lzop
   mailutils

+  moreutils

   make
   nano
   ncftp
   make
   nano
   ncftp


@@ -103,7 +105,6 @@ package %w[
   osmium-tool
   osmosis
   pandoc
   osmium-tool
   osmosis
   pandoc

-  pandoc

   pbzip2
   php-apcu
   php-cgi
   pbzip2
   php-apcu
   php-cgi


@@ -149,6 +150,7 @@ package %w[
   unrar
   unzip
   whois
   unrar
   unzip
   whois

+  xxd

   zip
   zlib1g-dev
 ]
   zip
   zlib1g-dev
 ]

diff --git a/cookbooks/dns/recipes/default.rb b/cookbooks/dns/recipes/default.rb
index d1b3d159e619132edefc54498e5d82f9d6dd2f7d..3a19d8a91662cfd83f7dd1bcbcb281e0bd532798 100644 (file)
--- a/cookbooks/dns/recipes/default.rb
+++ b/cookbooks/dns/recipes/default.rb
@@ -42,7 +42,7 @@ package %w[
 
 cache_dir = Chef::Config[:file_cache_path]
 
 
 cache_dir = Chef::Config[:file_cache_path]
 

-dnscontrol_version = "4.15.1"
+dnscontrol_version = "4.15.5"

 
 dnscontrol_arch = if arm?
                     "arm64"
 
 dnscontrol_arch = if arm?
                     "arm64"

diff --git a/cookbooks/exim/templates/default/exim4.conf.erb b/cookbooks/exim/templates/default/exim4.conf.erb
index ffc8be609f4d9f2f360cd37dd9015a063db908ee..3558af884317b3907656de655d1b105ad2d9a2d4 100644 (file)
--- a/cookbooks/exim/templates/default/exim4.conf.erb
+++ b/cookbooks/exim/templates/default/exim4.conf.erb
@@ -753,6 +753,9 @@ signed_smtp:
   multi_domain = false
   hosts_try_dane =
   tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION
   multi_domain = false
   hosts_try_dane =
   tls_require_ciphers = <%= node[:ssl][:gnutls_ciphers] %>:%LATEST_RECORD_VERSION

+<% if node[:exim][:external_interface] -%>
+  interface = <%= node[:exim][:external_interface] %>
+<% end -%>

 
 
 # This transport is used for handling pipe deliveries generated by alias or
 
 
 # This transport is used for handling pipe deliveries generated by alias or

diff --git a/cookbooks/imagery/recipes/tiler.rb b/cookbooks/imagery/recipes/tiler.rb
index ec7daa835ebc5594e27751f756e7369e375e13b4..319cb5794a749e11a70fc90665def050786b8f11 100644 (file)
--- a/cookbooks/imagery/recipes/tiler.rb
+++ b/cookbooks/imagery/recipes/tiler.rb
@@ -39,9 +39,7 @@ podman_service "titiler" do
   image container_image
   volume :"/store/imagery"       => "/store/imagery",
          :"/srv/imagery/sockets" => "/sockets"
   image container_image
   volume :"/store/imagery"       => "/store/imagery",
          :"/srv/imagery/sockets" => "/sockets"

-  environment :BIND                                => "unix:/sockets/titiler.sock",
-              :WORKERS_PER_CORE                    => 1,
-              :GDAL_CACHEMAX                       => 200,
+  environment :GDAL_CACHEMAX                       => 200,

               :GDAL_BAND_BLOCK_CACHE               => "HASHSET",
               :GDAL_DISABLE_READDIR_ON_OPEN        => "EMPTY_DIR",
               :GDAL_INGESTED_BYTES_AT_OPEN         => 32768,
               :GDAL_BAND_BLOCK_CACHE               => "HASHSET",
               :GDAL_DISABLE_READDIR_ON_OPEN        => "EMPTY_DIR",
               :GDAL_INGESTED_BYTES_AT_OPEN         => 32768,


@@ -52,6 +50,7 @@ podman_service "titiler" do
               :VSI_CACHE_SIZE                      => 5000000,
               :TITILER_API_ROOT_PATH               => "/api/v1/titiler",
               :FORWARDED_ALLOW_IPS                 => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips
               :VSI_CACHE_SIZE                      => 5000000,
               :TITILER_API_ROOT_PATH               => "/api/v1/titiler",
               :FORWARDED_ALLOW_IPS                 => "*" # https://docs.gunicorn.org/en/latest/settings.html#forwarded-allow-ips

+  command "gunicorn -k uvicorn.workers.UvicornWorker titiler.application.main:app --bind unix:/sockets/titiler.sock --workers #{node.cpu_cores}"

 end
 
 systemd_service "titiler-restart" do
 end
 
 systemd_service "titiler-restart" do

diff --git a/cookbooks/planet/files/default/replication-bin/replicate-changesets b/cookbooks/planet/files/default/replication-bin/replicate-changesets
index 0c53856d984f1aae58f754cc10a135ba8db27d83..bb4544f68f2eeba005634632f1d1f605665bf4cf 100755 (executable)
--- a/cookbooks/planet/files/default/replication-bin/replicate-changesets
+++ b/cookbooks/planet/files/default/replication-bin/replicate-changesets
@@ -96,7 +96,7 @@ class ChangesetBuilder
 
   def add_comments(xml, cs)
     # grab the visible changeset comments as well
 
   def add_comments(xml, cs)
     # grab the visible changeset comments as well

-    res = @conn.exec("select cc.author_id, u.display_name as author, cc.body, cc.created_at from changeset_comments cc join users u on cc.author_id=u.id where cc.changeset_id=#{cs.id} and cc.visible order by cc.created_at asc")
+    res = @conn.exec("select cc.id, cc.author_id, u.display_name as author, cc.body, (cc.created_at at time zone 'utc') as created_at from changeset_comments cc join users u on cc.author_id=u.id where cc.changeset_id=#{cs.id} and cc.visible order by cc.created_at asc")

     xml["comments_count"] = res.num_tuples.to_s
 
     # early return if there aren't any comments
     xml["comments_count"] = res.num_tuples.to_s
 
     # early return if there aren't any comments


@@ -105,6 +105,7 @@ class ChangesetBuilder
     discussion = XML::Node.new("discussion")
     res.each do |row|
       comment = XML::Node.new("comment")
     discussion = XML::Node.new("discussion")
     res.each do |row|
       comment = XML::Node.new("comment")

+      comment["id"] = row["id"]

       comment["uid"] = row["author_id"]
       comment["user"] = xml_sanitize(row["author"])
       comment["date"] = Time.parse(row["created_at"]).getutc.xmlschema
       comment["uid"] = row["author_id"]
       comment["user"] = xml_sanitize(row["author"])
       comment["date"] = Time.parse(row["created_at"]).getutc.xmlschema


@@ -127,7 +128,7 @@ end
 # sync a directory to guarantee it's on disk. have to recurse to the root
 # to guarantee sync for newly created directories.
 def fdirsync(d)
 # sync a directory to guarantee it's on disk. have to recurse to the root
 # to guarantee sync for newly created directories.
 def fdirsync(d)

-  while d != "/"
+  while d != "/" && d != "."

     fsync(d)
     d = File.dirname(d)
   end
     fsync(d)
     d = File.dirname(d)
   end


@@ -139,7 +140,7 @@ end
 class Replicator
   def initialize(config)
     @config = YAML.safe_load(File.read(config))
 class Replicator
   def initialize(config)
     @config = YAML.safe_load(File.read(config))

-    @state = YAML.safe_load(File.read(@config["state_file"]), [Time])
+    @state = YAML.safe_load(File.read(@config["state_file"]), :permitted_classes => [Time], :fallback => {})

     @conn = PG::Connection.connect(@config["db"])
     # get current time from the database rather than the current system
     @now = @conn.exec("select now() as now").map { |row| Time.parse(row["now"]) }[0]
     @conn = PG::Connection.connect(@config["db"])
     # get current time from the database rather than the current system
     @now = @conn.exec("select now() as now").map { |row| Time.parse(row["now"]) }[0]


@@ -153,7 +154,7 @@ class Replicator
     # for us to look at anything that was closed recently, and filter from
     # there.
     changesets = @conn
     # for us to look at anything that was closed recently, and filter from
     # there.
     changesets = @conn

-                 .exec("select id, created_at, closed_at, num_changes from changesets where closed_at > ((now() at time zone 'utc') - '1 hour'::interval)")
+                 .exec("select id, (created_at at time zone 'utc') as created_at, (closed_at at time zone 'utc') as closed_at, num_changes from changesets where (closed_at at time zone 'utc') > ((now() at time zone 'utc') - '1 hour'::interval)")

                  .map { |row| Changeset.new(row) }
                  .select { |cs| cs.activity_between?(last_run, @now) }
 
                  .map { |row| Changeset.new(row) }
                  .select { |cs| cs.activity_between?(last_run, @now) }
 


@@ -162,13 +163,13 @@ class Replicator
 
     # but also add any changesets which have new comments
     new_ids = @conn
 
     # but also add any changesets which have new comments
     new_ids = @conn

-              .exec("select distinct changeset_id from changeset_comments where created_at >= '#{last_run}' and created_at < '#{@now}' and visible")
+              .exec("select distinct changeset_id from changeset_comments where (created_at at time zone 'utc') >= '#{last_run}' and (created_at at time zone 'utc') < '#{@now}' and visible")

               .map { |row| row["changeset_id"].to_i }
               .reject { |c_id| cs_ids.include?(c_id) }
 
     new_ids.each do |id|
       @conn
               .map { |row| row["changeset_id"].to_i }
               .reject { |c_id| cs_ids.include?(c_id) }
 
     new_ids.each do |id|
       @conn

-        .exec("select id, created_at, closed_at, num_changes from changesets where id=#{id}")
+        .exec("select id, (created_at at time zone 'utc') as created_at, (closed_at at time zone 'utc') as closed_at, num_changes from changesets where id=#{id}")

         .map { |row| Changeset.new(row) }
         .each { |cs| changesets << cs }
     end
         .map { |row| Changeset.new(row) }
         .each { |cs| changesets << cs }
     end

diff --git a/cookbooks/podman/resources/service.rb b/cookbooks/podman/resources/service.rb
index 5ef0b5885f2d90476fc75103ed86572ae2d32d97..d9a328192eb6d86b24a596a0a6a704ed2715c406 100644 (file)
--- a/cookbooks/podman/resources/service.rb
+++ b/cookbooks/podman/resources/service.rb
@@ -27,6 +27,7 @@ property :image, String, :required => true
 property :ports, Hash, :default => {}
 property :environment, Hash, :default => {}
 property :volume, Hash, :default => {}
 property :ports, Hash, :default => {}
 property :environment, Hash, :default => {}
 property :volume, Hash, :default => {}

+property :command, String, :default => ""

 
 action :create do
   systemd_service new_resource.service do
 
 action :create do
   systemd_service new_resource.service do


@@ -35,7 +36,11 @@ action :create do
     notify_access "all"
     environment "PODMAN_SYSTEMD_UNIT" => "%n"
     exec_start_pre "/bin/rm --force %t/%n.ctr-id"
     notify_access "all"
     environment "PODMAN_SYSTEMD_UNIT" => "%n"
     exec_start_pre "/bin/rm --force %t/%n.ctr-id"

-    exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --userns=auto --label=io.containers.autoupdate=registry --pids-limit=-1 #{publish_options} #{environment_options} #{volume_options} --rm --sdnotify=conmon --detach --replace --name=%N #{new_resource.image}"
+    exec_start "/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon "\
+               "--userns=auto --label=io.containers.autoupdate=registry "\
+               "--pids-limit=-1 #{publish_options} #{environment_options} "\
+               "#{volume_options} --rm --sdnotify=conmon --detach --replace "\
+               "--name=%N #{new_resource.image} #{new_resource.command}"

     exec_stop "/usr/bin/podman stop --ignore --time=10 --cidfile=%t/%n.ctr-id"
     exec_stop_post "/usr/bin/podman rm --force --ignore --cidfile=%t/%n.ctr-id"
     timeout_start_sec 180
     exec_stop "/usr/bin/podman stop --ignore --time=10 --cidfile=%t/%n.ctr-id"
     exec_stop_post "/usr/bin/podman rm --force --ignore --cidfile=%t/%n.ctr-id"
     timeout_start_sec 180

diff --git a/cookbooks/taginfo/recipes/default.rb b/cookbooks/taginfo/recipes/default.rb
index 080d119241af8a96647d670f2971d5900d73053a..e4a6da00fe0867304cf5700a3389689aa0218d6d 100644 (file)
--- a/cookbooks/taginfo/recipes/default.rb
+++ b/cookbooks/taginfo/recipes/default.rb
@@ -163,7 +163,7 @@ node[:taginfo][:sites].each do |site|
     settings["opensearch"]["contact"] = "webmaster@openstreetmap.org"
     settings["paths"]["bin_dir"] = "#{directory}/build/src"
     settings["sources"]["download"] = ""
     settings["opensearch"]["contact"] = "webmaster@openstreetmap.org"
     settings["paths"]["bin_dir"] = "#{directory}/build/src"
     settings["sources"]["download"] = ""

-    settings["sources"]["create"] = "db languages projects wiki wikidata chronology"
+    settings["sources"]["create"] = "db languages projects wiki wikidata chronology sw"

     settings["sources"]["db"]["planetfile"] = "/var/lib/planet/planet.osh.pbf"
     settings["sources"]["chronology"]["osm_history_file"] = "/var/lib/planet/planet.osh.pbf"
     settings["tagstats"]["geodistribution"] = "DenseMmapArray"
     settings["sources"]["db"]["planetfile"] = "/var/lib/planet/planet.osh.pbf"
     settings["sources"]["chronology"]["osm_history_file"] = "/var/lib/planet/planet.osh.pbf"
     settings["tagstats"]["geodistribution"] = "DenseMmapArray"

diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb
index 48b60f3c80bb1add7c8b89938a528c533f103813..a8a78044c18fa72e3375f3a0bb4c35b8aac04012 100644 (file)
--- a/cookbooks/vectortile/recipes/default.rb
+++ b/cookbooks/vectortile/recipes/default.rb
@@ -250,7 +250,7 @@ template "/usr/local/bin/vector-update" do
   owner "root"
   group "root"
   mode "755"
   owner "root"
   group "root"
   mode "755"

-  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :post_processing => "/usr/local/bin/tiles-rerender"
+  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :post_processing => "/usr/local/bin/tiles-rerender"

 end
 
 template "/usr/local/bin/tiles-rerender" do
 end
 
 template "/usr/local/bin/tiles-rerender" do


@@ -258,7 +258,7 @@ template "/usr/local/bin/tiles-rerender" do
   owner "root"
   group "root"
   mode "755"
   owner "root"
   group "root"
   mode "755"

-  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :update_threads => 4
+  variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :expiry_dir => "/srv/vector.openstreetmap.org/data/", :update_threads => 4

 end
 
 systemd_service "replicate" do
 end
 
 systemd_service "replicate" do

diff --git a/cookbooks/vectortile/templates/default/import-planet.erb b/cookbooks/vectortile/templates/default/import-planet.erb
index bffa34a3a5df0df40e545221fc78326196987e2b..3d6110181d18589ade657116f6c5752e8bf34901 100644 (file)
--- a/cookbooks/vectortile/templates/default/import-planet.erb
+++ b/cookbooks/vectortile/templates/default/import-planet.erb
@@ -7,7 +7,7 @@
 
 set -e
 
 
 set -e
 

-export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;'
+export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;'

 
 # Import the osm2pgsql file specified as an argument, using the locations for spirit
 osm2pgsql \
 
 # Import the osm2pgsql file specified as an argument, using the locations for spirit
 osm2pgsql \

diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb
index f46b6f1f616a7ae8029a3a61ead1234ea7abb481..6c01b27f6f4a7fec493fd2704e09ffc2b5bb3456 100644 (file)
--- a/cookbooks/vectortile/templates/default/tiles-rerender.erb
+++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb
@@ -1,7 +1,11 @@
 #!/bin/sh
 set -eu
 #!/bin/sh
 set -eu

-<%= @tilekiln_bin %> generate tiles \
+
+cd "<%= @expiry_dir %>"
+
+wc -l z*.txt
+cat z*.txt | <%= @tilekiln_bin %> generate tiles \

 --source-dbname "<%= @source_database %>" \
 --storage-dbname "<%= @storage_database %>" \
 --num-threads "<%= node[:vectortile][:replication][:threads] %>" \
 --source-dbname "<%= @source_database %>" \
 --storage-dbname "<%= @storage_database %>" \
 --num-threads "<%= node[:vectortile][:replication][:threads] %>" \

---config <%= @config_path %> < <%= @tiles_file %>
+--config <%= @config_path %>

diff --git a/cookbooks/vectortile/templates/default/vector-update-notile.erb b/cookbooks/vectortile/templates/default/vector-update-notile.erb
index cd718789ca58b3aeecb93df11b54415f5c665abb..844d9d4faa0f64fa264cf79fea97da6473d61806 100644 (file)
--- a/cookbooks/vectortile/templates/default/vector-update-notile.erb
+++ b/cookbooks/vectortile/templates/default/vector-update-notile.erb
@@ -1,12 +1,13 @@
 #!/bin/sh
 
 # Usage
 #!/bin/sh
 
 # Usage

-# sudo -u tilekiln vector-update
+# sudo -u tileupdate vector-update

 
 set -eu
 
 
 set -eu
 

-export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;'
+export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;'

 
 

+cd "<%= @expiry_dir %>"

 osm2pgsql-replication update \
   -d "<%= @source_database %>" \
   --max-diff-size "<%= @diff_size %>"
 osm2pgsql-replication update \
   -d "<%= @source_database %>" \
   --max-diff-size "<%= @diff_size %>"

diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb
index e59923b27ed00a20d069b28302443c679bac5dd6..ba708c9b836690dbdcdd74ff551742c4dff2715d 100644 (file)
--- a/cookbooks/vectortile/templates/default/vector-update-tile.erb
+++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb
@@ -5,11 +5,10 @@
 
 set -eu
 
 
 set -eu
 

-export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;'
+export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;;'

 
 

+cd "<%= @expiry_dir %>"

 osm2pgsql-replication update \
   -d "<%= @source_database %>" \
   --max-diff-size "<%= @diff_size %>" \
 osm2pgsql-replication update \
   -d "<%= @source_database %>" \
   --max-diff-size "<%= @diff_size %>" \

-  --post-processing "<%= @post_processing %>" \
-  -- --expire-tiles=10-14 \
-  --expire-output="<%= @tiles_file %>"
+  --post-processing "<%= @post_processing %>"

diff --git a/cookbooks/wiki/recipes/default.rb b/cookbooks/wiki/recipes/default.rb
index e7a2b3e57b5ea6067bd41b3347baae9d695e89c1..e774d046fd45f4677f039edb73838a7dfa5a6e95 100644 (file)
--- a/cookbooks/wiki/recipes/default.rb
+++ b/cookbooks/wiki/recipes/default.rb
@@ -153,6 +153,7 @@ systemd_service "wiki-dump" do
   exec_start "/usr/bin/php w/maintenance/dumpBackup.php --full --quiet --output=gzip:dump/dump.xml.gz"
   working_directory "/srv/wiki.openstreetmap.org"
   user "wiki"
   exec_start "/usr/bin/php w/maintenance/dumpBackup.php --full --quiet --output=gzip:dump/dump.xml.gz"
   working_directory "/srv/wiki.openstreetmap.org"
   user "wiki"

+  nice 19

   sandbox :enable_network => true
   memory_deny_write_execute false
   restrict_address_families "AF_UNIX"
   sandbox :enable_network => true
   memory_deny_write_execute false
   restrict_address_families "AF_UNIX"


@@ -161,7 +162,7 @@ end
 
 systemd_timer "wiki-dump" do
   description "Wiki dump"
 
 systemd_timer "wiki-dump" do
   description "Wiki dump"

-  on_calendar "02:00"
+  on_calendar "Sun 02:30"

 end
 
 service "wiki-dump.timer" do
 end
 
 service "wiki-dump.timer" do

diff --git a/roles/aarnet.rb b/roles/aarnet.rb
deleted file mode 100644 (file)
index 8cab0ea..0000000
--- a/roles/aarnet.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-name "aarnet"
-description "Role applied to all servers at AARNet"
-
-default_attributes(
-  :accounts => {
-    :users => {
-      :chm => { :status => :administrator },
-      :bclifford => { :status => :administrator }
-    }
-  },
-  :hosted_by => "AARNet",
-  :location => "Carlton, Victoria, Australia",
-  :timezone => "Australia/Melbourne"
-)
-
-override_attributes(
-  :networking => {
-    :nameservers => ["202.158.207.1", "202.158.207.2"]
-  },
-  :ntp => {
-    :servers => ["0.au.pool.ntp.org", "1.au.pool.ntp.org", "oceania.pool.ntp.org"]
-  }
-)
-
-run_list(
-  "role[au]"
-)

diff --git a/roles/balerion.rb b/roles/balerion.rb
deleted file mode 100644 (file)
index 4aef19a..0000000
--- a/roles/balerion.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-name "balerion"
-description "Master role applied to balerion"
-
-default_attributes(
-  :networking => {
-    :interfaces => {
-      :external => {
-        :interface => "bond0",
-        :role => :external,
-        :inet => {
-          :address => "138.44.68.134",
-          :prefix => "30",
-          :gateway => "138.44.68.133"
-        },
-        :bond => {
-          :slaves => %w[ens14f0np0 ens14f1np1]
-        }
-      }
-    }
-  },
-  :postgresql => {
-    :settings => {
-      :defaults => {
-        :effective_cache_size => "16GB"
-      }
-    }
-  },
-  :sysctl => {
-    :postgres => {
-      :comment => "Increase shared memory for postgres",
-      :parameters => {
-        "kernel.shmmax" => 9 * 1024 * 1024 * 1024,
-        "kernel.shmall" => 9 * 1024 * 1024 * 1024 / 4096
-      }
-    }
-  },
-  :tile => {
-    :database => {
-      :cluster => "16/main",
-      :postgis => "3"
-    },
-    :mapnik => "3.1",
-    :replication => {
-      :directory => "/store/replication"
-    },
-    :styles => {
-      :default => {
-        :tile_directories => [
-          { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 }
-        ]
-      }
-    }
-  }
-)
-
-run_list(
-  "role[aarnet]",
-  "role[geodns]",
-  "role[tile]"
-)

diff --git a/roles/bowser.rb b/roles/bowser.rb
deleted file mode 100644 (file)
index eca0fa2..0000000
--- a/roles/bowser.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-name "bowser"
-description "Master role applied to bowser"
-
-default_attributes(
-  :networking => {
-    :interfaces => {
-      :external => {
-        :interface => "bond0",
-        :role => :external,
-        :inet => {
-          :address => "138.44.68.106",
-          :prefix => "30",
-          :gateway => "138.44.68.105"
-        },
-        :bond => {
-          :slaves => %w[ens14f0np0 ens14f1np1]
-        }
-      }
-    }
-  },
-  :postgresql => {
-    :settings => {
-      :defaults => {
-        :effective_cache_size => "16GB"
-      }
-    }
-  },
-  :sysctl => {
-    :postgres => {
-      :comment => "Increase shared memory for postgres",
-      :parameters => {
-        "kernel.shmmax" => 9 * 1024 * 1024 * 1024,
-        "kernel.shmall" => 9 * 1024 * 1024 * 1024 / 4096
-      }
-    }
-  },
-  :tile => {
-    :database => {
-      :cluster => "16/main",
-      :postgis => "3"
-    },
-    :mapnik => "3.1",
-    :replication => {
-      :directory => "/store/replication"
-    },
-    :styles => {
-      :default => {
-        :tile_directories => [
-          { :name => "/store/tiles/default", :min_zoom => 0, :max_zoom => 19 }
-        ]
-      }
-    }
-  }
-)
-
-run_list(
-  "role[aarnet]",
-  "role[tile]"
-)

diff --git a/roles/equinix-dub.rb b/roles/equinix-dub.rb
index 6561d30193bec5aa45adad36296b8234de8e223e..9652c2c8c19485bc2f13b304f2da5204706cb99a 100644 (file)
--- a/roles/equinix-dub.rb
+++ b/roles/equinix-dub.rb
@@ -24,6 +24,13 @@ default_attributes(
       },
       :external => {
         :zone => "dub",
       },
       :external => {
         :zone => "dub",

+        :inet => {
+          :rules => [
+            { :to => "10.0.0.0/8", :table => "main", :priority => 50 },
+            { :to => "172.16.0.0/12", :table => "main", :priority => 50 },
+            { :to => "192.168.0.0/16", :table => "main", :priority => 50 }
+          ]
+        },

         :inet6 => {
           :rules => [
             { :to => "2600:9000::/28", :table => 150, :priority => 100 }
         :inet6 => {
           :rules => [
             { :to => "2600:9000::/28", :table => 150, :priority => 100 }

diff --git a/roles/fafnir.rb b/roles/fafnir.rb
index bb6b7b667e4a9928015d9e545e8b07a6fd1288f7..d3bfcc1157ea1f1c6eecb85411223958a2c97aec 100644 (file)
--- a/roles/fafnir.rb
+++ b/roles/fafnir.rb
@@ -7,6 +7,7 @@ default_attributes(
     :last_address => "10.0.79.254"
   },
   :exim => {
     :last_address => "10.0.79.254"
   },
   :exim => {

+    :external_interface => "<;${if <{${randint:100}}{90} {184.104.226.98;2001:470:1:b3b::2}{87.252.214.98;2001:4d78:fe03:1c::2}}",

     :routes => {
       :openstreetmap => {
         :comment => "openstreetmap.org",
     :routes => {
       :openstreetmap => {
         :comment => "openstreetmap.org",

diff --git a/roles/ovh.rb b/roles/ovh.rb
deleted file mode 100644 (file)
index f8391a2..0000000
--- a/roles/ovh.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-name "ovh"
-description "Role applied to all servers at OVH"
-
-default_attributes(
-  :hosted_by => "OVH",
-  :location => "Roubaix, France"
-)
-
-override_attributes(
-  :networking => {
-    :nameservers => ["213.186.33.99"]
-  },
-  :ntp => {
-    :servers => ["0.fr.pool.ntp.org", "1.fr.pool.ntp.org", "europe.pool.ntp.org"]
-  }
-)
-
-run_list(
-  "role[fr]"
-)

diff --git a/roles/scorch.rb b/roles/scorch.rb
deleted file mode 100644 (file)
index 0649374..0000000
--- a/roles/scorch.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-name "scorch"
-description "Master role applied to scorch"
-
-default_attributes(
-  :devices => {
-    :ssd_system => {
-      :comment => "Tune scheduler for system disk",
-      :type => "block",
-      :bus => "scsi",
-      :serial => "3600605b009bbf5601fc3206407a43546",
-      :attrs => {
-        "queue/scheduler" => "noop",
-        "queue/nr_requests" => "256",
-        "queue/read_ahead_kb" => "2048"
-      }
-    }
-  },
-  :networking => {
-    :interfaces => {
-      :external => {
-        :interface => "eth0",
-        :role => :external,
-        :inet => {
-          :address => "176.31.235.79",
-          :prefix => "24",
-          :gateway => "176.31.235.254"
-        },
-        :inet6 => {
-          :address => "2001:41d0:2:fc4f::1",
-          :prefix => "64",
-          :gateway => "2001:41d0:2:fcff:ff:ff:ff:ff"
-        }
-      }
-    }
-  }
-)
-
-run_list(
-  "role[ovh]"
-)