]> git.openstreetmap.org Git - chef.git/commitdiff
add overpass cookbook
authorSarah Hoffmann <lonvia@denofr.de>
Sat, 23 Oct 2021 20:26:55 +0000 (22:26 +0200)
committerSarah Hoffmann <lonvia@denofr.de>
Mon, 25 Oct 2021 07:38:09 +0000 (09:38 +0200)
Adds a cookbook to run an updateable overpass. The cookbook
is in theory able to install a fully-featured overpass API.
The new role 'overpass-query' sets up a reduced version only
suitable for running queries for the query feature on the
main site.

Sets role up on gorwen.

cookbooks/overpass/README.md [new file with mode: 0644]
cookbooks/overpass/attributes/default.rb [new file with mode: 0644]
cookbooks/overpass/metadata.rb [new file with mode: 0644]
cookbooks/overpass/recipes/default.rb [new file with mode: 0644]
cookbooks/overpass/templates/default/apache.erb [new file with mode: 0644]
cookbooks/overpass/templates/default/overpass-import-db.erb [new file with mode: 0644]
cookbooks/overpass/templates/default/overpass-update-areas.erb [new file with mode: 0644]
cookbooks/overpass/templates/default/overpass-update-db.erb [new file with mode: 0644]
roles/gorwen.rb
roles/overpass-query.rb [new file with mode: 0644]

diff --git a/cookbooks/overpass/README.md b/cookbooks/overpass/README.md
new file mode 100644 (file)
index 0000000..baac18b
--- /dev/null
@@ -0,0 +1,5 @@
+# Overpass Cookbook
+
+This cookbook installs and configures an Overpass API. It can be configured
+to install a stripped down version that can only be used to serve requests
+from the "Query Feature" of the main OSM website.
diff --git a/cookbooks/overpass/attributes/default.rb b/cookbooks/overpass/attributes/default.rb
new file mode 100644 (file)
index 0000000..ed1d28f
--- /dev/null
@@ -0,0 +1,14 @@
+default[:overpass][:fqdn] = "overpass.openstreetmap.org"
+default[:overpass][:version] = "0.7.57"
+# One of: no, meta, attic
+default[:overpass][:meta_mode] = "attic"
+# One of: no, gz, lz4
+default[:overpass][:compression_mode] = "lz4"
+default[:overpass][:rate_limit] = 2
+default[:overpass][:dispatcher_space] = 10 * 1024 * 1024 * 1024
+default[:overpass][:clone_url] = "http://dev.overpass-api.de/api_drolbr"
+default[:overpass][:replication_url] = "https://planet.openstreetmap.org/replication/minute/"
+# If true only provide an API for the query feature on the website
+default[:overpass][:restricted_api] = true
+
+default[:overpass][:logdir] = "/var/log/overpass"
diff --git a/cookbooks/overpass/metadata.rb b/cookbooks/overpass/metadata.rb
new file mode 100644 (file)
index 0000000..e9755fe
--- /dev/null
@@ -0,0 +1,12 @@
+name              "overpass"
+maintainer        "OpenStreetMap Administrators"
+maintainer_email  "admins@openstreetmap.org"
+license           "Apache-2.0"
+description       "Installs and configures an Overpass server"
+
+version           "1.0.0"
+supports          "ubuntu"
+depends           "accounts"
+depends           "munin"
+depends           "apache"
+depends           "systemd"
diff --git a/cookbooks/overpass/recipes/default.rb b/cookbooks/overpass/recipes/default.rb
new file mode 100644 (file)
index 0000000..d807eab
--- /dev/null
@@ -0,0 +1,190 @@
+#
+# Cookbook:: overpass
+# Recipe:: default
+#
+# Copyright:: 2021, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "accounts"
+include_recipe "munin"
+include_recipe "apache"
+
+username = "overpass"
+basedir = data_bag_item("accounts", username)["home"]
+
+%w[bin site diffs db src].each do |dirname|
+  directory "#{basedir}/#{dirname}" do
+    owner username
+    group username
+    mode "755"
+    recursive true
+  end
+end
+
+## Install overpass from source
+
+srcdir = "#{basedir}/src/osm-3s_v#{node[:overpass][:version]}"
+
+package %w[
+  build-essential
+  libexpat1-dev
+  zlib1g-dev
+  liblz4-dev
+  pyosmium
+  osmium-tool
+]
+
+remote_file "#{srcdir}.tar.gz" do
+  action :create
+  source "https://dev.overpass-api.de/releases/osm-3s_v#{node[:overpass][:version]}.tar.gz"
+  owner username
+  group username
+  mode "644"
+end
+
+execute "source_tarball" do
+  cwd "#{basedir}/src"
+  command "tar -xf #{srcdir}.tar.gz"
+  user username
+  notifies :run, "execute[install_overpass]"
+  not_if { ::File.exist?(srcdir) }
+end
+
+execute "install_overpass" do
+  action :nothing
+  user username
+  cwd srcdir
+  command "./configure --enable-lz4 --prefix=#{basedir} && make install"
+end
+
+## Setup Apache
+
+ssl_certificate node[:fqdn] do
+  domains [node[:fqdn],
+           node[:overpass][:fqdn]]
+  notifies :reload, "service[apache2]"
+end
+
+apache_module "cgi"
+apache_module "headers"
+
+apache_site "#{node[:overpass][:fqdn]}" do
+  template "apache.erb"
+  directory "#{basedir}/site"
+  variables :script_directory => "#{basedir}/cgi-bin"
+end
+
+## Overpass deamons
+
+meta_map_short = {
+  "no" => "",
+  "meta" => "--meta",
+  "attic" => "--attic"
+}
+
+logdir = node[:overpass][:logdir]
+
+directory logdir do
+  owner username
+  group username
+  mode "755"
+  recursive true
+end
+
+%w[overpass-update-db overpass-update-areas].each do |fname|
+  template "#{basedir}/bin/#{fname}" do
+    source "#{fname}.erb"
+    owner "overpass"
+    group "overpass"
+    mode "700"
+    variables :basedir => basedir, :srcdir => srcdir
+  end
+end
+
+template "#{basedir}/bin/overpass-import-db" do
+  source "overpass-import-db.erb"
+  owner "root"
+  group "root"
+  mode "755"
+  variables :basedir => basedir, :username => username, :srcdir => srcdir
+end
+
+systemd_service "overpass-dispatcher" do
+  description "Overpass Main Dispatcher"
+  working_directory basedir
+  exec_start "#{basedir}/bin/dispatcher --osm-base #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db --rate-limit=#{node[:overpass][:rate_limit]} --space=#{node[:overpass][:dispatcher_space]}"
+  exec_stop "#{basedir}/bin/dispatcher --osm-base --terminate"
+  standard_output "append:#{logdir}/osm_base.log"
+  user username
+end
+
+service "overpass-dispatcher" do
+  action [:enable]
+end
+
+systemd_service "overpass-area-dispatcher" do
+  description "Overpass Area Dispatcher"
+  after ["overpass-dispatcher"]
+  working_directory basedir
+  exec_start "#{basedir}/bin/dispatcher --areas #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db"
+  exec_stop "#{basedir}/bin/dispatcher --areas --terminate"
+  standard_output "append:#{logdir}/areas.log"
+  user username
+end
+
+service "overpass-area-dispatcher" do
+  action [:enable]
+end
+
+systemd_service "overpass-update" do
+  description "Overpass Update Application"
+  after ["overpass-dispatcher"]
+  working_directory basedir
+  exec_start "#{basedir}/bin/overpass-update-db"
+  standard_output "append:#{logdir}/update.log"
+  user username
+end
+
+if node[:overpass][:meta_mode] == "attic"
+  systemd_service "overpass-area-processor" do
+    description "Overpass Area Processor"
+    after ["overpass-area-dispatcher"]
+    working_directory basedir
+    exec_start "#{basedir}/bin/overpass-update-areas"
+    standard_output "append:#{logdir}/area-processor.log"
+    nice 19
+    user username
+  end
+else
+  systemd_service "overpass-area-processor" do
+    description "Overpass Area Processor"
+    after ["overpass-area-dispatcher"]
+    working_directory basedir
+    exec_start "#{basedir}/bin/osm3s_query --progress --rules"
+    standard_input "file:#{srcdir}/rules/areas.osm3s"
+    standard_output "append:#{logdir}/area-processor.log"
+    nice 19
+    user username
+  end
+end
+
+systemd_timer "overpass-area-processor" do
+  description "Update areas in Overpass"
+  on_calendar "*-*-* *:*:00"
+end
+
+service "overpass-area-processor" do
+  action [:enable]
+end
diff --git a/cookbooks/overpass/templates/default/apache.erb b/cookbooks/overpass/templates/default/apache.erb
new file mode 100644 (file)
index 0000000..fbf82cf
--- /dev/null
@@ -0,0 +1,53 @@
+# DO NOT EDIT - This file is being maintained by Chef
+
+<VirtualHost *:80>
+        ServerName <%= node[:fqdn] %>
+        ServerAlias <%= node[:overpass][:fqdn] %>
+        ServerAdmin webmaster@openstreetmap.org
+
+        CustomLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-access.log combined
+        ErrorLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-error.log
+
+        DocumentRoot <%= @directory %>
+
+        RedirectPermanent /.well-known/acme-challenge/ http://acme.openstreetmap.org/.well-known/acme-challenge/
+        RedirectPermanent / https://<%= @name %>/
+</VirtualHost>
+
+
+<VirtualHost *:443>
+        ServerName <%= node[:fqdn] %>
+        ServerAlias <%= node[:overpass][:fqdn] %>
+        ServerAdmin webmaster@openstreetmap.org
+
+        CustomLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-access.log combined
+        ErrorLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-error.log
+
+        SSLEngine on
+        SSLCertificateFile /etc/ssl/certs/<%= node[:fqdn] %>.pem
+        SSLCertificateKeyFile /etc/ssl/private/<%= node[:fqdn] %>.key
+
+        DocumentRoot <%= @directory %>
+
+<% if node[:overpass][:restricted_api] -%>
+        ScriptAlias /query-features <%= @script_directory %>/interpreter
+        SetEnvIf Origin "http.*(osm.org|openstreetmap.org).*" AccessControlAllowOrigin=$0
+        # Remove Origin so Overpass does not interfere.
+        RequestHeader unset Origin
+        Header always add Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=AccessControlAllowOrigin
+<% else -%>
+        ScriptAlias /api/ <%= @script_directory %>/
+<% end -%>
+</VirtualHost>
+
+<Directory "<%= @directory %>">
+        Require all granted
+</Directory>
+
+<Directory "<%= @script_directory %>">
+        SetOutputFilter DEFLATE
+
+        AllowOverride None
+        Options +ExecCGI -MultiViews +FollowSymLinks
+        Require all granted
+</Directory>
diff --git a/cookbooks/overpass/templates/default/overpass-import-db.erb b/cookbooks/overpass/templates/default/overpass-import-db.erb
new file mode 100644 (file)
index 0000000..22cfe64
--- /dev/null
@@ -0,0 +1,74 @@
+#!/bin/bash -e
+
+FNAME=$1
+
+if [[ "x$FNAME" == "x" ]]; then
+  echo "Usage: overpass-import-db.sh <OSM file>"
+  exit 1
+fi
+
+case "$FNAME" in
+  *.gz) UNPACKER='gunzip -c' ;;
+  *.bz2) UNPACKER='bunzip2 -c' ;;
+  *) UNPACKER='osmium cat -o - -f xml' ;;
+esac
+
+<% if node[:overpass][:meta_mode] == "meta" -%>
+META=--meta
+<% elsif node[:overpass][:meta_mode] == "attic" -%>
+META=--keep-attic
+<% else -%>
+META=
+<% end -%>
+
+sudo systemctl stop overpass-area-processor.timer
+sudo systemctl stop overpass-update
+sudo systemctl stop overpass-area-dispatcher
+sudo systemctl stop overpass-dispatcher
+
+sleep 2
+
+# Remove old database
+sudo -u <%= @username %> rm -rf <%= @basedir %>/db/*
+
+$UNPACKER $FNAME | sudo -u <%= @username %> <%= @basedir %>/bin/update_database --db-dir='<%= @basedir %>/db' --compression-method=<%= node[:overpass][:compression_mode] %> --map-compression-method=<%= node[:overpass][:compression_mode] %> $META
+
+sudo -u <%= @username %> ln -s <%= @srcdir %>/rules <%= @basedir %>/db/rules
+
+echo "Import finished. Catching up with new changes."
+
+sudo systemctl start overpass-dispatcher
+sudo systemctl start overpass-area-dispatcher
+
+PYOSMIUM="sudo -u <%= @username %> pyosmium-get-changes --server <%= node[:overpass][:replication_url] %> -f <%= @basedir %>/db/replicate-id"
+<% if node[:overpass][:meta_mode] == "attic" -%>
+PYOSMIUM="$PYOSMIUM --no-deduplicate"
+<% end -%>
+
+# Get the replication id
+$PYOSMIUM -v -O $FNAME --ignore-osmosis-headers
+
+sudo -u <%= @username %> rm -f <%= @basedir %>/diffs/*
+
+while $PYOSMIUM -v -s 1000 -o <%= @basedir %>/diffs/latest.osc; do
+  if [ ! -f <%= @basedir %>/db/replicate-id ]; then
+    echo "Replication ID not written."
+    exit 1
+  fi
+  DATA_VERSION=`osmium fileinfo -e -g data.timestamp.last <%= @basedir %>/diffs/latest.osc`
+  echo "Downloaded up to timestamp $DATA_VERSION"
+  while ! sudo -u <%= @username %> <%= @basedir %>/bin/update_from_dir --osc-dir=<%= @basedir %>/diffs --version=$DATA_VERSION $META --flush-size=0; do
+    echo "Error while updating. Retry in 1 min."
+    sleep 60
+  done
+  sudo -u <%= @username %> rm <%= @basedir %>/diffs/latest.osc
+  echo "Finished up to $DATA_VERSION."
+done
+
+echo "DB up-to-date. Processing areas."
+
+sudo -u <%= @username %> <%= @basedir %>/bin/osm3s_query --progress --rules <<%= @srcdir %>/rules/areas.osm3s
+
+echo "All updates done."
+
+sudo systemctl start overpass-area-processor.timer
diff --git a/cookbooks/overpass/templates/default/overpass-update-areas.erb b/cookbooks/overpass/templates/default/overpass-update-areas.erb
new file mode 100644 (file)
index 0000000..e1aa23f
--- /dev/null
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "`date '+%F %T'`: update started"
+
+if [[ -a <%= @basedir %>/db/area_version ]]; then
+  sed "s/{{area_version}}/$(cat <%= @basedir %>/db/area_version)/g" <%= @srcdir %>/rules/areas_delta.osm3s | <%= @basedir %>/bin/osm3s_query --progress --rules
+else
+  cat <%= @srcdir %>/rules/areas.osm3s | <%= @basedir %>/bin/osm3s_query --progress --rules
+fi
+
+echo "`date '+%F %T'`: update finished"
diff --git a/cookbooks/overpass/templates/default/overpass-update-db.erb b/cookbooks/overpass/templates/default/overpass-update-db.erb
new file mode 100644 (file)
index 0000000..5ac8d15
--- /dev/null
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+PYOSMIUM="pyosmium-get-changes --server <%= node[:overpass][:replication_url] %> -f <%= @basedir %>/db/replicate-id"
+<% if node[:overpass][:meta_mode] == "attic" -%>
+PYOSMIUM="$PYOSMIUM --no-deduplicate"
+<% end -%>
+
+<% if node[:overpass][:meta_mode] == "meta" -%>
+META=--meta
+<% elsif node[:overpass][:meta_mode] == "attic" -%>
+META=--keep-attic
+<% else -%>
+META=
+<% end -%>
+
+while true; do
+  status=3 # make it sleep on issues
+
+  if [ -f <%= @basedir %>/db/replicate-id ]; then
+    # first apply any pending updates
+    if [ -f <%= @basedir %>/diffs/latest.osc ]; then
+      DATA_VERSION=`osmium fileinfo -e -g data.timestamp.last <%= @basedir %>/diffs/latest.osc`
+      if [ "x$DATA_VERSION" != "x" ]; then
+        echo "Downloaded up to timestamp $DATA_VERSION"
+        while ! <%= @basedir %>/bin/update_from_dir --osc-dir=<%= @basedir %>/diffs --version=$DATA_VERSION $META --flush-size=0; do
+          echo "Error while updating. Retry in 1 min."
+          sleep 60
+        done
+      fi
+      rm <%= @basedir %>/diffs/latest.osc
+    fi
+
+    $PYOSMIUM -v -s 1000 -o <%= @basedir %>/diffs/latest.osc
+    status=$?
+  fi
+
+  if [ $status -eq 0 ]; then
+    echo "Downloaded next batch."
+  elif [ $status -eq 3 ]; then
+    rm <%= @basedir %>/diffs/latest.osc
+    echo "No new data, sleeping for a minute."
+    sleep 60
+  else
+    echo "Fatal error, stopping updates."
+    exit $status
+  fi
+done
index a5452e40ba39d4c646ea1ce5455917de95a5ad87..3f9a1f3284e79155b48bc396fc39de44ab25a824 100644 (file)
@@ -34,5 +34,6 @@ default_attributes(
 
 run_list(
   "role[equinix-dub]",
 
 run_list(
   "role[equinix-dub]",
-  "role[hp-dl360e-g8]"
+  "role[hp-dl360e-g8]",
+  "role[overpass-query]"
 )
 )
diff --git a/roles/overpass-query.rb b/roles/overpass-query.rb
new file mode 100644 (file)
index 0000000..cb0dfb4
--- /dev/null
@@ -0,0 +1,24 @@
+name "overpass-query"
+description "Role applied to overpass servers for the query feature."
+
+default_attributes(
+  :accounts => {
+    :users => {
+      :lonvia => { :status => :administrator },
+      :overpass => {
+        :status => :role,
+        :members => [:lonvia, :tomh]
+      }
+    }
+  },
+  :overpass => {
+    :fqdn => "query.openstreetmap.org",
+    :meta_mode => "no",
+    :compression_mode => "no",
+    :restricted_api => true
+  }
+)
+
+run_list(
+  "recipe[overpass::default]"
+)