From 3b563e12600d3519011cde414ee0465698a7814f Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 23 Oct 2021 22:26:55 +0200 Subject: [PATCH 1/1] add overpass cookbook Adds a cookbook to run an updateable overpass. The cookbook is in theory able to install a fully-featured overpass API. The new role 'overpass-query' sets up a reduced version only suitable for running queries for the query feature on the main site. Sets role up on gorwen. --- cookbooks/overpass/README.md | 5 + cookbooks/overpass/attributes/default.rb | 14 ++ cookbooks/overpass/metadata.rb | 12 ++ cookbooks/overpass/recipes/default.rb | 190 ++++++++++++++++++ .../overpass/templates/default/apache.erb | 53 +++++ .../templates/default/overpass-import-db.erb | 74 +++++++ .../default/overpass-update-areas.erb | 11 + .../templates/default/overpass-update-db.erb | 47 +++++ roles/gorwen.rb | 3 +- roles/overpass-query.rb | 24 +++ 10 files changed, 432 insertions(+), 1 deletion(-) create mode 100644 cookbooks/overpass/README.md create mode 100644 cookbooks/overpass/attributes/default.rb create mode 100644 cookbooks/overpass/metadata.rb create mode 100644 cookbooks/overpass/recipes/default.rb create mode 100644 cookbooks/overpass/templates/default/apache.erb create mode 100644 cookbooks/overpass/templates/default/overpass-import-db.erb create mode 100644 cookbooks/overpass/templates/default/overpass-update-areas.erb create mode 100644 cookbooks/overpass/templates/default/overpass-update-db.erb create mode 100644 roles/overpass-query.rb diff --git a/cookbooks/overpass/README.md b/cookbooks/overpass/README.md new file mode 100644 index 000000000..baac18ba0 --- /dev/null +++ b/cookbooks/overpass/README.md @@ -0,0 +1,5 @@ +# Overpass Cookbook + +This cookbook installs and configures an Overpass API. It can be configured +to install a stripped down version that can only be used to serve requests +from the "Query Feature" of the main OSM website. diff --git a/cookbooks/overpass/attributes/default.rb b/cookbooks/overpass/attributes/default.rb new file mode 100644 index 000000000..ed1d28f97 --- /dev/null +++ b/cookbooks/overpass/attributes/default.rb @@ -0,0 +1,14 @@ +default[:overpass][:fqdn] = "overpass.openstreetmap.org" +default[:overpass][:version] = "0.7.57" +# One of: no, meta, attic +default[:overpass][:meta_mode] = "attic" +# One of: no, gz, lz4 +default[:overpass][:compression_mode] = "lz4" +default[:overpass][:rate_limit] = 2 +default[:overpass][:dispatcher_space] = 10 * 1024 * 1024 * 1024 +default[:overpass][:clone_url] = "http://dev.overpass-api.de/api_drolbr" +default[:overpass][:replication_url] = "https://planet.openstreetmap.org/replication/minute/" +# If true only provide an API for the query feature on the website +default[:overpass][:restricted_api] = true + +default[:overpass][:logdir] = "/var/log/overpass" diff --git a/cookbooks/overpass/metadata.rb b/cookbooks/overpass/metadata.rb new file mode 100644 index 000000000..e9755fe5d --- /dev/null +++ b/cookbooks/overpass/metadata.rb @@ -0,0 +1,12 @@ +name "overpass" +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache-2.0" +description "Installs and configures an Overpass server" + +version "1.0.0" +supports "ubuntu" +depends "accounts" +depends "munin" +depends "apache" +depends "systemd" diff --git a/cookbooks/overpass/recipes/default.rb b/cookbooks/overpass/recipes/default.rb new file mode 100644 index 000000000..d807eab69 --- /dev/null +++ b/cookbooks/overpass/recipes/default.rb @@ -0,0 +1,190 @@ +# +# Cookbook:: overpass +# Recipe:: default +# +# Copyright:: 2021, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "accounts" +include_recipe "munin" +include_recipe "apache" + +username = "overpass" +basedir = data_bag_item("accounts", username)["home"] + +%w[bin site diffs db src].each do |dirname| + directory "#{basedir}/#{dirname}" do + owner username + group username + mode "755" + recursive true + end +end + +## Install overpass from source + +srcdir = "#{basedir}/src/osm-3s_v#{node[:overpass][:version]}" + +package %w[ + build-essential + libexpat1-dev + zlib1g-dev + liblz4-dev + pyosmium + osmium-tool +] + +remote_file "#{srcdir}.tar.gz" do + action :create + source "https://dev.overpass-api.de/releases/osm-3s_v#{node[:overpass][:version]}.tar.gz" + owner username + group username + mode "644" +end + +execute "source_tarball" do + cwd "#{basedir}/src" + command "tar -xf #{srcdir}.tar.gz" + user username + notifies :run, "execute[install_overpass]" + not_if { ::File.exist?(srcdir) } +end + +execute "install_overpass" do + action :nothing + user username + cwd srcdir + command "./configure --enable-lz4 --prefix=#{basedir} && make install" +end + +## Setup Apache + +ssl_certificate node[:fqdn] do + domains [node[:fqdn], + node[:overpass][:fqdn]] + notifies :reload, "service[apache2]" +end + +apache_module "cgi" +apache_module "headers" + +apache_site "#{node[:overpass][:fqdn]}" do + template "apache.erb" + directory "#{basedir}/site" + variables :script_directory => "#{basedir}/cgi-bin" +end + +## Overpass deamons + +meta_map_short = { + "no" => "", + "meta" => "--meta", + "attic" => "--attic" +} + +logdir = node[:overpass][:logdir] + +directory logdir do + owner username + group username + mode "755" + recursive true +end + +%w[overpass-update-db overpass-update-areas].each do |fname| + template "#{basedir}/bin/#{fname}" do + source "#{fname}.erb" + owner "overpass" + group "overpass" + mode "700" + variables :basedir => basedir, :srcdir => srcdir + end +end + +template "#{basedir}/bin/overpass-import-db" do + source "overpass-import-db.erb" + owner "root" + group "root" + mode "755" + variables :basedir => basedir, :username => username, :srcdir => srcdir +end + +systemd_service "overpass-dispatcher" do + description "Overpass Main Dispatcher" + working_directory basedir + exec_start "#{basedir}/bin/dispatcher --osm-base #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db --rate-limit=#{node[:overpass][:rate_limit]} --space=#{node[:overpass][:dispatcher_space]}" + exec_stop "#{basedir}/bin/dispatcher --osm-base --terminate" + standard_output "append:#{logdir}/osm_base.log" + user username +end + +service "overpass-dispatcher" do + action [:enable] +end + +systemd_service "overpass-area-dispatcher" do + description "Overpass Area Dispatcher" + after ["overpass-dispatcher"] + working_directory basedir + exec_start "#{basedir}/bin/dispatcher --areas #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db" + exec_stop "#{basedir}/bin/dispatcher --areas --terminate" + standard_output "append:#{logdir}/areas.log" + user username +end + +service "overpass-area-dispatcher" do + action [:enable] +end + +systemd_service "overpass-update" do + description "Overpass Update Application" + after ["overpass-dispatcher"] + working_directory basedir + exec_start "#{basedir}/bin/overpass-update-db" + standard_output "append:#{logdir}/update.log" + user username +end + +if node[:overpass][:meta_mode] == "attic" + systemd_service "overpass-area-processor" do + description "Overpass Area Processor" + after ["overpass-area-dispatcher"] + working_directory basedir + exec_start "#{basedir}/bin/overpass-update-areas" + standard_output "append:#{logdir}/area-processor.log" + nice 19 + user username + end +else + systemd_service "overpass-area-processor" do + description "Overpass Area Processor" + after ["overpass-area-dispatcher"] + working_directory basedir + exec_start "#{basedir}/bin/osm3s_query --progress --rules" + standard_input "file:#{srcdir}/rules/areas.osm3s" + standard_output "append:#{logdir}/area-processor.log" + nice 19 + user username + end +end + +systemd_timer "overpass-area-processor" do + description "Update areas in Overpass" + on_calendar "*-*-* *:*:00" +end + +service "overpass-area-processor" do + action [:enable] +end diff --git a/cookbooks/overpass/templates/default/apache.erb b/cookbooks/overpass/templates/default/apache.erb new file mode 100644 index 000000000..fbf82cf04 --- /dev/null +++ b/cookbooks/overpass/templates/default/apache.erb @@ -0,0 +1,53 @@ +# DO NOT EDIT - This file is being maintained by Chef + + + ServerName <%= node[:fqdn] %> + ServerAlias <%= node[:overpass][:fqdn] %> + ServerAdmin webmaster@openstreetmap.org + + CustomLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-access.log combined + ErrorLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-error.log + + DocumentRoot <%= @directory %> + + RedirectPermanent /.well-known/acme-challenge/ http://acme.openstreetmap.org/.well-known/acme-challenge/ + RedirectPermanent / https://<%= @name %>/ + + + + + ServerName <%= node[:fqdn] %> + ServerAlias <%= node[:overpass][:fqdn] %> + ServerAdmin webmaster@openstreetmap.org + + CustomLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-access.log combined + ErrorLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-error.log + + SSLEngine on + SSLCertificateFile /etc/ssl/certs/<%= node[:fqdn] %>.pem + SSLCertificateKeyFile /etc/ssl/private/<%= node[:fqdn] %>.key + + DocumentRoot <%= @directory %> + +<% if node[:overpass][:restricted_api] -%> + ScriptAlias /query-features <%= @script_directory %>/interpreter + SetEnvIf Origin "http.*(osm.org|openstreetmap.org).*" AccessControlAllowOrigin=$0 + # Remove Origin so Overpass does not interfere. + RequestHeader unset Origin + Header always add Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=AccessControlAllowOrigin +<% else -%> + ScriptAlias /api/ <%= @script_directory %>/ +<% end -%> + + +"> + Require all granted + + +"> + SetOutputFilter DEFLATE + + AllowOverride None + Options +ExecCGI -MultiViews +FollowSymLinks + Require all granted + diff --git a/cookbooks/overpass/templates/default/overpass-import-db.erb b/cookbooks/overpass/templates/default/overpass-import-db.erb new file mode 100644 index 000000000..22cfe642e --- /dev/null +++ b/cookbooks/overpass/templates/default/overpass-import-db.erb @@ -0,0 +1,74 @@ +#!/bin/bash -e + +FNAME=$1 + +if [[ "x$FNAME" == "x" ]]; then + echo "Usage: overpass-import-db.sh " + exit 1 +fi + +case "$FNAME" in + *.gz) UNPACKER='gunzip -c' ;; + *.bz2) UNPACKER='bunzip2 -c' ;; + *) UNPACKER='osmium cat -o - -f xml' ;; +esac + +<% if node[:overpass][:meta_mode] == "meta" -%> +META=--meta +<% elsif node[:overpass][:meta_mode] == "attic" -%> +META=--keep-attic +<% else -%> +META= +<% end -%> + +sudo systemctl stop overpass-area-processor.timer +sudo systemctl stop overpass-update +sudo systemctl stop overpass-area-dispatcher +sudo systemctl stop overpass-dispatcher + +sleep 2 + +# Remove old database +sudo -u <%= @username %> rm -rf <%= @basedir %>/db/* + +$UNPACKER $FNAME | sudo -u <%= @username %> <%= @basedir %>/bin/update_database --db-dir='<%= @basedir %>/db' --compression-method=<%= node[:overpass][:compression_mode] %> --map-compression-method=<%= node[:overpass][:compression_mode] %> $META + +sudo -u <%= @username %> ln -s <%= @srcdir %>/rules <%= @basedir %>/db/rules + +echo "Import finished. Catching up with new changes." + +sudo systemctl start overpass-dispatcher +sudo systemctl start overpass-area-dispatcher + +PYOSMIUM="sudo -u <%= @username %> pyosmium-get-changes --server <%= node[:overpass][:replication_url] %> -f <%= @basedir %>/db/replicate-id" +<% if node[:overpass][:meta_mode] == "attic" -%> +PYOSMIUM="$PYOSMIUM --no-deduplicate" +<% end -%> + +# Get the replication id +$PYOSMIUM -v -O $FNAME --ignore-osmosis-headers + +sudo -u <%= @username %> rm -f <%= @basedir %>/diffs/* + +while $PYOSMIUM -v -s 1000 -o <%= @basedir %>/diffs/latest.osc; do + if [ ! -f <%= @basedir %>/db/replicate-id ]; then + echo "Replication ID not written." + exit 1 + fi + DATA_VERSION=`osmium fileinfo -e -g data.timestamp.last <%= @basedir %>/diffs/latest.osc` + echo "Downloaded up to timestamp $DATA_VERSION" + while ! sudo -u <%= @username %> <%= @basedir %>/bin/update_from_dir --osc-dir=<%= @basedir %>/diffs --version=$DATA_VERSION $META --flush-size=0; do + echo "Error while updating. Retry in 1 min." + sleep 60 + done + sudo -u <%= @username %> rm <%= @basedir %>/diffs/latest.osc + echo "Finished up to $DATA_VERSION." +done + +echo "DB up-to-date. Processing areas." + +sudo -u <%= @username %> <%= @basedir %>/bin/osm3s_query --progress --rules <<%= @srcdir %>/rules/areas.osm3s + +echo "All updates done." + +sudo systemctl start overpass-area-processor.timer diff --git a/cookbooks/overpass/templates/default/overpass-update-areas.erb b/cookbooks/overpass/templates/default/overpass-update-areas.erb new file mode 100644 index 000000000..e1aa23fdf --- /dev/null +++ b/cookbooks/overpass/templates/default/overpass-update-areas.erb @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "`date '+%F %T'`: update started" + +if [[ -a <%= @basedir %>/db/area_version ]]; then + sed "s/{{area_version}}/$(cat <%= @basedir %>/db/area_version)/g" <%= @srcdir %>/rules/areas_delta.osm3s | <%= @basedir %>/bin/osm3s_query --progress --rules +else + cat <%= @srcdir %>/rules/areas.osm3s | <%= @basedir %>/bin/osm3s_query --progress --rules +fi + +echo "`date '+%F %T'`: update finished" diff --git a/cookbooks/overpass/templates/default/overpass-update-db.erb b/cookbooks/overpass/templates/default/overpass-update-db.erb new file mode 100644 index 000000000..5ac8d1570 --- /dev/null +++ b/cookbooks/overpass/templates/default/overpass-update-db.erb @@ -0,0 +1,47 @@ +#!/bin/bash + +PYOSMIUM="pyosmium-get-changes --server <%= node[:overpass][:replication_url] %> -f <%= @basedir %>/db/replicate-id" +<% if node[:overpass][:meta_mode] == "attic" -%> +PYOSMIUM="$PYOSMIUM --no-deduplicate" +<% end -%> + +<% if node[:overpass][:meta_mode] == "meta" -%> +META=--meta +<% elsif node[:overpass][:meta_mode] == "attic" -%> +META=--keep-attic +<% else -%> +META= +<% end -%> + +while true; do + status=3 # make it sleep on issues + + if [ -f <%= @basedir %>/db/replicate-id ]; then + # first apply any pending updates + if [ -f <%= @basedir %>/diffs/latest.osc ]; then + DATA_VERSION=`osmium fileinfo -e -g data.timestamp.last <%= @basedir %>/diffs/latest.osc` + if [ "x$DATA_VERSION" != "x" ]; then + echo "Downloaded up to timestamp $DATA_VERSION" + while ! <%= @basedir %>/bin/update_from_dir --osc-dir=<%= @basedir %>/diffs --version=$DATA_VERSION $META --flush-size=0; do + echo "Error while updating. Retry in 1 min." + sleep 60 + done + fi + rm <%= @basedir %>/diffs/latest.osc + fi + + $PYOSMIUM -v -s 1000 -o <%= @basedir %>/diffs/latest.osc + status=$? + fi + + if [ $status -eq 0 ]; then + echo "Downloaded next batch." + elif [ $status -eq 3 ]; then + rm <%= @basedir %>/diffs/latest.osc + echo "No new data, sleeping for a minute." + sleep 60 + else + echo "Fatal error, stopping updates." + exit $status + fi +done diff --git a/roles/gorwen.rb b/roles/gorwen.rb index a5452e40b..3f9a1f328 100644 --- a/roles/gorwen.rb +++ b/roles/gorwen.rb @@ -34,5 +34,6 @@ default_attributes( run_list( "role[equinix-dub]", - "role[hp-dl360e-g8]" + "role[hp-dl360e-g8]", + "role[overpass-query]" ) diff --git a/roles/overpass-query.rb b/roles/overpass-query.rb new file mode 100644 index 000000000..cb0dfb4a2 --- /dev/null +++ b/roles/overpass-query.rb @@ -0,0 +1,24 @@ +name "overpass-query" +description "Role applied to overpass servers for the query feature." + +default_attributes( + :accounts => { + :users => { + :lonvia => { :status => :administrator }, + :overpass => { + :status => :role, + :members => [:lonvia, :tomh] + } + } + }, + :overpass => { + :fqdn => "query.openstreetmap.org", + :meta_mode => "no", + :compression_mode => "no", + :restricted_api => true + } +) + +run_list( + "recipe[overpass::default]" +) -- 2.39.5