--- /dev/null
+# Overpass Cookbook
+
+This cookbook installs and configures an Overpass API. It can be configured
+to install a stripped down version that can only be used to serve requests
+from the "Query Feature" of the main OSM website.
--- /dev/null
+default[:overpass][:fqdn] = "overpass.openstreetmap.org"
+default[:overpass][:version] = "0.7.57"
+# One of: no, meta, attic
+default[:overpass][:meta_mode] = "attic"
+# One of: no, gz, lz4
+default[:overpass][:compression_mode] = "lz4"
+default[:overpass][:rate_limit] = 2
+default[:overpass][:dispatcher_space] = 10 * 1024 * 1024 * 1024
+default[:overpass][:clone_url] = "http://dev.overpass-api.de/api_drolbr"
+default[:overpass][:replication_url] = "https://planet.openstreetmap.org/replication/minute/"
+# If true only provide an API for the query feature on the website
+default[:overpass][:restricted_api] = true
+
+default[:overpass][:logdir] = "/var/log/overpass"
--- /dev/null
+name "overpass"
+maintainer "OpenStreetMap Administrators"
+maintainer_email "admins@openstreetmap.org"
+license "Apache-2.0"
+description "Installs and configures an Overpass server"
+
+version "1.0.0"
+supports "ubuntu"
+depends "accounts"
+depends "munin"
+depends "apache"
+depends "systemd"
--- /dev/null
+#
+# Cookbook:: overpass
+# Recipe:: default
+#
+# Copyright:: 2021, OpenStreetMap Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "accounts"
+include_recipe "munin"
+include_recipe "apache"
+
+username = "overpass"
+basedir = data_bag_item("accounts", username)["home"]
+
+%w[bin site diffs db src].each do |dirname|
+ directory "#{basedir}/#{dirname}" do
+ owner username
+ group username
+ mode "755"
+ recursive true
+ end
+end
+
+## Install overpass from source
+
+srcdir = "#{basedir}/src/osm-3s_v#{node[:overpass][:version]}"
+
+package %w[
+ build-essential
+ libexpat1-dev
+ zlib1g-dev
+ liblz4-dev
+ pyosmium
+ osmium-tool
+]
+
+remote_file "#{srcdir}.tar.gz" do
+ action :create
+ source "https://dev.overpass-api.de/releases/osm-3s_v#{node[:overpass][:version]}.tar.gz"
+ owner username
+ group username
+ mode "644"
+end
+
+execute "source_tarball" do
+ cwd "#{basedir}/src"
+ command "tar -xf #{srcdir}.tar.gz"
+ user username
+ notifies :run, "execute[install_overpass]"
+ not_if { ::File.exist?(srcdir) }
+end
+
+execute "install_overpass" do
+ action :nothing
+ user username
+ cwd srcdir
+ command "./configure --enable-lz4 --prefix=#{basedir} && make install"
+end
+
+## Setup Apache
+
+ssl_certificate node[:fqdn] do
+ domains [node[:fqdn],
+ node[:overpass][:fqdn]]
+ notifies :reload, "service[apache2]"
+end
+
+apache_module "cgi"
+apache_module "headers"
+
+apache_site "#{node[:overpass][:fqdn]}" do
+ template "apache.erb"
+ directory "#{basedir}/site"
+ variables :script_directory => "#{basedir}/cgi-bin"
+end
+
+## Overpass deamons
+
+meta_map_short = {
+ "no" => "",
+ "meta" => "--meta",
+ "attic" => "--attic"
+}
+
+logdir = node[:overpass][:logdir]
+
+directory logdir do
+ owner username
+ group username
+ mode "755"
+ recursive true
+end
+
+%w[overpass-update-db overpass-update-areas].each do |fname|
+ template "#{basedir}/bin/#{fname}" do
+ source "#{fname}.erb"
+ owner "overpass"
+ group "overpass"
+ mode "700"
+ variables :basedir => basedir, :srcdir => srcdir
+ end
+end
+
+template "#{basedir}/bin/overpass-import-db" do
+ source "overpass-import-db.erb"
+ owner "root"
+ group "root"
+ mode "755"
+ variables :basedir => basedir, :username => username, :srcdir => srcdir
+end
+
+systemd_service "overpass-dispatcher" do
+ description "Overpass Main Dispatcher"
+ working_directory basedir
+ exec_start "#{basedir}/bin/dispatcher --osm-base #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db --rate-limit=#{node[:overpass][:rate_limit]} --space=#{node[:overpass][:dispatcher_space]}"
+ exec_stop "#{basedir}/bin/dispatcher --osm-base --terminate"
+ standard_output "append:#{logdir}/osm_base.log"
+ user username
+end
+
+service "overpass-dispatcher" do
+ action [:enable]
+end
+
+systemd_service "overpass-area-dispatcher" do
+ description "Overpass Area Dispatcher"
+ after ["overpass-dispatcher"]
+ working_directory basedir
+ exec_start "#{basedir}/bin/dispatcher --areas #{meta_map_short[node[:overpass][:meta_mode]]} --db-dir=#{basedir}/db"
+ exec_stop "#{basedir}/bin/dispatcher --areas --terminate"
+ standard_output "append:#{logdir}/areas.log"
+ user username
+end
+
+service "overpass-area-dispatcher" do
+ action [:enable]
+end
+
+systemd_service "overpass-update" do
+ description "Overpass Update Application"
+ after ["overpass-dispatcher"]
+ working_directory basedir
+ exec_start "#{basedir}/bin/overpass-update-db"
+ standard_output "append:#{logdir}/update.log"
+ user username
+end
+
+if node[:overpass][:meta_mode] == "attic"
+ systemd_service "overpass-area-processor" do
+ description "Overpass Area Processor"
+ after ["overpass-area-dispatcher"]
+ working_directory basedir
+ exec_start "#{basedir}/bin/overpass-update-areas"
+ standard_output "append:#{logdir}/area-processor.log"
+ nice 19
+ user username
+ end
+else
+ systemd_service "overpass-area-processor" do
+ description "Overpass Area Processor"
+ after ["overpass-area-dispatcher"]
+ working_directory basedir
+ exec_start "#{basedir}/bin/osm3s_query --progress --rules"
+ standard_input "file:#{srcdir}/rules/areas.osm3s"
+ standard_output "append:#{logdir}/area-processor.log"
+ nice 19
+ user username
+ end
+end
+
+systemd_timer "overpass-area-processor" do
+ description "Update areas in Overpass"
+ on_calendar "*-*-* *:*:00"
+end
+
+service "overpass-area-processor" do
+ action [:enable]
+end
--- /dev/null
+# DO NOT EDIT - This file is being maintained by Chef
+
+<VirtualHost *:80>
+ ServerName <%= node[:fqdn] %>
+ ServerAlias <%= node[:overpass][:fqdn] %>
+ ServerAdmin webmaster@openstreetmap.org
+
+ CustomLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-access.log combined
+ ErrorLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-error.log
+
+ DocumentRoot <%= @directory %>
+
+ RedirectPermanent /.well-known/acme-challenge/ http://acme.openstreetmap.org/.well-known/acme-challenge/
+ RedirectPermanent / https://<%= @name %>/
+</VirtualHost>
+
+
+<VirtualHost *:443>
+ ServerName <%= node[:fqdn] %>
+ ServerAlias <%= node[:overpass][:fqdn] %>
+ ServerAdmin webmaster@openstreetmap.org
+
+ CustomLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-access.log combined
+ ErrorLog /var/log/apache2/<%= node[:overpass][:fqdn] %>-error.log
+
+ SSLEngine on
+ SSLCertificateFile /etc/ssl/certs/<%= node[:fqdn] %>.pem
+ SSLCertificateKeyFile /etc/ssl/private/<%= node[:fqdn] %>.key
+
+ DocumentRoot <%= @directory %>
+
+<% if node[:overpass][:restricted_api] -%>
+ ScriptAlias /query-features <%= @script_directory %>/interpreter
+ SetEnvIf Origin "http.*(osm.org|openstreetmap.org).*" AccessControlAllowOrigin=$0
+ # Remove Origin so Overpass does not interfere.
+ RequestHeader unset Origin
+ Header always add Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=AccessControlAllowOrigin
+<% else -%>
+ ScriptAlias /api/ <%= @script_directory %>/
+<% end -%>
+</VirtualHost>
+
+<Directory "<%= @directory %>">
+ Require all granted
+</Directory>
+
+<Directory "<%= @script_directory %>">
+ SetOutputFilter DEFLATE
+
+ AllowOverride None
+ Options +ExecCGI -MultiViews +FollowSymLinks
+ Require all granted
+</Directory>
--- /dev/null
+#!/bin/bash -e
+
+FNAME=$1
+
+if [[ "x$FNAME" == "x" ]]; then
+ echo "Usage: overpass-import-db.sh <OSM file>"
+ exit 1
+fi
+
+case "$FNAME" in
+ *.gz) UNPACKER='gunzip -c' ;;
+ *.bz2) UNPACKER='bunzip2 -c' ;;
+ *) UNPACKER='osmium cat -o - -f xml' ;;
+esac
+
+<% if node[:overpass][:meta_mode] == "meta" -%>
+META=--meta
+<% elsif node[:overpass][:meta_mode] == "attic" -%>
+META=--keep-attic
+<% else -%>
+META=
+<% end -%>
+
+sudo systemctl stop overpass-area-processor.timer
+sudo systemctl stop overpass-update
+sudo systemctl stop overpass-area-dispatcher
+sudo systemctl stop overpass-dispatcher
+
+sleep 2
+
+# Remove old database
+sudo -u <%= @username %> rm -rf <%= @basedir %>/db/*
+
+$UNPACKER $FNAME | sudo -u <%= @username %> <%= @basedir %>/bin/update_database --db-dir='<%= @basedir %>/db' --compression-method=<%= node[:overpass][:compression_mode] %> --map-compression-method=<%= node[:overpass][:compression_mode] %> $META
+
+sudo -u <%= @username %> ln -s <%= @srcdir %>/rules <%= @basedir %>/db/rules
+
+echo "Import finished. Catching up with new changes."
+
+sudo systemctl start overpass-dispatcher
+sudo systemctl start overpass-area-dispatcher
+
+PYOSMIUM="sudo -u <%= @username %> pyosmium-get-changes --server <%= node[:overpass][:replication_url] %> -f <%= @basedir %>/db/replicate-id"
+<% if node[:overpass][:meta_mode] == "attic" -%>
+PYOSMIUM="$PYOSMIUM --no-deduplicate"
+<% end -%>
+
+# Get the replication id
+$PYOSMIUM -v -O $FNAME --ignore-osmosis-headers
+
+sudo -u <%= @username %> rm -f <%= @basedir %>/diffs/*
+
+while $PYOSMIUM -v -s 1000 -o <%= @basedir %>/diffs/latest.osc; do
+ if [ ! -f <%= @basedir %>/db/replicate-id ]; then
+ echo "Replication ID not written."
+ exit 1
+ fi
+ DATA_VERSION=`osmium fileinfo -e -g data.timestamp.last <%= @basedir %>/diffs/latest.osc`
+ echo "Downloaded up to timestamp $DATA_VERSION"
+ while ! sudo -u <%= @username %> <%= @basedir %>/bin/update_from_dir --osc-dir=<%= @basedir %>/diffs --version=$DATA_VERSION $META --flush-size=0; do
+ echo "Error while updating. Retry in 1 min."
+ sleep 60
+ done
+ sudo -u <%= @username %> rm <%= @basedir %>/diffs/latest.osc
+ echo "Finished up to $DATA_VERSION."
+done
+
+echo "DB up-to-date. Processing areas."
+
+sudo -u <%= @username %> <%= @basedir %>/bin/osm3s_query --progress --rules <<%= @srcdir %>/rules/areas.osm3s
+
+echo "All updates done."
+
+sudo systemctl start overpass-area-processor.timer
--- /dev/null
+#!/bin/bash
+
+echo "`date '+%F %T'`: update started"
+
+if [[ -a <%= @basedir %>/db/area_version ]]; then
+ sed "s/{{area_version}}/$(cat <%= @basedir %>/db/area_version)/g" <%= @srcdir %>/rules/areas_delta.osm3s | <%= @basedir %>/bin/osm3s_query --progress --rules
+else
+ cat <%= @srcdir %>/rules/areas.osm3s | <%= @basedir %>/bin/osm3s_query --progress --rules
+fi
+
+echo "`date '+%F %T'`: update finished"
--- /dev/null
+#!/bin/bash
+
+PYOSMIUM="pyosmium-get-changes --server <%= node[:overpass][:replication_url] %> -f <%= @basedir %>/db/replicate-id"
+<% if node[:overpass][:meta_mode] == "attic" -%>
+PYOSMIUM="$PYOSMIUM --no-deduplicate"
+<% end -%>
+
+<% if node[:overpass][:meta_mode] == "meta" -%>
+META=--meta
+<% elsif node[:overpass][:meta_mode] == "attic" -%>
+META=--keep-attic
+<% else -%>
+META=
+<% end -%>
+
+while true; do
+ status=3 # make it sleep on issues
+
+ if [ -f <%= @basedir %>/db/replicate-id ]; then
+ # first apply any pending updates
+ if [ -f <%= @basedir %>/diffs/latest.osc ]; then
+ DATA_VERSION=`osmium fileinfo -e -g data.timestamp.last <%= @basedir %>/diffs/latest.osc`
+ if [ "x$DATA_VERSION" != "x" ]; then
+ echo "Downloaded up to timestamp $DATA_VERSION"
+ while ! <%= @basedir %>/bin/update_from_dir --osc-dir=<%= @basedir %>/diffs --version=$DATA_VERSION $META --flush-size=0; do
+ echo "Error while updating. Retry in 1 min."
+ sleep 60
+ done
+ fi
+ rm <%= @basedir %>/diffs/latest.osc
+ fi
+
+ $PYOSMIUM -v -s 1000 -o <%= @basedir %>/diffs/latest.osc
+ status=$?
+ fi
+
+ if [ $status -eq 0 ]; then
+ echo "Downloaded next batch."
+ elif [ $status -eq 3 ]; then
+ rm <%= @basedir %>/diffs/latest.osc
+ echo "No new data, sleeping for a minute."
+ sleep 60
+ else
+ echo "Fatal error, stopping updates."
+ exit $status
+ fi
+done
run_list(
"role[equinix-dub]",
- "role[hp-dl360e-g8]"
+ "role[hp-dl360e-g8]",
+ "role[overpass-query]"
)
--- /dev/null
+name "overpass-query"
+description "Role applied to overpass servers for the query feature."
+
+default_attributes(
+ :accounts => {
+ :users => {
+ :lonvia => { :status => :administrator },
+ :overpass => {
+ :status => :role,
+ :members => [:lonvia, :tomh]
+ }
+ }
+ },
+ :overpass => {
+ :fqdn => "query.openstreetmap.org",
+ :meta_mode => "no",
+ :compression_mode => "no",
+ :restricted_api => true
+ }
+)
+
+run_list(
+ "recipe[overpass::default]"
+)
projects / chef.git / commitdiff