From 5a75898a87099ed06d5ec5c52f7751689d102388 Mon Sep 17 00:00:00 2001 From: Tom Hughes Date: Sun, 26 Nov 2023 10:41:02 +0000 Subject: [PATCH] Drop support for old style tile server replication --- cookbooks/tile/attributes/default.rb | 1 - .../files/default/bin/expire-tiles-single | 144 ------------ cookbooks/tile/files/default/ruby/expire.rb | 191 ---------------- cookbooks/tile/recipes/default.rb | 210 ++++++------------ .../templates/default/expire-tiles-custom.erb | 15 -- ...e-tiles-osm2pgsql.erb => expire-tiles.erb} | 0 .../templates/default/replicate-custom.erb | 104 --------- ...{replicate-osm2pgsql.erb => replicate.erb} | 0 8 files changed, 65 insertions(+), 600 deletions(-) delete mode 100644 cookbooks/tile/files/default/bin/expire-tiles-single delete mode 100755 cookbooks/tile/files/default/ruby/expire.rb delete mode 100644 cookbooks/tile/templates/default/expire-tiles-custom.erb rename cookbooks/tile/templates/default/{expire-tiles-osm2pgsql.erb => expire-tiles.erb} (100%) delete mode 100644 cookbooks/tile/templates/default/replicate-custom.erb rename cookbooks/tile/templates/default/{replicate-osm2pgsql.erb => replicate.erb} (100%) diff --git a/cookbooks/tile/attributes/default.rb b/cookbooks/tile/attributes/default.rb index 4efb899cf..776e274b0 100644 --- a/cookbooks/tile/attributes/default.rb +++ b/cookbooks/tile/attributes/default.rb @@ -8,7 +8,6 @@ default[:tile][:database][:tag_transform_script] = nil default[:tile][:mapnik] = "3.1" -default[:tile][:replication][:engine] = "osm2pgsql" default[:tile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute" default[:tile][:data] = {} diff --git a/cookbooks/tile/files/default/bin/expire-tiles-single b/cookbooks/tile/files/default/bin/expire-tiles-single deleted file mode 100644 index 431021b2e..000000000 --- a/cookbooks/tile/files/default/bin/expire-tiles-single +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python3 -""" -Expire meta tiles from a OSM change file by resetting their modified time. -""" - -import argparse -import os -import osmium as o -import pyproj - -EXPIRY_TIME = 946681200 # 2000-01-01 00:00:00 -# width/height of the spherical mercator projection -SIZE = 40075016.6855784 - -proj_transformer = pyproj.Transformer.from_crs('epsg:4326', 'epsg:3857', always_xy = True) - -class TileCollector(o.SimpleHandler): - - def __init__(self, node_cache, zoom): - super(TileCollector, self).__init__() - self.node_cache = o.index.create_map("dense_file_array," + node_cache) - self.done_nodes = set() - self.tile_set = set() - self.zoom = zoom - - def add_tile_from_node(self, location): - if not location.valid(): - return - - lat = max(-85, min(85.0, location.lat)) - x, y = proj_transformer.transform(location.lon, lat) - - # renormalise into unit space [0,1] - x = 0.5 + x / SIZE - y = 0.5 - y / SIZE - # transform into tile space - x = x * 2**self.zoom - y = y * 2**self.zoom - # chop of the fractional parts - self.tile_set.add((int(x), int(y), self.zoom)) - - def node(self, node): - # we put all the nodes into the hash, as it doesn't matter whether the node was - # added, deleted or modified - the tile will need updating anyway. - self.done_nodes.add(node.id) - self.add_tile_from_node(node.location) - - def way(self, way): - for n in way.nodes: - if not n.ref in self.done_nodes: - self.done_nodes.add(n.ref) - try: - self.add_tile_from_node(self.node_cache.get(n.ref)) - except KeyError: - pass # no coordinate - - -def xyz_to_meta(x, y, z, meta_size): - """ Return the file name of a meta tile. - This must match the definition of xyz to meta in mod_tile. - """ - # mask off the final few bits - x = x & ~(meta_size - 1) - y = y & ~(meta_size - 1) - - # generate the path - path = None - for i in range(0, 5): - part = str(((x & 0x0f) << 4) | (y & 0x0f)) - x = x >> 4 - y = y >> 4 - if path is None: - path = (part + ".meta") - else: - path = os.path.join(part, path) - - return os.path.join(str(z), path) - - -def expire_meta(meta): - """Expire the meta tile by setting the modified time back. - """ - exists = os.path.exists(meta) - if exists: - os.utime(meta, (EXPIRY_TIME, EXPIRY_TIME)) - return exists - - -def expire_meta_tiles(options): - proc = TileCollector(options.node_cache, options.max_zoom) - proc.apply_file(options.inputfile) - - tile_set = proc.tile_set - - # turn all the tiles into expires, putting them in the set - # so that we don't expire things multiple times - for z in range(options.min_zoom, options.max_zoom + 1): - meta_set = set() - new_set = set() - for xy in tile_set: - meta = xyz_to_meta(xy[0], xy[1], xy[2], options.meta_size) - - for tile_dir in options.tile_dir: - meta_set.add(os.path.join(tile_dir, meta)) - - # add the parent into the set for the next round - new_set.add((int(xy[0]/2), int(xy[1]/2), xy[2] - 1)) - - # expire all meta tiles - expired = 0 - for meta in meta_set: - if expire_meta(meta): - expired += 1 - print("Expired {0} tiles at zoom {1}".format(expired, z)) - - # continue with parent tiles - tile_set = new_set - -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - usage='%(prog)s [options] ') - parser.add_argument('--min', action='store', dest='min_zoom', default=13, - type=int, - help='Minimum zoom for expiry.') - parser.add_argument('--max', action='store', dest='max_zoom', default=20, - type=int, - help='Maximum zoom for expiry.') - parser.add_argument('-t', action='append', dest='tile_dir', default=None, - required=True, - help='Tile directory (repeat for multiple directories).') - parser.add_argument('--meta-tile-size', action='store', dest='meta_size', - default=8, type=int, - help='The size of the meta tile blocks.') - parser.add_argument('--node-cache', action='store', dest='node_cache', - default='/store/database/nodes', - help='osm2pgsql flatnode file.') - parser.add_argument('inputfile', - help='OSC input file.') - - options = parser.parse_args() - - expire_meta_tiles(options) diff --git a/cookbooks/tile/files/default/ruby/expire.rb b/cookbooks/tile/files/default/ruby/expire.rb deleted file mode 100755 index 3d693ad5b..000000000 --- a/cookbooks/tile/files/default/ruby/expire.rb +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/ruby - -require "rubygems" -require "proj4" -require "xml/libxml" -require "set" -require "time" - -module Expire - # projection object to go from latlon -> spherical mercator - PROJ = Proj4::Projection.new(["+proj=merc", "+a=6378137", "+b=6378137", - "+lat_ts=0.0", "+lon_0=0.0", "+x_0=0.0", - "+y_0=0", "+k=1.0", "+units=m", - "+nadgrids=@null", "+no_defs +over"]) - - # width/height of the spherical mercator projection - SIZE = 40075016.6855784 - # the size of the meta tile blocks - METATILE = 8 - # the directory root for meta tiles - HASH_ROOT = "/tiles/default/".freeze - # node cache file - NODE_CACHE_FILE = "/store/database/nodes".freeze - - # turns a spherical mercator coord into a tile coord - def self.tile_from_merc(point, zoom) - # renormalise into unit space [0,1] - point.x = 0.5 + point.x / SIZE - point.y = 0.5 - point.y / SIZE - # transform into tile space - point.x = point.x * 2**zoom - point.y = point.y * 2**zoom - # chop of the fractional parts - [point.x.to_int, point.y.to_int, zoom] - end - - # turns a latlon -> tile x,y given a zoom level - def self.tile_from_latlon(latlon, zoom) - # first convert to spherical mercator - point = PROJ.forward(latlon) - tile_from_merc(point, zoom) - end - - # this must match the definition of xyz_to_meta in mod_tile - def self.xyz_to_meta(x, y, z) - # mask off the final few bits - x &= ~(METATILE - 1) - y &= ~(METATILE - 1) - # generate the path - hash_path = (0..4).collect do |i| - (((x >> 4 * i) & 0xf) << 4) | ((y >> 4 * i) & 0xf) - end.reverse.join("/") - z.to_s + "/" + hash_path + ".meta" - end - - # time to reset to, some very stupidly early time, before OSM started - EXPIRY_TIME = Time.parse("2000-01-01 00:00:00") - - # expire the meta tile by setting the modified time back - def self.expire_meta(meta) - puts "Expiring #{meta}" - File.utime(EXPIRY_TIME, EXPIRY_TIME, meta) - end - - def self.expire(change_file, min_zoom, max_zoom, tile_dirs) - do_expire(change_file, min_zoom, max_zoom) do |set| - new_set = Set.new - meta_set = Set.new - - # turn all the tiles into expires, putting them in the set - # so that we don't expire things multiple times - set.each do |xy| - # this has to match the routine in mod_tile - meta = xyz_to_meta(xy[0], xy[1], xy[2]) - - # check each style working out what needs expiring - tile_dirs.each do |tile_dir| - meta_set.add(tile_dir + "/" + meta) if File.exist?(tile_dir + "/" + meta) - end - - # add the parent into the set for the next round - new_set.add([xy[0] / 2, xy[1] / 2, xy[2] - 1]) - end - - # expire all meta tiles - meta_set.each do |meta| - expire_meta(meta) - end - - # return the new set, consisting of all the parents - new_set - end - end - - def self.do_expire(change_file, min_zoom, max_zoom, &_) - # read in the osm change file - doc = XML::Document.file(change_file) - - # hash map to contain all the nodes - nodes = {} - - # we put all the nodes into the hash, as it doesn't matter whether the node was - # added, deleted or modified - the tile will need updating anyway. - doc.find("//node").each do |node| - lat = node["lat"].to_f - lat = -85 if lat < -85 - lat = 85 if lat > 85 - point = Proj4::Point.new(Math::PI * node["lon"].to_f / 180, - Math::PI * lat / 180) - nodes[node["id"].to_i] = tile_from_latlon(point, max_zoom) - end - - # now we look for all the ways that have changed and put all of their nodes into - # the hash too. this will add too many nodes, as it is possible a long way will be - # changed at only a portion of its length. however, due to the non-local way that - # mapnik does text placement, it may stil not be enough. - # - # also, we miss cases where nodes are deleted from ways where that node is not - # itself deleted and the coverage of the point set isn't enough to encompass the - # change. - node_cache = NodeCache.new(NODE_CACHE_FILE) - doc.find("//way/nd").each do |node| - node_id = node["ref"].to_i - - next if nodes.include? node_id - - # this is a node referenced but not added, modified or deleted, so it should - # still be in the node cache. - if (entry = node_cache[node_id]) - point = Proj4::Point.new(entry.lon, entry.lat) - nodes[node_id] = tile_from_merc(point, max_zoom) - end - end - node_cache.close - - # create a set of all the tiles at the maximum zoom level which are touched by - # any of the nodes we've collected. we'll create the tiles at other zoom levels - # by a simple recursion. - set = Set.new nodes.values - - # expire tiles and shrink to the set of parents - max_zoom.downto(min_zoom) do |_| - # allow the block to work on the set, returning the set at the next - # zoom level - set = yield set - end - end - - # wrapper to access the osm2pgsql node cache - class NodeCache - # node cache entry - class Node - attr_reader :lon, :lat - - def initialize(lon, lat) - @lat = lat.to_f / 100.0 - @lon = lon.to_f / 100.0 - end - end - - # open the cache - def initialize(filename) - @cache = File.new(filename, "r") - - throw "Unexpected format" unless @cache.sysread(4).unpack("l").first == 1 - throw "Unexpected ID size" unless @cache.sysread(4).unpack("l").first == 8 - - @max_id = @cache.sysread(8).unpack("q").first - end - - # close the cache - def close - @cache.close - end - - # lookup a node - def [](id) - if id <= @max_id - offset = 16 + id * 8 - - @cache.sysseek(offset) - - lon, lat = @cache.sysread(8).unpack("ll") - - node = Node.new(lon, lat) if lon != -2147483648 && lat != -2147483648 - end - - node - end - end -end diff --git a/cookbooks/tile/recipes/default.rb b/cookbooks/tile/recipes/default.rb index 3966954f8..d302c79ce 100644 --- a/cookbooks/tile/recipes/default.rb +++ b/cookbooks/tile/recipes/default.rb @@ -509,174 +509,94 @@ package %w[ osm2pgsql osmium-tool pyosmium - python3-pyproj ] -remote_directory "/usr/local/bin" do - source "bin" - owner "root" - group "root" - mode "755" - files_owner "root" - files_group "root" - files_mode "755" -end - directory "/var/lib/replicate" do owner "tile" group "tile" mode "755" end -if node[:tile][:replication][:engine] == "custom" - template "/usr/local/bin/expire-tiles" do - source "expire-tiles-custom.erb" - owner "root" - group "root" - mode "755" - end - - directory "/var/lib/replicate/expire-queue" do - owner "tile" - group "_renderd" - mode "775" - end - - template "/usr/local/bin/replicate" do - source "replicate-custom.erb" - owner "root" - group "root" - mode "755" - variables :postgresql_version => postgresql_version.to_f - end - - systemd_service "expire-tiles" do - description "Tile dirtying service" - type "simple" - user "_renderd" - exec_start "/usr/local/bin/expire-tiles" - nice 10 - sandbox true - read_write_paths tile_directories + [ - "/store/database/nodes", - "/var/lib/replicate/expire-queue", - "/var/log/tile" - ] - end - - systemd_path "expire-tiles" do - description "Tile dirtying trigger" - directory_not_empty "/var/lib/replicate/expire-queue" - end - - service "expire-tiles.path" do - action [:enable, :start] - subscribes :restart, "systemd_path[expire-tiles]" - end - - systemd_service "replicate" do - description "Rendering database replication service" - after "postgresql.service" - wants "postgresql.service" - user "tile" - exec_start "/usr/local/bin/replicate" - sandbox :enable_network => true - restrict_address_families "AF_UNIX" - read_write_paths [ - "/store/database/nodes", - "/var/lib/replicate", - "/var/log/tile" - ] - restart "on-failure" - end - - service "replicate" do - action [:enable, :start] - subscribes :restart, "template[/usr/local/bin/replicate]" - subscribes :restart, "systemd_service[replicate]" - end -elsif node[:tile][:replication][:engine] == "osm2pgsql" - template "/usr/local/bin/expire-tiles" do - source "expire-tiles-osm2pgsql.erb" - owner "root" - group "root" - mode "755" - end +template "/usr/local/bin/expire-tiles" do + source "expire-tiles.erb" + owner "root" + group "root" + mode "755" +end - directory "/var/lib/replicate/expire-queue" do - owner "tile" - group "_renderd" - mode "775" - end +directory "/var/lib/replicate/expire-queue" do + owner "tile" + group "_renderd" + mode "775" +end - template "/usr/local/bin/replicate" do - source "replicate-osm2pgsql.erb" - owner "root" - group "root" - mode "755" - end +template "/usr/local/bin/replicate" do + source "replicate.erb" + owner "root" + group "root" + mode "755" +end - systemd_service "expire-tiles" do - description "Tile dirtying service" - type "simple" - user "_renderd" - exec_start "/usr/local/bin/expire-tiles" - nice 10 - sandbox true - restrict_address_families "AF_UNIX" - read_write_paths tile_directories + [ - "/var/lib/replicate/expire-queue" - ] - end +systemd_service "expire-tiles" do + description "Tile dirtying service" + type "simple" + user "_renderd" + exec_start "/usr/local/bin/expire-tiles" + nice 10 + sandbox true + restrict_address_families "AF_UNIX" + read_write_paths tile_directories + [ + "/var/lib/replicate/expire-queue" + ] +end - systemd_path "expire-tiles" do - description "Tile dirtying trigger" - directory_not_empty "/var/lib/replicate/expire-queue" - end +systemd_path "expire-tiles" do + description "Tile dirtying trigger" + directory_not_empty "/var/lib/replicate/expire-queue" +end - service "expire-tiles.path" do - action [:enable, :start] - subscribes :restart, "systemd_path[expire-tiles]" - end +service "expire-tiles.path" do + action [:enable, :start] + subscribes :restart, "systemd_path[expire-tiles]" +end - template "/usr/local/bin/replicate-post" do - source "replicate-post.erb" - owner "root" - group "root" - mode "755" - end +template "/usr/local/bin/replicate-post" do + source "replicate-post.erb" + owner "root" + group "root" + mode "755" +end - osm2pgsql_arguments = %w[ +osm2pgsql_arguments = %w[ --number-processes=1 --log-progress=false --expire-tiles=13-19 --expire-output=/var/lib/replicate/dirty-tiles.txt ] - osm2pgsql_arguments.append("--multi-geometry") if node[:tile][:database][:multi_geometry] - osm2pgsql_arguments.append("--hstore") if node[:tile][:database][:hstore] - osm2pgsql_arguments.append("--tag-transform-script=#{node[:tile][:database][:tag_transform_script]}") if node[:tile][:database][:tag_transform_script] +osm2pgsql_arguments.append("--multi-geometry") if node[:tile][:database][:multi_geometry] +osm2pgsql_arguments.append("--hstore") if node[:tile][:database][:hstore] +osm2pgsql_arguments.append("--tag-transform-script=#{node[:tile][:database][:tag_transform_script]}") if node[:tile][:database][:tag_transform_script] - systemd_service "replicate" do - description "Rendering database replication service" - after "postgresql.service" - wants "postgresql.service" - user "tile" - exec_start "/usr/local/bin/replicate" - sandbox :enable_network => true - restrict_address_families "AF_UNIX" - read_write_paths [ - "/store/database/nodes", - "/var/lib/replicate" - ] - restart "on-failure" - end +systemd_service "replicate" do + description "Rendering database replication service" + after "postgresql.service" + wants "postgresql.service" + user "tile" + exec_start "/usr/local/bin/replicate" + sandbox :enable_network => true + restrict_address_families "AF_UNIX" + read_write_paths [ + "/store/database/nodes", + "/var/lib/replicate" + ] + restart "on-failure" +end - service "replicate" do - action [:enable, :start] - subscribes :restart, "template[/usr/local/bin/replicate]" - subscribes :restart, "systemd_service[replicate]" - end +service "replicate" do + action [:enable, :start] + subscribes :restart, "template[/usr/local/bin/replicate]" + subscribes :restart, "systemd_service[replicate]" end template "/usr/local/bin/render-lowzoom" do diff --git a/cookbooks/tile/templates/default/expire-tiles-custom.erb b/cookbooks/tile/templates/default/expire-tiles-custom.erb deleted file mode 100644 index 3a023d6b2..000000000 --- a/cookbooks/tile/templates/default/expire-tiles-custom.erb +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/ruby - -# DO NOT EDIT - This file is being maintained by Chef - -args = [ -<% node[:tile][:styles].each do |name,details| -%> - "-t", "/srv/tile.openstreetmap.org/tiles/<%= name %>", -<% end -%> - "--min", "13", - "--max", "<%= node[:tile][:styles].collect { |n,d| d[:max_zoom] }.max %>" -] - -Dir.glob("/var/lib/replicate/expire-queue/changes-*.gz").sort.each do |f| - system("/usr/local/bin/expire-tiles-single", *args, f) && File::unlink(f) -end diff --git a/cookbooks/tile/templates/default/expire-tiles-osm2pgsql.erb b/cookbooks/tile/templates/default/expire-tiles.erb similarity index 100% rename from cookbooks/tile/templates/default/expire-tiles-osm2pgsql.erb rename to cookbooks/tile/templates/default/expire-tiles.erb diff --git a/cookbooks/tile/templates/default/replicate-custom.erb b/cookbooks/tile/templates/default/replicate-custom.erb deleted file mode 100644 index 4612f5f68..000000000 --- a/cookbooks/tile/templates/default/replicate-custom.erb +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -# DO NOT EDIT - This file is being maintained by Chef - -# Before running updates, the replication needs to be set up with the timestamp -# set to the day of the latest planet dump. Setting to midnight ensures we get -# conistent data after first run. osmosis --read-replication-interval-init is -# used to initially create the state file - -# Define exit handler -function onexit { - [ -f sequence-prev.txt ] && mv sequence-prev.txt sequence.txt -} - -# Change to the replication state directory -cd /var/lib/replicate - -# Install exit handler -trap onexit EXIT - -# Loop indefinitely -while true -do - # Work out the name of the next file - file="changes-$(cat sequence.txt).osc.gz" - - # Save sequence file so we can rollback if an error occurs - cp sequence.txt sequence-prev.txt - - # Fetch the next set of changes - pyosmium-get-changes --server=<%= node[:tile][:replication][:url] %> --sequence-file=sequence.txt --outfile=${file} - - # Save exit status - status=$? - - # Check for errors - if [ $status -eq 0 ] - then - # Enable exit on error - set -e - - # Log the new data - echo "Fetched new data from $(cat sequence-prev.txt) to $(cat sequence.txt) into ${file}" - - # Apply the changes to the database - osm2pgsql --database gis --slim --append --number-processes=1 --log-progress=false \ -<% if node[:tile][:database][:node_file] -%> - --flat-nodes=<%= node[:tile][:database][:node_file] %> \ -<% end -%> -<% if node[:tile][:database][:multi_geometry] -%> - --multi-geometry \ -<% end -%> -<% if node[:tile][:database][:hstore] -%> - --hstore \ -<% end -%> -<% if node[:tile][:database][:style_file] -%> - --style=<%= node[:tile][:database][:style_file] %> \ -<% end -%> -<% if node[:tile][:database][:tag_transform_script] -%> - --tag-transform-script=<%= node[:tile][:database][:tag_transform_script] %> \ -<% end -%> - ${file} - - # No need to rollback now - rm sequence-prev.txt - - # Get buffer count - buffers=$(osmium fileinfo --extended --get=data.buffers.count ${file}) - - # If this diff has content mark it as the latest diff - if [ $buffers -gt 0 ] - then - ln -f ${file} changes-latest.osc.gz - fi - - # Queue these changes for expiry processing - ln ${file} expire-queue/${file} - - # Delete old downloads - find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \; - - # Disable exit on error - set +e - elif [ $status -eq 3 ] - then - # Log the lack of data - echo "No new data available. Sleeping..." - - # Remove file, it will just be an empty changeset - rm ${file} - - # Sleep for a short while - sleep 30 - else - # Log our failure to fetch changes - echo "Failed to fetch changes - waiting a few minutes before retry" - - # Remove any output that was produced - rm -f ${file} - - # Wait five minutes and have another go - sleep 300 - fi -done diff --git a/cookbooks/tile/templates/default/replicate-osm2pgsql.erb b/cookbooks/tile/templates/default/replicate.erb similarity index 100% rename from cookbooks/tile/templates/default/replicate-osm2pgsql.erb rename to cookbooks/tile/templates/default/replicate.erb -- 2.39.5