From 04743763991d4bd0b536ceaec43439d6b3b4842c Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Wed, 3 Jul 2024 17:03:00 -0700 Subject: [PATCH] Create a new vectortile role This role is for tilekiln based vector tile serving --- .kitchen.yml | 3 + cookbooks/postgresql/libraries/postgresql.rb | 18 +- cookbooks/postgresql/resources/schema.rb | 114 +++++++ cookbooks/vectortile/README.md | 9 + cookbooks/vectortile/attributes/default.rb | 15 + .../files/default/html/clientaccesspolicy.xml | 13 + .../files/default/html/crossdomain.xml | 5 + .../vectortile/files/default/html/favicon.ico | Bin 0 -> 15086 bytes .../vectortile/files/default/html/robots.txt | 13 + cookbooks/vectortile/metadata.rb | 16 + cookbooks/vectortile/recipes/default.rb | 282 ++++++++++++++++++ .../templates/default/import-planet.erb | 24 ++ .../templates/default/index.html.erb | 19 ++ .../vectortile/templates/default/nginx.erb | 46 +++ .../default/tilekiln-storage-init.erb | 8 + .../templates/default/tiles-rerender.erb | 7 + .../default/vector-update-notile.erb | 12 + .../templates/default/vector-update-tile.erb | 15 + roles/vectortile.rb | 72 +++++ test/data_bags/accounts/tilekiln.json | 6 + test/data_bags/accounts/tileupdate.json | 6 + .../vectortile/inspec/import_spec.rb | 3 + .../vectortile/inspec/nginx_spec.rb | 26 ++ .../vectortile/inspec/osm2pgsql_spec.rb | 4 + .../vectortile/inspec/postgresql_spec.rb | 13 + .../vectortile/inspec/storage_spec.rb | 3 + .../vectortile/inspec/themepark_spec.rb | 7 + .../vectortile/inspec/tilekiln_spec.rb | 30 ++ 28 files changed, 788 insertions(+), 1 deletion(-) create mode 100644 cookbooks/postgresql/resources/schema.rb create mode 100644 cookbooks/vectortile/README.md create mode 100644 cookbooks/vectortile/attributes/default.rb create mode 100644 cookbooks/vectortile/files/default/html/clientaccesspolicy.xml create mode 100644 cookbooks/vectortile/files/default/html/crossdomain.xml create mode 100644 cookbooks/vectortile/files/default/html/favicon.ico create mode 100644 cookbooks/vectortile/files/default/html/robots.txt create mode 100644 cookbooks/vectortile/metadata.rb create mode 100644 cookbooks/vectortile/recipes/default.rb create mode 100644 cookbooks/vectortile/templates/default/import-planet.erb create mode 100644 cookbooks/vectortile/templates/default/index.html.erb create mode 100644 cookbooks/vectortile/templates/default/nginx.erb create mode 100644 cookbooks/vectortile/templates/default/tilekiln-storage-init.erb create mode 100644 cookbooks/vectortile/templates/default/tiles-rerender.erb create mode 100644 cookbooks/vectortile/templates/default/vector-update-notile.erb create mode 100644 cookbooks/vectortile/templates/default/vector-update-tile.erb create mode 100644 roles/vectortile.rb create mode 100644 test/data_bags/accounts/tilekiln.json create mode 100644 test/data_bags/accounts/tileupdate.json create mode 100644 test/integration/vectortile/inspec/import_spec.rb create mode 100644 test/integration/vectortile/inspec/nginx_spec.rb create mode 100644 test/integration/vectortile/inspec/osm2pgsql_spec.rb create mode 100644 test/integration/vectortile/inspec/postgresql_spec.rb create mode 100644 test/integration/vectortile/inspec/storage_spec.rb create mode 100644 test/integration/vectortile/inspec/themepark_spec.rb create mode 100644 test/integration/vectortile/inspec/tilekiln_spec.rb diff --git a/.kitchen.yml b/.kitchen.yml index 0af8c80de..f7cfa5d38 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -416,6 +416,9 @@ suites: - name: trac run_list: - recipe[trac::default] + - name: vectortile + run_list: + - recipe[vectortile::default] - name: web-cgimap run_list: - recipe[web::cgimap] diff --git a/cookbooks/postgresql/libraries/postgresql.rb b/cookbooks/postgresql/libraries/postgresql.rb index 1a39da25a..b2df4aed6 100644 --- a/cookbooks/postgresql/libraries/postgresql.rb +++ b/cookbooks/postgresql/libraries/postgresql.rb @@ -4,6 +4,10 @@ module OpenStreetMap class PostgreSQL include Chef::Mixin::ShellOut + SCHEMA_PRIVILEGES = [ + :create, :usage + ].freeze + TABLE_PRIVILEGES = [ :select, :insert, :update, :delete, :truncate, :references, :trigger ].freeze @@ -115,9 +119,21 @@ module OpenStreetMap end end + def schemas(database) + @schemas ||= {} + @schemas[database] ||= query("SELECT n.nspname, pg_catalog.pg_get_userbyid(n.nspowner) AS usename, n.nspacl FROM pg_namespace AS n WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema'", :database => database).each_with_object({}) do |schema, schemas| + name = "#{schema[:nspname]}" + + schemas[name] = { + :owner => schema[:usename], + :permissions => parse_acl(schema[:nspacl] || "{}") + } + end + end + def tables(database) @tables ||= {} - @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind = 'r'", :database => database).each_with_object({}) do |table, tables| + @tables[database] ||= query("SELECT n.nspname, c.relname, u.usename, c.relacl FROM pg_class AS c INNER JOIN pg_user AS u ON c.relowner = u.usesysid INNER JOIN pg_namespace AS n ON c.relnamespace = n.oid WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind IN ('r', 'p')", :database => database).each_with_object({}) do |table, tables| name = "#{table[:nspname]}.#{table[:relname]}" tables[name] = { diff --git a/cookbooks/postgresql/resources/schema.rb b/cookbooks/postgresql/resources/schema.rb new file mode 100644 index 000000000..a7bf0ebdb --- /dev/null +++ b/cookbooks/postgresql/resources/schema.rb @@ -0,0 +1,114 @@ +# +# Cookbook:: postgresql +# Resource:: postgresql_schema +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +unified_mode true + +default_action :create + +property :schema, :kind_of => String, :name_property => true +property :cluster, :kind_of => String, :required => true +property :database, :kind_of => String, :required => true +property :owner, :kind_of => String, :required => [:create] +property :permissions, :kind_of => Hash, :default => {} + +action :create do + if schemas.include?(qualified_name) + # Handle the case of an existing schema + if new_resource.owner != schemas[qualified_name][:owner] + converge_by("set owner for #{new_resource} to #{new_resource.owner}") do + Chef::Log.info("Setting owner for #{new_resource} to #{new_resource.owner}") + cluster.execute(:command => "ALTER SCHEMA #{qualified_name} OWNER TO \"#{new_resource.owner}\"", :database => new_resource.database) + end + end + + schemas[qualified_name][:permissions].each_key do |user| + next if new_resource.permissions[user] + # Remove permissions from users who no longer have them + converge_by("revoke all for #{user} on #{new_resource}") do + Chef::Log.info("Revoking all for #{user} on #{new_resource}") + cluster.execute(:command => "REVOKE ALL ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database) + end + end + new_resource.permissions.each do |user, new_privileges| + current_privileges = schemas[qualified_name][:permissions][user] || {} + new_privileges = Array(new_privileges) + + if new_privileges.include?(:all) + new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES + end + + OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege| + if new_privileges.include?(privilege) + unless current_privileges.include?(privilege) + converge_by("grant #{privilege} for #{user} on #{new_resource}") do + Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}") + cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database) + end + end + elsif current_privileges.include?(privilege) + converge_by("revoke #{privilege} for #{user} on #{new_resource}") do + Chef::Log.info("Revoking #{privilege} for #{user} on #{new_resource}") + cluster.execute(:command => "REVOKE #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} FROM \"#{user}\"", :database => new_resource.database) + end + end + end + end + else + converge_by "create schema #{new_resource.schema}" do + cluster.execute(:command => "CREATE SCHEMA #{new_resource.schema} AUTHORIZATION #{new_resource.owner}", :database => new_resource.database) + # Because the schema is new, we don't have to worry about revoking or checking current permissions + new_resource.permissions.each do |user, new_privileges| + new_privileges = Array(new_privileges) + if new_privileges.include?(:all) + new_privileges |= OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES + end + OpenStreetMap::PostgreSQL::SCHEMA_PRIVILEGES.each do |privilege| + next unless new_privileges.include?(privilege) + converge_by("grant #{privilege} for #{user} on #{new_resource}") do + Chef::Log.info("Granting #{privilege} for #{user} on #{new_resource}") + cluster.execute(:command => "GRANT #{privilege.to_s.upcase} ON SCHEMA #{qualified_name} TO \"#{user}\"", :database => new_resource.database) + end + end + end + end + end +end + +action :drop do + if schemas.include?(qualified_name) + converge_by("drop #{new_resource}") do + Chef::Log.info("Dropping #{new_resource}") + cluster.execute(:command => "DROP SCHEMA #{qualified_name}", :database => new_resource.database) + end + end +end + +action_class do + def cluster + @cluster ||= OpenStreetMap::PostgreSQL.new(new_resource.cluster) + end + + def schemas + @schemas ||= cluster.schemas(new_resource.database) + end + + def qualified_name + "#{new_resource.name}" + end +end diff --git a/cookbooks/vectortile/README.md b/cookbooks/vectortile/README.md new file mode 100644 index 000000000..659ee7aa4 --- /dev/null +++ b/cookbooks/vectortile/README.md @@ -0,0 +1,9 @@ +# vectortile cookbook + +This cookbook installs and configures the tilekiln based tileservers + +## Accounts +The following accounts are used +- `www-data` for nginx serving static files and proxying +- `tilekiln` for the process serving tiles +- `tileupdate` for the process running osm2pgsql and tilekiln on updates diff --git a/cookbooks/vectortile/attributes/default.rb b/cookbooks/vectortile/attributes/default.rb new file mode 100644 index 000000000..0df4249e7 --- /dev/null +++ b/cookbooks/vectortile/attributes/default.rb @@ -0,0 +1,15 @@ +default[:vectortile][:database][:cluster] = "16/main" +default[:vectortile][:database][:postgis] = "3" +default[:vectortile][:database][:nodes_store] = :flat +default[:vectortile][:serve][:threads] = node.cpu_cores +default[:vectortile][:serve][:mode] = :live +default[:vectortile][:replication][:url] = "https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute" +default[:vectortile][:replication][:status] = :enabled +default[:vectortile][:replication][:tileupdate] = :enabled +default[:vectortile][:replication][:threads] = node.cpu_cores + +default[:postgresql][:versions] |= [node[:vectortile][:database][:cluster].split("/").first] +default[:postgresql][:monitor_database] = "tiles" +default[:postgresql][:settings][:defaults][:max_connections] = (node.cpu_cores * 4 + 20).to_s +default[:accounts][:users][:tileupdate][:status] = :role +default[:accounts][:users][:tilekiln][:status] = :role diff --git a/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml b/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml new file mode 100644 index 000000000..d60ec1d3f --- /dev/null +++ b/cookbooks/vectortile/files/default/html/clientaccesspolicy.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/cookbooks/vectortile/files/default/html/crossdomain.xml b/cookbooks/vectortile/files/default/html/crossdomain.xml new file mode 100644 index 000000000..35fc43447 --- /dev/null +++ b/cookbooks/vectortile/files/default/html/crossdomain.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/cookbooks/vectortile/files/default/html/favicon.ico b/cookbooks/vectortile/files/default/html/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..975e1cb0d9a56a11d5843a5bc851bd8ed2f6b0ca GIT binary patch literal 15086 zcmd6O2XLF$u`WG+FP>yFac;>Z&LnYillUfaY{`-(%d(1P$#xva&o4dkbsXDq;@GL~ zPF&QB)w|e9kRaH51HlFoBtej91VD6xs3ZUaBuJ{zvi9x!&QHRZ;Z^c7Z|1)HG_yKC z{!KNi>1_xx8L3!YYz;OFYqt2bW0 zeEB0+E?>TBVPWCF7H$xXUrsY`v zj=y!BzF&Mt=u*5x9H-Bx<^EgmxZbt&^WvSMVDJlFogI&zzHt7%pSHW=tIjnptY5b# z`r501R2)Bcc)y}V{#Adu3JS_vO{)FITr#$%-{i)v~6_!(}y<&o(qR zJlfUOwMDPj-zDV3bDtK^i+82@7IS~SV<9(qxV@pF;oAy@;zwB>+R8%>-E{Ze_t2_4 zR?|1Xemni^ZC|B3SKWT~&u8~tEwOfxY`lkZM_Oa&+sk|M8_G_WR+K%ZR4R7}KB1eh z8;!E8l(38vb+EK|VWN$3+1R7y5g-095SPT)EKLW{}sF9Vsj< z{8?^6{`0xDiqi$!re1|llPaGaqKfDcS4M}q-`>GZp+T;k9Oj~6A0M|?^L}eSs31*r zXt*GDx=YsF+^>2Lw)IcxdB5H$${x!Bu6$J%rFhrjp{uf6L|x_ixCyl%}p zzWc6qwEE6f{LOEC9WvZSzd!j7pB}8{B7YAR`gbMLf zco%$Ef!fD4WHu|gHf8`BsVJ!7J;prVYtEx(GSC5AAukbm1HP|`HE})r5$A-BS;ifY z$;Xsb43ayQl-*fLPyFghTDNvJt-t$j+OTc|-2;1CyZSD=^PArwA;WKv{+UkpRYp)ZRe2@oP*8O$Bo<&H*sU!%=zPOyl*6rcN=qgmoaC_j{d@yxNTJPnpmUPvz;{m zVhvOWyF=ig%0eC79Mf~#q?w9_TKJhiKF1q3+)ejw*ub0a-Nc(VY=SP<^P1In^7;*H z>6g1-;@z?mF7S3yj&6_(or5g%^-{01iyI<3@U5hNx0*5M;N3)N-kYXIs6J}oqxMoE z1MM8n<~`e5~NTzS0 zU%dJ}ZQ6JbZNBe5+PY~gZP~b$?p?nD>ti+DbMIPu^2I;Up|(QGhwYr}ShSr=e?N`) z*OJ<*qROz2YA5wn>o-%W&O((Y3)PQVs3m40d8nU`J1S_ekU^gfIkKct$Rup)gsEf< z{`pp6=V^YIk7>R5r$WCO2^d%jeJjLT4k)>1asWCva-p%6pL^p4-m+-}-@kb?Z`-mB z<2LAGBd@=E4Xs^LS+X^#>11KMlO z6}n)|z3``$J=V&a-qr(%)3=CND|k)wy<87-^Yb6;Pq@w9NjsT+h%u8UYDrkBDFHj3 zGEw#9Fy&d5^s9G%PuuU`Ob>3`Mh|U&2;)O^|E4Xpal?AreE+?)D{DXX^psOqS1&30 zEmY^Xkl?#Mq9=v7m$J;wbZVfIj;R%Nq*G4sIfn5$+ETvDT|$jYrK5VJ(-% zhFAgLlxJ+XoW74W@fAua32 zM_-6ePTn#+oxh=4x z9wQZ)dg#Ts-yk8w!#j4+OW9@g%fd$5vUwx@;OQUI?+(97zXAPv-z)S+?m^mDpG}7= z&e4(V_b8{}ILTxusZ?iJ4%Po?xVeVM<{=&QSeOV4f%ATK9|n*R8Wbto6KSN z??R!V$P@ob#Ixn=K;*Z`6aPhN)Vy1Qxi5vSq)85&$%hWkSvvS=otg)&D%gdd%;Dp(b`k zhS?6gvf>>UkQK+lX%B~^W9;@0b7|RmKC<^UZfPvw5?uoyR~GZ>j#Bc3$F~app@Yu| z9W3Wx*Wn3EO3FPFd%l}2@}i0J5C+c0OlI=N)5mvqixR zrf6bjj5GuF^!mX+QHQZ>FqNA5L?jaWypUmfI(c(?4G7y2@vUDwu&FexO(|eMi!>;W z7y6JBICLy@ArJJk!r#yJQ)V9U^|Qy+#yW2w8{EC@)6{Wu;Yr^4@~?Qyee3wCpFYlh zlafPDH4jhdc_>Yob20DI@10$EG8!H>&vPsrMQM|AKeW zt*{rtzZ477{Hx7_&w+2u{UWd6SF)c|%6`P+#bfz6;!#LP4T%x%L=5T(^ike;BNroH zk7(+s{MwP`x~h^pdwv&73Cd$mo}_lP0H#VirG47a-@xnkXN!5$zEVX z7$w45fJ`(D-Ua`Hj8UVIgT`jO6q*at#C(LNE=DPd@!XjM@sSbJIx+t%?EePg`>Vr4 zUIlC-&8g+%qQyxxnR6k#09R5>qRcfosl8@Zh=j;v!+4DiVN>?-i3oa zLkHDtfel6RBx_`e9=Y&1k6AtKNoFj8#p1L^6{(zaz zoH}-PVj}X{OUvT0@I&F$)H>lK^L?0azXtZu&!yO>Rw0&EO&K`fs-?_ctrXX=UnJp^ z1U2LS^WzPi<7{AM>3KT7|4oiKdwI&Gp(yhGu)T{7$U`v45>I-pd%-v6S@2CUT%Q{4 z;`nG6`LIretUBZw&=(6bBR_RbyRk0DIhqQB|NU_h!&dr#ODgzQRa{Rs2@};WkvJAq z0XxlB8A%b+66T!*|5<(|W%^r5iP$Ea(9oIDIyzHzjxhH$e_e@C@ET-MGJ-qLy}YdElGDe^0Q7+I&iC$NsDr$HJz32@iF5*HWLR z=UZvtpUxkGSCT)jPmXYX61gF$ZW)z@v7U_@g!Cw+Q$C7tX@xk7av+CpcQup2(a!mK=Vj@6 zPv<{k>`u8H&C|%2|AH)34g0BqZxFJT4%?{IYoHU(8p8b30Y@S7VHve4^Xd5UcW3~) zOTeQ6?6Il1cX9b`?O{niL>__Qs=9ZCK9tJLulX zEBLnw{)HYSd4xV@g-p11X(~X_0Y*RWu~E`H2tTA|hg(hRK=+dUz(>G#TAYpCG+aq$ z_^Z5}vxRBC(|Qp6&o5m5Y{!g!p?$_qX;R_1DQYIww4EB4#u}4@DiL=|B3e4_Xb@|G z_Ly?$&Dw*cC^|u7Uig-{ho+{yluCsu7I)FbnFvkCy)+*ilk8|wKF9)jQUO6DSQkce zdeo#6HnU_mZN3�#j=ksUfAVnr!e>MY6m%(|o7p5d4cgLN_zk-I;Rmhcx5BUd+NB z7?lYdE2nH+XL16+F*Ep=@V`@z20rGj;CIw#`S6K%cm(?{|BR1C4iLW-<*^wz&s>>i z_q3CzF2#9jHq4i2qkLsH#&fW%OVeSV#M*Wv|Is6UrhSg^Kd^glcGPq2U>U0o&D;e2 z>h(jL1iylo^Dl5j@65yt-Lp=XDCHzEw&A^EF zh5oSb6}Gs#*_Uura5ez`W8mLI68}jDPhUwQH*oX(`*WOHm?2jJKIw9TgR|qvpTq2d zj~xj2Nj^u|jndsr4TfrN8K`E1qnl11+Z_>k+sgSDxv1D1CsNa&2i|Z+4UutP`;y|&T4w3oVqj$Tyx4GVzfj>z&6ixmr-8+5pssiWS_K=BW5RGa*RT= zK^jYXD0z8`ywfh4T}V;lavbOV6uUSnk&wZibkYd&sy=+L&)-KK);8q-71V8PA%j^( z#YOq=F6TWx|00JI{7=p%ZWp16*EY7*v4SkHK#@GeB@=Uu#-pQ8u z2=X2q@**>KcsDEx%~*8h@8SbJMk3uFN?e{$suGY z#$GJbpypEzT9!xpsS3DIZ31yRVI~o`MVu2cLXI5juqKc99DJRElTIFsxp)F=cmg>X zbimPz6XZ#Nckl~7$(FFPb;`mPV2OhF5%@TB*g!+x0j}#S6TH))3z%$uF4rpS52fdO zxek`gGng97MqWmW=%BO*6S~M$>*zw4j`9qHguIRlkyBSr3{e&KP1W#2b%@OZAIfqy zQnt4qw$(_xYmU=}tV0xvxyT>$kU!xgPuwl-v0Mo!#V^IB92skboXC${Aj^cAjQ%0g zy9a2<0oyS(kj~RV>fTD)b#^}`-k;mCobUAUhx`-W(2y{Dq<#BRekp9|LKpV(OSU9* zQ575JrfI~oX)9u#6?ruBQ}~I3K(mxf%L1*O?`h&U%J%cAvwL|wVr3t6;hAu-Ytqi6 z_&qk|W$%=mUGO`O$x*fhOl(99Gk6BsjB&)TVmli0V+z?Xs=ru3*mD8YXd>M^Y(54Ld; z{2uUAI^vY`5p@Z_weK}DnmgDZb+9L5CokmiBK~?JE^>wJZ1bDhGHxJiz{su%D|@3> zvimfA{`>(xTyl=OrXAcjkRQpyy;Lx!rkBp_qNAsF(XhUqq6shd z=zfZT!pU(8;CsYt{!x1Ey;rFWb+)b%ypx=jc?5QLfcKx?%lisXaJH$EF%PW7epm$@ zrF+K7lMAUQ({sOkOwWI0Vdmj-!TZ!ONuwl#qzhq7RI0bSofjv6Klj0vuSegX{l4dN z?4WTz*sV!Cs}rb>=-3-XlKsF{V`bc+}|+KV@4d~f;p(>%<7aOKOs`Phrbxj=_{#<`$Qxmb{C zKB!uGe_94{{?fJgKQlBxo~@tvpG6M6+j}|oY~ua-wczb0asBTl!PoTrbGK`gwydU5 zf0kx)wiT!*pM3Hs>htGLzoSwqzk~Y0SN>Mdh~Qn+5JhhZ`ZE5-@ArRhsb}S*;#!)o z>&>6IChkRV$k$rC)lVPnu}rVsvV(8?%9rVZ2ex=#-}_eI*{Z_&;+nEI+uPe85cfr+ z(SH{FVEmNbZof_RgH$!vJ+0_!$g0sc_ZRdvwv=`?9II(-cvf6XKO^+Jk_Yj9liT`e zaZh#4p^_}!{SSW&{UK|3)g7zQ7jhf_^o6JRJZeg!CnV3=nJOG;=_#$P+gVXj@jZD> z`Adaeb!El2&an#gQYf*%YexTv=>0fh7rh?^e85mJeOz78EAOb<8w>|;72jLVgZOO^ zc^**&+BtK$od4~`rx>+sUcYvo)T1K$I==ghAM**+=wzZN1$95UTEiuN&1F$bLobBX zOVEh@g5K6HdN+_$^-FyNOLfx4-V<8^cY3w2Uhn&pQ_)*X{HTQo)LdchLSMiy(c7>V zJq`DQkM)>~Z}Q_$|A3FE6nwt7k1q@W=Xau)z~3i*TLrld_;zDdFZD0TLu#qcyvva8 zTUhLWsT^&~5o>C>{Nh;buWLg+A<>`Eh`g{GILmXdy}$4abiZll~?k3g^s{gpM)-3ozed|kYG0Q?v#K66QL2eIfxe|QH zu>XG%`_3KPwxX8&uvFW>@7@i3;KUwQkD!ksXkgU7r8R;+3*L{p*{#UuJr#NUO65L& zt>z%Vt2)K!tqt4^EVIecUlotXJ_Wz{mU+Q#*xy)3i# z^Y0J50Uqv`YT7?IlE*)J{Y^etlf^qL&+@C;`*~;eIX;1!@PU#XK2V&+hjHyd+j%}? zsNgf2O3YV2muYL!AEAq)PWSocbw=SAgpZJUdLpt=KNn&B2p%#~KbEz3aXo5Wg@Z%< z+R@$oR-TN1_VSzPb=bm>{p2xz8oe9OzVad$R%WuYtDMbPpHAS4o=NPRMtZp#wTtZH zQ(V_3=Tq$p^l7N9;`^djE9OARE_|m>KeQ3DlMMKvAUSL?TRX%#sIzyVHl;#6qyacU zOGM8_S%>*s4?f7-&};G7cfQS~vQumYmYl9%iaLl9`{mGVh@;6EPXcqQs?Fn^sw`HU zdW%u#xMitUAo*rdqwF$if3sNITdeVA4Prl!+KC)BJ?w?q3JhG-^BkxZ_t)g|uYdj{ z)K9mw5i;9=O{}cL2p+7!mjcNkPb5&Yo1WxuO+B~uw!baDA3@J-I=+gy(c;z>rt5P; z_6vOjEQc>GgB{kQm#8NMTf}~_!K>oN{1g1pwvF7^EaQl+1K&m+2;W;tCd|PoaJj&& zpC{)gcq$d+=BCoP)jG09)M}x3sa7s(%vGVGF;SlqwM^0XhZ+y|r#krZel7`AI}$Y}J!;#7u$_MRm65nb!XI0d<(GvH(taqqklG>DRTgm@ zf!(mejXnUIL8`|cgD*YiD&;!t`_Jm@xIliIr+{fE-CCXqnWg%PsG)cxM*jD@NjK^y zU6>y=qvrD=cB#R!)ze4)eZ7yS>1Csp5)5vERF#apG7S|)B)xv zoQt?W>Hq@&pN}}D`V8tYQr%@3_LTm9D|~Pt@V5?C^PiS~-*sv70I&lI-){r9-vrD{ zkihv%EJoC}y6}A&ztew)i{vLb;T-}tKPuH7mau$*)q@|qMDvp#sTLHpcXE$^5wjP# zYm>E>b?DdX>`*?J{=Tq-*u|Mm0^1h*e}M8-O{g(KlhUS=TK8zVLzl zDUV0s4!}>aXE#gzWhEv93tJHDK;%232D`iD6dO^~0&dIbZIG~YKQKMqti3 z4m0puc3}N?p=Q_NQ*pDYp1br-Twb0%mbQJtgII?mCbq||sx%fQ^1Wijft+FBvN#t0 zK;&8#=+i6@4Dk7aW9$WH=3c^Gy-7EFfxm|5f)e)X0S+6S3oK%oh|TSm7Ow9vW9>*Q zceJ-YpO$^;`!`%!Shxuo(zN)#AUSH+7qoqxHH^L}_`^EXHwBg|dM%4!3;UYSaHqPK z1HkLTsUU~p>wVCHzT=}TnTJO!UVA<<>7WN9B<(gV1aUf z8=v)5^3g6i=PJ&kW^Uy1s2g~uN5TLdAq(5XBRm$hvAVyO&&bbkYZ$flxyjutzb$Zu zmZWubIX0GWMz1D(aU*=8ut%{rS`w)FM|$9!J6I7=@o|@evoQ*r++TfxbKsY%8syxf ztl;`4#QJ&}pRdZ~(;69SEvOYGMj7+`@XGH`zMtAI@S)}SmNGs%;F_9x^5O>z{}OwD z_J0D`I%J#=^a8h8C=YgWDe~047Ub2oLe9cmRK*N2{ha@JVqyOF2zV%XgH*m6M-JCB6e&~0zz`s`Bll~1~{qgaO6HhmV`#UQA zU6tynagRF{d>H5dgH4HN#P`7e4=nScYUicN%=CMfjW}HVH;W;)HNfhdGHz7^Edkw} zCO?irH?1IX{U*?j*CX6};}YG3@k3JMT|Meg+qGPjo+fLcpcxn>0b1EW^t~rkE{HeYc=q1-RK9cME)c7YGd!3 zYp?4SIjWFHjH-w>SQYH$-|l`D`?5{EdFv*AyYMWRVLzsJskwR#K4N@?kJ~DxcN{Pk zq-Ihx0%4P6v`mfVw=xRac6fWKmO>$tSCOihNV2xikJ~WEx$w4 z$Rke64~0%de5efSe8q@^`8MRG=sl?mYq-9>nB|!Vc@#ZwBL5Wmp9%XgTO98|J$FFc zRgeC(o9C}y{=A4C)$rfN_F*mz^>8up+?<9Sjw6?Hqb_Gf9v4QxTnxFJ0sVC7kKtjL z<{Gf5Tdtu7)Rr8bZbF|z4f3o)XFs2`m$0lkhZCsFOe0RG(1UaH=R^=C-= z4~y7_o+8LQ_k9csygEvz*n-yB9CEB1PInAgq6Gv7jA z&gbBlzHgrOWfl2aTs5KIdhey!Qy+mr~m)} literal 0 HcmV?d00001 diff --git a/cookbooks/vectortile/files/default/html/robots.txt b/cookbooks/vectortile/files/default/html/robots.txt new file mode 100644 index 000000000..bfb50b0ae --- /dev/null +++ b/cookbooks/vectortile/files/default/html/robots.txt @@ -0,0 +1,13 @@ +User-agent: * +Disallow: /12/ +Disallow: /13/ +Disallow: /14/ +Disallow: /15/ +Disallow: /16/ +Disallow: /17/ +Disallow: /18/ +Disallow: /19/ +Disallow: /20/ +Disallow: /21/ +Disallow: /22/ +Disallow: /23/ diff --git a/cookbooks/vectortile/metadata.rb b/cookbooks/vectortile/metadata.rb new file mode 100644 index 000000000..d3b44e37e --- /dev/null +++ b/cookbooks/vectortile/metadata.rb @@ -0,0 +1,16 @@ +name "vectortile" +maintainer "OpenStreetMap Administrators" +maintainer_email "admins@openstreetmap.org" +license "Apache-2.0" +description "Installs and configures vector tile servers" + +version "1.0.0" +supports "ubuntu" +depends "accounts" +depends "git" +depends "nginx" +depends "postgresql" +depends "prometheus" +depends "python" +depends "systemd" +depends "tools" diff --git a/cookbooks/vectortile/recipes/default.rb b/cookbooks/vectortile/recipes/default.rb new file mode 100644 index 000000000..37fe86b43 --- /dev/null +++ b/cookbooks/vectortile/recipes/default.rb @@ -0,0 +1,282 @@ +# +# Cookbook:: vectortile +# Recipe:: default +# +# Copyright:: 2024, OpenStreetMap Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "accounts" +include_recipe "git" +include_recipe "nginx" +include_recipe "postgresql" +include_recipe "prometheus" +include_recipe "python" +include_recipe "tools" + +directory "/srv/vector.openstreetmap.org" do + user "tileupdate" + group "tileupdate" + mode "755" +end + +nginx_site "default" do + action [:delete] +end + +nginx_site "vector.openstreetmap.org" do + template "nginx.erb" +end + +ssl_certificate node[:fqdn] do + domains [node[:fqdn], "vector.openstreetmap.org"] + notifies :reload, "service[nginx]" +end + +remote_directory "/srv/vector.openstreetmap.org/html" do + source "html" + owner "www-data" + group "www-data" + mode "755" + files_owner "www-data" + files_group "www-data" + files_mode "644" +end + +template "/srv/vector.openstreetmap.org/html/index.html" do + source "index.html.erb" + owner "www-data" + group "www-data" + mode "644" +end + +postgresql_version = node[:vectortile][:database][:cluster].split("/").first +postgis_version = node[:vectortile][:database][:postgis] + +package "postgresql-#{postgresql_version}-postgis-#{postgis_version}" + +# Spirit dependencies +package %w[ + osm2pgsql + gdal-bin + python3-yaml + python3-psycopg2 +] + +style_directory = "/srv/vector.openstreetmap.org/spirit" +git style_directory do + repository "https://github.com/pnorman/spirit.git" + user "tileupdate" + group "tileupdate" +end + +shortbread_config = "#{style_directory}/shortbread.yaml" + +themepark_directory = "/srv/vector.openstreetmap.org/osm2pgsql-themepark" +git themepark_directory do + repository "https://github.com/osm2pgsql-dev/osm2pgsql-themepark.git" + user "tileupdate" + group "tileupdate" +end + +tilekiln_directory = "/opt/tilekiln" + +python_virtualenv tilekiln_directory do + interpreter "/usr/bin/python3" +end + +python_package "tilekiln" do + python_virtualenv tilekiln_directory + python_version "3" + version "0.5.1" +end + +template "/srv/vector.openstreetmap.org/html/index.html" do + source "index.html.erb" + owner "www-data" + group "www-data" + mode "644" +end + +directory "/srv/vector.openstreetmap.org/data" do + user "tileupdate" + group "tileupdate" + mode "755" +end + +node_store_options = node[:vectortile][:database][:nodes_store] == :flat ? "--flat-nodes '/srv/vector.openstreetmap.org/data/nodes.bin'" : "" +template "/usr/local/bin/import-planet" do + source "import-planet.erb" + owner "root" + group "root" + mode "755" + variables :node_store_options => "#{node_store_options}" +end + +template "/usr/local/bin/tilekiln-storage-init" do + source "tilekiln-storage-init.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :storage_database => "tiles", :config_path => "#{shortbread_config}" +end + +postgresql_user "tomh" do + cluster node[:vectortile][:database][:cluster] + superuser true +end + +postgresql_user "pnorman" do + cluster node[:vectortile][:database][:cluster] + superuser true +end + +postgresql_user "tilekiln" do + cluster node[:vectortile][:database][:cluster] +end + +postgresql_user "tileupdate" do + cluster node[:vectortile][:database][:cluster] +end + +postgresql_database "tiles" do + cluster node[:vectortile][:database][:cluster] + owner "tileupdate" +end + +postgresql_database "spirit" do + cluster node[:vectortile][:database][:cluster] + owner "tileupdate" +end + +postgresql_extension "postgis" do + cluster node[:vectortile][:database][:cluster] + database "spirit" +end + +postgresql_schema "tilekiln" do + cluster node[:vectortile][:database][:cluster] + database "tiles" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => :usage + notifies :run, "execute[tilekiln-storage-init]", :immediately +end + +execute "tilekiln-storage-init" do + action :nothing + command "/usr/local/bin/tilekiln-storage-init" + user "tileupdate" +end + +%w[metadata shortbread_v1].each do |table| + postgresql_table table do + cluster node[:vectortile][:database][:cluster] + database "tiles" + schema "tilekiln" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => :select + end +end + +(0..14).each do |zoom| + postgresql_table "shortbread_v1_z#{zoom}" do + cluster node[:vectortile][:database][:cluster] + database "tiles" + schema "tilekiln" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => node[:vectortile][:serve][:mode] == :live ? [:select, :insert, :update] : :select + end +end + +%w[addresses aerialways aeroways boundaries boundary_labels bridges buildings +dam_lines dam_polygons ferries land pier_lines pier_polygons place_labels +planet_osm_nodes planet_osm_rels planet_osm_ways pois public_transport railways +road_routes roads sites street_polygons streets_labels_points +streets_polygons_labels water_area_labels water_areas water_lines water_lines_labels].each do |table| + postgresql_table table do + cluster node[:vectortile][:database][:cluster] + database "spirit" + schema "public" + owner "tileupdate" + permissions "tileupdate" => :all, "tilekiln" => :select + end +end + +tilekiln_mode = node[:vectortile][:serve][:mode] == :live ? "live --config #{shortbread_config} --source-dbname spirit" : "static" + +systemd_service "tilekiln" do + description "Tilekiln vector tile server" + user "tilekiln" + after "postgresql.service" + wants "postgresql.service" + sandbox :enable_network => true + restrict_address_families "AF_UNIX" + exec_start "#{tilekiln_directory}/bin/tilekiln serve #{tilekiln_mode} --storage-dbname tiles --num-threads #{node[:vectortile][:serve][:threads]}" +end + +service "tilekiln" do + action [:enable, :start] +end + +execute "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py" do + command "/srv/vector.openstreetmap.org/spirit/scripts/get-external-data.py -R tilekiln" + cwd "/srv/vector.openstreetmap.org/spirit" + user "tileupdate" + group "tileupdate" + ignore_failure true +end + +template "/usr/local/bin/vector-update" do + source node[:vectortile][:replication][:tileupdate] == :enabled ? "vector-update-tile.erb" : "vector-update-notile.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :config_path => "#{shortbread_config}", :diff_size => "1000", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :post_processing => "/usr/local/bin/tiles-rerender" +end + +template "/usr/local/bin/tiles-rerender" do + source "tiles-rerender.erb" + owner "root" + group "root" + mode "755" + variables :tilekiln_bin => "#{tilekiln_directory}/bin/tilekiln", :source_database => "spirit", :storage_database => "tiles", :config_path => "#{shortbread_config}", :tiles_file => "/srv/vector.openstreetmap.org/data/tiles.txt", :update_threads => 4 +end + +systemd_service "replicate" do + description "Get replication updates" + user "tileupdate" + after "postgresql.service" + wants "postgresql.service" + sandbox :enable_network => true + restrict_address_families "AF_UNIX" + read_write_paths ["/srv/vector.openstreetmap.org/data/"] + exec_start "/usr/local/bin/vector-update" +end + +systemd_timer "replicate" do + description "Get replication updates" + on_boot_sec 60 + on_unit_active_sec 30 + accuracy_sec 5 +end + +if node[:vectortile][:replication][:status] == :enabled + service "replicate.timer" do + action [:enable, :start] + end +else + service "replicate.timer" do + action [:stop, :disable] + end +end diff --git a/cookbooks/vectortile/templates/default/import-planet.erb b/cookbooks/vectortile/templates/default/import-planet.erb new file mode 100644 index 000000000..bffa34a3a --- /dev/null +++ b/cookbooks/vectortile/templates/default/import-planet.erb @@ -0,0 +1,24 @@ +#!/bin/sh + +# Usage +# sudo -u tileupdate import-planet osmfile.osm.pbf +# sudo -u tileupdate import-planet --cache 50000 osmfile.osm.pbf +# This script sets the appropriate Lua and style paths for the osm2pgsql import + +set -e + +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' + +# Import the osm2pgsql file specified as an argument, using the locations for spirit +osm2pgsql \ + --output flex \ + --style '/srv/vector.openstreetmap.org/spirit/shortbread.lua' \ + --slim \ + <%= @node_store_options %> \ + -d spirit \ + $@ + +# Set up replication. This doesn't specify the replication server, so it will use planet.osm.org on extracts +osm2pgsql-replication init \ + -d spirit \ + --server '<%= node[:vectortile][:replication][:url] %>' diff --git a/cookbooks/vectortile/templates/default/index.html.erb b/cookbooks/vectortile/templates/default/index.html.erb new file mode 100644 index 000000000..feb7f36fa --- /dev/null +++ b/cookbooks/vectortile/templates/default/index.html.erb @@ -0,0 +1,19 @@ + + + +vector.openstreetmap.org + + + + + + +You've reached the OpenStreetMap.org vector tile server (<%= node['fqdn'] %>)
+
+
If you are a user...
+
You probably want OpenStreetMap itself.
+
If you are a developer...
+
Please be aware of the tile usage policy.
+
+ + diff --git a/cookbooks/vectortile/templates/default/nginx.erb b/cookbooks/vectortile/templates/default/nginx.erb new file mode 100644 index 000000000..838d30a4f --- /dev/null +++ b/cookbooks/vectortile/templates/default/nginx.erb @@ -0,0 +1,46 @@ +upstream tilekiln { + server 127.0.0.1:8000; +} + +server { + listen 80 default_server; + listen [::]:80 default_server; + + location /nginx_status { + stub_status on; + access_log off; + allow 127.0.0.1; + allow ::1; + deny all; + } + + rewrite ^/\.well-known/acme-challenge/(.*)$ http://acme.openstreetmap.org/.well-known/acme-challenge/$1 permanent; + + location / { + return 301 https://$host$request_uri; + } +} + +server { + # IPv4 + listen 443 ssl default_server; + # IPv6 + listen [::]:443 ssl default_server; + http2 on; + server_name localhost; + + ssl_certificate /etc/ssl/certs/<%= node[:fqdn] %>.pem; + ssl_certificate_key /etc/ssl/private/<%= node[:fqdn] %>.key; + + location /nginx_status { + stub_status on; + access_log off; + allow 127.0.0.1; + allow ::1; + deny all; + } + location /shortbread_v1/ { + proxy_pass http://tilekiln; + } + root /srv/vector.openstreetmap.org/html; +} diff --git a/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb new file mode 100644 index 000000000..972d0e488 --- /dev/null +++ b/cookbooks/vectortile/templates/default/tilekiln-storage-init.erb @@ -0,0 +1,8 @@ +#!/bin/sh + +# Usage +# sudo -u tileupdate tilekiln-storage-init + +<%= @tilekiln_bin %> storage init \ + --storage-dbname <%= @storage_database %> \ + --config <%= @config_path %> diff --git a/cookbooks/vectortile/templates/default/tiles-rerender.erb b/cookbooks/vectortile/templates/default/tiles-rerender.erb new file mode 100644 index 000000000..f46b6f1f6 --- /dev/null +++ b/cookbooks/vectortile/templates/default/tiles-rerender.erb @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +<%= @tilekiln_bin %> generate tiles \ +--source-dbname "<%= @source_database %>" \ +--storage-dbname "<%= @storage_database %>" \ +--num-threads "<%= node[:vectortile][:replication][:threads] %>" \ +--config <%= @config_path %> < <%= @tiles_file %> diff --git a/cookbooks/vectortile/templates/default/vector-update-notile.erb b/cookbooks/vectortile/templates/default/vector-update-notile.erb new file mode 100644 index 000000000..cd718789c --- /dev/null +++ b/cookbooks/vectortile/templates/default/vector-update-notile.erb @@ -0,0 +1,12 @@ +#!/bin/sh + +# Usage +# sudo -u tilekiln vector-update + +set -eu + +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' + +osm2pgsql-replication update \ + -d "<%= @source_database %>" \ + --max-diff-size "<%= @diff_size %>" diff --git a/cookbooks/vectortile/templates/default/vector-update-tile.erb b/cookbooks/vectortile/templates/default/vector-update-tile.erb new file mode 100644 index 000000000..e59923b27 --- /dev/null +++ b/cookbooks/vectortile/templates/default/vector-update-tile.erb @@ -0,0 +1,15 @@ +#!/bin/sh + +# Usage +# sudo -u tilekiln vector-update + +set -eu + +export LUA_PATH='/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/?.lua;/srv/vector.openstreetmap.org/spirit/?.lua;;' + +osm2pgsql-replication update \ + -d "<%= @source_database %>" \ + --max-diff-size "<%= @diff_size %>" \ + --post-processing "<%= @post_processing %>" \ + -- --expire-tiles=10-14 \ + --expire-output="<%= @tiles_file %>" diff --git a/roles/vectortile.rb b/roles/vectortile.rb new file mode 100644 index 000000000..189cddb8d --- /dev/null +++ b/roles/vectortile.rb @@ -0,0 +1,72 @@ +name "vectortile" +description "Role applied to all vector tile servers" + +default_attributes( + :accounts => { + :users => { + :pnorman => { :status => :administrator }, + :tile => { + :members => [:tomh, :pnorman] + } + } + }, + :postgresql => { + :settings => { + :defaults => { + :max_connections => "250", + :shared_buffers => "16GB", + :work_mem => "128MB", + :maintenance_work_mem => "8GB", + :max_parallel_workers_per_gather => "0", + :wal_level => "minimal", + :wal_buffers => "1024kB", + :wal_writer_delay => "500ms", + :checkpoint_timeout => "60min", + :commit_delay => "10000", + :max_wal_size => "10GB", + :max_wal_senders => "0", + :jit => "off", + :track_activity_query_size => "16384", + :autovacuum_vacuum_scale_factor => "0.05", + :autovacuum_analyze_scale_factor => "0.02" + } + } + }, + :sysctl => { + :sockets => { + :comment => "Increase size of connection queue", + :parameters => { + "net.core.somaxconn" => 10000 + } + }, + :network_conntrack_time_wait => { + :comment => "Only track completed connections for 30 seconds", + :parameters => { + "net.netfilter.nf_conntrack_tcp_timeout_time_wait" => "30" + } + }, + :network_conntrack_max => { + :comment => "Increase max number of connections tracked", + :parameters => { + "net.netfilter.nf_conntrack_max" => "524288" + } + }, + :no_tcp_slow_start => { + :comment => "Disable TCP slow start", + :parameters => { + "net.ipv4.tcp_slow_start_after_idle" => "0" + } + }, + :tcp_use_bbr => { + :comment => "Use TCP BBR Congestion Control", + :parameters => { + "net.core.default_qdisc" => "fq", + "net.ipv4.tcp_congestion_control" => "bbr" + } + } + } +) + +run_list( + "recipe[vectortile]" +) diff --git a/test/data_bags/accounts/tilekiln.json b/test/data_bags/accounts/tilekiln.json new file mode 100644 index 000000000..100e01771 --- /dev/null +++ b/test/data_bags/accounts/tilekiln.json @@ -0,0 +1,6 @@ +{ + "id": "tilekiln", + "uid": "532", + "comment": "vectortile.openstreetmap.org", + "manage_home": false +} diff --git a/test/data_bags/accounts/tileupdate.json b/test/data_bags/accounts/tileupdate.json new file mode 100644 index 000000000..8b6ba87c6 --- /dev/null +++ b/test/data_bags/accounts/tileupdate.json @@ -0,0 +1,6 @@ +{ + "id": "tileupdate", + "uid": "531", + "comment": "vectortile.openstreetmap.org", + "manage_home": false +} diff --git a/test/integration/vectortile/inspec/import_spec.rb b/test/integration/vectortile/inspec/import_spec.rb new file mode 100644 index 000000000..13ff57e87 --- /dev/null +++ b/test/integration/vectortile/inspec/import_spec.rb @@ -0,0 +1,3 @@ +describe file("/usr/local/bin/import-planet") do + it { should be_executable.by_user("tileupdate") } +end diff --git a/test/integration/vectortile/inspec/nginx_spec.rb b/test/integration/vectortile/inspec/nginx_spec.rb new file mode 100644 index 000000000..75c1e16b8 --- /dev/null +++ b/test/integration/vectortile/inspec/nginx_spec.rb @@ -0,0 +1,26 @@ +describe package("nginx") do + it { should be_installed } +end + +describe service("nginx") do + it { should be_enabled } + it { should be_running } +end + +describe port(80) do + it { should be_listening } + its("protocols") { should cmp %w[tcp tcp6] } +end + +describe port(443) do + it { should be_listening } + its("protocols") { should cmp %w[tcp tcp6] } +end + +describe http("http://localhost") do + its("status") { should cmp 301 } +end + +describe http("https://localhost", :ssl_verify => false) do + its("status") { should cmp 200 } +end diff --git a/test/integration/vectortile/inspec/osm2pgsql_spec.rb b/test/integration/vectortile/inspec/osm2pgsql_spec.rb new file mode 100644 index 000000000..e5efe3340 --- /dev/null +++ b/test/integration/vectortile/inspec/osm2pgsql_spec.rb @@ -0,0 +1,4 @@ +describe package("osm2pgsql") do + it { should be_installed } + its("version") { should cmp >= "1.11.0" } +end diff --git a/test/integration/vectortile/inspec/postgresql_spec.rb b/test/integration/vectortile/inspec/postgresql_spec.rb new file mode 100644 index 000000000..925d6746b --- /dev/null +++ b/test/integration/vectortile/inspec/postgresql_spec.rb @@ -0,0 +1,13 @@ +describe package("postgresql-16") do + it { should be_installed } +end + +describe service("postgresql@16-main") do + it { should be_enabled } + it { should be_running } +end + +describe port(5432) do + it { should be_listening } + its("protocols") { should cmp %w[tcp tcp6] } +end diff --git a/test/integration/vectortile/inspec/storage_spec.rb b/test/integration/vectortile/inspec/storage_spec.rb new file mode 100644 index 000000000..9498695d1 --- /dev/null +++ b/test/integration/vectortile/inspec/storage_spec.rb @@ -0,0 +1,3 @@ +describe file("/usr/local/bin/tilekiln-storage-init") do + it { should be_executable.by_user("tileupdate") } +end diff --git a/test/integration/vectortile/inspec/themepark_spec.rb b/test/integration/vectortile/inspec/themepark_spec.rb new file mode 100644 index 000000000..eb7a97013 --- /dev/null +++ b/test/integration/vectortile/inspec/themepark_spec.rb @@ -0,0 +1,7 @@ +describe file("/srv/vector.openstreetmap.org/osm2pgsql-themepark/lua/themepark.lua") do + it { should exist } +end + +describe file("/srv/vector.openstreetmap.org/spirit/spirit.lua") do + it { should exist } +end diff --git a/test/integration/vectortile/inspec/tilekiln_spec.rb b/test/integration/vectortile/inspec/tilekiln_spec.rb new file mode 100644 index 000000000..2887571db --- /dev/null +++ b/test/integration/vectortile/inspec/tilekiln_spec.rb @@ -0,0 +1,30 @@ +describe pip("tilekiln", "/opt/tilekiln/bin/pip") do + it { should be_installed } + its("version") { should cmp >= "0.5.0" } +end + +describe service("tilekiln") do + it { should be_enabled } + it { should be_running } +end + +describe port(8000) do + it { should be_listening } + its("protocols") { should cmp %w[tcp] } +end + +describe http("http://localhost:8000") do + its("status") { should cmp 404 } +end + +describe http("https://localhost/shortbread_v1/tilejson.json", :ssl_verify => false) do + its("status") { should cmp 200 } +end + +# There are no tiles so everything should return a 404 +describe http("https://localhost/shortbread_v1/0/0/0.mvt", :ssl_verify => false) do + its("status") { should cmp 404 } +end +describe http("https://localhost/shortbread_v1/16/0/0.mvt", :ssl_verify => false) do + its("status") { should cmp 404 } +end -- 2.39.5