From: Grant Slater <github@firefishy.com>
Date: Tue, 1 Nov 2022 12:01:14 +0000 (+0000)
Subject: foundation: add wiki robots.txt
X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/92b965efcd8f94ac50d057336aa0a4a5e79ed378

foundation: add wiki robots.txt
---

diff --git a/cookbooks/foundation/recipes/wiki.rb b/cookbooks/foundation/recipes/wiki.rb
index 6d5a9746f..9764a0934 100644
--- a/cookbooks/foundation/recipes/wiki.rb
+++ b/cookbooks/foundation/recipes/wiki.rb
@@ -63,3 +63,10 @@ cookbook_file "/srv/wiki.osmfoundation.org/Wiki.png" do
   group node[:mediawiki][:group]
   mode "644"
 end
+
+template "/srv/wiki.osmfoundation.org/robots.txt" do
+  owner node[:mediawiki][:user]
+  group node[:mediawiki][:group]
+  mode "644"
+  source "robots.txt.erb"
+end
diff --git a/cookbooks/foundation/templates/default/robots.txt.erb b/cookbooks/foundation/templates/default/robots.txt.erb
new file mode 100644
index 000000000..bea20e8a7
--- /dev/null
+++ b/cookbooks/foundation/templates/default/robots.txt.erb
@@ -0,0 +1,21 @@
+User-agent: ia_archiver
+Allow: /
+
+User-agent: 008
+Disallow: /
+
+User-agent: *
+Disallow: /index.php/
+Disallow: /wiki/Spam
+Disallow: /wiki/Donate/International_Bank_Transfers
+Allow: /w/load.php
+Allow: /w/images/
+Allow: /w/skins/
+Disallow: /w/
+Disallow: /wiki/Special:Collection
+Disallow: /wiki/Special:Random
+Disallow: /wiki/Special%3ARandom
+Disallow: /wiki/Special:Search
+
+User-agent: Exabot
+Crawl-delay: 60