foundation: add wiki robots.txt

diff --git a/cookbooks/foundation/recipes/wiki.rb b/cookbooks/foundation/recipes/wiki.rb
index 6d5a9746fc745ccf5c7be1c96f26954800e3576b..9764a0934221370f83ce6aa803a0bdc979492015 100644 (file)
--- a/cookbooks/foundation/recipes/wiki.rb
+++ b/cookbooks/foundation/recipes/wiki.rb
@@ -63,3 +63,10 @@ cookbook_file "/srv/wiki.osmfoundation.org/Wiki.png" do
   group node[:mediawiki][:group]
   mode "644"
 end
+
+template "/srv/wiki.osmfoundation.org/robots.txt" do
+  owner node[:mediawiki][:user]
+  group node[:mediawiki][:group]
+  mode "644"
+  source "robots.txt.erb"
+end

diff --git a/cookbooks/foundation/templates/default/robots.txt.erb b/cookbooks/foundation/templates/default/robots.txt.erb
new file mode 100644 (file)
index 0000000..bea20e8
--- /dev/null
+++ b/cookbooks/foundation/templates/default/robots.txt.erb
@@ -0,0 +1,21 @@
+User-agent: ia_archiver
+Allow: /
+
+User-agent: 008
+Disallow: /
+
+User-agent: *
+Disallow: /index.php/
+Disallow: /wiki/Spam
+Disallow: /wiki/Donate/International_Bank_Transfers
+Allow: /w/load.php
+Allow: /w/images/
+Allow: /w/skins/
+Disallow: /w/
+Disallow: /wiki/Special:Collection
+Disallow: /wiki/Special:Random
+Disallow: /wiki/Special%3ARandom
+Disallow: /wiki/Special:Search
+
+User-agent: Exabot
+Crawl-delay: 60