wiki: Delay abusive scrapers

diff --git a/cookbooks/wiki/templates/default/robots.txt.erb b/cookbooks/wiki/templates/default/robots.txt.erb
index de552f04968ad9b22cffee3658752b18e325d965..932d07ebacda4aedd0da02162a9a5a9404ddea59 100644 (file)
--- a/cookbooks/wiki/templates/default/robots.txt.erb
+++ b/cookbooks/wiki/templates/default/robots.txt.erb
@@ -19,6 +19,15 @@ Disallow: /wiki/Special:Search
 User-agent: Exabot
 Crawl-delay: 60
 
 User-agent: Exabot
 Crawl-delay: 60
 

+User-Agent: ImagesiftBot
+Crawl-delay: 60
+
+User-agent: SemrushBot
+Crawl-delay: 60
+
+User-agent: SeznamBot
+Crawl-delay: 60
+

 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
 
 Host: wiki.openstreetmap.org
 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
 
 Host: wiki.openstreetmap.org