X-Git-Url: https://git.openstreetmap.org./chef.git/blobdiff_plain/d055ba8cab7e97e0a8524aa0bf694ce8001875b2..d5f8a18658e100abeba9f8fcdf7d9e4cbcaea3a6:/cookbooks/wiki/templates/default/robots.txt.erb diff --git a/cookbooks/wiki/templates/default/robots.txt.erb b/cookbooks/wiki/templates/default/robots.txt.erb index 616a0a5cd..932d07eba 100644 --- a/cookbooks/wiki/templates/default/robots.txt.erb +++ b/cookbooks/wiki/templates/default/robots.txt.erb @@ -19,6 +19,15 @@ Disallow: /wiki/Special:Search User-agent: Exabot Crawl-delay: 60 -Sitemap: http://wiki.openstreetmap.org/sitemap-index-wiki.xml +User-Agent: ImagesiftBot +Crawl-delay: 60 + +User-agent: SemrushBot +Crawl-delay: 60 + +User-agent: SeznamBot +Crawl-delay: 60 + +Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml Host: wiki.openstreetmap.org