cookbooks/wiki/templates/default/robots.txt.erb

   1 User-agent: ia_archiver
   2 Allow: /
   3
   4 User-agent: 008
   5 Disallow: /
   6
   7 User-agent: *
   8 Disallow: /index.php/
   9 Disallow: /wiki/Spam
  10 Allow: /w/load.php
  11 Allow: /w/images/
  12 Allow: /w/skins/
  13 Disallow: /w/
  14 Disallow: /wiki/Special:Collection
  15 Disallow: /wiki/Special:Random
  16 Disallow: /wiki/Special%3ARandom
  17 Disallow: /wiki/Special:Search
  18
  19 User-agent: Exabot
  20 Crawl-delay: 60
  21
  22 User-Agent: ImagesiftBot
  23 Crawl-delay: 60
  24
  25 User-agent: SemrushBot
  26 Crawl-delay: 60
  27
  28 User-agent: SeznamBot
  29 Crawl-delay: 60
  30
  31 Sitemap: https://wiki.openstreetmap.org/sitemap-index-wiki.xml
  32
  33 Host: wiki.openstreetmap.org