website/robots.txt

   1 User-agent: ia_archiver
   2 Allow: /
   3
   4 User-agent: *
   5 Disallow: /search.php
   6 Disallow: /search
   7 Disallow: /details.php
   8 Disallow: /details
   9 Disallow: /reverse.php
  10 Disallow: /reverse
  11 Disallow: /hierarchy
  12 Disallow: /hierarchy.php
  13 Disallow: /lookup
  14 Disallow: /lookup.php