X-Git-Url: https://git.openstreetmap.org./osqa.git/blobdiff_plain/3133f7ff69649e991283ccc6c5de3e52d4f921b0..15eabbf86190e4bc8ae9dc747d853b0300976b12:/forum_modules/robotstxt/settings.py

diff --git a/forum_modules/robotstxt/settings.py b/forum_modules/robotstxt/settings.py
index 634eab7..44d576a 100644
--- a/forum_modules/robotstxt/settings.py
+++ b/forum_modules/robotstxt/settings.py
@@ -1,15 +1,21 @@
 from forum.settings.base import Setting, SettingSet
 from django.forms.widgets import Textarea
+from django.core.urlresolvers import reverse
+from forum.settings import APP_URL
 
 ROBOTS_SET = SettingSet('robots', 'Robots txt', "Set up the robots.txt file.", 3000)
 
 ROBOTS_FILE = Setting('ROBOTS_FILE',
-"""
+"""Sitemap: %s/sitemap.xml
+
 User-Agent: *
 Disallow: /accounts/
 Disallow: /users/
 Disallow: /revisions/
-""", ROBOTS_SET, dict(
+Disallow: /search
+Disallow: /matching_tags
+Disallow: /*sort=
+""" % APP_URL, ROBOTS_SET, dict(
 label = "Robots.txt file",
 help_text = """
 The robots.txt file search engine spiders will see.