closes OSQA-376 - updates the default robots.txt to the file we use on meta.osqa

git-svn-id: http://svn.osqa.net/svnroot/osqa/trunk@541 0cfe37f9-358a-4d5e-be75-b63607b5c754

diff --git a/forum_modules/robotstxt/settings.py b/forum_modules/robotstxt/settings.py
index 634eab7ba44a189a7b4acd3caf610969985b3fb8..c2620a26c65d31468121a0fb344ecd2c6d1685c2 100644 (file)
--- a/forum_modules/robotstxt/settings.py
+++ b/forum_modules/robotstxt/settings.py
@@ -9,6 +9,9 @@ User-Agent: *
 Disallow: /accounts/
 Disallow: /users/
 Disallow: /revisions/
 Disallow: /accounts/
 Disallow: /users/
 Disallow: /revisions/

+Disallow: /search
+Disallow: /matching_tags
+Disallow: /*sort=

 """, ROBOTS_SET, dict(
 label = "Robots.txt file",
 help_text = """
 """, ROBOTS_SET, dict(
 label = "Robots.txt file",
 help_text = """

diff --git a/forum_modules/robotstxt/templates/robots.txt b/forum_modules/robotstxt/templates/robots.txt
index 574fc315134c95db548e50173725609cee2c48a8..a93756d62f1a1a5b4ce2816a066be23af2a6390e 100644 (file)
--- a/forum_modules/robotstxt/templates/robots.txt
+++ b/forum_modules/robotstxt/templates/robots.txt
@@ -1,2 +1,7 @@
-User-agent: *
-Disallow: / 
\ No newline at end of file
+User-Agent: *
+Disallow: /accounts/
+Disallow: /users/
+Disallow: /revisions/
+Disallow: /search
+Disallow: /matching_tags
+Disallow: /*sort=
\ No newline at end of file