From: claycarpenter <claycarpenter@0cfe37f9-358a-4d5e-be75-b63607b5c754>
Date: Tue, 1 Feb 2011 16:18:44 +0000 (+0000)
Subject: Accepting incoming merge (branches/carpenter->trunk).  Incorporates fixes to the... 
X-Git-Tag: live~456
X-Git-Url: https://git.openstreetmap.org./osqa.git/commitdiff_plain/886fb5136dfb9ff258b8948c981a68bbf3550208

Accepting incoming merge (branches/carpenter->trunk).  Incorporates fixes to the importer component.

git-svn-id: http://svn.osqa.net/svnroot/osqa/trunk@724 0cfe37f9-358a-4d5e-be75-b63607b5c754
---

diff --git a/forum/__init__.py b/forum/__init__.py
index 8b13789..e69de29 100644
--- a/forum/__init__.py
+++ b/forum/__init__.py
@@ -1 +0,0 @@
-
diff --git a/forum/management/__init__.py b/forum/management/__init__.py
index b654caa..629678e 100644
--- a/forum/management/__init__.py
+++ b/forum/management/__init__.py
@@ -1,3 +1,42 @@
 from forum.modules import get_modules_script
 
-get_modules_script('management')
\ No newline at end of file
+get_modules_script('management')
+
+from django.db.models.signals import post_syncdb
+import forum.models
+
+def post_syncdb_callback(sender, **kwargs):
+    # Import the needed libraries to use the database and detect the
+    # DB engine/sever type being currently employed.
+    from django.db import connection, connections
+
+    verbosity = int(kwargs["verbosity"])
+
+    # Get the DB engine being used for persistence for this app.
+    current_db_engine = connections.databases[connection.alias]['ENGINE']
+
+    # Make sure the updates are only executed for a MySQL DB.
+    if current_db_engine.find("mysql") > 0:
+        # Ok, mysql was found in the engine description.  Go ahead
+        # and attempt to execute the alter table statements.
+        cursor = connection.cursor()
+
+        # Pair the table names with the columns that need to be updated.
+        updatable_table_columns = {
+            "forum_tag": "name",
+            "auth_user": "username"
+        }
+
+        # Update each column in turn.
+        for table_name, column_name in updatable_table_columns.iteritems():
+            alter_table_statement = "ALTER TABLE %(table_name)s MODIFY %(column_name)s varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;" % {
+                "table_name": table_name, "column_name": column_name}
+            log_status(verbosity,"Updating MySQL column with this statement: " + alter_table_statement)
+            cursor.execute(alter_table_statement)
+
+def log_status(verbosity, message):
+    if verbosity == 2:
+        print "[DEBUG] " + str(message)
+
+# Link the callback to the post_syncdb signal.
+post_syncdb.connect(post_syncdb_callback, sender=forum.models)
diff --git a/forum_modules/exporter/importer.py b/forum_modules/exporter/importer.py
index f371a70..2fd72e9 100644
--- a/forum_modules/exporter/importer.py
+++ b/forum_modules/exporter/importer.py
@@ -16,6 +16,22 @@ import commands, settings
 
 NO_DEFAULT = object()
 
+import string
+
+class SafeReader():
+    def __init__(self, loc):
+        self.base = open(loc)
+
+    def read(self, *args):
+        return "".join(c for c in self.base.read(*args) if c in string.printable)
+
+    def readLine(self, *args):
+        return "".join(c for c in self.base.readLine(*args) if c in string.printable)
+
+    def close(self):
+        self.base.close()
+
+
 class ContentElement():
     def __init__(self, content):
         self._content = content
@@ -296,7 +312,7 @@ def file_handler(file_name, root_tag, el_tag, name, args_handler=None, pre_callb
             parser.setContentHandler(handler)
             #parser.setErrorHandler(SaxErrorHandler())
 
-            parser.parse(os.path.join(location, file_name))
+            parser.parse(SafeReader(os.path.join(location, file_name)))
 
             if post_callback:
                 post_callback()
@@ -570,10 +586,19 @@ def actions_import(row, nodes, users, actions_map):
 
 
 
+# Record of all persisted votes.
+persisted_votes = []
 @post_action('voteup', 'votedown', 'voteupcomment')
 def vote_action(row, action, users, nodes, actions):
-    orm.Vote(user_id=action.user_id, node_id=action.node_id, action=action,
-             voted_at=action.action_date, value=(action.action_type != 'votedown') and 1 or -1).save()
+    # Check to see if the vote has already been registered.
+    if not (action.user_id, action.node_id) in persisted_votes:
+        # Persist the vote action.
+        orm.Vote(user_id=action.user_id, node_id=action.node_id, action=action,
+                 voted_at=action.action_date, value=(action.action_type != 'votedown') and 1 or -1).save()
+
+        # Record the vote action.  This will help us avoid duplicates.
+        persisted_votes.append((action.user_id, action.node_id))
+
 
 def state_action(state):
     def fn(row, action, users, nodes, actions):