Merging jambazov --> trunk.

[osqa.git] / forum_modules / sximporter / importer.py

diff --git a/forum_modules/sximporter/importer.py b/forum_modules/sximporter/importer.py
index d63541c03c64eb86bf92c610d05d542533c66156..0f7f812626e23968ff072493092218a5d984a383 100644 (file)
--- a/forum_modules/sximporter/importer.py
+++ b/forum_modules/sximporter/importer.py
@@ -169,7 +169,8 @@ openidre = re.compile('^https?\:\/\/')
 def userimport(path, options):
 #users = readTable(dump, "Users")
 
 def userimport(path, options):
 #users = readTable(dump, "Users")
 

-    user_by_name = {}
+    usernames = []
+    openids = set()

     uidmapper = IdMapper()
     #merged_users = []
 
     uidmapper = IdMapper()
     #merged_users = []
 


@@ -186,24 +187,30 @@ def userimport(path, options):
         #print "\n".join(["%s : %s" % i for i in sxu.items()])
         if int(sxu.get('id')) == int(owneruid):
             osqau = orm.User.objects.get(id=1)
         #print "\n".join(["%s : %s" % i for i in sxu.items()])
         if int(sxu.get('id')) == int(owneruid):
             osqau = orm.User.objects.get(id=1)

+            for assoc in orm.AuthKeyUserAssociation.objects.filter(user=osqau):
+                openids.add(assoc.key)

             uidmapper[owneruid] = 1
             uidmapper[-1] = 1
             create = False
         else:
             uidmapper[owneruid] = 1
             uidmapper[-1] = 1
             create = False
         else:

-            username = sxu.get('displayname',
-                               sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu))))
+            username = unicode(sxu.get('displayname',
+                               sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu)))))[:30]

 
 

-            if not isinstance(username, UnknownUser) and username in user_by_name:
+            if username in usernames:

             #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email:
             #    osqau = user_by_name[username]
             #    create = False
             #    uidmapper[sxu.get('id')] = osqau.id
             #else:
             #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email:
             #    osqau = user_by_name[username]
             #    create = False
             #    uidmapper[sxu.get('id')] = osqau.id
             #else:

-                inc = 1
-                while ("%s %d" % (username, inc)) in user_by_name:
+                inc = 0
+
+                while True:

                     inc += 1
                     inc += 1

+                    totest = "%s %d" % (username[:29 - len(str(inc))], inc)

 
 

-                username = "%s %d" % (username, inc)
+                    if not totest in usernames:
+                        username = totest
+                        break          

 
         sxbadges = sxu.get('badgesummary', None)
         badges = {'1':'0', '2':'0', '3':'0'}
 
         sxbadges = sxu.get('badgesummary', None)
         badges = {'1':'0', '2':'0', '3':'0'}


@@ -214,7 +221,7 @@ def userimport(path, options):
         if create:
             osqau = orm.User(
                     id           = sxu.get('id'),
         if create:
             osqau = orm.User(
                     id           = sxu.get('id'),

-                    username     = unicode(username),
+                    username     = username,

                     password     = '!',
                     email        = sxu.get('email', ''),
                     is_superuser = sxu.get('usertypeid') == '5',
                     password     = '!',
                     email        = sxu.get('email', ''),
                     is_superuser = sxu.get('usertypeid') == '5',


@@ -230,7 +237,7 @@ def userimport(path, options):
                     gold          = int(badges['1']),
                     silver        = int(badges['2']),
                     bronze        = int(badges['3']),
                     gold          = int(badges['1']),
                     silver        = int(badges['2']),
                     bronze        = int(badges['3']),

-                    real_name     = sxu.get('realname', ''),
+                    real_name     = sxu.get('realname', '')[:30],

                     location      = sxu.get('location', ''),
                     )
 
                     location      = sxu.get('location', ''),
                     )
 


@@ -283,12 +290,19 @@ def userimport(path, options):
             #merged_users.append(osqau.id)
             osqau.save()
 
             #merged_users.append(osqau.id)
             osqau.save()
 

-        user_by_name[osqau.username] = osqau
+        usernames.append(osqau.username)

 
         openid = sxu.get('openid', None)
 
         openid = sxu.get('openid', None)

-        if openid and openidre.match(openid):
+        if openid and openidre.match(openid) and (not openid in openids):

             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl")
             assoc.save()
             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl")
             assoc.save()

+            openids.add(openid)
+
+        openidalt = sxu.get('openidalt', None)
+        if openidalt and openidre.match(openidalt) and (not openidalt in openids):
+            assoc = orm.AuthKeyUserAssociation(user=osqau, key=openidalt, provider="openidurl")
+            assoc.save()
+            openids.add(openidalt)

 
     readTable(path, "Users", callback)
 
 
     readTable(path, "Users", callback)
 


@@ -418,6 +432,7 @@ def postimport(dump, uidmap, tagmap):
         post.save()
 
         all.append(int(post.id))
         post.save()
 
         all.append(int(post.id))

+        create_and_activate_revision(post)

 
         del post
 
 
         del post
 


@@ -466,6 +481,8 @@ def comment_import(dump, uidmap, posts):
                 action_date = oc.added_at
                 )
 
                 action_date = oc.added_at
                 )
 

+        create_and_activate_revision(oc)
+

         create_action.save()
         oc.save()
 
         create_action.save()
         oc.save()
 


@@ -480,7 +497,6 @@ def add_tags_to_post(post, tagmap):
     tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag]
     post.tagnames = " ".join([t.name for t in tags]).strip()
     post.tags = tags
     tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag]
     post.tagnames = " ".join([t.name for t in tags]).strip()
     post.tags = tags

-    create_and_activate_revision(post)

 
 
 def create_and_activate_revision(post):
 
 
 def create_and_activate_revision(post):


@@ -759,6 +775,8 @@ def pages_import(dump, currid):
                 author_id = 1
                 )
 
                 author_id = 1
                 )
 

+        create_and_activate_revision(page)
+

         page.save()
         registry[sxp['url'][1:]] = page.id
 
         page.save()
         registry[sxp['url'][1:]] = page.id
 


@@ -787,12 +805,13 @@ def pages_import(dump, currid):
 
 sx2osqa_set_map = {
 u'theme.html.name': 'APP_TITLE',
 
 sx2osqa_set_map = {
 u'theme.html.name': 'APP_TITLE',

-u'theme.html.footer': 'USE_CUSTOM_FOOTER',
+u'theme.html.footer': 'CUSTOM_FOOTER',

 u'theme.html.sidebar': 'SIDEBAR_UPPER_TEXT',
 u'theme.html.sidebar-low': 'SIDEBAR_LOWER_TEXT',
 u'theme.html.welcome': 'APP_INTRO',
 u'theme.html.head': 'CUSTOM_HEAD',
 u'theme.html.sidebar': 'SIDEBAR_UPPER_TEXT',
 u'theme.html.sidebar-low': 'SIDEBAR_LOWER_TEXT',
 u'theme.html.welcome': 'APP_INTRO',
 u'theme.html.head': 'CUSTOM_HEAD',

-u'theme.html.header': 'CUSTOM_HEADER'
+u'theme.html.header': 'CUSTOM_HEADER',
+u'theme.css': 'CUSTOM_CSS',

 }
 
 html_codes = (
 }
 
 html_codes = (


@@ -818,10 +837,14 @@ def static_import(dump):
 
     def callback(set):
         if unicode(set['name']) in sx2osqa_set_map:
 
     def callback(set):
         if unicode(set['name']) in sx2osqa_set_map:

-            kv = orm.KeyValue(
-                    key = sx2osqa_set_map[set['name']],
-                    value = dbsafe_encode(html_decode(set['value']))
-                    )
+            try:
+                kv = orm.KeyValue.objects.get(key=sx2osqa_set_map[set['name']])
+                kv.value = dbsafe_encode(html_decode(set['value']))
+            except:
+                kv = orm.KeyValue(
+                        key = sx2osqa_set_map[set['name']],
+                        value = dbsafe_encode(html_decode(set['value']))
+                        )

 
             kv.save()
         else:
 
             kv.save()
         else:


@@ -853,9 +876,21 @@ def reset_sequences():
         db.execute_many(PG_SEQUENCE_RESETS)
         db.commit_transaction()
 
         db.execute_many(PG_SEQUENCE_RESETS)
         db.commit_transaction()
 

+def reindex_fts():
+    from south.db import db
+    if db.backend_name == "postgres":
+        db.start_transaction()
+        db.execute_many("UPDATE forum_noderevision set id = id WHERE TRUE;")
+        db.commit_transaction()
+

 
 def sximport(dump, options):
 
 def sximport(dump, options):

-    disable_triggers()
+    try:
+        disable_triggers()
+        triggers_disabled = True
+    except:
+        triggers_disabled = False
+

     uidmap = userimport(dump, options)
     tagmap = tagsimport(dump, uidmap)
     gc.collect()
     uidmap = userimport(dump, options)
     tagmap = tagsimport(dump, uidmap)
     gc.collect()


@@ -882,7 +917,10 @@ def sximport(dump, options):
     db.commit_transaction()
 
     reset_sequences()
     db.commit_transaction()
 
     reset_sequences()

-    enable_triggers()
+
+    if triggers_disabled:
+        enable_triggers()
+        reindex_fts()

 
 
 PG_DISABLE_TRIGGERS = """
 
 
 PG_DISABLE_TRIGGERS = """


@@ -965,4 +1003,4 @@ SELECT setval('"forum_openidassociation_id_seq"', coalesce(max("id"), 1) + 2, ma
 
 
     
 
 
     

-    
\ No newline at end of file
+