forum_modules/sximporter/importer.py

   1 # -*- coding: utf-8 -*-
   2
   3 from xml.dom import minidom
   4 from datetime import datetime, timedelta
   5 import time
   6 import re
   7 import os
   8 import gc
   9 from django.utils.translation import ugettext as _
  10 from django.template.defaultfilters import slugify
  11 from forum.models.utils import dbsafe_encode
  12 from orm import orm
  13
  14 from django.utils.encoding import force_unicode
  15 from django.db.utils import IntegrityError
  16
  17 try:
  18     from cPickle import loads, dumps
  19 except ImportError:
  20     from pickle import loads, dumps
  21
  22 from copy import deepcopy
  23 from base64 import b64encode, b64decode
  24 from zlib import compress, decompress
  25
  26 from xml.sax import make_parser
  27 from xml.sax.handler import ContentHandler
  28
  29 class SXTableHandler(ContentHandler):
  30     def __init__(self, fname, callback):
  31         self.in_row = False
  32         self.el_data = {}
  33         self.ch_data = ''
  34
  35         self.fname = fname.lower()
  36         self.callback = callback
  37
  38     def startElement(self, name, attrs):
  39         if name.lower() == self.fname:
  40             pass
  41         elif name.lower() == "row":
  42             self.in_row = True
  43
  44     def characters(self, ch):
  45         self.ch_data += ch
  46
  47     def endElement(self, name):
  48         if name.lower() == self.fname:
  49             pass
  50         elif name.lower() == "row":
  51             self.callback(self.el_data)
  52
  53             self.in_row = False
  54             del self.el_data
  55             self.el_data = {}
  56         elif self.in_row:
  57             self.el_data[name.lower()] = self.ch_data.strip()
  58             del self.ch_data
  59             self.ch_data = ''
  60
  61
  62 def readTable(path, name, callback):
  63     parser = make_parser()
  64     handler = SXTableHandler(name, callback)
  65     parser.setContentHandler(handler)
  66
  67     f = os.path.join(path, "%s.xml" % name)
  68     parser.parse(f)
  69
  70
  71 def dbsafe_encode(value):
  72     return force_unicode(b64encode(compress(dumps(deepcopy(value)))))
  73
  74 def getText(el):
  75     rc = ""
  76     for node in el.childNodes:
  77         if node.nodeType == node.TEXT_NODE:
  78             rc = rc + node.data
  79     return rc.strip()
  80
  81 msstrip = re.compile(r'^(.*)\.\d+')
  82 def readTime(ts):
  83     noms = msstrip.match(ts)
  84     if noms:
  85         ts = noms.group(1)
  86
  87     return datetime(*time.strptime(ts, '%Y-%m-%dT%H:%M:%S')[0:6])
  88
  89 #def readEl(el):
  90 #    return dict([(n.tagName.lower(), getText(n)) for n in el.childNodes if n.nodeType == el.ELEMENT_NODE])
  91
  92 #def readTable(dump, name):
  93 #    for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row'):
  94 #        yield readEl(e)
  95 #return [readEl(e) for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row')]
  96
  97 google_accounts_lookup = re.compile(r'^https?://www.google.com/accounts/')
  98 yahoo_accounts_lookup = re.compile(r'^https?://me.yahoo.com/a/')
  99
 100 openid_lookups = [
 101         re.compile(r'^https?://www.google.com/profiles/(?P<uname>\w+(\.\w+)*)/?$'),
 102         re.compile(r'^https?://me.yahoo.com/(?P<uname>\w+(\.\w+)*)/?$'),
 103         re.compile(r'^https?://openid.aol.com/(?P<uname>\w+(\.\w+)*)/?$'),
 104         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).myopenid.com/?$'),
 105         re.compile(r'^https?://flickr.com/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 106         re.compile(r'^https?://technorati.com/people/technorati/(?P<uname>\w+(\.\w+)*)/?$'),
 107         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).wordpress.com/?$'),
 108         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).blogspot.com/?$'),
 109         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).livejournal.com/?$'),
 110         re.compile(r'^https?://claimid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 111         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).pip.verisignlabs.com/?$'),
 112         re.compile(r'^https?://getopenid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 113         re.compile(r'^https?://[\w\.]+/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 114         re.compile(r'^https?://(?P<uname>[\w\.]+)/?$'),
 115         ]
 116
 117 def final_username_attempt(sxu):
 118     openid = sxu.get('openid', None)
 119
 120     if openid:
 121         if google_accounts_lookup.search(openid):
 122             return UnknownGoogleUser(sxu.get('id'))
 123         if yahoo_accounts_lookup.search(openid):
 124             return UnknownYahooUser(sxu.get('id'))
 125
 126         for lookup in openid_lookups:
 127             if lookup.search(openid):
 128                 return lookup.search(openid).group('uname')
 129
 130     return UnknownUser(sxu.get('id'))
 131
 132 class UnknownUser(object):
 133     def __init__(self, id):
 134         self._id = id
 135
 136     def __str__(self):
 137         return _("user-%(id)s") % {'id': self._id}
 138
 139     def __unicode__(self):
 140         return self.__str__()
 141
 142     def encode(self, *args):
 143         return self.__str__()
 144
 145 class UnknownGoogleUser(UnknownUser):
 146     def __str__(self):
 147         return _("user-%(id)s (google)") % {'id': self._id}
 148
 149 class UnknownYahooUser(UnknownUser):
 150     def __str__(self):
 151         return _("user-%(id)s (yahoo)") % {'id': self._id}
 152
 153
 154 class IdMapper(dict):
 155     def __getitem__(self, key):
 156         key = int(key)
 157         return super(IdMapper, self).get(key, 1)
 158
 159     def __setitem__(self, key, value):
 160         super(IdMapper, self).__setitem__(int(key), int(value))
 161
 162 class IdIncrementer():
 163     def __init__(self, initial):
 164         self.value = initial
 165
 166     def inc(self):
 167         self.value += 1
 168
 169 openidre = re.compile('^https?\:\/\/')
 170 def userimport(path, options):
 171 #users = readTable(dump, "Users")
 172
 173     usernames = []
 174     openids = set()
 175     uidmapper = IdMapper()
 176     #merged_users = []
 177
 178     owneruid = options.get('owneruid', None)
 179     #check for empty values
 180     if not owneruid:
 181         owneruid = None
 182
 183     def callback(sxu):
 184         create = True
 185
 186         if sxu.get('id') == '-1':
 187             return
 188         #print "\n".join(["%s : %s" % i for i in sxu.items()])
 189         if int(sxu.get('id')) == int(owneruid):
 190             osqau = orm.User.objects.get(id=1)
 191             for assoc in orm.AuthKeyUserAssociation.objects.filter(user=osqau):
 192                 openids.add(assoc.key)
 193             uidmapper[owneruid] = 1
 194             uidmapper[-1] = 1
 195             create = False
 196         else:
 197             username = unicode(sxu.get('displayname',
 198                                sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu)))))[:30]
 199
 200             if username in usernames:
 201             #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email:
 202             #    osqau = user_by_name[username]
 203             #    create = False
 204             #    uidmapper[sxu.get('id')] = osqau.id
 205             #else:
 206                 inc = 0
 207
 208                 while True:
 209                     inc += 1
 210                     totest = "%s %d" % (username[:29 - len(str(inc))], inc)
 211
 212                     if not totest in usernames:
 213                         username = totest
 214                         break
 215
 216         sxbadges = sxu.get('badgesummary', None)
 217         badges = {'1':'0', '2':'0', '3':'0'}
 218
 219         if sxbadges:
 220             badges.update(dict([b.split('=') for b in sxbadges.split()]))
 221
 222         if create:
 223             osqau = orm.User(
 224                     id           = sxu.get('id'),
 225                     username     = username,
 226                     password     = '!',
 227                     email        = sxu.get('email', ''),
 228                     is_superuser = sxu.get('usertypeid') == '5',
 229                     is_staff     = sxu.get('usertypeid') == '4',
 230                     is_active    = True,
 231                     date_joined  = readTime(sxu.get('creationdate')),
 232                     last_seen    = readTime(sxu.get('lastaccessdate')),
 233                     about         = sxu.get('aboutme', ''),
 234                     date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None,
 235                     email_isvalid = int(sxu.get('usertypeid')) > 2,
 236                     website       = sxu.get('websiteurl', ''),
 237                     reputation    = int(sxu.get('reputation')),
 238                     gold          = int(badges['1']),
 239                     silver        = int(badges['2']),
 240                     bronze        = int(badges['3']),
 241                     real_name     = sxu.get('realname', '')[:30],
 242                     location      = sxu.get('location', ''),
 243                     )
 244
 245             osqau.save()
 246
 247             user_joins = orm.Action(
 248                     action_type = "userjoins",
 249                     action_date = osqau.date_joined,
 250                     user = osqau
 251                     )
 252             user_joins.save()
 253
 254             rep = orm.ActionRepute(
 255                     value = 1,
 256                     user = osqau,
 257                     date = osqau.date_joined,
 258                     action = user_joins
 259                     )
 260             rep.save()
 261
 262             try:
 263                 orm.SubscriptionSettings.objects.get(user=osqau)
 264             except:
 265                 s = orm.SubscriptionSettings(user=osqau)
 266                 s.save()
 267
 268             uidmapper[osqau.id] = osqau.id
 269         else:
 270             new_about = sxu.get('aboutme', None)
 271             if new_about and osqau.about != new_about:
 272                 if osqau.about:
 273                     osqau.about = "%s\n|\n%s" % (osqau.about, new_about)
 274                 else:
 275                     osqau.about = new_about
 276
 277             osqau.username = sxu.get('displayname',
 278                                      sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu))))
 279             osqau.email = sxu.get('email', '')
 280             osqau.reputation += int(sxu.get('reputation'))
 281             osqau.gold += int(badges['1'])
 282             osqau.silver += int(badges['2'])
 283             osqau.bronze += int(badges['3'])
 284
 285             osqau.date_joined = readTime(sxu.get('creationdate'))
 286             osqau.website = sxu.get('websiteurl', '')
 287             osqau.date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None
 288             osqau.location = sxu.get('location', '')
 289             osqau.real_name = sxu.get('realname', '')
 290
 291             #merged_users.append(osqau.id)
 292             osqau.save()
 293
 294         usernames.append(osqau.username)
 295
 296         openid = sxu.get('openid', None)
 297         if openid and openidre.match(openid) and (not openid in openids):
 298             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl")
 299             assoc.save()
 300             openids.add(openid)
 301
 302         openidalt = sxu.get('openidalt', None)
 303         if openidalt and openidre.match(openidalt) and (not openidalt in openids):
 304             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openidalt, provider="openidurl")
 305             assoc.save()
 306             openids.add(openidalt)
 307
 308     readTable(path, "Users", callback)
 309
 310     if uidmapper[-1] == -1:
 311         uidmapper[-1] = 1
 312
 313     return uidmapper
 314
 315 def tagsimport(dump, uidmap):
 316 #tags = readTable(dump, "Tags")
 317
 318     tagmap = {}
 319
 320     def callback(sxtag):
 321         otag = orm.Tag(
 322                 id = int(sxtag['id']),
 323                 name = sxtag['name'],
 324                 used_count = int(sxtag['count']),
 325                 created_by_id = uidmap[sxtag.get('userid', 1)],
 326                 )
 327         otag.save()
 328
 329         tagmap[otag.name] = otag
 330
 331     readTable(dump, "Tags", callback)
 332
 333     return tagmap
 334
 335 def add_post_state(name, post, action):
 336     if not "(%s)" % name in post.state_string:
 337         post.state_string = "%s(%s)" % (post.state_string, name)
 338         post.save()
 339
 340     try:
 341         state = orm.NodeState.objects.get(node=post, state_type=name)
 342         state.action = action
 343         state.save()
 344     except:
 345         state = orm.NodeState(node=post, state_type=name, action=action)
 346         state.save()
 347
 348 def remove_post_state(name, post):
 349     if "(%s)" % name in post.state_string:
 350         try:
 351             state = orm.NodeState.objects.get(state_type=name, post=post)
 352             state.delete()
 353         except:
 354             pass
 355     post.state_string = "".join("(%s)" % s for s in re.findall('\w+', post.state_string) if s != name)
 356
 357 def postimport(dump, uidmap, tagmap):
 358 #history = {}
 359 #accepted = {}
 360     all = []
 361
 362     #for h in readTable(dump, "PostHistory"):
 363     #    if not history.get(h.get('postid'), None):
 364     #        history[h.get('postid')] = []
 365     #
 366     #    history[h.get('postid')].append(h)
 367
 368     #posts = readTable(dump, "Posts")
 369
 370     def callback(sxpost):
 371         nodetype = (sxpost.get('posttypeid') == '1') and "nodetype" or "answer"
 372
 373         post = orm.Node(
 374                 node_type = nodetype,
 375                 id = sxpost['id'],
 376                 added_at = readTime(sxpost['creationdate']),
 377                 body = sxpost['body'],
 378                 score = sxpost.get('score', 0),
 379                 author_id = sxpost.get('deletiondate', None) and 1 or uidmap[sxpost.get('owneruserid', 1)]
 380                 )
 381
 382         post.save()
 383
 384         create_action = orm.Action(
 385                 action_type = (nodetype == "nodetype") and "ask" or "answer",
 386                 user_id = post.author_id,
 387                 node = post,
 388                 action_date = post.added_at
 389                 )
 390
 391         create_action.save()
 392
 393         if sxpost.get('lasteditoruserid', None):
 394             revise_action = orm.Action(
 395                     action_type = "revise",
 396                     user_id = uidmap[sxpost.get('lasteditoruserid')],
 397                     node = post,
 398                     action_date = readTime(sxpost['lasteditdate']),
 399                     )
 400
 401             revise_action.save()
 402             post.last_edited = revise_action
 403
 404         if sxpost.get('communityowneddate', None):
 405             wikify_action = orm.Action(
 406                     action_type = "wikify",
 407                     user_id = 1,
 408                     node = post,
 409                     action_date = readTime(sxpost['communityowneddate'])
 410                     )
 411
 412             wikify_action.save()
 413             add_post_state("wiki", post, wikify_action)
 414
 415         if sxpost.get('lastactivityuserid', None):
 416             post.last_activity_by_id = uidmap[sxpost['lastactivityuserid']]
 417             post.last_activity_at = readTime(sxpost['lastactivitydate'])
 418
 419         if sxpost.get('posttypeid') == '1': #question
 420             post.node_type = "question"
 421             post.title = sxpost['title']
 422
 423             tagnames = sxpost['tags'].replace(u'ö', '-').replace(u'é', '').replace(u'à', '')
 424             post.tagnames = tagnames
 425
 426             post.extra_count = sxpost.get('viewcount', 0)
 427
 428             add_tags_to_post(post, tagmap)
 429
 430         else:
 431             post.parent_id = sxpost['parentid']
 432
 433         post.save()
 434
 435         all.append(int(post.id))
 436
 437         del post
 438
 439     readTable(dump, "Posts", callback)
 440
 441     return all
 442
 443 def comment_import(dump, uidmap, posts):
 444 #comments = readTable(dump, "PostComments")
 445     currid = IdIncrementer(max(posts))
 446     mapping = {}
 447
 448     def callback(sxc):
 449         currid.inc()
 450         oc = orm.Node(
 451                 id = currid.value,
 452                 node_type = "comment",
 453                 added_at = readTime(sxc['creationdate']),
 454                 author_id = uidmap[sxc.get('userid', 1)],
 455                 body = sxc['text'],
 456                 parent_id = sxc.get('postid'),
 457                 )
 458
 459         if sxc.get('deletiondate', None):
 460             delete_action = orm.Action(
 461                     action_type = "delete",
 462                     user_id = uidmap[sxc['deletionuserid']],
 463                     action_date = readTime(sxc['deletiondate'])
 464                     )
 465
 466             oc.author_id = uidmap[sxc['deletionuserid']]
 467             oc.save()
 468
 469             delete_action.node = oc
 470             delete_action.save()
 471
 472             add_post_state("deleted", oc, delete_action)
 473         else:
 474             oc.author_id = uidmap[sxc.get('userid', 1)]
 475             oc.save()
 476
 477         create_action = orm.Action(
 478                 action_type = "comment",
 479                 user_id = oc.author_id,
 480                 node = oc,
 481                 action_date = oc.added_at
 482                 )
 483
 484         create_action.save()
 485         oc.save()
 486
 487         posts.append(int(oc.id))
 488         mapping[int(sxc['id'])] = int(oc.id)
 489
 490     readTable(dump, "PostComments", callback)
 491     return posts, mapping
 492
 493
 494 def add_tags_to_post(post, tagmap):
 495     tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag]
 496     post.tagnames = " ".join([t.name for t in tags]).strip()
 497     post.tags = tags
 498     create_and_activate_revision(post)
 499
 500
 501 def create_and_activate_revision(post):
 502     rev = orm.NodeRevision(
 503             author_id = post.author_id,
 504             body = post.body,
 505             node_id = post.id,
 506             revised_at = post.added_at,
 507             revision = 1,
 508             summary = 'Initial revision',
 509             tagnames = post.tagnames,
 510             title = post.title,
 511             )
 512
 513     rev.save()
 514     post.active_revision_id = rev.id
 515     post.save()
 516
 517 def post_vote_import(dump, uidmap, posts):
 518 #votes = readTable(dump, "Posts2Votes")
 519     close_reasons = {}
 520
 521     def close_callback(r):
 522         close_reasons[r['id']] = r['name']
 523
 524     readTable(dump, "CloseReasons", close_callback)
 525
 526     user2vote = []
 527
 528     def callback(sxv):
 529         action = orm.Action(
 530                 user_id=uidmap[sxv['userid']],
 531                 action_date = readTime(sxv['creationdate']),
 532                 )
 533
 534         if not int(sxv['postid']) in posts: return
 535         node = orm.Node.objects.get(id=sxv['postid'])
 536         action.node = node
 537
 538         if sxv['votetypeid'] == '1':
 539             answer = node
 540             question = orm.Node.objects.get(id=answer.parent_id)
 541
 542             action.action_type = "acceptanswer"
 543             action.save()
 544
 545             answer.marked = True
 546
 547             question.extra_ref_id = answer.id
 548
 549             answer.save()
 550             question.save()
 551
 552         elif sxv['votetypeid'] in ('2', '3'):
 553             if not (action.node.id, action.user_id) in user2vote:
 554                 user2vote.append((action.node.id, action.user_id))
 555
 556                 action.action_type = (sxv['votetypeid'] == '2') and "voteup" or "votedown"
 557                 action.save()
 558
 559                 ov = orm.Vote(
 560                         node_id = action.node.id,
 561                         user_id = action.user_id,
 562                         voted_at = action.action_date,
 563                         value = sxv['votetypeid'] == '2' and 1 or -1,
 564                         action = action
 565                         )
 566                 ov.save()
 567             else:
 568                 action.action_type = "unknown"
 569                 action.save()
 570
 571         elif sxv['votetypeid'] in ('4', '12', '13'):
 572             action.action_type = "flag"
 573             action.save()
 574
 575             of = orm.Flag(
 576                     node = action.node,
 577                     user_id = action.user_id,
 578                     flagged_at = action.action_date,
 579                     reason = '',
 580                     action = action
 581                     )
 582
 583             of.save()
 584
 585         elif sxv['votetypeid'] == '5':
 586             action.action_type = "favorite"
 587             action.save()
 588
 589         elif sxv['votetypeid'] == '6':
 590             action.action_type = "close"
 591             action.extra = dbsafe_encode(close_reasons[sxv['comment']])
 592             action.save()
 593
 594             node.marked = True
 595             node.save()
 596
 597         elif sxv['votetypeid'] == '7':
 598             action.action_type = "unknown"
 599             action.save()
 600
 601             node.marked = False
 602             node.save()
 603
 604             remove_post_state("closed", node)
 605
 606         elif sxv['votetypeid'] == '10':
 607             action.action_type = "delete"
 608             action.save()
 609
 610         elif sxv['votetypeid'] == '11':
 611             action.action_type = "unknown"
 612             action.save()
 613
 614             remove_post_state("deleted", node)
 615
 616         else:
 617             action.action_type = "unknown"
 618             action.save()
 619
 620         if sxv.get('targetrepchange', None):
 621             rep = orm.ActionRepute(
 622                     action = action,
 623                     date = action.action_date,
 624                     user_id = uidmap[sxv['targetuserid']],
 625                     value = int(sxv['targetrepchange'])
 626                     )
 627
 628             rep.save()
 629
 630         if sxv.get('voterrepchange', None):
 631             rep = orm.ActionRepute(
 632                     action = action,
 633                     date = action.action_date,
 634                     user_id = uidmap[sxv['userid']],
 635                     value = int(sxv['voterrepchange'])
 636                     )
 637
 638             rep.save()
 639
 640         if action.action_type in ("acceptanswer", "delete", "close"):
 641             state = {"acceptanswer": "accepted", "delete": "deleted", "close": "closed"}[action.action_type]
 642             add_post_state(state, node, action)
 643
 644     readTable(dump, "Posts2Votes", callback)
 645
 646
 647 def comment_vote_import(dump, uidmap, comments):
 648 #votes = readTable(dump, "Comments2Votes")
 649     user2vote = []
 650     comments2score = {}
 651
 652     def callback(sxv):
 653         if sxv['votetypeid'] == "2":
 654             comment_id = comments[int(sxv['postcommentid'])]
 655             user_id = uidmap[sxv['userid']]
 656
 657             if not (comment_id, user_id) in user2vote:
 658                 user2vote.append((comment_id, user_id))
 659
 660                 action = orm.Action(
 661                         action_type = "voteupcomment",
 662                         user_id = user_id,
 663                         action_date = readTime(sxv['creationdate']),
 664                         node_id = comment_id
 665                         )
 666                 action.save()
 667
 668                 ov = orm.Vote(
 669                         node_id = comment_id,
 670                         user_id = user_id,
 671                         voted_at = action.action_date,
 672                         value = 1,
 673                         action = action
 674                         )
 675
 676                 ov.save()
 677
 678                 if not comment_id in comments2score:
 679                     comments2score[comment_id] = 1
 680                 else:
 681                     comments2score[comment_id] += 1
 682
 683     readTable(dump, "Comments2Votes", callback)
 684
 685     for cid, score in comments2score.items():
 686         orm.Node.objects.filter(id=cid).update(score=score)
 687
 688
 689 def badges_import(dump, uidmap, post_list):
 690 #node_ctype = orm['contenttypes.contenttype'].objects.get(name='node')
 691
 692     sxbadges = {}
 693
 694     def sxcallback(b):
 695         sxbadges[int(b['id'])] = b
 696
 697     readTable(dump, "Badges", sxcallback)
 698
 699     obadges = dict([(b.cls, b) for b in orm.Badge.objects.all()])
 700     user_badge_count = {}
 701
 702     sx_to_osqa = {}
 703
 704     for id, sxb in sxbadges.items():
 705         cls = "".join(sxb['name'].replace('&', 'And').split(' '))
 706
 707         if cls in obadges:
 708             sx_to_osqa[id] = obadges[cls]
 709         else:
 710             osqab = orm.Badge(
 711                     cls = cls,
 712                     awarded_count = 0,
 713                     type = sxb['class']
 714                     )
 715             osqab.save()
 716             sx_to_osqa[id] = osqab
 717
 718     osqaawards = []
 719
 720     def callback(sxa):
 721         badge = sx_to_osqa[int(sxa['badgeid'])]
 722
 723         user_id = uidmap[sxa['userid']]
 724         if not user_badge_count.get(user_id, None):
 725             user_badge_count[user_id] = 0
 726
 727         action = orm.Action(
 728                 action_type = "award",
 729                 user_id = user_id,
 730                 action_date = readTime(sxa['date'])
 731                 )
 732
 733         action.save()
 734
 735         osqaa = orm.Award(
 736                 user_id = uidmap[sxa['userid']],
 737                 badge = badge,
 738                 node_id = post_list[user_badge_count[user_id]],
 739                 awarded_at = action.action_date,
 740                 action = action
 741                 )
 742
 743         osqaa.save()
 744         badge.awarded_count += 1
 745         user_badge_count[user_id] += 1
 746
 747     readTable(dump, "Users2Badges", callback)
 748
 749     for badge in obadges.values():
 750         badge.save()
 751
 752 def pages_import(dump, currid):
 753     currid = IdIncrementer(currid)
 754     registry = {}
 755     #sx_pages = readTable(dump, "FlatPages")
 756
 757     def callback(sxp):
 758         currid.inc()
 759         page = orm.Node(
 760                 id = currid.value,
 761                 node_type = "page",
 762                 title = sxp['name'],
 763                 body = b64decode(sxp['value']),
 764                 extra = dbsafe_encode({
 765                 'path': sxp['url'][1:],
 766                 'mimetype': sxp['contenttype'],
 767                 'template': (sxp['usemaster'] == "true") and "default" or "none",
 768                 'render': "html",
 769                 'sidebar': "",
 770                 'sidebar_wrap': True,
 771                 'sidebar_render': "html",
 772                 'comments': False
 773                 }),
 774                 author_id = 1
 775                 )
 776
 777         page.save()
 778         registry[sxp['url'][1:]] = page.id
 779
 780         create_action = orm.Action(
 781                 action_type = "newpage",
 782                 user_id = page.author_id,
 783                 node = page
 784                 )
 785
 786         create_action.save()
 787
 788         if sxp['active'] == "true" and sxp['contenttype'] == "text/html":
 789             pub_action = orm.Action(
 790                     action_type = "publish",
 791                     user_id = page.author_id,
 792                     node = page
 793                     )
 794
 795             pub_action.save()
 796             add_post_state("published", page, pub_action)
 797
 798     readTable(dump, "FlatPages", callback)
 799
 800     kv = orm.KeyValue(key='STATIC_PAGE_REGISTRY', value=dbsafe_encode(registry))
 801     kv.save()
 802
 803 sx2osqa_set_map = {
 804 u'theme.html.name': 'APP_TITLE',
 805 u'theme.html.footer': 'CUSTOM_FOOTER',
 806 u'theme.html.sidebar': 'SIDEBAR_UPPER_TEXT',
 807 u'theme.html.sidebar-low': 'SIDEBAR_LOWER_TEXT',
 808 u'theme.html.welcome': 'APP_INTRO',
 809 u'theme.html.head': 'CUSTOM_HEAD',
 810 u'theme.html.header': 'CUSTOM_HEADER',
 811 u'theme.css': 'CUSTOM_CSS',
 812 }
 813
 814 html_codes = (
 815 ('&amp;', '&'),
 816 ('&lt;', '<'),
 817 ('&gt;', '>'),
 818 ('&quot;', '"'),
 819 ('&#39;', "'"),
 820 )
 821
 822 def html_decode(html):
 823     html = force_unicode(html)
 824
 825     for args in html_codes:
 826         html = html.replace(*args)
 827
 828     return html
 829
 830
 831 def static_import(dump):
 832 #sx_sets = readTable(dump, "ThemeTextResources")
 833     sx_unknown = {}
 834
 835     def callback(set):
 836         if unicode(set['name']) in sx2osqa_set_map:
 837             try:
 838                 kv = orm.KeyValue.objects.get(key=sx2osqa_set_map[set['name']])
 839                 kv.value = dbsafe_encode(html_decode(set['value']))
 840             except:
 841                 kv = orm.KeyValue(
 842                         key = sx2osqa_set_map[set['name']],
 843                         value = dbsafe_encode(html_decode(set['value']))
 844                         )
 845
 846             kv.save()
 847         else:
 848             sx_unknown[set['name']] = html_decode(set['value'])
 849
 850     readTable(dump, "ThemeTextResources", callback)
 851
 852     unknown = orm.KeyValue(key='SXIMPORT_UNKNOWN_SETS', value=dbsafe_encode(sx_unknown))
 853     unknown.save()
 854
 855 def disable_triggers():
 856     from south.db import db
 857     if db.backend_name == "postgres":
 858         db.execute_many(PG_DISABLE_TRIGGERS)
 859         db.commit_transaction()
 860         db.start_transaction()
 861
 862 def enable_triggers():
 863     from south.db import db
 864     if db.backend_name == "postgres":
 865         db.start_transaction()
 866         db.execute_many(PG_ENABLE_TRIGGERS)
 867         db.commit_transaction()
 868
 869 def reset_sequences():
 870     from south.db import db
 871     if db.backend_name == "postgres":
 872         db.start_transaction()
 873         db.execute_many(PG_SEQUENCE_RESETS)
 874         db.commit_transaction()
 875
 876 def reindex_fts():
 877     from south.db import db
 878     if db.backend_name == "postgres":
 879         db.start_transaction()
 880         db.execute_many("UPDATE forum_noderevision set id = id WHERE TRUE;")
 881         db.commit_transaction()
 882
 883
 884 def sximport(dump, options):
 885     try:
 886         disable_triggers()
 887         triggers_disabled = True
 888     except:
 889         triggers_disabled = False
 890
 891     uidmap = userimport(dump, options)
 892     tagmap = tagsimport(dump, uidmap)
 893     gc.collect()
 894
 895     posts = postimport(dump, uidmap, tagmap)
 896     gc.collect()
 897
 898     posts, comments = comment_import(dump, uidmap, posts)
 899     gc.collect()
 900
 901     post_vote_import(dump, uidmap, posts)
 902     gc.collect()
 903
 904     comment_vote_import(dump, uidmap, comments)
 905     gc.collect()
 906
 907     badges_import(dump, uidmap, posts)
 908
 909     pages_import(dump, max(posts))
 910     static_import(dump)
 911     gc.collect()
 912
 913     from south.db import db
 914     db.commit_transaction()
 915
 916     reset_sequences()
 917
 918     if triggers_disabled:
 919         enable_triggers()
 920         reindex_fts()
 921
 922
 923 PG_DISABLE_TRIGGERS = """
 924 ALTER table auth_user DISABLE TRIGGER ALL;
 925 ALTER table auth_user_groups DISABLE TRIGGER ALL;
 926 ALTER table auth_user_user_permissions DISABLE TRIGGER ALL;
 927 ALTER table forum_keyvalue DISABLE TRIGGER ALL;
 928 ALTER table forum_action DISABLE TRIGGER ALL;
 929 ALTER table forum_actionrepute DISABLE TRIGGER ALL;
 930 ALTER table forum_subscriptionsettings DISABLE TRIGGER ALL;
 931 ALTER table forum_validationhash DISABLE TRIGGER ALL;
 932 ALTER table forum_authkeyuserassociation DISABLE TRIGGER ALL;
 933 ALTER table forum_tag DISABLE TRIGGER ALL;
 934 ALTER table forum_markedtag DISABLE TRIGGER ALL;
 935 ALTER table forum_node DISABLE TRIGGER ALL;
 936 ALTER table forum_nodestate DISABLE TRIGGER ALL;
 937 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 938 ALTER table forum_noderevision DISABLE TRIGGER ALL;
 939 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 940 ALTER table forum_questionsubscription DISABLE TRIGGER ALL;
 941 ALTER table forum_vote DISABLE TRIGGER ALL;
 942 ALTER table forum_flag DISABLE TRIGGER ALL;
 943 ALTER table forum_badge DISABLE TRIGGER ALL;
 944 ALTER table forum_award DISABLE TRIGGER ALL;
 945 ALTER table forum_openidnonce DISABLE TRIGGER ALL;
 946 ALTER table forum_openidassociation DISABLE TRIGGER ALL;
 947 """
 948
 949 PG_ENABLE_TRIGGERS = """
 950 ALTER table auth_user ENABLE TRIGGER ALL;
 951 ALTER table auth_user_groups ENABLE TRIGGER ALL;
 952 ALTER table auth_user_user_permissions ENABLE TRIGGER ALL;
 953 ALTER table forum_keyvalue ENABLE TRIGGER ALL;
 954 ALTER table forum_action ENABLE TRIGGER ALL;
 955 ALTER table forum_actionrepute ENABLE TRIGGER ALL;
 956 ALTER table forum_subscriptionsettings ENABLE TRIGGER ALL;
 957 ALTER table forum_validationhash ENABLE TRIGGER ALL;
 958 ALTER table forum_authkeyuserassociation ENABLE TRIGGER ALL;
 959 ALTER table forum_tag ENABLE TRIGGER ALL;
 960 ALTER table forum_markedtag ENABLE TRIGGER ALL;
 961 ALTER table forum_node ENABLE TRIGGER ALL;
 962 ALTER table forum_nodestate ENABLE TRIGGER ALL;
 963 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 964 ALTER table forum_noderevision ENABLE TRIGGER ALL;
 965 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 966 ALTER table forum_questionsubscription ENABLE TRIGGER ALL;
 967 ALTER table forum_vote ENABLE TRIGGER ALL;
 968 ALTER table forum_flag ENABLE TRIGGER ALL;
 969 ALTER table forum_badge ENABLE TRIGGER ALL;
 970 ALTER table forum_award ENABLE TRIGGER ALL;
 971 ALTER table forum_openidnonce ENABLE TRIGGER ALL;
 972 ALTER table forum_openidassociation ENABLE TRIGGER ALL;
 973 """
 974
 975 PG_SEQUENCE_RESETS = """
 976 SELECT setval('"auth_user_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user";
 977 SELECT setval('"auth_user_groups_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_groups";
 978 SELECT setval('"auth_user_user_permissions_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_user_permissions";
 979 SELECT setval('"forum_keyvalue_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_keyvalue";
 980 SELECT setval('"forum_action_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_action";
 981 SELECT setval('"forum_actionrepute_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_actionrepute";
 982 SELECT setval('"forum_subscriptionsettings_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_subscriptionsettings";
 983 SELECT setval('"forum_validationhash_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_validationhash";
 984 SELECT setval('"forum_authkeyuserassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_authkeyuserassociation";
 985 SELECT setval('"forum_tag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_tag";
 986 SELECT setval('"forum_markedtag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_markedtag";
 987 SELECT setval('"forum_node_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node";
 988 SELECT setval('"forum_nodestate_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_nodestate";
 989 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
 990 SELECT setval('"forum_noderevision_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_noderevision";
 991 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
 992 SELECT setval('"forum_questionsubscription_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_questionsubscription";
 993 SELECT setval('"forum_vote_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_vote";
 994 SELECT setval('"forum_flag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_flag";
 995 SELECT setval('"forum_badge_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_badge";
 996 SELECT setval('"forum_award_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_award";
 997 SELECT setval('"forum_openidnonce_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidnonce";
 998 SELECT setval('"forum_openidassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidassociation";
 999 """
1000
1001
1002
1003