forum_modules/sximporter/importer.py

   1 # -*- coding: utf-8 -*-
   2
   3 from datetime import datetime
   4 import time
   5 import re
   6 import os
   7 import gc
   8 from django.utils.translation import ugettext as _
   9 from orm import orm
  10
  11 from django.utils.encoding import force_unicode
  12
  13 try:
  14     from cPickle import loads, dumps
  15 except ImportError:
  16     from pickle import loads, dumps
  17
  18 from copy import deepcopy
  19 from base64 import b64encode, b64decode
  20 from zlib import compress, decompress
  21
  22 from xml.sax import make_parser
  23 from xml.sax.handler import ContentHandler
  24
  25 class SXTableHandler(ContentHandler):
  26     def __init__(self, fname, callback):
  27         self.in_row = False
  28         self.el_data = {}
  29         self.ch_data = ''
  30
  31         self.fname = fname.lower()
  32         self.callback = callback
  33
  34     def startElement(self, name, attrs):
  35         if name.lower() == self.fname:
  36             pass
  37         elif name.lower() == "row":
  38             self.in_row = True
  39
  40     def characters(self, ch):
  41         self.ch_data += ch
  42
  43     def endElement(self, name):
  44         if name.lower() == self.fname:
  45             pass
  46         elif name.lower() == "row":
  47             self.callback(self.el_data)
  48
  49             self.in_row = False
  50             del self.el_data
  51             self.el_data = {}
  52         elif self.in_row:
  53             self.el_data[name.lower()] = self.ch_data.strip()
  54             del self.ch_data
  55             self.ch_data = ''
  56
  57
  58 def readTable(path, name, callback):
  59     parser = make_parser()
  60     handler = SXTableHandler(name, callback)
  61     parser.setContentHandler(handler)
  62
  63     f = os.path.join(path, "%s.xml" % name)
  64     parser.parse(f)
  65
  66
  67 def dbsafe_encode(value):
  68     return force_unicode(b64encode(compress(dumps(deepcopy(value)))))
  69
  70 def getText(el):
  71     rc = ""
  72     for node in el.childNodes:
  73         if node.nodeType == node.TEXT_NODE:
  74             rc = rc + node.data
  75     return rc.strip()
  76
  77 msstrip = re.compile(r'^(.*)\.\d+')
  78 def readTime(ts):
  79     noms = msstrip.match(ts)
  80     if noms:
  81         ts = noms.group(1)
  82
  83     return datetime(*time.strptime(ts, '%Y-%m-%dT%H:%M:%S')[0:6])
  84
  85 #def readEl(el):
  86 #    return dict([(n.tagName.lower(), getText(n)) for n in el.childNodes if n.nodeType == el.ELEMENT_NODE])
  87
  88 #def readTable(dump, name):
  89 #    for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row'):
  90 #        yield readEl(e)
  91 #return [readEl(e) for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row')]
  92
  93 google_accounts_lookup = re.compile(r'^https?://www.google.com/accounts/')
  94 yahoo_accounts_lookup = re.compile(r'^https?://me.yahoo.com/a/')
  95
  96 openid_lookups = [
  97         re.compile(r'^https?://www.google.com/profiles/(?P<uname>\w+(\.\w+)*)/?$'),
  98         re.compile(r'^https?://me.yahoo.com/(?P<uname>\w+(\.\w+)*)/?$'),
  99         re.compile(r'^https?://openid.aol.com/(?P<uname>\w+(\.\w+)*)/?$'),
 100         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).myopenid.com/?$'),
 101         re.compile(r'^https?://flickr.com/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 102         re.compile(r'^https?://technorati.com/people/technorati/(?P<uname>\w+(\.\w+)*)/?$'),
 103         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).wordpress.com/?$'),
 104         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).blogspot.com/?$'),
 105         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).livejournal.com/?$'),
 106         re.compile(r'^https?://claimid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 107         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).pip.verisignlabs.com/?$'),
 108         re.compile(r'^https?://getopenid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 109         re.compile(r'^https?://[\w\.]+/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 110         re.compile(r'^https?://(?P<uname>[\w\.]+)/?$'),
 111         ]
 112
 113 def final_username_attempt(sxu):
 114     openid = sxu.get('openid', None)
 115
 116     if openid:
 117         if google_accounts_lookup.search(openid):
 118             return UnknownGoogleUser(sxu.get('id'))
 119         if yahoo_accounts_lookup.search(openid):
 120             return UnknownYahooUser(sxu.get('id'))
 121
 122         for lookup in openid_lookups:
 123             if lookup.search(openid):
 124                 return lookup.search(openid).group('uname')
 125
 126     return UnknownUser(sxu.get('id'))
 127
 128 class UnknownUser(object):
 129     def __init__(self, id):
 130         self._id = id
 131
 132     def __str__(self):
 133         return _("user-%(id)s") % {'id': self._id}
 134
 135     def __unicode__(self):
 136         return self.__str__()
 137
 138     def encode(self, *args):
 139         return self.__str__()
 140
 141 class UnknownGoogleUser(UnknownUser):
 142     def __str__(self):
 143         return _("user-%(id)s (google)") % {'id': self._id}
 144
 145 class UnknownYahooUser(UnknownUser):
 146     def __str__(self):
 147         return _("user-%(id)s (yahoo)") % {'id': self._id}
 148
 149
 150 class IdMapper(dict):
 151     def __getitem__(self, key):
 152         key = int(key)
 153         return super(IdMapper, self).get(key, 1)
 154
 155     def __setitem__(self, key, value):
 156         super(IdMapper, self).__setitem__(int(key), int(value))
 157
 158 class IdIncrementer():
 159     def __init__(self, initial):
 160         self.value = initial
 161
 162     def inc(self):
 163         self.value += 1
 164
 165 openidre = re.compile('^https?\:\/\/')
 166 def userimport(path, options):
 167
 168     usernames = []
 169     openids = set()
 170     uidmapper = IdMapper()
 171
 172     authenticated_user = options.get('authenticated_user', None)
 173     owneruid = options.get('owneruid', None)
 174     #check for empty values
 175     if not owneruid:
 176         owneruid = None
 177
 178     def callback(sxu):
 179         create = True
 180
 181         if sxu.get('id') == '-1':
 182             return
 183         #print "\n".join(["%s : %s" % i for i in sxu.items()])
 184
 185         if int(sxu.get('id')) == int(owneruid):
 186             if authenticated_user:
 187                 osqau = orm.User.objects.get(id=authenticated_user.id)
 188
 189                 for assoc in orm.AuthKeyUserAssociation.objects.filter(user=osqau):
 190                     openids.add(assoc.key)
 191
 192                 uidmapper[owneruid] = osqau.id
 193                 uidmapper[-1] = osqau.id
 194                 create = False
 195             else:
 196                 uidmapper[owneruid] = int(owneruid)
 197                 uidmapper[-1] = int(owneruid)
 198
 199
 200         sxbadges = sxu.get('badgesummary', None)
 201         badges = {'1':'0', '2':'0', '3':'0'}
 202
 203         if sxbadges:
 204             badges.update(dict([b.split('=') for b in sxbadges.split()]))
 205
 206         if create:
 207             username = unicode(sxu.get('displayname',
 208                                sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu)))))[:30]
 209
 210             if username in usernames:
 211             #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email:
 212             #    osqau = user_by_name[username]
 213             #    create = False
 214             #    uidmapper[sxu.get('id')] = osqau.id
 215             #else:
 216                 inc = 0
 217
 218                 while True:
 219                     inc += 1
 220                     totest = "%s %d" % (username[:29 - len(str(inc))], inc)
 221
 222                     if not totest in usernames:
 223                         username = totest
 224                         break
 225
 226             osqau = orm.User(
 227                     id           = sxu.get('id'),
 228                     username     = username,
 229                     password     = '!',
 230                     email        = sxu.get('email', ''),
 231                     is_superuser = sxu.get('usertypeid') == '5',
 232                     is_staff     = sxu.get('usertypeid') == '4',
 233                     is_active    = True,
 234                     date_joined  = readTime(sxu.get('creationdate')),
 235                     last_seen    = readTime(sxu.get('lastaccessdate')),
 236                     about         = sxu.get('aboutme', ''),
 237                     date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None,
 238                     email_isvalid = int(sxu.get('usertypeid')) > 2,
 239                     website       = sxu.get('websiteurl', ''),
 240                     reputation    = int(sxu.get('reputation')),
 241                     gold          = int(badges['1']),
 242                     silver        = int(badges['2']),
 243                     bronze        = int(badges['3']),
 244                     real_name     = sxu.get('realname', '')[:30],
 245                     location      = sxu.get('location', ''),
 246                     )
 247
 248             osqau.save()
 249
 250             user_joins = orm.Action(
 251                     action_type = "userjoins",
 252                     action_date = osqau.date_joined,
 253                     user = osqau
 254                     )
 255             user_joins.save()
 256
 257             rep = orm.ActionRepute(
 258                     value = 1,
 259                     user = osqau,
 260                     date = osqau.date_joined,
 261                     action = user_joins
 262                     )
 263             rep.save()
 264
 265             try:
 266                 orm.SubscriptionSettings.objects.get(user=osqau)
 267             except:
 268                 s = orm.SubscriptionSettings(user=osqau)
 269                 s.save()
 270
 271             uidmapper[osqau.id] = osqau.id
 272         else:
 273             new_about = sxu.get('aboutme', None)
 274             if new_about and osqau.about != new_about:
 275                 if osqau.about:
 276                     osqau.about = "%s\n|\n%s" % (osqau.about, new_about)
 277                 else:
 278                     osqau.about = new_about
 279
 280             osqau.username = sxu.get('displayname',
 281                                      sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu))))
 282             osqau.email = sxu.get('email', '')
 283             osqau.reputation += int(sxu.get('reputation'))
 284             osqau.gold += int(badges['1'])
 285             osqau.silver += int(badges['2'])
 286             osqau.bronze += int(badges['3'])
 287
 288             osqau.date_joined = readTime(sxu.get('creationdate'))
 289             osqau.website = sxu.get('websiteurl', '')
 290             osqau.date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None
 291             osqau.location = sxu.get('location', '')
 292             osqau.real_name = sxu.get('realname', '')
 293
 294             #merged_users.append(osqau.id)
 295             osqau.save()
 296
 297         usernames.append(osqau.username)
 298
 299         openid = sxu.get('openid', None)
 300         if openid and openidre.match(openid) and (not openid in openids):
 301             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl")
 302             assoc.save()
 303             openids.add(openid)
 304
 305         openidalt = sxu.get('openidalt', None)
 306         if openidalt and openidre.match(openidalt) and (not openidalt in openids):
 307             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openidalt, provider="openidurl")
 308             assoc.save()
 309             openids.add(openidalt)
 310
 311     readTable(path, "Users", callback)
 312
 313     if uidmapper[-1] == -1:
 314         uidmapper[-1] = 1
 315
 316     return uidmapper
 317
 318 def tagsimport(dump, uidmap):
 319
 320     tagmap = {}
 321
 322     def callback(sxtag):
 323         otag = orm.Tag(
 324                 id = int(sxtag['id']),
 325                 name = sxtag['name'],
 326                 used_count = int(sxtag['count']),
 327                 created_by_id = uidmap[sxtag.get('userid', 1)],
 328                 )
 329         otag.save()
 330
 331         tagmap[otag.name] = otag
 332
 333     readTable(dump, "Tags", callback)
 334
 335     return tagmap
 336
 337 def add_post_state(name, post, action):
 338     if not "(%s)" % name in post.state_string:
 339         post.state_string = "%s(%s)" % (post.state_string, name)
 340         post.save()
 341
 342     try:
 343         state = orm.NodeState.objects.get(node=post, state_type=name)
 344         state.action = action
 345         state.save()
 346     except:
 347         state = orm.NodeState(node=post, state_type=name, action=action)
 348         state.save()
 349
 350 def remove_post_state(name, post):
 351     if "(%s)" % name in post.state_string:
 352         try:
 353             state = orm.NodeState.objects.get(state_type=name, post=post)
 354             state.delete()
 355         except:
 356             pass
 357     post.state_string = "".join("(%s)" % s for s in re.findall('\w+', post.state_string) if s != name)
 358
 359 def postimport(dump, uidmap, tagmap):
 360     all = []
 361
 362     def callback(sxpost):
 363         nodetype = (sxpost.get('posttypeid') == '1') and "nodetype" or "answer"
 364
 365         post = orm.Node(
 366                 node_type = nodetype,
 367                 id = sxpost['id'],
 368                 added_at = readTime(sxpost['creationdate']),
 369                 body = sxpost['body'],
 370                 score = sxpost.get('score', 0),
 371                 author_id = sxpost.get('deletiondate', None) and 1 or uidmap[sxpost.get('owneruserid', 1)]
 372                 )
 373
 374         post.save()
 375
 376         create_action = orm.Action(
 377                 action_type = (nodetype == "nodetype") and "ask" or "answer",
 378                 user_id = post.author_id,
 379                 node = post,
 380                 action_date = post.added_at
 381                 )
 382
 383         create_action.save()
 384
 385         if sxpost.get('lasteditoruserid', None):
 386             revise_action = orm.Action(
 387                     action_type = "revise",
 388                     user_id = uidmap[sxpost.get('lasteditoruserid')],
 389                     node = post,
 390                     action_date = readTime(sxpost['lasteditdate']),
 391                     )
 392
 393             revise_action.save()
 394             post.last_edited = revise_action
 395
 396         if sxpost.get('communityowneddate', None):
 397             wikify_action = orm.Action(
 398                     action_type = "wikify",
 399                     user_id = 1,
 400                     node = post,
 401                     action_date = readTime(sxpost['communityowneddate'])
 402                     )
 403
 404             wikify_action.save()
 405             add_post_state("wiki", post, wikify_action)
 406
 407         if sxpost.get('lastactivityuserid', None):
 408             post.last_activity_by_id = uidmap[sxpost['lastactivityuserid']]
 409             post.last_activity_at = readTime(sxpost['lastactivitydate'])
 410
 411         if sxpost.get('posttypeid') == '1': #question
 412             post.node_type = "question"
 413             post.title = sxpost['title']
 414
 415             tagnames = sxpost['tags'].replace(u'ö', '-').replace(u'é', '').replace(u'à', '')
 416             post.tagnames = tagnames
 417
 418             post.extra_count = sxpost.get('viewcount', 0)
 419
 420             add_tags_to_post(post, tagmap)
 421
 422         else:
 423             post.parent_id = sxpost['parentid']
 424
 425         post.save()
 426
 427         all.append(int(post.id))
 428         create_and_activate_revision(post)
 429
 430         del post
 431
 432     readTable(dump, "Posts", callback)
 433
 434     return all
 435
 436 def comment_import(dump, uidmap, posts):
 437     currid = IdIncrementer(max(posts))
 438     mapping = {}
 439
 440     def callback(sxc):
 441         currid.inc()
 442         oc = orm.Node(
 443                 id = currid.value,
 444                 node_type = "comment",
 445                 added_at = readTime(sxc['creationdate']),
 446                 author_id = uidmap[sxc.get('userid', 1)],
 447                 body = sxc['text'],
 448                 parent_id = sxc.get('postid'),
 449                 )
 450
 451         if sxc.get('deletiondate', None):
 452             delete_action = orm.Action(
 453                     action_type = "delete",
 454                     user_id = uidmap[sxc['deletionuserid']],
 455                     action_date = readTime(sxc['deletiondate'])
 456                     )
 457
 458             oc.author_id = uidmap[sxc['deletionuserid']]
 459             oc.save()
 460
 461             delete_action.node = oc
 462             delete_action.save()
 463
 464             add_post_state("deleted", oc, delete_action)
 465         else:
 466             oc.author_id = uidmap[sxc.get('userid', 1)]
 467             oc.save()
 468
 469         create_action = orm.Action(
 470                 action_type = "comment",
 471                 user_id = oc.author_id,
 472                 node = oc,
 473                 action_date = oc.added_at
 474                 )
 475
 476         create_and_activate_revision(oc)
 477
 478         create_action.save()
 479         oc.save()
 480
 481         posts.append(int(oc.id))
 482         mapping[int(sxc['id'])] = int(oc.id)
 483
 484     readTable(dump, "PostComments", callback)
 485     return posts, mapping
 486
 487
 488 def add_tags_to_post(post, tagmap):
 489     tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag]
 490     post.tagnames = " ".join([t.name for t in tags]).strip()
 491     post.tags = tags
 492
 493
 494 def create_and_activate_revision(post):
 495     rev = orm.NodeRevision(
 496             author_id = post.author_id,
 497             body = post.body,
 498             node_id = post.id,
 499             revised_at = post.added_at,
 500             revision = 1,
 501             summary = 'Initial revision',
 502             tagnames = post.tagnames,
 503             title = post.title,
 504             )
 505
 506     rev.save()
 507     post.active_revision_id = rev.id
 508     post.save()
 509
 510 def post_vote_import(dump, uidmap, posts):
 511     close_reasons = {}
 512
 513     def close_callback(r):
 514         close_reasons[r['id']] = r['name']
 515
 516     readTable(dump, "CloseReasons", close_callback)
 517
 518     user2vote = []
 519
 520     def callback(sxv):
 521         action = orm.Action(
 522                 user_id=uidmap[sxv['userid']],
 523                 action_date = readTime(sxv['creationdate']),
 524                 )
 525
 526         if not int(sxv['postid']) in posts: return
 527         node = orm.Node.objects.get(id=sxv['postid'])
 528         action.node = node
 529
 530         if sxv['votetypeid'] == '1':
 531             answer = node
 532             question = orm.Node.objects.get(id=answer.parent_id)
 533
 534             action.action_type = "acceptanswer"
 535             action.save()
 536
 537             answer.marked = True
 538
 539             question.extra_ref_id = answer.id
 540
 541             answer.save()
 542             question.save()
 543
 544         elif sxv['votetypeid'] in ('2', '3'):
 545             if not (action.node.id, action.user_id) in user2vote:
 546                 user2vote.append((action.node.id, action.user_id))
 547
 548                 action.action_type = (sxv['votetypeid'] == '2') and "voteup" or "votedown"
 549                 action.save()
 550
 551                 ov = orm.Vote(
 552                         node_id = action.node.id,
 553                         user_id = action.user_id,
 554                         voted_at = action.action_date,
 555                         value = sxv['votetypeid'] == '2' and 1 or -1,
 556                         action = action
 557                         )
 558                 ov.save()
 559             else:
 560                 action.action_type = "unknown"
 561                 action.save()
 562
 563         elif sxv['votetypeid'] in ('4', '12', '13'):
 564             action.action_type = "flag"
 565             action.save()
 566
 567             of = orm.Flag(
 568                     node = action.node,
 569                     user_id = action.user_id,
 570                     flagged_at = action.action_date,
 571                     reason = '',
 572                     action = action
 573                     )
 574
 575             of.save()
 576
 577         elif sxv['votetypeid'] == '5':
 578             action.action_type = "favorite"
 579             action.save()
 580
 581         elif sxv['votetypeid'] == '6':
 582             action.action_type = "close"
 583             action.extra = dbsafe_encode(close_reasons[sxv['comment']])
 584             action.save()
 585
 586             node.marked = True
 587             node.save()
 588
 589         elif sxv['votetypeid'] == '7':
 590             action.action_type = "unknown"
 591             action.save()
 592
 593             node.marked = False
 594             node.save()
 595
 596             remove_post_state("closed", node)
 597
 598         elif sxv['votetypeid'] == '10':
 599             action.action_type = "delete"
 600             action.save()
 601
 602         elif sxv['votetypeid'] == '11':
 603             action.action_type = "unknown"
 604             action.save()
 605
 606             remove_post_state("deleted", node)
 607
 608         else:
 609             action.action_type = "unknown"
 610             action.save()
 611
 612         if sxv.get('targetrepchange', None):
 613             rep = orm.ActionRepute(
 614                     action = action,
 615                     date = action.action_date,
 616                     user_id = uidmap[sxv['targetuserid']],
 617                     value = int(sxv['targetrepchange'])
 618                     )
 619
 620             rep.save()
 621
 622         if sxv.get('voterrepchange', None):
 623             rep = orm.ActionRepute(
 624                     action = action,
 625                     date = action.action_date,
 626                     user_id = uidmap[sxv['userid']],
 627                     value = int(sxv['voterrepchange'])
 628                     )
 629
 630             rep.save()
 631
 632         if action.action_type in ("acceptanswer", "delete", "close"):
 633             state = {"acceptanswer": "accepted", "delete": "deleted", "close": "closed"}[action.action_type]
 634             add_post_state(state, node, action)
 635
 636     readTable(dump, "Posts2Votes", callback)
 637
 638
 639 def comment_vote_import(dump, uidmap, comments):
 640     user2vote = []
 641     comments2score = {}
 642
 643     def callback(sxv):
 644         if sxv['votetypeid'] == "2":
 645             comment_id = comments[int(sxv['postcommentid'])]
 646             user_id = uidmap[sxv['userid']]
 647
 648             if not (comment_id, user_id) in user2vote:
 649                 user2vote.append((comment_id, user_id))
 650
 651                 action = orm.Action(
 652                         action_type = "voteupcomment",
 653                         user_id = user_id,
 654                         action_date = readTime(sxv['creationdate']),
 655                         node_id = comment_id
 656                         )
 657                 action.save()
 658
 659                 ov = orm.Vote(
 660                         node_id = comment_id,
 661                         user_id = user_id,
 662                         voted_at = action.action_date,
 663                         value = 1,
 664                         action = action
 665                         )
 666
 667                 ov.save()
 668
 669                 if not comment_id in comments2score:
 670                     comments2score[comment_id] = 1
 671                 else:
 672                     comments2score[comment_id] += 1
 673
 674     readTable(dump, "Comments2Votes", callback)
 675
 676     for cid, score in comments2score.items():
 677         orm.Node.objects.filter(id=cid).update(score=score)
 678
 679
 680 def badges_import(dump, uidmap, post_list):
 681
 682     sxbadges = {}
 683
 684     def sxcallback(b):
 685         sxbadges[int(b['id'])] = b
 686
 687     readTable(dump, "Badges", sxcallback)
 688
 689     obadges = dict([(b.cls, b) for b in orm.Badge.objects.all()])
 690     user_badge_count = {}
 691
 692     sx_to_osqa = {}
 693
 694     for id, sxb in sxbadges.items():
 695         cls = "".join(sxb['name'].replace('&', 'And').split(' '))
 696
 697         if cls in obadges:
 698             sx_to_osqa[id] = obadges[cls]
 699         else:
 700             osqab = orm.Badge(
 701                     cls = cls,
 702                     awarded_count = 0,
 703                     type = sxb['class']
 704                     )
 705             osqab.save()
 706             sx_to_osqa[id] = osqab
 707
 708     osqaawards = []
 709
 710     def callback(sxa):
 711         badge = sx_to_osqa[int(sxa['badgeid'])]
 712
 713         user_id = uidmap[sxa['userid']]
 714         if not user_badge_count.get(user_id, None):
 715             user_badge_count[user_id] = 0
 716
 717         action = orm.Action(
 718                 action_type = "award",
 719                 user_id = user_id,
 720                 action_date = readTime(sxa['date'])
 721                 )
 722
 723         action.save()
 724
 725         osqaa = orm.Award(
 726                 user_id = uidmap[sxa['userid']],
 727                 badge = badge,
 728                 node_id = post_list[user_badge_count[user_id]],
 729                 awarded_at = action.action_date,
 730                 action = action
 731                 )
 732
 733         osqaa.save()
 734         badge.awarded_count += 1
 735         user_badge_count[user_id] += 1
 736
 737     readTable(dump, "Users2Badges", callback)
 738
 739     for badge in obadges.values():
 740         badge.save()
 741
 742 def save_setting(k, v):
 743     try:
 744         kv = orm.KeyValue.objects.get(key=k)
 745         kv.value = v
 746     except:
 747         kv = orm.KeyValue(key = k, value = v)
 748
 749     kv.save()
 750
 751
 752 def pages_import(dump, currid):
 753     currid = IdIncrementer(currid)
 754     registry = {}
 755
 756     def callback(sxp):
 757         currid.inc()
 758         page = orm.Node(
 759                 id = currid.value,
 760                 node_type = "page",
 761                 title = sxp['name'],
 762                 body = b64decode(sxp['value']),
 763                 extra = dbsafe_encode({
 764                 'path': sxp['url'][1:],
 765                 'mimetype': sxp['contenttype'],
 766                 'template': (sxp['usemaster'] == "true") and "default" or "none",
 767                 'render': "html",
 768                 'sidebar': "",
 769                 'sidebar_wrap': True,
 770                 'sidebar_render': "html",
 771                 'comments': False
 772                 }),
 773                 author_id = 1
 774                 )
 775
 776         create_and_activate_revision(page)
 777
 778         page.save()
 779         registry[sxp['url'][1:]] = page.id
 780
 781         create_action = orm.Action(
 782                 action_type = "newpage",
 783                 user_id = page.author_id,
 784                 node = page
 785                 )
 786
 787         create_action.save()
 788
 789         if sxp['active'] == "true" and sxp['contenttype'] == "text/html":
 790             pub_action = orm.Action(
 791                     action_type = "publish",
 792                     user_id = page.author_id,
 793                     node = page
 794                     )
 795
 796             pub_action.save()
 797             add_post_state("published", page, pub_action)
 798
 799     readTable(dump, "FlatPages", callback)
 800
 801     save_setting('STATIC_PAGE_REGISTRY', dbsafe_encode(registry))
 802
 803 sx2osqa_set_map = {
 804 u'theme.html.name': 'APP_TITLE',
 805 u'theme.html.footer': 'CUSTOM_FOOTER',
 806 u'theme.html.sidebar': 'SIDEBAR_UPPER_TEXT',
 807 u'theme.html.sidebar-low': 'SIDEBAR_LOWER_TEXT',
 808 u'theme.html.welcome': 'APP_INTRO',
 809 u'theme.html.head': 'CUSTOM_HEAD',
 810 u'theme.html.header': 'CUSTOM_HEADER',
 811 u'theme.css': 'CUSTOM_CSS',
 812 }
 813
 814 html_codes = (
 815 ('&amp;', '&'),
 816 ('&lt;', '<'),
 817 ('&gt;', '>'),
 818 ('&quot;', '"'),
 819 ('&#39;', "'"),
 820 )
 821
 822 def html_decode(html):
 823     html = force_unicode(html)
 824
 825     for args in html_codes:
 826         html = html.replace(*args)
 827
 828     return html
 829
 830
 831 def static_import(dump):
 832     sx_unknown = {}
 833
 834     def callback(set):
 835         if unicode(set['name']) in sx2osqa_set_map:
 836             save_setting(sx2osqa_set_map[set['name']], dbsafe_encode(html_decode(set['value'])))
 837         else:
 838             sx_unknown[set['name']] = html_decode(set['value'])
 839
 840     readTable(dump, "ThemeTextResources", callback)
 841
 842     save_setting('SXIMPORT_UNKNOWN_SETS', dbsafe_encode(sx_unknown))
 843
 844 def disable_triggers():
 845     from south.db import db
 846     if db.backend_name == "postgres":
 847         db.execute_many(PG_DISABLE_TRIGGERS)
 848         db.commit_transaction()
 849         db.start_transaction()
 850
 851 def enable_triggers():
 852     from south.db import db
 853     if db.backend_name == "postgres":
 854         db.start_transaction()
 855         db.execute_many(PG_ENABLE_TRIGGERS)
 856         db.commit_transaction()
 857
 858 def reset_sequences():
 859     from south.db import db
 860     if db.backend_name == "postgres":
 861         db.start_transaction()
 862         db.execute_many(PG_SEQUENCE_RESETS)
 863         db.commit_transaction()
 864
 865 def reindex_fts():
 866     from south.db import db
 867     if db.backend_name == "postgres":
 868         db.start_transaction()
 869         db.execute_many("UPDATE forum_noderevision set id = id WHERE TRUE;")
 870         db.commit_transaction()
 871
 872
 873 def sximport(dump, options):
 874     try:
 875         disable_triggers()
 876         triggers_disabled = True
 877     except:
 878         triggers_disabled = False
 879
 880     uidmap = userimport(dump, options)
 881     tagmap = tagsimport(dump, uidmap)
 882     gc.collect()
 883
 884     posts = postimport(dump, uidmap, tagmap)
 885     gc.collect()
 886
 887     posts, comments = comment_import(dump, uidmap, posts)
 888     gc.collect()
 889
 890     post_vote_import(dump, uidmap, posts)
 891     gc.collect()
 892
 893     comment_vote_import(dump, uidmap, comments)
 894     gc.collect()
 895
 896     badges_import(dump, uidmap, posts)
 897
 898     pages_import(dump, max(posts))
 899     static_import(dump)
 900     gc.collect()
 901
 902     from south.db import db
 903     db.commit_transaction()
 904
 905     reset_sequences()
 906
 907     if triggers_disabled:
 908         enable_triggers()
 909         reindex_fts()
 910
 911
 912 PG_DISABLE_TRIGGERS = """
 913 ALTER table auth_user DISABLE TRIGGER ALL;
 914 ALTER table auth_user_groups DISABLE TRIGGER ALL;
 915 ALTER table auth_user_user_permissions DISABLE TRIGGER ALL;
 916 ALTER table forum_keyvalue DISABLE TRIGGER ALL;
 917 ALTER table forum_action DISABLE TRIGGER ALL;
 918 ALTER table forum_actionrepute DISABLE TRIGGER ALL;
 919 ALTER table forum_subscriptionsettings DISABLE TRIGGER ALL;
 920 ALTER table forum_validationhash DISABLE TRIGGER ALL;
 921 ALTER table forum_authkeyuserassociation DISABLE TRIGGER ALL;
 922 ALTER table forum_tag DISABLE TRIGGER ALL;
 923 ALTER table forum_markedtag DISABLE TRIGGER ALL;
 924 ALTER table forum_node DISABLE TRIGGER ALL;
 925 ALTER table forum_nodestate DISABLE TRIGGER ALL;
 926 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 927 ALTER table forum_noderevision DISABLE TRIGGER ALL;
 928 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 929 ALTER table forum_questionsubscription DISABLE TRIGGER ALL;
 930 ALTER table forum_vote DISABLE TRIGGER ALL;
 931 ALTER table forum_flag DISABLE TRIGGER ALL;
 932 ALTER table forum_badge DISABLE TRIGGER ALL;
 933 ALTER table forum_award DISABLE TRIGGER ALL;
 934 ALTER table forum_openidnonce DISABLE TRIGGER ALL;
 935 ALTER table forum_openidassociation DISABLE TRIGGER ALL;
 936 """
 937
 938 PG_ENABLE_TRIGGERS = """
 939 ALTER table auth_user ENABLE TRIGGER ALL;
 940 ALTER table auth_user_groups ENABLE TRIGGER ALL;
 941 ALTER table auth_user_user_permissions ENABLE TRIGGER ALL;
 942 ALTER table forum_keyvalue ENABLE TRIGGER ALL;
 943 ALTER table forum_action ENABLE TRIGGER ALL;
 944 ALTER table forum_actionrepute ENABLE TRIGGER ALL;
 945 ALTER table forum_subscriptionsettings ENABLE TRIGGER ALL;
 946 ALTER table forum_validationhash ENABLE TRIGGER ALL;
 947 ALTER table forum_authkeyuserassociation ENABLE TRIGGER ALL;
 948 ALTER table forum_tag ENABLE TRIGGER ALL;
 949 ALTER table forum_markedtag ENABLE TRIGGER ALL;
 950 ALTER table forum_node ENABLE TRIGGER ALL;
 951 ALTER table forum_nodestate ENABLE TRIGGER ALL;
 952 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 953 ALTER table forum_noderevision ENABLE TRIGGER ALL;
 954 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 955 ALTER table forum_questionsubscription ENABLE TRIGGER ALL;
 956 ALTER table forum_vote ENABLE TRIGGER ALL;
 957 ALTER table forum_flag ENABLE TRIGGER ALL;
 958 ALTER table forum_badge ENABLE TRIGGER ALL;
 959 ALTER table forum_award ENABLE TRIGGER ALL;
 960 ALTER table forum_openidnonce ENABLE TRIGGER ALL;
 961 ALTER table forum_openidassociation ENABLE TRIGGER ALL;
 962 """
 963
 964 PG_SEQUENCE_RESETS = """
 965 SELECT setval('"auth_user_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user";
 966 SELECT setval('"auth_user_groups_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_groups";
 967 SELECT setval('"auth_user_user_permissions_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_user_permissions";
 968 SELECT setval('"forum_keyvalue_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_keyvalue";
 969 SELECT setval('"forum_action_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_action";
 970 SELECT setval('"forum_actionrepute_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_actionrepute";
 971 SELECT setval('"forum_subscriptionsettings_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_subscriptionsettings";
 972 SELECT setval('"forum_validationhash_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_validationhash";
 973 SELECT setval('"forum_authkeyuserassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_authkeyuserassociation";
 974 SELECT setval('"forum_tag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_tag";
 975 SELECT setval('"forum_markedtag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_markedtag";
 976 SELECT setval('"forum_node_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node";
 977 SELECT setval('"forum_nodestate_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_nodestate";
 978 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
 979 SELECT setval('"forum_noderevision_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_noderevision";
 980 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
 981 SELECT setval('"forum_questionsubscription_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_questionsubscription";
 982 SELECT setval('"forum_vote_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_vote";
 983 SELECT setval('"forum_flag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_flag";
 984 SELECT setval('"forum_badge_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_badge";
 985 SELECT setval('"forum_award_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_award";
 986 SELECT setval('"forum_openidnonce_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidnonce";
 987 SELECT setval('"forum_openidassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidassociation";
 988 """
 989
 990
 991
 992