forum_modules/sximporter/importer.py

   1 # -*- coding: utf-8 -*-
   2
   3 from datetime import datetime
   4 import time
   5 import re
   6 import os
   7 import gc
   8 from django.utils.translation import ugettext as _
   9
  10 from django.utils.encoding import force_unicode
  11
  12 try:
  13     from cPickle import loads, dumps
  14 except ImportError:
  15     from pickle import loads, dumps
  16
  17 from copy import deepcopy
  18 from base64 import b64encode, b64decode
  19 from zlib import compress, decompress
  20
  21 from xml.sax import make_parser
  22 from xml.sax.handler import ContentHandler
  23
  24 def create_orm():
  25     from django.conf import settings
  26     from south.orm import FakeORM
  27
  28     get_migration_number_re = re.compile(r'^((\d+)_.*)\.py$')
  29
  30     migrations_folder = os.path.join(settings.SITE_SRC_ROOT, 'forum/migrations')
  31
  32     highest_number = 0
  33     highest_file = None
  34
  35     for f in os.listdir(migrations_folder):
  36         if os.path.isfile(os.path.join(migrations_folder, f)):
  37             m = get_migration_number_re.match(f)
  38
  39             if m:
  40                 found = int(m.group(2))
  41
  42                 if found > highest_number:
  43                     highest_number = found
  44                     highest_file = m.group(1)
  45
  46     mod = __import__('forum.migrations.%s' % highest_file, globals(), locals(), ['forum.migrations'])
  47     return FakeORM(getattr(mod, 'Migration'), "forum")
  48
  49 orm = create_orm()
  50
  51 class SXTableHandler(ContentHandler):
  52     def __init__(self, fname, callback):
  53         self.in_row = False
  54         self.el_data = {}
  55         self.ch_data = ''
  56
  57         self.fname = fname.lower()
  58         self.callback = callback
  59
  60     def startElement(self, name, attrs):
  61         if name.lower() == self.fname:
  62             pass
  63         elif name.lower() == "row":
  64             self.in_row = True
  65
  66     def characters(self, ch):
  67         self.ch_data += ch
  68
  69     def endElement(self, name):
  70         if name.lower() == self.fname:
  71             pass
  72         elif name.lower() == "row":
  73             self.callback(self.el_data)
  74
  75             self.in_row = False
  76             del self.el_data
  77             self.el_data = {}
  78         elif self.in_row:
  79             self.el_data[name.lower()] = self.ch_data.strip()
  80             del self.ch_data
  81             self.ch_data = ''
  82
  83
  84 def readTable(path, name, callback):
  85     parser = make_parser()
  86     handler = SXTableHandler(name, callback)
  87     parser.setContentHandler(handler)
  88
  89     f = os.path.join(path, "%s.xml" % name)
  90     parser.parse(f)
  91
  92
  93 def dbsafe_encode(value):
  94     return force_unicode(b64encode(compress(dumps(deepcopy(value)))))
  95
  96 def getText(el):
  97     rc = ""
  98     for node in el.childNodes:
  99         if node.nodeType == node.TEXT_NODE:
 100             rc = rc + node.data
 101     return rc.strip()
 102
 103 msstrip = re.compile(r'^(.*)\.\d+')
 104 def readTime(ts):
 105     noms = msstrip.match(ts)
 106     if noms:
 107         ts = noms.group(1)
 108
 109     return datetime(*time.strptime(ts, '%Y-%m-%dT%H:%M:%S')[0:6])
 110
 111 #def readEl(el):
 112 #    return dict([(n.tagName.lower(), getText(n)) for n in el.childNodes if n.nodeType == el.ELEMENT_NODE])
 113
 114 #def readTable(dump, name):
 115 #    for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row'):
 116 #        yield readEl(e)
 117 #return [readEl(e) for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row')]
 118
 119 google_accounts_lookup = re.compile(r'^https?://www.google.com/accounts/')
 120 yahoo_accounts_lookup = re.compile(r'^https?://me.yahoo.com/a/')
 121
 122 openid_lookups = [
 123         re.compile(r'^https?://www.google.com/profiles/(?P<uname>\w+(\.\w+)*)/?$'),
 124         re.compile(r'^https?://me.yahoo.com/(?P<uname>\w+(\.\w+)*)/?$'),
 125         re.compile(r'^https?://openid.aol.com/(?P<uname>\w+(\.\w+)*)/?$'),
 126         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).myopenid.com/?$'),
 127         re.compile(r'^https?://flickr.com/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 128         re.compile(r'^https?://technorati.com/people/technorati/(?P<uname>\w+(\.\w+)*)/?$'),
 129         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).wordpress.com/?$'),
 130         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).blogspot.com/?$'),
 131         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).livejournal.com/?$'),
 132         re.compile(r'^https?://claimid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 133         re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).pip.verisignlabs.com/?$'),
 134         re.compile(r'^https?://getopenid.com/(?P<uname>\w+(\.\w+)*)/?$'),
 135         re.compile(r'^https?://[\w\.]+/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),
 136         re.compile(r'^https?://(?P<uname>[\w\.]+)/?$'),
 137         ]
 138
 139 def final_username_attempt(sxu):
 140     openid = sxu.get('openid', None)
 141
 142     if openid:
 143         if google_accounts_lookup.search(openid):
 144             return UnknownGoogleUser(sxu.get('id'))
 145         if yahoo_accounts_lookup.search(openid):
 146             return UnknownYahooUser(sxu.get('id'))
 147
 148         for lookup in openid_lookups:
 149             if lookup.search(openid):
 150                 return lookup.search(openid).group('uname')
 151
 152     return UnknownUser(sxu.get('id'))
 153
 154 class UnknownUser(object):
 155     def __init__(self, id):
 156         self._id = id
 157
 158     def __str__(self):
 159         return _("user-%(id)s") % {'id': self._id}
 160
 161     def __unicode__(self):
 162         return self.__str__()
 163
 164     def encode(self, *args):
 165         return self.__str__()
 166
 167 class UnknownGoogleUser(UnknownUser):
 168     def __str__(self):
 169         return _("user-%(id)s (google)") % {'id': self._id}
 170
 171 class UnknownYahooUser(UnknownUser):
 172     def __str__(self):
 173         return _("user-%(id)s (yahoo)") % {'id': self._id}
 174
 175
 176 class IdMapper(dict):
 177
 178     def __init__(self):
 179         self.default = 1
 180
 181     def __getitem__(self, key):
 182         key = int(key)
 183         return super(IdMapper, self).get(key, self.default)
 184
 185     def __setitem__(self, key, value):
 186         super(IdMapper, self).__setitem__(int(key), int(value))
 187
 188 class IdIncrementer():
 189     def __init__(self, initial):
 190         self.value = initial
 191
 192     def inc(self):
 193         self.value += 1
 194
 195 openidre = re.compile('^https?\:\/\/')
 196 def userimport(path, options):
 197
 198     usernames = []
 199     openids = set()
 200     uidmapper = IdMapper()
 201
 202     authenticated_user = options.get('authenticated_user', None)
 203     owneruid = options.get('owneruid', None)
 204     #check for empty values
 205     if not owneruid:
 206         owneruid = None
 207     else:
 208         owneruid = int(owneruid)
 209
 210     def callback(sxu):
 211         create = True
 212         set_mapper_defaults = False
 213
 214         if sxu.get('id') == '-1':
 215             return
 216         #print "\n".join(["%s : %s" % i for i in sxu.items()])
 217
 218         if (owneruid and (int(sxu.get('id')) == owneruid)) or (
 219             (not owneruid) and len(uidmapper)):
 220
 221             set_mapper_defaults = True
 222
 223             if authenticated_user:
 224                 osqau = orm.User.objects.get(id=authenticated_user.id)
 225
 226                 for assoc in orm.AuthKeyUserAssociation.objects.filter(user=osqau):
 227                     openids.add(assoc.key)
 228
 229                 uidmapper[owneruid] = osqau.id
 230                 create = False
 231
 232         sxbadges = sxu.get('badgesummary', None)
 233         badges = {'1':'0', '2':'0', '3':'0'}
 234
 235         if sxbadges:
 236             badges.update(dict([b.split('=') for b in sxbadges.split()]))
 237
 238         if create:
 239             username = unicode(sxu.get('displayname',
 240                                sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu)))))[:30]
 241
 242             if username in usernames:
 243             #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email:
 244             #    osqau = user_by_name[username]
 245             #    create = False
 246             #    uidmapper[sxu.get('id')] = osqau.id
 247             #else:
 248                 inc = 0
 249
 250                 while True:
 251                     inc += 1
 252                     totest = "%s %d" % (username[:29 - len(str(inc))], inc)
 253
 254                     if not totest in usernames:
 255                         username = totest
 256                         break
 257
 258             osqau = orm.User(
 259                     id           = sxu.get('id'),
 260                     username     = username,
 261                     password     = '!',
 262                     email        = sxu.get('email', ''),
 263                     is_superuser = sxu.get('usertypeid') == '5',
 264                     is_staff     = sxu.get('usertypeid') == '4',
 265                     is_active    = True,
 266                     date_joined  = readTime(sxu.get('creationdate')),
 267                     last_seen    = readTime(sxu.get('lastaccessdate')),
 268                     about         = sxu.get('aboutme', ''),
 269                     date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None,
 270                     email_isvalid = int(sxu.get('usertypeid')) > 2,
 271                     website       = sxu.get('websiteurl', ''),
 272                     reputation    = int(sxu.get('reputation')),
 273                     gold          = int(badges['1']),
 274                     silver        = int(badges['2']),
 275                     bronze        = int(badges['3']),
 276                     real_name     = sxu.get('realname', '')[:30],
 277                     location      = sxu.get('location', ''),
 278                     )
 279
 280             osqau.save()
 281
 282             user_joins = orm.Action(
 283                     action_type = "userjoins",
 284                     action_date = osqau.date_joined,
 285                     user = osqau
 286                     )
 287             user_joins.save()
 288
 289             rep = orm.ActionRepute(
 290                     value = 1,
 291                     user = osqau,
 292                     date = osqau.date_joined,
 293                     action = user_joins
 294                     )
 295             rep.save()
 296
 297             try:
 298                 orm.SubscriptionSettings.objects.get(user=osqau)
 299             except:
 300                 s = orm.SubscriptionSettings(user=osqau)
 301                 s.save()
 302
 303             uidmapper[osqau.id] = osqau.id
 304         else:
 305             new_about = sxu.get('aboutme', None)
 306             if new_about and osqau.about != new_about:
 307                 if osqau.about:
 308                     osqau.about = "%s\n|\n%s" % (osqau.about, new_about)
 309                 else:
 310                     osqau.about = new_about
 311
 312             osqau.username = sxu.get('displayname',
 313                                      sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu))))
 314             osqau.email = sxu.get('email', '')
 315             osqau.reputation += int(sxu.get('reputation'))
 316             osqau.gold += int(badges['1'])
 317             osqau.silver += int(badges['2'])
 318             osqau.bronze += int(badges['3'])
 319
 320             osqau.date_joined = readTime(sxu.get('creationdate'))
 321             osqau.website = sxu.get('websiteurl', '')
 322             osqau.date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None
 323             osqau.location = sxu.get('location', '')
 324             osqau.real_name = sxu.get('realname', '')
 325
 326             #merged_users.append(osqau.id)
 327             osqau.save()
 328
 329         if set_mapper_defaults:
 330             uidmapper[-1] = osqau.id
 331             uidmapper.default = osqau.id
 332
 333         usernames.append(osqau.username)
 334
 335         openid = sxu.get('openid', None)
 336         if openid and openidre.match(openid) and (not openid in openids):
 337             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl")
 338             assoc.save()
 339             openids.add(openid)
 340
 341         openidalt = sxu.get('openidalt', None)
 342         if openidalt and openidre.match(openidalt) and (not openidalt in openids):
 343             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openidalt, provider="openidurl")
 344             assoc.save()
 345             openids.add(openidalt)
 346
 347     readTable(path, "Users", callback)
 348
 349     #if uidmapper[-1] == -1:
 350     #    uidmapper[-1] = 1
 351
 352     return uidmapper
 353
 354 def tagsimport(dump, uidmap):
 355
 356     tagmap = {}
 357
 358     def callback(sxtag):
 359         otag = orm.Tag(
 360                 id = int(sxtag['id']),
 361                 name = sxtag['name'],
 362                 used_count = int(sxtag['count']),
 363                 created_by_id = uidmap[sxtag.get('userid', 1)],
 364                 )
 365         otag.save()
 366
 367         tagmap[otag.name] = otag
 368
 369     readTable(dump, "Tags", callback)
 370
 371     return tagmap
 372
 373 def add_post_state(name, post, action):
 374     if not "(%s)" % name in post.state_string:
 375         post.state_string = "%s(%s)" % (post.state_string, name)
 376         post.save()
 377
 378     try:
 379         state = orm.NodeState.objects.get(node=post, state_type=name)
 380         state.action = action
 381         state.save()
 382     except:
 383         state = orm.NodeState(node=post, state_type=name, action=action)
 384         state.save()
 385
 386 def remove_post_state(name, post):
 387     if "(%s)" % name in post.state_string:
 388         try:
 389             state = orm.NodeState.objects.get(state_type=name, post=post)
 390             state.delete()
 391         except:
 392             pass
 393     post.state_string = "".join("(%s)" % s for s in re.findall('\w+', post.state_string) if s != name)
 394
 395 def postimport(dump, uidmap, tagmap):
 396     all = {}
 397
 398     def callback(sxpost):
 399         nodetype = (sxpost.get('posttypeid') == '1') and "nodetype" or "answer"
 400
 401         post = orm.Node(
 402                 node_type = nodetype,
 403                 id = sxpost['id'],
 404                 added_at = readTime(sxpost['creationdate']),
 405                 body = sxpost['body'],
 406                 score = sxpost.get('score', 0),
 407                 author_id = sxpost.get('deletiondate', None) and 1 or uidmap[sxpost.get('owneruserid', 1)]
 408                 )
 409
 410         post.save()
 411
 412         create_action = orm.Action(
 413                 action_type = (nodetype == "nodetype") and "ask" or "answer",
 414                 user_id = post.author_id,
 415                 node = post,
 416                 action_date = post.added_at
 417                 )
 418
 419         create_action.save()
 420
 421         if sxpost.get('lasteditoruserid', None):
 422             revise_action = orm.Action(
 423                     action_type = "revise",
 424                     user_id = uidmap[sxpost.get('lasteditoruserid')],
 425                     node = post,
 426                     action_date = readTime(sxpost['lasteditdate']),
 427                     )
 428
 429             revise_action.save()
 430             post.last_edited = revise_action
 431
 432         if sxpost.get('communityowneddate', None):
 433             wikify_action = orm.Action(
 434                     action_type = "wikify",
 435                     user_id = 1,
 436                     node = post,
 437                     action_date = readTime(sxpost['communityowneddate'])
 438                     )
 439
 440             wikify_action.save()
 441             add_post_state("wiki", post, wikify_action)
 442
 443         if sxpost.get('lastactivityuserid', None):
 444             post.last_activity_by_id = uidmap[sxpost['lastactivityuserid']]
 445             post.last_activity_at = readTime(sxpost['lastactivitydate'])
 446
 447         if sxpost.get('posttypeid') == '1': #question
 448             post.node_type = "question"
 449             post.title = sxpost['title']
 450
 451             tagnames = sxpost['tags'].replace(u'ö', '-').replace(u'é', '').replace(u'à', '')
 452             post.tagnames = tagnames
 453
 454             post.extra_count = sxpost.get('viewcount', 0)
 455
 456             add_tags_to_post(post, tagmap)
 457             all[int(post.id)] = int(post.id)
 458
 459         else:
 460             post.parent_id = sxpost['parentid']
 461             post.abs_parent_id = sxpost['parentid']
 462             all[int(post.id)] = int(sxpost['parentid'])
 463
 464         post.save()
 465
 466         create_and_activate_revision(post)
 467
 468         del post
 469
 470     readTable(dump, "Posts", callback)
 471
 472     return all
 473
 474 def comment_import(dump, uidmap, absparent_map):
 475     posts = absparent_map.keys()
 476
 477     currid = IdIncrementer(max(posts))
 478     mapping = {}
 479
 480     def callback(sxc):
 481         currid.inc()
 482         oc = orm.Node(
 483                 id = currid.value,
 484                 node_type = "comment",
 485                 added_at = readTime(sxc['creationdate']),
 486                 author_id = uidmap[sxc.get('userid', 1)],
 487                 body = sxc['text'],
 488                 parent_id = sxc.get('postid'),
 489                 abs_parent_id = absparent_map.get(int(sxc.get('postid')), sxc.get('postid'))
 490                 )
 491
 492         if sxc.get('deletiondate', None):
 493             delete_action = orm.Action(
 494                     action_type = "delete",
 495                     user_id = uidmap[sxc['deletionuserid']],
 496                     action_date = readTime(sxc['deletiondate'])
 497                     )
 498
 499             oc.author_id = uidmap[sxc['deletionuserid']]
 500             oc.save()
 501
 502             delete_action.node = oc
 503             delete_action.save()
 504
 505             add_post_state("deleted", oc, delete_action)
 506         else:
 507             oc.author_id = uidmap[sxc.get('userid', 1)]
 508             oc.save()
 509
 510         create_action = orm.Action(
 511                 action_type = "comment",
 512                 user_id = oc.author_id,
 513                 node = oc,
 514                 action_date = oc.added_at
 515                 )
 516
 517         create_and_activate_revision(oc)
 518
 519         create_action.save()
 520         oc.save()
 521
 522         posts.append(int(oc.id))
 523         mapping[int(sxc['id'])] = int(oc.id)
 524
 525     readTable(dump, "PostComments", callback)
 526     return posts, mapping
 527
 528
 529 def add_tags_to_post(post, tagmap):
 530     tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag]
 531     post.tagnames = " ".join([t.name for t in tags]).strip()
 532     post.tags = tags
 533
 534
 535 def create_and_activate_revision(post):
 536     rev = orm.NodeRevision(
 537             author_id = post.author_id,
 538             body = post.body,
 539             node_id = post.id,
 540             revised_at = post.added_at,
 541             revision = 1,
 542             summary = 'Initial revision',
 543             tagnames = post.tagnames,
 544             title = post.title,
 545             )
 546
 547     rev.save()
 548     post.active_revision_id = rev.id
 549     post.save()
 550
 551 def post_vote_import(dump, uidmap, posts):
 552     close_reasons = {}
 553
 554     def close_callback(r):
 555         close_reasons[r['id']] = r['name']
 556
 557     readTable(dump, "CloseReasons", close_callback)
 558
 559     user2vote = []
 560
 561     def callback(sxv):
 562         action = orm.Action(
 563                 user_id=uidmap[sxv['userid']],
 564                 action_date = readTime(sxv['creationdate']),
 565                 )
 566
 567         if not int(sxv['postid']) in posts: return
 568         node = orm.Node.objects.get(id=sxv['postid'])
 569         action.node = node
 570
 571         if sxv['votetypeid'] == '1':
 572             answer = node
 573             question = orm.Node.objects.get(id=answer.parent_id)
 574
 575             action.action_type = "acceptanswer"
 576             action.save()
 577
 578             answer.marked = True
 579
 580             question.extra_ref_id = answer.id
 581
 582             answer.save()
 583             question.save()
 584
 585         elif sxv['votetypeid'] in ('2', '3'):
 586             if not (action.node.id, action.user_id) in user2vote:
 587                 user2vote.append((action.node.id, action.user_id))
 588
 589                 action.action_type = (sxv['votetypeid'] == '2') and "voteup" or "votedown"
 590                 action.save()
 591
 592                 ov = orm.Vote(
 593                         node_id = action.node.id,
 594                         user_id = action.user_id,
 595                         voted_at = action.action_date,
 596                         value = sxv['votetypeid'] == '2' and 1 or -1,
 597                         action = action
 598                         )
 599                 ov.save()
 600             else:
 601                 action.action_type = "unknown"
 602                 action.save()
 603
 604         elif sxv['votetypeid'] in ('4', '12', '13'):
 605             action.action_type = "flag"
 606             action.save()
 607
 608             of = orm.Flag(
 609                     node = action.node,
 610                     user_id = action.user_id,
 611                     flagged_at = action.action_date,
 612                     reason = '',
 613                     action = action
 614                     )
 615
 616             of.save()
 617
 618         elif sxv['votetypeid'] == '5':
 619             action.action_type = "favorite"
 620             action.save()
 621
 622         elif sxv['votetypeid'] == '6':
 623             action.action_type = "close"
 624             action.extra = dbsafe_encode(close_reasons[sxv['comment']])
 625             action.save()
 626
 627             node.marked = True
 628             node.save()
 629
 630         elif sxv['votetypeid'] == '7':
 631             action.action_type = "unknown"
 632             action.save()
 633
 634             node.marked = False
 635             node.save()
 636
 637             remove_post_state("closed", node)
 638
 639         elif sxv['votetypeid'] == '10':
 640             action.action_type = "delete"
 641             action.save()
 642
 643         elif sxv['votetypeid'] == '11':
 644             action.action_type = "unknown"
 645             action.save()
 646
 647             remove_post_state("deleted", node)
 648
 649         else:
 650             action.action_type = "unknown"
 651             action.save()
 652
 653         if sxv.get('targetrepchange', None):
 654             rep = orm.ActionRepute(
 655                     action = action,
 656                     date = action.action_date,
 657                     user_id = uidmap[sxv['targetuserid']],
 658                     value = int(sxv['targetrepchange'])
 659                     )
 660
 661             rep.save()
 662
 663         if sxv.get('voterrepchange', None):
 664             rep = orm.ActionRepute(
 665                     action = action,
 666                     date = action.action_date,
 667                     user_id = uidmap[sxv['userid']],
 668                     value = int(sxv['voterrepchange'])
 669                     )
 670
 671             rep.save()
 672
 673         if action.action_type in ("acceptanswer", "delete", "close"):
 674             state = {"acceptanswer": "accepted", "delete": "deleted", "close": "closed"}[action.action_type]
 675             add_post_state(state, node, action)
 676
 677     readTable(dump, "Posts2Votes", callback)
 678
 679
 680 def comment_vote_import(dump, uidmap, comments):
 681     user2vote = []
 682     comments2score = {}
 683
 684     def callback(sxv):
 685         if sxv['votetypeid'] == "2":
 686             comment_id = comments[int(sxv['postcommentid'])]
 687             user_id = uidmap[sxv['userid']]
 688
 689             if not (comment_id, user_id) in user2vote:
 690                 user2vote.append((comment_id, user_id))
 691
 692                 action = orm.Action(
 693                         action_type = "voteupcomment",
 694                         user_id = user_id,
 695                         action_date = readTime(sxv['creationdate']),
 696                         node_id = comment_id
 697                         )
 698                 action.save()
 699
 700                 ov = orm.Vote(
 701                         node_id = comment_id,
 702                         user_id = user_id,
 703                         voted_at = action.action_date,
 704                         value = 1,
 705                         action = action
 706                         )
 707
 708                 ov.save()
 709
 710                 if not comment_id in comments2score:
 711                     comments2score[comment_id] = 1
 712                 else:
 713                     comments2score[comment_id] += 1
 714
 715     readTable(dump, "Comments2Votes", callback)
 716
 717     for cid, score in comments2score.items():
 718         orm.Node.objects.filter(id=cid).update(score=score)
 719
 720
 721 def badges_import(dump, uidmap, post_list):
 722
 723     sxbadges = {}
 724
 725     def sxcallback(b):
 726         sxbadges[int(b['id'])] = b
 727
 728     readTable(dump, "Badges", sxcallback)
 729
 730     obadges = dict([(b.cls, b) for b in orm.Badge.objects.all()])
 731     user_badge_count = {}
 732
 733     sx_to_osqa = {}
 734
 735     for id, sxb in sxbadges.items():
 736         cls = "".join(sxb['name'].replace('&', 'And').split(' '))
 737
 738         if cls in obadges:
 739             sx_to_osqa[id] = obadges[cls]
 740         else:
 741             osqab = orm.Badge(
 742                     cls = cls,
 743                     awarded_count = 0,
 744                     type = sxb['class']
 745                     )
 746             osqab.save()
 747             sx_to_osqa[id] = osqab
 748
 749     osqaawards = []
 750
 751     def callback(sxa):
 752         badge = sx_to_osqa[int(sxa['badgeid'])]
 753
 754         user_id = uidmap[sxa['userid']]
 755         if not user_badge_count.get(user_id, None):
 756             user_badge_count[user_id] = 0
 757
 758         action = orm.Action(
 759                 action_type = "award",
 760                 user_id = user_id,
 761                 action_date = readTime(sxa['date'])
 762                 )
 763
 764         action.save()
 765
 766         osqaa = orm.Award(
 767                 user_id = uidmap[sxa['userid']],
 768                 badge = badge,
 769                 node_id = post_list[user_badge_count[user_id]],
 770                 awarded_at = action.action_date,
 771                 action = action
 772                 )
 773
 774         osqaa.save()
 775         badge.awarded_count += 1
 776
 777         user_badge_count[user_id] += 1
 778
 779     readTable(dump, "Users2Badges", callback)
 780
 781     for badge in obadges.values():
 782         badge.save()
 783
 784 def save_setting(k, v):
 785     try:
 786         kv = orm.KeyValue.objects.get(key=k)
 787         kv.value = v
 788     except:
 789         kv = orm.KeyValue(key = k, value = v)
 790
 791     kv.save()
 792
 793
 794 def pages_import(dump, currid, owner):
 795     currid = IdIncrementer(currid)
 796     registry = {}
 797
 798     def callback(sxp):
 799         currid.inc()
 800         page = orm.Node(
 801                 id = currid.value,
 802                 node_type = "page",
 803                 title = sxp['name'],
 804                 body = b64decode(sxp['value']),
 805                 extra = dbsafe_encode({
 806                 'path': sxp['url'][1:],
 807                 'mimetype': sxp['contenttype'],
 808                 'template': (sxp['usemaster'] == "true") and "default" or "none",
 809                 'render': "html",
 810                 'sidebar': "",
 811                 'sidebar_wrap': True,
 812                 'sidebar_render': "html",
 813                 'comments': False
 814                 }),
 815                 author_id = owner
 816                 )
 817
 818         create_and_activate_revision(page)
 819
 820         page.save()
 821         registry[sxp['url'][1:]] = page.id
 822
 823         create_action = orm.Action(
 824                 action_type = "newpage",
 825                 user_id = page.author_id,
 826                 node = page
 827                 )
 828
 829         create_action.save()
 830
 831         if sxp['active'] == "true" and sxp['contenttype'] == "text/html":
 832             pub_action = orm.Action(
 833                     action_type = "publish",
 834                     user_id = page.author_id,
 835                     node = page
 836                     )
 837
 838             pub_action.save()
 839             add_post_state("published", page, pub_action)
 840
 841     readTable(dump, "FlatPages", callback)
 842
 843     save_setting('STATIC_PAGE_REGISTRY', dbsafe_encode(registry))
 844
 845 sx2osqa_set_map = {
 846 u'theme.html.name': 'APP_TITLE',
 847 u'theme.html.footer': 'CUSTOM_FOOTER',
 848 u'theme.html.sidebar': 'SIDEBAR_UPPER_TEXT',
 849 u'theme.html.sidebar-low': 'SIDEBAR_LOWER_TEXT',
 850 u'theme.html.welcome': 'APP_INTRO',
 851 u'theme.html.head': 'CUSTOM_HEAD',
 852 u'theme.html.header': 'CUSTOM_HEADER',
 853 u'theme.css': 'CUSTOM_CSS',
 854 }
 855
 856 html_codes = (
 857 ('&amp;', '&'),
 858 ('&lt;', '<'),
 859 ('&gt;', '>'),
 860 ('&quot;', '"'),
 861 ('&#39;', "'"),
 862 )
 863
 864 def html_decode(html):
 865     html = force_unicode(html)
 866
 867     for args in html_codes:
 868         html = html.replace(*args)
 869
 870     return html
 871
 872
 873 def static_import(dump):
 874     sx_unknown = {}
 875
 876     def callback(set):
 877         if unicode(set['name']) in sx2osqa_set_map:
 878             save_setting(sx2osqa_set_map[set['name']], dbsafe_encode(html_decode(set['value'])))
 879         else:
 880             sx_unknown[set['name']] = html_decode(set['value'])
 881
 882     readTable(dump, "ThemeTextResources", callback)
 883
 884     save_setting('SXIMPORT_UNKNOWN_SETS', dbsafe_encode(sx_unknown))
 885
 886 def disable_triggers():
 887     from south.db import db
 888     if db.backend_name == "postgres":
 889         db.execute_many(PG_DISABLE_TRIGGERS)
 890         db.commit_transaction()
 891         db.start_transaction()
 892
 893 def enable_triggers():
 894     from south.db import db
 895     if db.backend_name == "postgres":
 896         db.start_transaction()
 897         db.execute_many(PG_ENABLE_TRIGGERS)
 898         db.commit_transaction()
 899
 900 def reset_sequences():
 901     from south.db import db
 902     if db.backend_name == "postgres":
 903         db.start_transaction()
 904         db.execute_many(PG_SEQUENCE_RESETS)
 905         db.commit_transaction()
 906
 907 def reindex_fts():
 908     from south.db import db
 909     if db.backend_name == "postgres":
 910         db.start_transaction()
 911         db.execute_many("UPDATE forum_noderevision set id = id WHERE TRUE;")
 912         db.commit_transaction()
 913
 914
 915 def sximport(dump, options):
 916     try:
 917         disable_triggers()
 918         triggers_disabled = True
 919     except:
 920         triggers_disabled = False
 921
 922     uidmap = userimport(dump, options)
 923     tagmap = tagsimport(dump, uidmap)
 924     gc.collect()
 925
 926     posts = postimport(dump, uidmap, tagmap)
 927     gc.collect()
 928
 929     posts, comments = comment_import(dump, uidmap, posts)
 930     gc.collect()
 931
 932     post_vote_import(dump, uidmap, posts)
 933     gc.collect()
 934
 935     comment_vote_import(dump, uidmap, comments)
 936     gc.collect()
 937
 938     badges_import(dump, uidmap, posts)
 939
 940     pages_import(dump, max(posts), uidmap.default)
 941     static_import(dump)
 942     gc.collect()
 943
 944     from south.db import db
 945     db.commit_transaction()
 946
 947     reset_sequences()
 948
 949     if triggers_disabled:
 950         enable_triggers()
 951         reindex_fts()
 952
 953
 954 PG_DISABLE_TRIGGERS = """
 955 ALTER table auth_user DISABLE TRIGGER ALL;
 956 ALTER table auth_user_groups DISABLE TRIGGER ALL;
 957 ALTER table auth_user_user_permissions DISABLE TRIGGER ALL;
 958 ALTER table forum_keyvalue DISABLE TRIGGER ALL;
 959 ALTER table forum_action DISABLE TRIGGER ALL;
 960 ALTER table forum_actionrepute DISABLE TRIGGER ALL;
 961 ALTER table forum_subscriptionsettings DISABLE TRIGGER ALL;
 962 ALTER table forum_validationhash DISABLE TRIGGER ALL;
 963 ALTER table forum_authkeyuserassociation DISABLE TRIGGER ALL;
 964 ALTER table forum_tag DISABLE TRIGGER ALL;
 965 ALTER table forum_markedtag DISABLE TRIGGER ALL;
 966 ALTER table forum_node DISABLE TRIGGER ALL;
 967 ALTER table forum_nodestate DISABLE TRIGGER ALL;
 968 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 969 ALTER table forum_noderevision DISABLE TRIGGER ALL;
 970 ALTER table forum_node_tags DISABLE TRIGGER ALL;
 971 ALTER table forum_questionsubscription DISABLE TRIGGER ALL;
 972 ALTER table forum_vote DISABLE TRIGGER ALL;
 973 ALTER table forum_flag DISABLE TRIGGER ALL;
 974 ALTER table forum_badge DISABLE TRIGGER ALL;
 975 ALTER table forum_award DISABLE TRIGGER ALL;
 976 ALTER table forum_openidnonce DISABLE TRIGGER ALL;
 977 ALTER table forum_openidassociation DISABLE TRIGGER ALL;
 978 """
 979
 980 PG_ENABLE_TRIGGERS = """
 981 ALTER table auth_user ENABLE TRIGGER ALL;
 982 ALTER table auth_user_groups ENABLE TRIGGER ALL;
 983 ALTER table auth_user_user_permissions ENABLE TRIGGER ALL;
 984 ALTER table forum_keyvalue ENABLE TRIGGER ALL;
 985 ALTER table forum_action ENABLE TRIGGER ALL;
 986 ALTER table forum_actionrepute ENABLE TRIGGER ALL;
 987 ALTER table forum_subscriptionsettings ENABLE TRIGGER ALL;
 988 ALTER table forum_validationhash ENABLE TRIGGER ALL;
 989 ALTER table forum_authkeyuserassociation ENABLE TRIGGER ALL;
 990 ALTER table forum_tag ENABLE TRIGGER ALL;
 991 ALTER table forum_markedtag ENABLE TRIGGER ALL;
 992 ALTER table forum_node ENABLE TRIGGER ALL;
 993 ALTER table forum_nodestate ENABLE TRIGGER ALL;
 994 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 995 ALTER table forum_noderevision ENABLE TRIGGER ALL;
 996 ALTER table forum_node_tags ENABLE TRIGGER ALL;
 997 ALTER table forum_questionsubscription ENABLE TRIGGER ALL;
 998 ALTER table forum_vote ENABLE TRIGGER ALL;
 999 ALTER table forum_flag ENABLE TRIGGER ALL;
1000 ALTER table forum_badge ENABLE TRIGGER ALL;
1001 ALTER table forum_award ENABLE TRIGGER ALL;
1002 ALTER table forum_openidnonce ENABLE TRIGGER ALL;
1003 ALTER table forum_openidassociation ENABLE TRIGGER ALL;
1004 """
1005
1006 PG_SEQUENCE_RESETS = """
1007 SELECT setval('"auth_user_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user";
1008 SELECT setval('"auth_user_groups_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_groups";
1009 SELECT setval('"auth_user_user_permissions_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_user_permissions";
1010 SELECT setval('"forum_keyvalue_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_keyvalue";
1011 SELECT setval('"forum_action_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_action";
1012 SELECT setval('"forum_actionrepute_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_actionrepute";
1013 SELECT setval('"forum_subscriptionsettings_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_subscriptionsettings";
1014 SELECT setval('"forum_validationhash_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_validationhash";
1015 SELECT setval('"forum_authkeyuserassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_authkeyuserassociation";
1016 SELECT setval('"forum_tag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_tag";
1017 SELECT setval('"forum_markedtag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_markedtag";
1018 SELECT setval('"forum_node_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node";
1019 SELECT setval('"forum_nodestate_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_nodestate";
1020 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
1021 SELECT setval('"forum_noderevision_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_noderevision";
1022 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_node_tags";
1023 SELECT setval('"forum_questionsubscription_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_questionsubscription";
1024 SELECT setval('"forum_vote_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_vote";
1025 SELECT setval('"forum_flag_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_flag";
1026 SELECT setval('"forum_badge_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_badge";
1027 SELECT setval('"forum_award_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_award";
1028 SELECT setval('"forum_openidnonce_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidnonce";
1029 SELECT setval('"forum_openidassociation_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "forum_openidassociation";
1030 """
1031
1032
1033
1034