forum_modules/sximporter/importer.py

   1 # -*- coding: utf-8 -*-\r
   2 \r
   3 from xml.dom import minidom\r
   4 from datetime import datetime, timedelta\r
   5 import time\r
   6 import re\r
   7 from django.utils.translation import ugettext as _\r
   8 from django.template.defaultfilters import slugify\r
   9 from forum.models.utils import dbsafe_encode\r
  10 from orm import orm\r
  11 \r
  12 def getText(el):\r
  13     rc = ""\r
  14     for node in el.childNodes:\r
  15         if node.nodeType == node.TEXT_NODE:\r
  16             rc = rc + node.data\r
  17     return rc.strip()\r
  18 \r
  19 msstrip = re.compile(r'^(.*)\.\d+')\r
  20 def readTime(ts):\r
  21     noms = msstrip.match(ts)\r
  22     if noms:\r
  23         ts = noms.group(1)\r
  24 \r
  25     return datetime(*time.strptime(ts, '%Y-%m-%dT%H:%M:%S')[0:6])\r
  26 \r
  27 def readEl(el):\r
  28     return dict([(n.tagName.lower(), getText(n)) for n in el.childNodes if n.nodeType == el.ELEMENT_NODE])\r
  29 \r
  30 def readTable(dump, name):\r
  31     return [readEl(e) for e in minidom.parseString(dump.read("%s.xml" % name)).getElementsByTagName('row')]\r
  32 \r
  33 google_accounts_lookup = re.compile(r'^https?://www.google.com/accounts/')\r
  34 yahoo_accounts_lookup = re.compile(r'^https?://me.yahoo.com/a/')\r
  35 \r
  36 openid_lookups = [\r
  37     re.compile(r'^https?://www.google.com/profiles/(?P<uname>\w+(\.\w+)*)/?$'),\r
  38     re.compile(r'^https?://me.yahoo.com/(?P<uname>\w+(\.\w+)*)/?$'),\r
  39     re.compile(r'^https?://openid.aol.com/(?P<uname>\w+(\.\w+)*)/?$'),\r
  40     re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).myopenid.com/?$'),\r
  41     re.compile(r'^https?://flickr.com/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),\r
  42     re.compile(r'^https?://technorati.com/people/technorati/(?P<uname>\w+(\.\w+)*)/?$'),\r
  43     re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).wordpress.com/?$'),\r
  44     re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).blogspot.com/?$'),\r
  45     re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).livejournal.com/?$'),\r
  46     re.compile(r'^https?://claimid.com/(?P<uname>\w+(\.\w+)*)/?$'),\r
  47     re.compile(r'^https?://(?P<uname>\w+(\.\w+)*).pip.verisignlabs.com/?$'),\r
  48     re.compile(r'^https?://getopenid.com/(?P<uname>\w+(\.\w+)*)/?$'),\r
  49     re.compile(r'^https?://[\w\.]+/(\w+/)*(?P<uname>\w+(\.\w+)*)/?$'),\r
  50     re.compile(r'^https?://(?P<uname>[\w\.]+)/?$'),\r
  51 ]\r
  52 \r
  53 def final_username_attempt(sxu):\r
  54     openid = sxu.get('openid', None)\r
  55 \r
  56     if openid:\r
  57         if google_accounts_lookup.search(openid):\r
  58             return UnknownGoogleUser()\r
  59         if yahoo_accounts_lookup.search(openid):\r
  60             return UnknownYahooUser()\r
  61 \r
  62         for lookup in openid_lookups:\r
  63             if lookup.search(openid):\r
  64                 return lookup.search(openid).group('uname')\r
  65 \r
  66     return UnknownUser()\r
  67 \r
  68 class UnknownUser(object):\r
  69     counter = 0\r
  70     def __init__(self):\r
  71         self.__class__.counter += 1\r
  72         self.number = self.__class__.counter\r
  73 \r
  74     def __str__(self):\r
  75         return _("Unknown user %(number)d") % {'number': self.number}\r
  76 \r
  77     def __unicode__(self):\r
  78         return self.__str__()\r
  79 \r
  80     def encode(self, *args):\r
  81         return self.__str__()\r
  82 \r
  83 class UnknownGoogleUser(UnknownUser):\r
  84     counter = 0\r
  85 \r
  86     def __str__(self):\r
  87         return _("Unknown google user %(number)d") % {'number': self.number}\r
  88 \r
  89 class UnknownYahooUser(UnknownUser):\r
  90     counter = 0\r
  91 \r
  92     def __str__(self):\r
  93         return _("Unknown yahoo user %(number)d") % {'number': self.number}\r
  94 \r
  95 \r
  96 class IdMapper(dict):\r
  97     def __getitem__(self, key):\r
  98         key = int(key)\r
  99         return super(IdMapper, self).get(key, 1)\r
 100 \r
 101     def __setitem__(self, key, value):\r
 102         super(IdMapper, self).__setitem__(int(key), int(value))\r
 103 \r
 104 openidre = re.compile('^https?\:\/\/')\r
 105 def userimport(dump, options):\r
 106     users = readTable(dump, "Users")\r
 107 \r
 108     user_by_name = {}\r
 109     uidmapper = IdMapper()\r
 110     merged_users = []\r
 111 \r
 112     owneruid = options.get('owneruid', None)\r
 113     #check for empty values\r
 114     if not owneruid:\r
 115         owneruid = None\r
 116 \r
 117     for sxu in users:\r
 118         create = True\r
 119 \r
 120         if sxu.get('id') == '-1':\r
 121             continue\r
 122 \r
 123         if int(sxu.get('id')) == int(owneruid):\r
 124             osqau = orm.User.objects.get(id=1)\r
 125             uidmapper[owneruid] = 1\r
 126             uidmapper[-1] = 1\r
 127             create = False\r
 128         else:\r
 129             username = sxu.get('displayname', sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu))))\r
 130 \r
 131             if not isinstance(username, UnknownUser) and username in user_by_name:\r
 132                 #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email:\r
 133                 #    osqau = user_by_name[username]\r
 134                 #    create = False\r
 135                 #    uidmapper[sxu.get('id')] = osqau.id\r
 136                 #else:\r
 137                 inc = 1\r
 138                 while ("%s %d" % (username, inc)) in user_by_name:\r
 139                     inc += 1\r
 140 \r
 141                 username = "%s %d" % (username, inc)\r
 142 \r
 143         sxbadges = sxu.get('badgesummary', None)\r
 144         badges = {'1':'0','2':'0','3':'0'}\r
 145 \r
 146         if sxbadges:\r
 147             badges.update(dict([b.split('=') for b in sxbadges.split()]))\r
 148 \r
 149         if create:\r
 150             osqau = orm.User(\r
 151                 id           = sxu.get('id'),\r
 152                 username     = unicode(username),\r
 153                 password     = '!',\r
 154                 email        = sxu.get('email', ''),\r
 155                 is_superuser = sxu.get('usertypeid') == '5',\r
 156                 is_staff     = sxu.get('usertypeid') == '4',\r
 157                 is_active    = True,\r
 158                 date_joined  = readTime(sxu.get('creationdate')),\r
 159                 last_seen    = readTime(sxu.get('lastaccessdate')),\r
 160                 about         = sxu.get('aboutme', ''),\r
 161                 date_of_birth = sxu.get('birthday', None) and readTime(sxu['birthday']) or None,\r
 162                 email_isvalid = int(sxu.get('usertypeid')) > 2,\r
 163                 website       = sxu.get('websiteurl', ''),\r
 164                 reputation    = int(sxu.get('reputation')),\r
 165                 gold          = int(badges['1']),\r
 166                 silver        = int(badges['2']),\r
 167                 bronze        = int(badges['3']),\r
 168                 real_name     = sxu.get('realname', ''),\r
 169             )\r
 170 \r
 171             osqau.save()\r
 172 \r
 173             user_joins = orm.Action(\r
 174                 action_type = "userjoins",\r
 175                 action_date = osqau.date_joined,\r
 176                 user = osqau\r
 177             )\r
 178             user_joins.save()\r
 179 \r
 180             rep = orm.ActionRepute(\r
 181                 value = 1,\r
 182                 user = osqau,\r
 183                 date = osqau.date_joined,\r
 184                 action = user_joins\r
 185             )\r
 186             rep.save()            \r
 187 \r
 188             try:\r
 189                 orm.SubscriptionSettings.objects.get(user=osqau)\r
 190             except:\r
 191                 s = orm.SubscriptionSettings(user=osqau)\r
 192                 s.save()\r
 193 \r
 194             uidmapper[osqau.id] = osqau.id\r
 195         else:\r
 196             new_about = sxu.get('aboutme', None)\r
 197             if new_about and osqau.about != new_about:\r
 198                 if osqau.about:\r
 199                     osqau.about = "%s\n|\n%s" % (osqau.about, new_about)\r
 200                 else:\r
 201                     osqau.about = new_about\r
 202 \r
 203             osqau.username = sxu.get('displayname', sxu.get('displaynamecleaned', sxu.get('realname', UnknownUser())))\r
 204             osqau.email = sxu.get('email', '')\r
 205             osqau.reputation += int(sxu.get('reputation'))\r
 206             osqau.gold += int(badges['1'])\r
 207             osqau.silver += int(badges['2'])\r
 208             osqau.bronze += int(badges['3'])\r
 209 \r
 210             merged_users.append(osqau.id)\r
 211             osqau.save()\r
 212 \r
 213         user_by_name[osqau.username] = osqau\r
 214 \r
 215         openid = sxu.get('openid', None)\r
 216         if openid and openidre.match(openid):\r
 217             assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl")\r
 218             assoc.save()\r
 219 \r
 220     if uidmapper[-1] == -1:\r
 221         uidmapper[-1] = 1\r
 222 \r
 223     return (uidmapper, merged_users)\r
 224 \r
 225 def tagsimport(dump, uidmap):\r
 226     tags = readTable(dump, "Tags")\r
 227 \r
 228     tagmap = {}\r
 229 \r
 230     for sxtag in tags:\r
 231         otag = orm.Tag(\r
 232             id = int(sxtag['id']),\r
 233             name = sxtag['name'],\r
 234             used_count = int(sxtag['count']),\r
 235             created_by_id = uidmap[sxtag.get('userid', 1)],\r
 236         )\r
 237         otag.save()\r
 238 \r
 239         tagmap[otag.name] = otag\r
 240 \r
 241     return tagmap\r
 242 \r
 243 def postimport(dump, uidmap, tagmap):\r
 244     history = {}\r
 245     accepted = {}\r
 246     all = {}\r
 247 \r
 248     for h in readTable(dump, "PostHistory"):\r
 249         if not history.get(h.get('postid'), None):\r
 250             history[h.get('postid')] = []\r
 251 \r
 252         history[h.get('postid')].append(h)\r
 253 \r
 254     posts = readTable(dump, "Posts")\r
 255 \r
 256     for sxpost in posts:\r
 257         nodetype = (sxpost.get('posttypeid') == '1') and "nodetype" or "answer"\r
 258 \r
 259         post = orm.Node(\r
 260             node_type = nodetype,\r
 261             id = sxpost['id'],\r
 262             added_at = readTime(sxpost['creationdate']),\r
 263             body = sxpost['body'],\r
 264             score = sxpost.get('score', 0),\r
 265             author_id = sxpost.get('deletiondate', None) and 1 or uidmap[sxpost['owneruserid']]\r
 266         )\r
 267 \r
 268         post.save()\r
 269 \r
 270         create_action = orm.Action(\r
 271             action_type = (nodetype == "nodetype") and "ask" or "answer",\r
 272             user_id = post.author_id,\r
 273             node = post,\r
 274             action_date = post.added_at\r
 275         )\r
 276 \r
 277         create_action.save()\r
 278 \r
 279         #if sxpost.get('deletiondate', None):\r
 280         #    delete_action = orm.Action(\r
 281         #        action_type = "delete",\r
 282         #        user_id = 1,\r
 283         #        node = post,\r
 284         #        action_date = readTime(sxpost['deletiondate'])\r
 285         #    )\r
 286 \r
 287         #    delete_action.save()\r
 288         #    post.deleted = delete_action\r
 289 \r
 290         if sxpost.get('lasteditoruserid', None):\r
 291             revise_action = orm.Action(\r
 292                 action_type = "revise",\r
 293                 user_id = uidmap[sxpost.get('lasteditoruserid')],\r
 294                 node = post,\r
 295                 action_date = readTime(sxpost['lasteditdate']),\r
 296             )\r
 297 \r
 298             revise_action.save()\r
 299             post.last_edited = revise_action\r
 300 \r
 301         if sxpost.get('communityowneddate', None):\r
 302             post.wiki = True\r
 303 \r
 304             wikify_action = orm.Action(\r
 305                 action_type = "wikify",\r
 306                 user_id = 1,\r
 307                 node = post,\r
 308                 action_date = readTime(sxpost['communityowneddate'])\r
 309             )\r
 310 \r
 311             wikify_action.save()\r
 312 \r
 313 \r
 314         if sxpost.get('lastactivityuserid', None):\r
 315             post.last_activity_by_id = uidmap[sxpost['lastactivityuserid']]\r
 316             post.last_activity_at = readTime(sxpost['lastactivitydate'])\r
 317 \r
 318             \r
 319         if sxpost.get('posttypeid') == '1': #question\r
 320             post.node_type = "question"\r
 321             post.title = sxpost['title']\r
 322 \r
 323             tagnames = sxpost['tags'].replace(u'ö', '-').replace(u'é', '').replace(u'à', '')\r
 324             post.tagnames = tagnames\r
 325 \r
 326             post.extra_count = sxpost.get('viewcount', 0)\r
 327 \r
 328             #if sxpost.get('closeddate', None):\r
 329             #    post.marked = True\r
 330             #\r
 331             #    close_action = orm.Action(\r
 332             #        action_type = "close",\r
 333             #        user_id = 1,\r
 334             #        node = post,\r
 335             #        action_date = datetime.now() - timedelta(days=7)\r
 336             #    )\r
 337             #\r
 338             #    close_action.save()\r
 339             #    post.extra_action = close_action\r
 340 \r
 341             #if sxpost.get('acceptedanswerid', None):\r
 342             #    accepted[int(sxpost.get('acceptedanswerid'))] = post\r
 343 \r
 344             #post.save()\r
 345 \r
 346         else:\r
 347             post.parent_id = sxpost['parentid']\r
 348 \r
 349             #if int(post.id) in accepted:\r
 350                 #post.marked = True\r
 351 \r
 352                 #accept_action = orm.Action(\r
 353                 #    action_type = "acceptanswer",\r
 354                 #    user_id = accepted[int(post.id)].author_id,\r
 355                 #    node = post,\r
 356                 #    action_date = datetime.now() - timedelta(days=7)\r
 357                 #)\r
 358 \r
 359                 #accept_action.save()\r
 360 \r
 361 \r
 362                 #post.accepted_at = datetime.now()\r
 363                 #post.accepted_by_id = accepted[int(post.id)].author_id\r
 364 \r
 365                 #accepted[int(post.id)].extra_ref = post\r
 366                 #accepted[int(post.id)].save()\r
 367 \r
 368         post.save()\r
 369 \r
 370         all[int(post.id)] = post\r
 371 \r
 372     return all\r
 373 \r
 374 def comment_import(dump, uidmap, posts):\r
 375     comments = readTable(dump, "PostComments")\r
 376     currid = max(posts.keys())\r
 377     mapping = {}\r
 378 \r
 379     for sxc in comments:\r
 380         currid += 1\r
 381         oc = orm.Node(\r
 382             id = currid,\r
 383             node_type = "comment",\r
 384             added_at = readTime(sxc['creationdate']),\r
 385             author_id = uidmap[sxc.get('userid', 1)],\r
 386             body = sxc['text'],\r
 387             parent_id = sxc.get('postid'),\r
 388         )\r
 389 \r
 390         if sxc.get('deletiondate', None):\r
 391             delete_action = orm.Action(\r
 392                 action_type = "delete",\r
 393                 user_id = uidmap[sxc['deletionuserid']],\r
 394                 action_date = readTime(sxc['deletiondate'])\r
 395             )\r
 396 \r
 397             oc.author_id = uidmap[sxc['deletionuserid']]\r
 398             oc.save()\r
 399 \r
 400             delete_action.node = oc\r
 401             delete_action.save()\r
 402 \r
 403             oc.deleted = delete_action\r
 404         else:\r
 405             oc.author_id = uidmap[sxc.get('userid', 1)]\r
 406             oc.save()\r
 407 \r
 408         create_action = orm.Action(\r
 409             action_type = "comment",\r
 410             user_id = oc.author_id,\r
 411             node = oc,\r
 412             action_date = oc.added_at\r
 413         )\r
 414 \r
 415         create_action.save()\r
 416         oc.save()\r
 417 \r
 418         posts[oc.id] = oc\r
 419         mapping[int(sxc['id'])] = int(oc.id)\r
 420 \r
 421     return posts, mapping\r
 422 \r
 423 \r
 424 def add_tags_to_posts(posts, tagmap):\r
 425     for post in posts.values():\r
 426         if post.node_type == "question":\r
 427             tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag]\r
 428             post.tagnames = " ".join([t.name for t in tags]).strip()\r
 429             post.tags = tags\r
 430 \r
 431         create_and_activate_revision(post)\r
 432 \r
 433 \r
 434 def create_and_activate_revision(post):\r
 435     rev = orm.NodeRevision(\r
 436         author_id = post.author_id,\r
 437         body = post.body,\r
 438         node_id = post.id,\r
 439         revised_at = post.added_at,\r
 440         revision = 1,\r
 441         summary = 'Initial revision',\r
 442         tagnames = post.tagnames,\r
 443         title = post.title,\r
 444     )\r
 445 \r
 446     rev.save()\r
 447     post.active_revision_id = rev.id\r
 448     post.save()\r
 449 \r
 450 def post_vote_import(dump, uidmap, posts):\r
 451     votes = readTable(dump, "Posts2Votes")\r
 452     close_reasons = dict([(r['id'], r['name']) for r in readTable(dump, "CloseReasons")])\r
 453 \r
 454     user2vote = []\r
 455 \r
 456     for sxv in votes:\r
 457         action = orm.Action(\r
 458             user_id=uidmap[sxv['userid']],\r
 459             action_date = readTime(sxv['creationdate']),\r
 460         )\r
 461 \r
 462         node = posts.get(int(sxv['postid']), None)\r
 463         if not node: continue\r
 464         action.node = node\r
 465 \r
 466         if sxv['votetypeid'] == '1':\r
 467             answer = node\r
 468             question = posts.get(int(answer.parent_id), None)\r
 469 \r
 470             action.action_type = "acceptanswer"\r
 471             action.save()\r
 472 \r
 473             answer.marked = True\r
 474             answer.extra_action = action\r
 475 \r
 476             question.extra_ref_id = answer.id\r
 477 \r
 478             answer.save()\r
 479             question.save()\r
 480 \r
 481         elif sxv['votetypeid'] in ('2', '3'):\r
 482             if not (action.node.id, action.user_id) in user2vote:\r
 483                 user2vote.append((action.node.id, action.user_id))\r
 484 \r
 485                 action.action_type = (sxv['votetypeid'] == '2') and "voteup" or "votedown"\r
 486                 action.save()\r
 487 \r
 488                 ov = orm.Vote(\r
 489                     node_id = action.node.id,\r
 490                     user_id = action.user_id,\r
 491                     voted_at = action.action_date,\r
 492                     value = sxv['votetypeid'] == '2' and 1 or -1,\r
 493                     action = action\r
 494                 )\r
 495                 ov.save()\r
 496             else:\r
 497                 action.action_type = "unknown"\r
 498                 action.save()\r
 499 \r
 500         elif sxv['votetypeid'] in ('4', '12', '13'):\r
 501             action.action_type = "flag"\r
 502             action.save()\r
 503 \r
 504             of = orm.Flag(\r
 505                 node = action.node,\r
 506                 user_id = action.user_id,\r
 507                 flagged_at = action.action_date,\r
 508                 reason = '',\r
 509                 action = action\r
 510             )\r
 511 \r
 512             of.save()\r
 513 \r
 514         elif sxv['votetypeid'] == '5':\r
 515             action.action_type = "favorite"\r
 516             action.save()\r
 517 \r
 518         elif sxv['votetypeid'] == '6':\r
 519             action.action_type = "close"\r
 520             action.extra = dbsafe_encode(close_reasons[sxv['comment']])\r
 521             action.save()\r
 522 \r
 523             node.marked = True\r
 524             node.extra_action = action\r
 525             node.save()\r
 526 \r
 527         elif sxv['votetypeid'] == '7':\r
 528             action.action_type = "unknown"\r
 529             action.save()\r
 530             \r
 531             node.marked = False\r
 532             node.extra_action = None\r
 533             node.save()\r
 534 \r
 535         elif sxv['votetypeid'] == '10':\r
 536             action.action_type = "delete"\r
 537             action.save()\r
 538 \r
 539             node.deleted = action\r
 540             node.save()\r
 541 \r
 542         elif sxv['votetypeid'] == '11':\r
 543             action.action_type = "unknown"\r
 544             action.save()\r
 545 \r
 546             node.deleted = None\r
 547             node.save()\r
 548 \r
 549         else:\r
 550             action.action_type = "unknown"\r
 551             action.save()\r
 552 \r
 553 \r
 554         if sxv.get('targetrepchange', None):\r
 555             rep = orm.ActionRepute(\r
 556                 action = action,\r
 557                 date = action.action_date,\r
 558                 user_id = uidmap[sxv['targetuserid']],\r
 559                 value = int(sxv['targetrepchange'])\r
 560             )\r
 561 \r
 562             rep.save()\r
 563 \r
 564         if sxv.get('voterrepchange', None):\r
 565             rep = orm.ActionRepute(\r
 566                 action = action,\r
 567                 date = action.action_date,\r
 568                 user_id = uidmap[sxv['userid']],\r
 569                 value = int(sxv['voterrepchange'])\r
 570             )\r
 571 \r
 572             rep.save()\r
 573 \r
 574 \r
 575 def comment_vote_import(dump, uidmap, comments, posts):\r
 576     votes = readTable(dump, "Comments2Votes")\r
 577     user2vote = []\r
 578 \r
 579     for sxv in votes:\r
 580         if sxv['votetypeid'] == "2":\r
 581             comment_id = comments[int(sxv['postcommentid'])]\r
 582             user_id = uidmap[sxv['userid']]\r
 583 \r
 584             if not (comment_id, user_id) in user2vote:\r
 585                 user2vote.append((comment_id, user_id))\r
 586 \r
 587                 action = orm.Action(\r
 588                     action_type = "voteupcomment",\r
 589                     user_id = user_id,\r
 590                     action_date = readTime(sxv['creationdate']),\r
 591                     node_id = comment_id\r
 592                 )\r
 593                 action.save()\r
 594 \r
 595                 ov = orm.Vote(\r
 596                     node_id = comment_id,\r
 597                     user_id = user_id,\r
 598                     voted_at = action.action_date,\r
 599                     value = 1,\r
 600                     action = action\r
 601                 )\r
 602 \r
 603                 ov.save()\r
 604 \r
 605                 posts[int(action.node_id)].score += 1\r
 606                 posts[int(action.node_id)].save()\r
 607 \r
 608 \r
 609 \r
 610 def badges_import(dump, uidmap, post_list):\r
 611     node_ctype = orm['contenttypes.contenttype'].objects.get(name='node')\r
 612     obadges = dict([(b.cls, b) for b in orm.Badge.objects.all()])\r
 613     sxbadges = dict([(int(b['id']), b) for b in readTable(dump, "Badges")])\r
 614     user_badge_count = {}\r
 615 \r
 616     sx_to_osqa = {}\r
 617 \r
 618     for id, sxb in sxbadges.items():\r
 619         cls = "".join(sxb['name'].replace('&', 'And').split(' '))\r
 620 \r
 621         if cls in obadges:\r
 622             sx_to_osqa[id] = obadges[cls]\r
 623         else:\r
 624             osqab = orm.Badge(\r
 625                 cls = cls,\r
 626                 awarded_count = 0,\r
 627                 type = sxb['class']                \r
 628             )\r
 629             osqab.save()\r
 630             sx_to_osqa[id] = osqab\r
 631 \r
 632     sxawards = readTable(dump, "Users2Badges")\r
 633     osqaawards = []\r
 634 \r
 635     for sxa in sxawards:\r
 636         badge = sx_to_osqa[int(sxa['badgeid'])]\r
 637 \r
 638         user_id = uidmap[sxa['userid']]\r
 639         if not user_badge_count.get(user_id, None):\r
 640             user_badge_count[user_id] = 0\r
 641 \r
 642         action = orm.Action(\r
 643             action_type = "award",\r
 644             user_id = user_id,\r
 645             action_date = readTime(sxa['date'])\r
 646         )\r
 647 \r
 648         action.save()\r
 649 \r
 650         osqaa = orm.Award(\r
 651             user_id = uidmap[sxa['userid']],\r
 652             badge = badge,\r
 653             node = post_list[user_badge_count[user_id]],\r
 654             awarded_at = action.action_date,\r
 655             action = action\r
 656         )\r
 657 \r
 658         osqaa.save()\r
 659         badge.awarded_count += 1\r
 660         user_badge_count[user_id] += 1\r
 661 \r
 662     for badge in obadges.values():\r
 663         badge.save()\r
 664 \r
 665 \r
 666 def reset_sequences():\r
 667     from south.db import db\r
 668     if db.backend_name == "postgres":\r
 669         db.start_transaction()\r
 670         db.execute_many(PG_SEQUENCE_RESETS)\r
 671         db.commit_transaction()\r
 672 \r
 673 def sximport(dump, options):\r
 674     uidmap, merged_users = userimport(dump, options)\r
 675     tagmap = tagsimport(dump, uidmap)\r
 676     posts = postimport(dump, uidmap, tagmap)\r
 677     posts, comments = comment_import(dump, uidmap, posts)\r
 678     add_tags_to_posts(posts, tagmap)\r
 679     post_vote_import(dump, uidmap, posts)\r
 680     comment_vote_import(dump, uidmap, comments, posts)\r
 681     badges_import(dump, uidmap, posts.values())\r
 682 \r
 683     from south.db import db\r
 684     db.commit_transaction()\r
 685 \r
 686     reset_sequences()\r
 687 \r
 688     \r
 689     \r
 690 PG_SEQUENCE_RESETS = """\r
 691 SELECT setval('"auth_user_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user";\r
 692 SELECT setval('"auth_user_groups_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_groups";\r
 693 SELECT setval('"auth_user_user_permissions_id_seq"', coalesce(max("id"), 1) + 2, max("id") IS NOT null) FROM "auth_user_user_permissions";\r
 694 SELECT setval('"forum_keyvalue_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_keyvalue";\r
 695 SELECT setval('"forum_action_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_action";\r
 696 SELECT setval('"forum_actionrepute_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_actionrepute";\r
 697 SELECT setval('"forum_subscriptionsettings_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_subscriptionsettings";\r
 698 SELECT setval('"forum_validationhash_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_validationhash";\r
 699 SELECT setval('"forum_authkeyuserassociation_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_authkeyuserassociation";\r
 700 SELECT setval('"forum_tag_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_tag";\r
 701 SELECT setval('"forum_markedtag_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_markedtag";\r
 702 SELECT setval('"forum_node_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_node";\r
 703 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_node_tags";\r
 704 SELECT setval('"forum_noderevision_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_noderevision";\r
 705 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_node_tags";\r
 706 SELECT setval('"forum_questionsubscription_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_questionsubscription";\r
 707 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_node_tags";\r
 708 SELECT setval('"forum_node_tags_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_node_tags";\r
 709 SELECT setval('"forum_vote_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_vote";\r
 710 SELECT setval('"forum_flag_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_flag";\r
 711 SELECT setval('"forum_badge_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_badge";\r
 712 SELECT setval('"forum_award_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_award";\r
 713 SELECT setval('"forum_openidnonce_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_openidnonce";\r
 714 SELECT setval('"forum_openidassociation_id_seq"', coalesce(max("id"), 1), max("id") IS NOT null) FROM "forum_openidassociation";\r
 715 """\r
 716 \r
 717 \r
 718     \r
 719