X-Git-Url: https://git.openstreetmap.org./osqa.git/blobdiff_plain/cae5bfd94d187b8258411467ad6231f24b84e4f0..c6c168511f18d8984901b2c828a47310a2d54d64:/forum_modules/sximporter/importer.py diff --git a/forum_modules/sximporter/importer.py b/forum_modules/sximporter/importer.py index d63541c..0f7f812 100644 --- a/forum_modules/sximporter/importer.py +++ b/forum_modules/sximporter/importer.py @@ -169,7 +169,8 @@ openidre = re.compile('^https?\:\/\/') def userimport(path, options): #users = readTable(dump, "Users") - user_by_name = {} + usernames = [] + openids = set() uidmapper = IdMapper() #merged_users = [] @@ -186,24 +187,30 @@ def userimport(path, options): #print "\n".join(["%s : %s" % i for i in sxu.items()]) if int(sxu.get('id')) == int(owneruid): osqau = orm.User.objects.get(id=1) + for assoc in orm.AuthKeyUserAssociation.objects.filter(user=osqau): + openids.add(assoc.key) uidmapper[owneruid] = 1 uidmapper[-1] = 1 create = False else: - username = sxu.get('displayname', - sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu)))) + username = unicode(sxu.get('displayname', + sxu.get('displaynamecleaned', sxu.get('realname', final_username_attempt(sxu)))))[:30] - if not isinstance(username, UnknownUser) and username in user_by_name: + if username in usernames: #if options.get('mergesimilar', False) and sxu.get('email', 'INVALID') == user_by_name[username].email: # osqau = user_by_name[username] # create = False # uidmapper[sxu.get('id')] = osqau.id #else: - inc = 1 - while ("%s %d" % (username, inc)) in user_by_name: + inc = 0 + + while True: inc += 1 + totest = "%s %d" % (username[:29 - len(str(inc))], inc) - username = "%s %d" % (username, inc) + if not totest in usernames: + username = totest + break sxbadges = sxu.get('badgesummary', None) badges = {'1':'0', '2':'0', '3':'0'} @@ -214,7 +221,7 @@ def userimport(path, options): if create: osqau = orm.User( id = sxu.get('id'), - username = unicode(username), + username = username, password = '!', email = sxu.get('email', ''), is_superuser = sxu.get('usertypeid') == '5', @@ -230,7 +237,7 @@ def userimport(path, options): gold = int(badges['1']), silver = int(badges['2']), bronze = int(badges['3']), - real_name = sxu.get('realname', ''), + real_name = sxu.get('realname', '')[:30], location = sxu.get('location', ''), ) @@ -283,12 +290,19 @@ def userimport(path, options): #merged_users.append(osqau.id) osqau.save() - user_by_name[osqau.username] = osqau + usernames.append(osqau.username) openid = sxu.get('openid', None) - if openid and openidre.match(openid): + if openid and openidre.match(openid) and (not openid in openids): assoc = orm.AuthKeyUserAssociation(user=osqau, key=openid, provider="openidurl") assoc.save() + openids.add(openid) + + openidalt = sxu.get('openidalt', None) + if openidalt and openidre.match(openidalt) and (not openidalt in openids): + assoc = orm.AuthKeyUserAssociation(user=osqau, key=openidalt, provider="openidurl") + assoc.save() + openids.add(openidalt) readTable(path, "Users", callback) @@ -418,6 +432,7 @@ def postimport(dump, uidmap, tagmap): post.save() all.append(int(post.id)) + create_and_activate_revision(post) del post @@ -466,6 +481,8 @@ def comment_import(dump, uidmap, posts): action_date = oc.added_at ) + create_and_activate_revision(oc) + create_action.save() oc.save() @@ -480,7 +497,6 @@ def add_tags_to_post(post, tagmap): tags = [tag for tag in [tagmap.get(name.strip()) for name in post.tagnames.split(u' ') if name] if tag] post.tagnames = " ".join([t.name for t in tags]).strip() post.tags = tags - create_and_activate_revision(post) def create_and_activate_revision(post): @@ -759,6 +775,8 @@ def pages_import(dump, currid): author_id = 1 ) + create_and_activate_revision(page) + page.save() registry[sxp['url'][1:]] = page.id @@ -787,12 +805,13 @@ def pages_import(dump, currid): sx2osqa_set_map = { u'theme.html.name': 'APP_TITLE', -u'theme.html.footer': 'USE_CUSTOM_FOOTER', +u'theme.html.footer': 'CUSTOM_FOOTER', u'theme.html.sidebar': 'SIDEBAR_UPPER_TEXT', u'theme.html.sidebar-low': 'SIDEBAR_LOWER_TEXT', u'theme.html.welcome': 'APP_INTRO', u'theme.html.head': 'CUSTOM_HEAD', -u'theme.html.header': 'CUSTOM_HEADER' +u'theme.html.header': 'CUSTOM_HEADER', +u'theme.css': 'CUSTOM_CSS', } html_codes = ( @@ -818,10 +837,14 @@ def static_import(dump): def callback(set): if unicode(set['name']) in sx2osqa_set_map: - kv = orm.KeyValue( - key = sx2osqa_set_map[set['name']], - value = dbsafe_encode(html_decode(set['value'])) - ) + try: + kv = orm.KeyValue.objects.get(key=sx2osqa_set_map[set['name']]) + kv.value = dbsafe_encode(html_decode(set['value'])) + except: + kv = orm.KeyValue( + key = sx2osqa_set_map[set['name']], + value = dbsafe_encode(html_decode(set['value'])) + ) kv.save() else: @@ -853,9 +876,21 @@ def reset_sequences(): db.execute_many(PG_SEQUENCE_RESETS) db.commit_transaction() +def reindex_fts(): + from south.db import db + if db.backend_name == "postgres": + db.start_transaction() + db.execute_many("UPDATE forum_noderevision set id = id WHERE TRUE;") + db.commit_transaction() + def sximport(dump, options): - disable_triggers() + try: + disable_triggers() + triggers_disabled = True + except: + triggers_disabled = False + uidmap = userimport(dump, options) tagmap = tagsimport(dump, uidmap) gc.collect() @@ -882,7 +917,10 @@ def sximport(dump, options): db.commit_transaction() reset_sequences() - enable_triggers() + + if triggers_disabled: + enable_triggers() + reindex_fts() PG_DISABLE_TRIGGERS = """ @@ -965,4 +1003,4 @@ SELECT setval('"forum_openidassociation_id_seq"', coalesce(max("id"), 1) + 2, ma - \ No newline at end of file +