Fixes comment edit, which was being performed against already parsed markdown.

[osqa.git] / forum_modules / exporter / exporter.py

diff --git a/forum_modules/exporter/exporter.py b/forum_modules/exporter/exporter.py
index a52bbb6a77ae08b97f9567ca0a32b7ee169bdde4..df1237f004f72b55201352b7292f3b6bd5fdbbf1 100644 (file)
--- a/forum_modules/exporter/exporter.py
+++ b/forum_modules/exporter/exporter.py
@@ -1,4 +1,4 @@
-import os, tarfile, datetime, logging
+import os, tarfile, datetime, logging, re, ConfigParser, shutil

 
 from django.core.cache import cache
 from django.utils.translation import ugettext as _
 
 from django.core.cache import cache
 from django.utils.translation import ugettext as _


@@ -8,6 +8,9 @@ from forum.templatetags.extra_tags import diff_date
 import xml.etree.ElementTree
 from xml.etree import ElementTree as ET
 from xml.etree.ElementTree import Comment, _encode, ProcessingInstruction, QName, fixtag, _escape_attrib, _escape_cdata
 import xml.etree.ElementTree
 from xml.etree import ElementTree as ET
 from xml.etree.ElementTree import Comment, _encode, ProcessingInstruction, QName, fixtag, _escape_attrib, _escape_cdata

+from forum import settings
+from django.conf import settings as djsettings
+import settings as selfsettings

 
 CACHE_KEY = "%s_exporter_state" % APP_URL
 EXPORT_STEPS = []
 
 CACHE_KEY = "%s_exporter_state" % APP_URL
 EXPORT_STEPS = []


@@ -15,6 +18,13 @@ EXPORT_STEPS = []
 TMP_FOLDER = os.path.join(os.path.dirname(__file__), 'tmp')
 LAST_BACKUP = os.path.join(TMP_FOLDER, 'backup.tar.gz')
 
 TMP_FOLDER = os.path.join(os.path.dirname(__file__), 'tmp')
 LAST_BACKUP = os.path.join(TMP_FOLDER, 'backup.tar.gz')
 

+DATE_AND_AUTHOR_INF_SECTION = 'DateAndAuthor'
+OPTIONS_INF_SECTION = 'Options'
+META_INF_SECTION = 'Meta'
+
+DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+DATE_FORMAT = "%Y-%m-%d"
+

 def Etree_pretty__write(self, file, node, encoding, namespaces,
                         level=0, identator="    "):
     tag = node.tag
 def Etree_pretty__write(self, file, node, encoding, namespaces,
                         level=0, identator="    "):
     tag = node.tag


@@ -47,10 +57,10 @@ def Etree_pretty__write(self, file, node, encoding, namespaces,
                         if xmlns: xmlns_items.append(xmlns)
                 except TypeError:
                     raise #_raise_serialization_error(v)
                         if xmlns: xmlns_items.append(xmlns)
                 except TypeError:
                     raise #_raise_serialization_error(v)

-                file.write(" %s=\"%s\"" % (_encode(k, encoding),
+                file.write(u" %s=\"%s\"" % (_encode(k, encoding),

                                             _escape_attrib(v, encoding)))
             for k, v in xmlns_items:
                                             _escape_attrib(v, encoding)))
             for k, v in xmlns_items:

-                file.write(" %s=\"%s\"" % (_encode(k, encoding),
+                file.write(u" %s=\"%s\"" % (_encode(k, encoding),

                                             _escape_attrib(v, encoding)))
         if node.text or len(node):
             file.write(">")
                                             _escape_attrib(v, encoding)))
         if node.text or len(node):
             file.write(">")


@@ -69,14 +79,15 @@ def Etree_pretty__write(self, file, node, encoding, namespaces,
     if node.tail:
         file.write(_escape_cdata(node.tail.replace("\n", level * identator + "\n"), encoding))
 
     if node.tail:
         file.write(_escape_cdata(node.tail.replace("\n", level * identator + "\n"), encoding))
 

-def _add_tag(el, name, content = None):
-    tag = ET.SubElement(el, name)
-    if content:
-        tag.text = content
-    return tag
+def make_date(date, with_time=True):
+    try:
+        return date.strftime(with_time and DATETIME_FORMAT or DATE_FORMAT)
+    except ValueError, e:
+        return date.replace(year=1900).strftime(with_time and DATETIME_FORMAT or DATE_FORMAT)
+

 
 

-def ET_Element_add_tag(el, name, content = None, **attrs):
-    tag = ET.SubElement(el, name)
+def ET_Element_add_tag(el, tag_name, content = None, **attrs):
+    tag = ET.SubElement(el, tag_name)

 
     if content:
         tag.text = unicode(content)
 
     if content:
         tag.text = unicode(content)


@@ -86,33 +97,120 @@ def ET_Element_add_tag(el, name, content = None, **attrs):
 
     return tag
 
 
     return tag
 

-def make_extra(el, extra):
-    if not extra:
+GOOD_TAG_NAME = re.compile("^\w+$")
+
+def make_extra(el, v):
+    if v is None:

         return
 
         return
 

-    if isinstance(extra, dict):
-        for k, v in extra.items():
-            make_extra(el.add(k), v)
+
+    if isinstance(v, (int, long, str, unicode, float, bool, dict, list, tuple)):
+        if isinstance(v, tuple):
+            t = 'list'
+        else:
+            t = v.__class__.__name__

     else:
     else:

-        el.text = unicode(extra)
+        t = 'unknown'
+
+    value = el.add('value', type=t)
+
+    if isinstance(v, (list, tuple)):
+        for i in v:
+            item = value.add('item')
+            make_extra(item, i)
+
+    elif isinstance(v, dict):
+        for k, i in v.items():
+            item = value.add('item', key=k)
+            make_extra(item, i)
+    else:
+        value.text = unicode(v)

 
 def write_to_file(root, tmp, filename):
     tree = ET.ElementTree(root)
     tree.write(os.path.join(tmp, filename), encoding='UTF-8')
 
 
 def write_to_file(root, tmp, filename):
     tree = ET.ElementTree(root)
     tree.write(os.path.join(tmp, filename), encoding='UTF-8')
 

-def create_targz(tmp, files):
+def create_targz(tmp, files, start_time, options, user, state, set_state):

     if os.path.exists(LAST_BACKUP):
         os.remove(LAST_BACKUP)
         
     t = tarfile.open(name=LAST_BACKUP, mode = 'w:gz')
 
     if os.path.exists(LAST_BACKUP):
         os.remove(LAST_BACKUP)
         
     t = tarfile.open(name=LAST_BACKUP, mode = 'w:gz')
 

+    state['overall']['status'] = _('Compressing xml files')
+    set_state()
+

     for f in files:
     for f in files:

-        t.add(os.path.join(tmp, f), arcname=f)
+        t.add(os.path.join(tmp, f), arcname="/%s" % f)
+
+    if options.get('uplodaded_files', False):
+        state['overall']['status'] = _('Importing uploaded files')
+        set_state()
+        export_upfiles(t)
+
+    if options.get('import_skins_folder', False):
+        state['overall']['status'] = _('Importing skins folder')
+        set_state()
+        export_skinsfolder(t)
+
+    state['overall']['status'] = _('Writing inf file.')
+    set_state()
+
+    now = datetime.datetime.now()
+    domain = re.match('[\w-]+\.[\w-]+(\.[\w-]+)*', djsettings.APP_URL)
+    if domain:
+        domain = '_'.join(domain.get(0).split('.'))
+    else:
+        domain = 'localhost'
+
+    fname = "%s-%s" % (domain, now.strftime('%Y%m%d%H%M'))
+
+    inf = ConfigParser.SafeConfigParser()
+
+    inf.add_section(DATE_AND_AUTHOR_INF_SECTION)
+
+    inf.set(DATE_AND_AUTHOR_INF_SECTION, 'file-name', "%s.tar.gz" % fname)
+    inf.set(DATE_AND_AUTHOR_INF_SECTION, 'author', unicode(user.id))
+    inf.set(DATE_AND_AUTHOR_INF_SECTION, 'site', djsettings.APP_URL)
+    inf.set(DATE_AND_AUTHOR_INF_SECTION, 'started', start_time.strftime(DATETIME_FORMAT))
+    inf.set(DATE_AND_AUTHOR_INF_SECTION, 'finished', now.strftime(DATETIME_FORMAT))
+
+    inf.add_section(OPTIONS_INF_SECTION)
+    inf.set(OPTIONS_INF_SECTION, 'anon-data', str(options.get('anon_data', False)))
+    inf.set(OPTIONS_INF_SECTION, 'with-upfiles', str(options.get('uplodaded_files', False)))
+    inf.set(OPTIONS_INF_SECTION, 'with-skins', str(options.get('import_skins_folder', False)))
+
+    inf.add_section(META_INF_SECTION)

 
 

+    for id, s in state.items():
+        inf.set(META_INF_SECTION, id, str(s['count']))
+
+    with open(os.path.join(tmp, 'backup.inf'), 'wb') as inffile:
+        inf.write(inffile)
+
+    t.add(os.path.join(tmp, '/backup.inf'), arcname='backup.inf')
+    state['overall']['status'] = _('Saving backup file')
+    set_state()

     t.close()
     t.close()

+    shutil.copyfile(LAST_BACKUP, os.path.join(selfsettings.EXPORTER_BACKUP_STORAGE, "%s.tar.gz" % fname))
+    shutil.copyfile(os.path.join(tmp, 'backup.inf'), os.path.join(selfsettings.EXPORTER_BACKUP_STORAGE, "%s.backup.inf" % fname))
+
+    
+
+def export_upfiles(tf):
+    folder = str(settings.UPFILES_FOLDER)
+
+    if os.path.exists(folder):
+        tf.add(folder, arcname='/upfiles')
+

 
 

+def export_skinsfolder(tf):
+    folder = djsettings.TEMPLATE_DIRS[0]

 
 

-def export(options):
+    if os.path.exists(folder):
+        tf.add(folder, arcname='/skins')
+
+
+def export(options, user):

     original__write = xml.etree.ElementTree.ElementTree._write
     xml.etree.ElementTree.ElementTree._write = Etree_pretty__write
     xml.etree.ElementTree._ElementInterface.add = ET_Element_add_tag
     original__write = xml.etree.ElementTree.ElementTree._write
     xml.etree.ElementTree.ElementTree._write = Etree_pretty__write
     xml.etree.ElementTree._ElementInterface.add = ET_Element_add_tag


@@ -174,7 +272,7 @@ def export(options):
         state['overall']['status'] = _('Compressing files')
         set_state()
 
         state['overall']['status'] = _('Compressing files')
         set_state()
 

-        create_targz(tmp, dump_files)
+        create_targz(tmp, dump_files, start_time, options, user, state, set_state)

         full_state['running'] = False
         full_state['errors'] = False
         state['overall']['status'] = _('Done')
         full_state['running'] = False
         full_state['errors'] = False
         state['overall']['status'] = _('Done')


@@ -191,6 +289,7 @@ def export(options):
         xml.etree.ElementTree.ElementTree._write = original__write
         del xml.etree.ElementTree._ElementInterface.add
 
         xml.etree.ElementTree.ElementTree._write = original__write
         del xml.etree.ElementTree._ElementInterface.add
 

+

 def exporter_step(queryset, root_tag_name, el_tag_name, name, date_lock=None, user_data=False):
 
     def decorator(fn):
 def exporter_step(queryset, root_tag_name, el_tag_name, name, date_lock=None, user_data=False):
 
     def decorator(fn):


@@ -202,7 +301,7 @@ def exporter_step(queryset, root_tag_name, el_tag_name, name, date_lock=None, us
         def decorated(ping, lock, anon_data):
             root = ET.Element(root_tag_name)
 
         def decorated(ping, lock, anon_data):
             root = ET.Element(root_tag_name)
 

-            for item in qs(lock).select_related():
+            for item in qs(lock).order_by('id').select_related():

                 el = root.add(el_tag_name)
                 fn(item, el, anon_data)
                 ping()
                 el = root.add(el_tag_name)
                 fn(item, el, anon_data)
                 ping()


@@ -236,15 +335,18 @@ def export_tags(t, el, anon_data):
 def export_users(u, el, anon_data):
     el.add('id', u.id)
     el.add('username', u.username)
 def export_users(u, el, anon_data):
     el.add('id', u.id)
     el.add('username', u.username)

+    el.add('password', u.password)

     el.add('email', u.email, validated=u.email_isvalid and 'true' or 'false')
     el.add('email', u.email, validated=u.email_isvalid and 'true' or 'false')

-    el.add('joindate', u.date_joined)
+    el.add('reputation', u.reputation)
+    el.add('badges', bronze=u.bronze, silver=u.silver, gold=u.gold)
+    el.add('joindate', make_date(u.date_joined))
+    el.add('active', u.is_active and 'true' or 'false')

 
 

-    el.add('firstname', u.first_name)
-    el.add('lastname', u.last_name)
+    el.add('realname', u.real_name)

     el.add('bio', u.about)
     el.add('location', u.location)
     el.add('website', u.website)
     el.add('bio', u.about)
     el.add('location', u.location)
     el.add('website', u.website)

-    el.add('birthdate', u.date_of_birth)
+    el.add('birthdate', u.date_of_birth and make_date(u.date_of_birth, with_time=False) or "")

 
     roles = el.add('roles')
 
 
     roles = el.add('roles')
 


@@ -254,7 +356,37 @@ def export_users(u, el, anon_data):
     if u.is_staff:
         roles.add('role', 'moderator')
 
     if u.is_staff:
         roles.add('role', 'moderator')
 

-    el.add('reputation', u.reputation)
+    auth = el.add('authKeys')
+    for a in u.auth_keys.all():
+        key = auth.add('key')
+        key.add('provider', a.provider)
+        key.add('key', a.key)
+
+
+    ss = u.subscription_settings
+
+    notify = el.add('notifications', enabled=ss.enable_notifications and 'true' or 'false')
+
+    notify.add('notify', **dict([(t, ss.__dict__.get(t, 'n') == 'i' and 'true' or 'false') for t in ['member_joins', 'new_question', 'new_question_watched_tags', 'subscribed_questions']]))
+
+    notify.add('autoSubscribe', **dict([(t, ss.__dict__.get(t, False) and 'true' or 'false') for t in [
+            'all_questions', 'all_questions_watched_tags', 'questions_asked', 'questions_answered', 'questions_commented', 'questions_viewed']]))
+
+    notify.add('notifyOnSubscribed', **dict([(t, ss.__dict__.get("notify_%s" % t, False) and 'true' or 'false') for t in [
+            'answers', 'reply_to_comments', 'comments_own_post', 'comments', 'accepted']]))
+
+    notify.add('digest', ss.send_digest and 'on' or 'off')
+
+    watched = el.add('watchedTags')
+    rejected = el.add('rejectedTags')
+
+    for m in u.tag_selections.all():
+        if m.reason == 'good':
+            watched.add('tag', m.tag.name)
+        else:
+            rejected.add('tag', m.tag.name)
+
+    

 
 @exporter_step(Node.objects.all(), 'nodes', 'node', _('Nodes'), 'added_at')
 def export_nodes(n, el, anon_data):
 
 @exporter_step(Node.objects.all(), 'nodes', 'node', _('Nodes'), 'added_at')
 def export_nodes(n, el, anon_data):


@@ -263,40 +395,54 @@ def export_nodes(n, el, anon_data):
 
     if not anon_data:
         el.add('author', n.author.id)
 
     if not anon_data:
         el.add('author', n.author.id)

-    el.add('date', n.added_at)
+    el.add('date', make_date(n.added_at))

     el.add('parent', n.parent and n.parent.id or "")
     el.add('parent', n.parent and n.parent.id or "")

+    el.add('absparent', n.abs_parent and n.abs_parent or "")
+
+    act = el.add('lastactivity')
+    act.add('by', n.last_activity_by and n.last_activity_by.id or "")
+    act.add('at', n.last_activity_at and make_date(n.last_activity_at) or "")

 
     el.add('title', n.title)
     el.add('body', n.body)
 
 
     el.add('title', n.title)
     el.add('body', n.body)
 

+    el.add('score', n.score)
+

     tags = el.add('tags')
 
     for t in n.tagname_list():
         tags.add('tag', t)
 
     tags = el.add('tags')
 
     for t in n.tagname_list():
         tags.add('tag', t)
 

-    revs = el.add('revisions', active=n.active_revision and n.active_revision or n.revisions.order_by('revision')[0])
+    try:
+        active = n.active_revision and n.active_revision.revision or n.revisions.order_by('revision')[0].revision
+    except IndexError:
+        active = 0
+
+    revs = el.add('revisions', active=active)

 
     for r in n.revisions.order_by('revision'):
 
     for r in n.revisions.order_by('revision'):

-        rev = _add_tag(revs, 'revision')
+        rev = revs.add('revision')

         rev.add('number', r.revision)
         rev.add('summary', r.summary)
         if not anon_data:
             rev.add('author', r.author.id)
         rev.add('number', r.revision)
         rev.add('summary', r.summary)
         if not anon_data:
             rev.add('author', r.author.id)

-        rev.add('date', r.revised_at)
+        rev.add('date', make_date(r.revised_at))

 
         rev.add('title', r.title)
         rev.add('body', r.body)
         rev.add('tags', ", ".join(r.tagname_list()))
 
 
         rev.add('title', r.title)
         rev.add('body', r.body)
         rev.add('tags', ", ".join(r.tagname_list()))
 

+    el.add('marked', n.marked and 'true' or 'false')

     el.add('extraRef', n.extra_ref and n.extra_ref.id or "")
     el.add('extraRef', n.extra_ref and n.extra_ref.id or "")

-    make_extra(el.add('exraData'), n.extra)
+    make_extra(el.add('extraData'), n.extra)
+    el.add('extraCount', n.extra_count and n.extra_count or "")

 
 
 @exporter_step(Action.objects.all(), 'actions', 'action', _('Actions'), 'action_date')
 def export_actions(a, el, anon_data):
     el.add('id', a.id)
     el.add('type', a.action_type)
 
 
 @exporter_step(Action.objects.all(), 'actions', 'action', _('Actions'), 'action_date')
 def export_actions(a, el, anon_data):
     el.add('id', a.id)
     el.add('type', a.action_type)

-    el.add('date', a.action_date)
+    el.add('date', make_date(a.action_date))

 
     if not anon_data:
         el.add('user', a.user.id)
 
     if not anon_data:
         el.add('user', a.user.id)


@@ -313,7 +459,7 @@ def export_actions(a, el, anon_data):
             canceled.add('user', a.canceled_by.id)
             canceled.add('ip', a.canceled_ip)
 
             canceled.add('user', a.canceled_by.id)
             canceled.add('ip', a.canceled_ip)
 

-        canceled.add('date', a.canceled_at)        
+        canceled.add('date', make_date(a.canceled_at))        

 
     if not anon_data:
         reputes = el.add('reputes')
 
     if not anon_data:
         reputes = el.add('reputes')


@@ -324,28 +470,33 @@ def export_actions(a, el, anon_data):
             repute.add('value', r.value)
 
 
             repute.add('value', r.value)
 
 

-@exporter_step(NodeState.objects.all(), 'states', 'state', _('Node states'), 'action__action_date')
-def export_states(s, el, anon_data):
-    el.add('type', s.state_type)
-    el.add('node', s.node.id)
-    el.add('trigger', s.action.id)
+#@exporter_step(NodeState.objects.all(), 'states', 'state', _('Node states'), 'action__action_date')
+#def export_states(s, el, anon_data):
+#    el.add('type', s.state_type)
+#    el.add('node', s.node.id)
+#    el.add('trigger', s.action.id)

 
 
 
 

-@exporter_step(Badge.objects.all(), 'badges', 'badge', _('Badges'), user_data=True)
-def export_badges(b, el, anon_data):
-    el.add('type', ["", 'gold', 'silver', 'bronze'][b.type])
-    el.add('name', b.cls)
-    el.add('count', b.awarded_count)
+#@exporter_step(Badge.objects.all(), 'badges', 'badge', _('Badges'), user_data=True)
+#def export_badges(b, el, anon_data):
+#    el.add('type', ["", 'gold', 'silver', 'bronze'][b.type])
+#    el.add('name', b.cls)
+#    el.add('count', b.awarded_count)

 
 
 @exporter_step(Award.objects.all(), 'awards', 'award', _('Awards'), 'awarded_at', True)
 def export_awards(a, el, anon_data):
     el.add('badge', a.badge.cls)
 
 
 @exporter_step(Award.objects.all(), 'awards', 'award', _('Awards'), 'awarded_at', True)
 def export_awards(a, el, anon_data):
     el.add('badge', a.badge.cls)

-    el.add('user', a.user)
+    el.add('user', a.user.id)

     el.add('node', a.node and a.node.id or "")
     el.add('trigger', a.trigger and a.trigger.id or "")
     el.add('action', a.action.id)
 
     el.add('node', a.node and a.node.id or "")
     el.add('trigger', a.trigger and a.trigger.id or "")
     el.add('action', a.action.id)
 

+@exporter_step(KeyValue.objects.all(), 'settings', 'setting', _('Settings'))
+def export_settings(s, el, anon_data):
+    el.add('key', s.key)
+    make_extra(el.add('value'), s.value)
+