forum/models/base.py

   1 import datetime
   2 import re
   3 try:
   4     from hashlib import md5
   5 except:
   6     from md5 import new as md5
   7 from urllib import quote_plus, urlencode
   8 from django.db import models, IntegrityError, connection, transaction
   9 from django.utils.http import urlquote  as django_urlquote
  10 from django.utils.html import strip_tags
  11 from django.core.urlresolvers import reverse
  12 from django.contrib.contenttypes import generic
  13 from django.contrib.contenttypes.models import ContentType
  14 from django.core.cache import cache
  15 from django.template.defaultfilters import slugify
  16 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
  17 from django.utils.translation import ugettext as _
  18 from django.utils.safestring import mark_safe
  19 from django.contrib.sitemaps import ping_google
  20 import django.dispatch
  21 from forum import settings
  22 import logging
  23
  24
  25 if not hasattr(cache, 'get_many'):
  26     #put django 1.2 code here
  27     pass
  28
  29 class LazyQueryList(object):
  30     def __init__(self, model, items):
  31         self.items = items
  32         self.model = model
  33
  34     def __getitem__(self, k):
  35         return self.model.objects.get(id=self.items[k][0])
  36
  37     def __iter__(self):
  38         for id in self.items:
  39             yield self.model.objects.get(id=id[0])
  40
  41     def __len__(self):
  42         return len(self.items)
  43
  44 class ToFetch(str):
  45     pass
  46
  47 class CachedQuerySet(models.query.QuerySet):
  48
  49     def lazy(self):
  50         if not len(self.query.aggregates):
  51             values_list = ['id']
  52
  53             if len(self.query.extra):
  54                 extra_keys = self.query.extra.keys()
  55                 values_list += extra_keys
  56
  57             return LazyQueryList(self.model, list(self.values_list(*values_list)))
  58         else:
  59             return self
  60
  61     def obj_from_datadict(self, datadict):
  62         obj = self.model()
  63         obj.__dict__.update(datadict)
  64
  65         if hasattr(obj, '_state'):
  66             obj._state.db = 'default'
  67
  68         return obj
  69
  70     def _base_clone(self):
  71         return self._clone(klass=models.query.QuerySet)
  72
  73     def get(self, *args, **kwargs):
  74         key = self.model.infer_cache_key(kwargs)
  75
  76         if key is not None:
  77             obj = cache.get(key)
  78
  79             if obj is None:
  80                 obj = self._base_clone().get(*args, **kwargs)
  81                 obj.cache()
  82             else:
  83                 obj = self.obj_from_datadict(obj)
  84                 obj.reset_original_state()
  85
  86             return obj
  87
  88         return self._base_clone().get(*args, **kwargs)
  89
  90     def _fetch_from_query_cache(self, key):
  91         invalidation_key = self.model._get_cache_query_invalidation_key()
  92         cached_result = cache.get_many([invalidation_key, key])
  93
  94         if not invalidation_key in cached_result:
  95             self.model._set_query_cache_invalidation_timestamp()
  96             return None
  97
  98         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
  99             return cached_result[key][1]
 100
 101         return None
 102
 103     def count(self):
 104         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
 105         result = self._fetch_from_query_cache(cache_key)
 106
 107         if result is not None:
 108             return result
 109
 110         result = super(CachedQuerySet, self).count()
 111         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
 112         return result
 113
 114     def iterator(self):
 115         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
 116         on_cache_query_attr = self.model.value_to_list_on_cache_query()
 117
 118         to_return = None
 119         to_cache = {}
 120
 121         key_list = self._fetch_from_query_cache(cache_key)
 122
 123         if key_list is None:
 124             if not len(self.query.aggregates):
 125                 values_list = [on_cache_query_attr]
 126
 127                 if len(self.query.extra):
 128                     values_list += self.query.extra.keys()
 129
 130                 key_list = [v[0] for v in self.values_list(*values_list)]
 131                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
 132             else:
 133                 to_return = list(super(CachedQuerySet, self).iterator())
 134                 to_cache[cache_key] = (datetime.datetime.now(), [row.__dict__[on_cache_query_attr] for row in to_return])
 135
 136         if (not to_return) and key_list:
 137             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
 138             cached = cache.get_many(row_keys)
 139
 140             to_return = [
 141                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(key_list[i]) for i, ck in enumerate(row_keys)
 142             ]
 143
 144             if len(cached) != len(row_keys):
 145                 to_fetch = [str(tr) for tr in to_return if isinstance(tr, ToFetch)]
 146
 147                 fetched = dict([(str(r.__dict__[on_cache_query_attr]), r) for r in
 148                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
 149
 150                 to_return = [(isinstance(tr, ToFetch) and fetched[str(tr)] or tr) for tr in to_return]
 151                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
 152
 153         if len(to_cache):
 154             cache.set_many(to_cache, 60 * 60)
 155
 156         if to_return:
 157             for row in to_return:
 158                 if hasattr(row, 'leaf'):
 159                     yield row.leaf
 160                 else:
 161                     yield row
 162
 163     def _get_query_hash(self):
 164         return md5(str(self.query)).hexdigest()
 165
 166
 167
 168 class CachedManager(models.Manager):
 169     use_for_related_fields = True
 170
 171     def get_query_set(self):
 172         return CachedQuerySet(self.model)
 173
 174     def get_or_create(self, *args, **kwargs):
 175         try:
 176             return self.get(*args, **kwargs)
 177         except:
 178             return super(CachedManager, self).get_or_create(*args, **kwargs)
 179
 180
 181 class DenormalizedField(object):
 182     def __init__(self, manager, *args, **kwargs):
 183         self.manager = manager
 184         self.filter = (args, kwargs)
 185
 186     def setup_class(self, cls, name):
 187         dict_name = '_%s_dencache_' % name
 188
 189         def getter(inst):
 190             val = inst.__dict__.get(dict_name, None)
 191
 192             if val is None:
 193                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
 194                 inst.__dict__[dict_name] = val
 195                 inst.cache()
 196
 197             return val
 198
 199         def reset_cache(inst):
 200             inst.__dict__.pop(dict_name, None)
 201             inst.uncache()
 202
 203         cls.add_to_class(name, property(getter))
 204         cls.add_to_class("reset_%s_cache" % name, reset_cache)
 205
 206
 207 class BaseMetaClass(models.Model.__metaclass__):
 208     to_denormalize = []
 209
 210     def __new__(cls, *args, **kwargs):
 211         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
 212
 213         BaseMetaClass.to_denormalize.extend(
 214             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
 215         )
 216
 217         return new_cls
 218
 219     @classmethod
 220     def setup_denormalizes(cls):
 221         for new_cls, name, field in BaseMetaClass.to_denormalize:
 222             field.setup_class(new_cls, name)
 223
 224
 225 class BaseModel(models.Model):
 226     __metaclass__ = BaseMetaClass
 227
 228     objects = CachedManager()
 229
 230     class Meta:
 231         abstract = True
 232         app_label = 'forum'
 233
 234     def __init__(self, *args, **kwargs):
 235         super(BaseModel, self).__init__(*args, **kwargs)
 236         self.reset_original_state(kwargs.keys())
 237
 238     def reset_original_state(self, reset_fields=None):
 239         self._original_state = self._as_dict()
 240
 241         if reset_fields:
 242             self._original_state.update(dict([(f, None) for f in reset_fields]))
 243
 244     def get_dirty_fields(self):
 245         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
 246
 247     def _as_dict(self):
 248         return dict([(name, getattr(self, name)) for name in
 249                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
 250         ])
 251
 252     def _get_update_kwargs(self):
 253         return dict([
 254             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
 255         ])
 256
 257     def save(self, full_save=False, *args, **kwargs):
 258         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
 259
 260         if self.id and not full_save:
 261             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
 262         else:
 263             super(BaseModel, self).save()
 264
 265         if put_back:
 266             try:
 267                 self.__dict__.update(
 268                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
 269                 )
 270             except:
 271                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
 272                 self.uncache()
 273
 274         self.reset_original_state()
 275         self._set_query_cache_invalidation_timestamp()
 276         self.cache()
 277
 278     @classmethod
 279     def _get_cache_query_invalidation_key(cls):
 280         return cls._generate_cache_key("INV_TS")
 281
 282     @classmethod
 283     def _set_query_cache_invalidation_timestamp(cls):
 284         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
 285
 286         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
 287             base._set_query_cache_invalidation_timestamp()
 288
 289     @classmethod
 290     def _generate_cache_key(cls, key, group=None):
 291         if group is None:
 292             group = cls.__name__
 293
 294         return '%s:%s:%s' % (settings.APP_URL, group, key)
 295
 296     def cache_key(self):
 297         return self._generate_cache_key(self.id)
 298
 299     @classmethod
 300     def value_to_list_on_cache_query(cls):
 301         return 'id'
 302
 303     @classmethod
 304     def infer_cache_key(cls, querydict):
 305         try:
 306             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
 307                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
 308
 309             return cls._generate_cache_key(pk)
 310         except:
 311             return None
 312
 313     def cache(self):
 314         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
 315
 316     def uncache(self):
 317         cache.delete(self.cache_key())
 318
 319     def delete(self):
 320         self.uncache()
 321         self._set_query_cache_invalidation_timestamp()
 322         super(BaseModel, self).delete()
 323
 324
 325 from user import User
 326 from node import Node, NodeRevision, NodeManager
 327 from action import Action
 328
 329
 330
 331
 332