forum/models/base.py

   1 import datetime
   2 import re
   3 try:
   4     from hashlib import md5
   5 except:
   6     from md5 import new as md5
   7 from urllib import quote_plus, urlencode
   8 from django.db import models, IntegrityError, connection, transaction
   9 from django.utils.http import urlquote  as django_urlquote
  10 from django.utils.html import strip_tags
  11 from django.conf import settings as django_settings
  12 from django.core.urlresolvers import reverse
  13 from django.contrib.contenttypes import generic
  14 from django.contrib.contenttypes.models import ContentType
  15 from django.core.cache import cache
  16 from django.template.defaultfilters import slugify
  17 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
  18 from django.utils.translation import ugettext as _
  19 from django.utils.safestring import mark_safe
  20 from django.utils.encoding import force_unicode
  21 from django.contrib.sitemaps import ping_google
  22 import django.dispatch
  23 from forum import settings
  24 import logging
  25
  26
  27 if not hasattr(cache, 'get_many'):
  28     #put django 1.2 code here
  29     pass
  30
  31 class LazyQueryList(object):
  32     def __init__(self, model, items):
  33         self.items = items
  34         self.model = model
  35
  36     def __getitem__(self, k):
  37         return self.model.objects.get(id=self.items[k][0])
  38
  39     def __iter__(self):
  40         for id in self.items:
  41             yield self.model.objects.get(id=id[0])
  42
  43     def __len__(self):
  44         return len(self.items)
  45
  46 class ToFetch(unicode):
  47     pass
  48
  49 class CachedQuerySet(models.query.QuerySet):
  50
  51     def lazy(self):
  52         if not len(self.query.aggregates):
  53             values_list = ['id']
  54
  55             if len(self.query.extra):
  56                 extra_keys = self.query.extra.keys()
  57                 values_list += extra_keys
  58
  59             return LazyQueryList(self.model, list(self.values_list(*values_list)))
  60         else:
  61             return self
  62
  63     def obj_from_datadict(self, datadict):
  64         obj = self.model()
  65         obj.__dict__.update(datadict)
  66
  67         if hasattr(obj, '_state'):
  68             obj._state.db = 'default'
  69
  70         return obj
  71
  72     def _base_clone(self):
  73         return self._clone(klass=models.query.QuerySet)
  74
  75     def get(self, *args, **kwargs):
  76         key = self.model.infer_cache_key(kwargs)
  77
  78         if key is not None:
  79             obj = cache.get(key)
  80
  81             if obj is None:
  82                 obj = self._base_clone().get(*args, **kwargs)
  83                 obj.cache()
  84             else:
  85                 obj = self.obj_from_datadict(obj)
  86                 obj.reset_original_state()
  87
  88             return obj
  89
  90         return self._base_clone().get(*args, **kwargs)
  91
  92     def _fetch_from_query_cache(self, key):
  93         invalidation_key = self.model._get_cache_query_invalidation_key()
  94         cached_result = cache.get_many([invalidation_key, key])
  95
  96         if not invalidation_key in cached_result:
  97             self.model._set_query_cache_invalidation_timestamp()
  98             return None
  99
 100         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
 101             return cached_result[key][1]
 102
 103         return None
 104
 105     def count(self):
 106         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
 107         result = self._fetch_from_query_cache(cache_key)
 108
 109         if result is not None:
 110             return result
 111
 112         result = super(CachedQuerySet, self).count()
 113         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
 114         return result
 115
 116     def iterator(self):
 117         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
 118         on_cache_query_attr = self.model.value_to_list_on_cache_query()
 119
 120         to_return = None
 121         to_cache = {}
 122
 123         with_aggregates = len(self.query.aggregates) > 0
 124         key_list = self._fetch_from_query_cache(cache_key)
 125
 126         if key_list is None:
 127             if not with_aggregates:
 128                 values_list = [on_cache_query_attr]
 129
 130                 if len(self.query.extra):
 131                     values_list += self.query.extra.keys()
 132
 133                 key_list = [v[0] for v in self.values_list(*values_list)]
 134                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
 135             else:
 136                 to_return = list(super(CachedQuerySet, self).iterator())
 137                 to_cache[cache_key] = (datetime.datetime.now(), [
 138                     (row.__dict__[on_cache_query_attr], dict([(k, row.__dict__[k]) for k in self.query.aggregates.keys()]))
 139                     for row in to_return])
 140         elif with_aggregates:
 141             tmp = key_list
 142             key_list = [k[0] for k in tmp]
 143             with_aggregates = [k[1] for k in tmp]
 144             del tmp
 145
 146         if (not to_return) and key_list:
 147             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
 148             cached = cache.get_many(row_keys)
 149
 150             to_return = [
 151                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(force_unicode(key_list[i])) for i, ck in enumerate(row_keys)
 152             ]
 153
 154             if len(cached) != len(row_keys):
 155                 to_fetch = [unicode(tr) for tr in to_return if isinstance(tr, ToFetch)]
 156
 157                 fetched = dict([(force_unicode(r.__dict__[on_cache_query_attr]), r) for r in
 158                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
 159
 160                 to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)] or tr) for tr in to_return]
 161                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
 162
 163             if with_aggregates:
 164                 for i, r in enumerate(to_return):
 165                     r.__dict__.update(with_aggregates[i])
 166
 167
 168         if len(to_cache):
 169             cache.set_many(to_cache, 60 * 60)
 170
 171         if to_return:
 172             for row in to_return:
 173                 if hasattr(row, 'leaf'):
 174                     yield row.leaf
 175                 else:
 176                     yield row
 177
 178     def _get_query_hash(self):
 179         return md5(unicode(self.query).encode("utf-8")).hexdigest()
 180
 181
 182
 183 class CachedManager(models.Manager):
 184     use_for_related_fields = True
 185
 186     def get_query_set(self):
 187         return CachedQuerySet(self.model)
 188
 189     def get_or_create(self, *args, **kwargs):
 190         try:
 191             return self.get(*args, **kwargs)
 192         except:
 193             return super(CachedManager, self).get_or_create(*args, **kwargs)
 194
 195
 196 class DenormalizedField(object):
 197     def __init__(self, manager, *args, **kwargs):
 198         self.manager = manager
 199         self.filter = (args, kwargs)
 200
 201     def setup_class(self, cls, name):
 202         dict_name = '_%s_dencache_' % name
 203
 204         def getter(inst):
 205             val = inst.__dict__.get(dict_name, None)
 206
 207             if val is None:
 208                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
 209                 inst.__dict__[dict_name] = val
 210                 inst.cache()
 211
 212             return val
 213
 214         def reset_cache(inst):
 215             inst.__dict__.pop(dict_name, None)
 216             inst.uncache()
 217
 218         cls.add_to_class(name, property(getter))
 219         cls.add_to_class("reset_%s_cache" % name, reset_cache)
 220
 221
 222 class BaseMetaClass(models.Model.__metaclass__):
 223     to_denormalize = []
 224
 225     def __new__(cls, *args, **kwargs):
 226         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
 227
 228         BaseMetaClass.to_denormalize.extend(
 229             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
 230         )
 231
 232         return new_cls
 233
 234     @classmethod
 235     def setup_denormalizes(cls):
 236         for new_cls, name, field in BaseMetaClass.to_denormalize:
 237             field.setup_class(new_cls, name)
 238
 239
 240 class BaseModel(models.Model):
 241     __metaclass__ = BaseMetaClass
 242
 243     objects = CachedManager()
 244
 245     class Meta:
 246         abstract = True
 247         app_label = 'forum'
 248
 249     def __init__(self, *args, **kwargs):
 250         super(BaseModel, self).__init__(*args, **kwargs)
 251         self.reset_original_state(kwargs.keys())
 252
 253     def reset_original_state(self, reset_fields=None):
 254         self._original_state = self._as_dict()
 255
 256         if reset_fields:
 257             self._original_state.update(dict([(f, None) for f in reset_fields]))
 258
 259     def get_dirty_fields(self):
 260         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
 261
 262     def _as_dict(self):
 263         return dict([(name, getattr(self, name)) for name in
 264                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
 265         ])
 266
 267     def _get_update_kwargs(self):
 268         return dict([
 269             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
 270         ])
 271
 272     def save(self, full_save=False, *args, **kwargs):
 273         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
 274
 275         if hasattr(self, '_state'):
 276             self._state.db = 'default'
 277
 278         if self.id and not full_save:
 279             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
 280         else:
 281             super(BaseModel, self).save()
 282
 283         if put_back:
 284             try:
 285                 self.__dict__.update(
 286                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
 287                 )
 288             except:
 289                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
 290                 self.uncache()
 291
 292         self.reset_original_state()
 293         self._set_query_cache_invalidation_timestamp()
 294         self.cache()
 295
 296     @classmethod
 297     def _get_cache_query_invalidation_key(cls):
 298         return cls._generate_cache_key("INV_TS")
 299
 300     @classmethod
 301     def _set_query_cache_invalidation_timestamp(cls):
 302         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
 303
 304         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
 305             base._set_query_cache_invalidation_timestamp()
 306
 307     @classmethod
 308     def _generate_cache_key(cls, key, group=None):
 309         if group is None:
 310             group = cls.__name__
 311
 312         return '%s:%s:%s' % (settings.APP_URL, group, key)
 313
 314     def cache_key(self):
 315         return self._generate_cache_key(self.id)
 316
 317     @classmethod
 318     def value_to_list_on_cache_query(cls):
 319         return 'id'
 320
 321     @classmethod
 322     def infer_cache_key(cls, querydict):
 323         try:
 324             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
 325                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
 326
 327             cache_key = cls._generate_cache_key(pk)
 328
 329             if len(cache_key) > django_settings.CACHE_MAX_KEY_LENGTH:
 330                 cache_key = cache_key[:django_settings.CACHE_MAX_KEY_LENGTH]
 331
 332             return cache_key
 333         except:
 334             return None
 335
 336     def cache(self):
 337         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
 338
 339     def uncache(self):
 340         cache.delete(self.cache_key())
 341
 342     def delete(self):
 343         self.uncache()
 344         self._set_query_cache_invalidation_timestamp()
 345         super(BaseModel, self).delete()
 346
 347
 348 from user import User
 349 from node import Node, NodeRevision, NodeManager
 350 from action import Action
 351
 352
 353
 354
 355