forum/models/base.py

   1 import datetime
   2 import re
   3 try:
   4     from hashlib import md5
   5 except:
   6     from md5 import new as md5
   7 from urllib import quote_plus, urlencode
   8 from django.db import models, IntegrityError, connection, transaction
   9 from django.utils.http import urlquote  as django_urlquote
  10 from django.utils.html import strip_tags
  11 from django.core.urlresolvers import reverse
  12 from django.contrib.contenttypes import generic
  13 from django.contrib.contenttypes.models import ContentType
  14 from django.core.cache import cache
  15 from django.template.defaultfilters import slugify
  16 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
  17 from django.utils.translation import ugettext as _
  18 from django.utils.safestring import mark_safe
  19 from django.contrib.sitemaps import ping_google
  20 import django.dispatch
  21 from forum import settings
  22 import logging
  23
  24
  25 if not hasattr(cache, 'get_many'):
  26     #put django 1.2 code here
  27     pass
  28
  29 class LazyQueryList(object):
  30     def __init__(self, model, items):
  31         self.items = items
  32         self.model = model
  33
  34     def __getitem__(self, k):
  35         return self.model.objects.get(id=self.items[k][0])
  36
  37     def __iter__(self):
  38         for id in self.items:
  39             yield self.model.objects.get(id=id[0])
  40
  41     def __len__(self):
  42         return len(self.items)
  43
  44 class ToFetch(str):
  45     pass
  46
  47 class CachedQuerySet(models.query.QuerySet):
  48
  49     def lazy(self):
  50         if not len(self.query.aggregates):
  51             values_list = ['id']
  52
  53             if len(self.query.extra):
  54                 extra_keys = self.query.extra.keys()
  55                 values_list += extra_keys
  56
  57             return LazyQueryList(self.model, list(self.values_list(*values_list)))
  58         else:
  59             return self
  60
  61     def obj_from_datadict(self, datadict):
  62         obj = self.model()
  63         obj.__dict__.update(datadict)
  64
  65         if hasattr(obj, '_state'):
  66             obj._state.db = 'default'
  67
  68         return obj
  69
  70     def _base_clone(self):
  71         return self._clone(klass=models.query.QuerySet)
  72
  73     def get(self, *args, **kwargs):
  74         key = self.model.infer_cache_key(kwargs)
  75
  76         if key is not None:
  77             obj = cache.get(key)
  78
  79             if obj is None:
  80                 obj = self._base_clone().get(*args, **kwargs)
  81                 obj.cache()
  82             else:
  83                 obj = self.obj_from_datadict(obj)
  84                 obj.reset_original_state()
  85
  86             return obj
  87
  88         return self._base_clone().get(*args, **kwargs)
  89
  90     def _fetch_from_query_cache(self, key):
  91         invalidation_key = self.model._get_cache_query_invalidation_key()
  92         cached_result = cache.get_many([invalidation_key, key])
  93
  94         if not invalidation_key in cached_result:
  95             self.model._set_query_cache_invalidation_timestamp()
  96             return None
  97
  98         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
  99             return cached_result[key][1]
 100
 101         return None
 102
 103     def count(self):
 104         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
 105         result = self._fetch_from_query_cache(cache_key)
 106
 107         if result is not None:
 108             return result
 109
 110         result = super(CachedQuerySet, self).count()
 111         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
 112         return result
 113
 114     def iterator(self):
 115         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
 116         on_cache_query_attr = self.model.value_to_list_on_cache_query()
 117
 118         to_return = None
 119         to_cache = {}
 120
 121         key_list = self._fetch_from_query_cache(cache_key)
 122
 123         if key_list is None:
 124             if not len(self.query.aggregates):
 125                 values_list = [on_cache_query_attr]
 126
 127                 if len(self.query.extra):
 128                     values_list += self.query.extra.keys()
 129
 130                 key_list = [v[0] for v in self.values_list(*values_list)]
 131                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
 132             else:
 133                 to_return = list(super(CachedQuerySet, self).iterator())
 134                 to_cache[cache_key] = (datetime.datetime.now(), [row.__dict__[on_cache_query_attr] for row in to_return])
 135
 136         if (not to_return) and key_list:
 137             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
 138             cached = cache.get_many(row_keys)
 139
 140             to_return = [
 141                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(key_list[i]) for i, ck in enumerate(row_keys)
 142             ]
 143
 144             if len(cached) != len(row_keys):
 145                 to_fetch = [str(tr) for tr in to_return if isinstance(tr, ToFetch)]
 146
 147                 fetched = dict([(str(r.__dict__[on_cache_query_attr]), r) for r in
 148                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
 149
 150                 to_return = [(isinstance(tr, ToFetch) and fetched[str(tr)] or tr) for tr in to_return]
 151                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
 152
 153
 154         if len(to_cache):
 155             cache.set_many(to_cache, 60 * 60)
 156
 157         if to_return:
 158             for row in to_return:
 159                 if hasattr(row, 'leaf'):
 160                     yield row.leaf
 161                 else:
 162                     yield row
 163
 164     def _get_query_hash(self):
 165         return md5(str(self.query)).hexdigest()
 166
 167
 168
 169 class CachedManager(models.Manager):
 170     use_for_related_fields = True
 171
 172     def get_query_set(self):
 173         return CachedQuerySet(self.model)
 174
 175     def get_or_create(self, *args, **kwargs):
 176         try:
 177             return self.get(*args, **kwargs)
 178         except:
 179             return super(CachedManager, self).get_or_create(*args, **kwargs)
 180
 181
 182 class DenormalizedField(object):
 183     def __init__(self, manager, *args, **kwargs):
 184         self.manager = manager
 185         self.filter = (args, kwargs)
 186
 187     def setup_class(self, cls, name):
 188         dict_name = '_%s_dencache_' % name
 189
 190         def getter(inst):
 191             val = inst.__dict__.get(dict_name, None)
 192
 193             if val is None:
 194                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
 195                 inst.__dict__[dict_name] = val
 196                 inst.cache()
 197
 198             return val
 199
 200         def reset_cache(inst):
 201             inst.__dict__.pop(dict_name, None)
 202             inst.uncache()
 203
 204         cls.add_to_class(name, property(getter))
 205         cls.add_to_class("reset_%s_cache" % name, reset_cache)
 206
 207
 208 class BaseMetaClass(models.Model.__metaclass__):
 209     to_denormalize = []
 210
 211     def __new__(cls, *args, **kwargs):
 212         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
 213
 214         BaseMetaClass.to_denormalize.extend(
 215             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
 216         )
 217
 218         return new_cls
 219
 220     @classmethod
 221     def setup_denormalizes(cls):
 222         for new_cls, name, field in BaseMetaClass.to_denormalize:
 223             field.setup_class(new_cls, name)
 224
 225
 226 class BaseModel(models.Model):
 227     __metaclass__ = BaseMetaClass
 228
 229     objects = CachedManager()
 230
 231     class Meta:
 232         abstract = True
 233         app_label = 'forum'
 234
 235     def __init__(self, *args, **kwargs):
 236         super(BaseModel, self).__init__(*args, **kwargs)
 237         self.reset_original_state(kwargs.keys())
 238
 239     def reset_original_state(self, reset_fields=None):
 240         self._original_state = self._as_dict()
 241
 242         if reset_fields:
 243             self._original_state.update(dict([(f, None) for f in reset_fields]))
 244
 245     def get_dirty_fields(self):
 246         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
 247
 248     def _as_dict(self):
 249         return dict([(name, getattr(self, name)) for name in
 250                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
 251         ])
 252
 253     def _get_update_kwargs(self):
 254         return dict([
 255             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
 256         ])
 257
 258     def save(self, full_save=False, *args, **kwargs):
 259         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
 260
 261         if hasattr(self, '_state'):
 262             self._state.db = 'default'
 263
 264         if self.id and not full_save:
 265             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
 266         else:
 267             super(BaseModel, self).save()
 268
 269         if put_back:
 270             try:
 271                 self.__dict__.update(
 272                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
 273                 )
 274             except:
 275                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
 276                 self.uncache()
 277
 278         self.reset_original_state()
 279         self._set_query_cache_invalidation_timestamp()
 280         self.cache()
 281
 282     @classmethod
 283     def _get_cache_query_invalidation_key(cls):
 284         return cls._generate_cache_key("INV_TS")
 285
 286     @classmethod
 287     def _set_query_cache_invalidation_timestamp(cls):
 288         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
 289
 290         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
 291             base._set_query_cache_invalidation_timestamp()
 292
 293     @classmethod
 294     def _generate_cache_key(cls, key, group=None):
 295         if group is None:
 296             group = cls.__name__
 297
 298         return '%s:%s:%s' % (settings.APP_URL, group, key)
 299
 300     def cache_key(self):
 301         return self._generate_cache_key(self.id)
 302
 303     @classmethod
 304     def value_to_list_on_cache_query(cls):
 305         return 'id'
 306
 307     @classmethod
 308     def infer_cache_key(cls, querydict):
 309         try:
 310             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
 311                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
 312
 313             return cls._generate_cache_key(pk)
 314         except:
 315             return None
 316
 317     def cache(self):
 318         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
 319
 320     def uncache(self):
 321         cache.delete(self.cache_key())
 322
 323     def delete(self):
 324         self.uncache()
 325         self._set_query_cache_invalidation_timestamp()
 326         super(BaseModel, self).delete()
 327
 328
 329 from user import User
 330 from node import Node, NodeRevision, NodeManager
 331 from action import Action
 332
 333
 334
 335
 336