forum/models/base.py

   1 import datetime
   2 import re
   3 try:
   4     from hashlib import md5
   5 except:
   6     from md5 import new as md5
   7 from urllib import quote_plus, urlencode
   8 from django.db import models, IntegrityError, connection, transaction
   9 from django.utils.http import urlquote  as django_urlquote
  10 from django.utils.html import strip_tags
  11 from django.conf import settings as django_settings
  12 from django.core.urlresolvers import reverse
  13 from django.contrib.contenttypes import generic
  14 from django.contrib.contenttypes.models import ContentType
  15 from django.core.cache import cache
  16 from django.template.defaultfilters import slugify
  17 from django.db.models.signals import post_delete, post_save, pre_save, pre_delete
  18 from django.utils.translation import ugettext as _
  19 from django.utils.safestring import mark_safe
  20 from django.utils.encoding import force_unicode
  21 from django.contrib.sitemaps import ping_google
  22 import django.dispatch
  23 from forum import settings
  24 import logging
  25
  26
  27 if not hasattr(cache, 'get_many'):
  28     #put django 1.2 code here
  29     pass
  30
  31 class LazyQueryList(object):
  32     def __init__(self, model, items):
  33         self.items = items
  34         self.model = model
  35
  36     def __getitem__(self, k):
  37         return self.model.objects.get(id=self.items[k][0])
  38
  39     def __iter__(self):
  40         for id in self.items:
  41             yield self.model.objects.get(id=id[0])
  42
  43     def __len__(self):
  44         return len(self.items)
  45
  46 class ToFetch(unicode):
  47     pass
  48
  49 class CachedQuerySet(models.query.QuerySet):
  50
  51     def lazy(self):
  52         if not len(self.query.aggregates):
  53             values_list = ['id']
  54
  55             if len(self.query.extra):
  56                 extra_keys = self.query.extra.keys()
  57                 values_list += extra_keys
  58
  59             return LazyQueryList(self.model, list(self.values_list(*values_list)))
  60         else:
  61             return self
  62
  63     def obj_from_datadict(self, datadict):
  64         obj = self.model()
  65         obj.__dict__.update(datadict)
  66
  67         if hasattr(obj, '_state'):
  68             obj._state.db = 'default'
  69
  70         return obj
  71
  72     def _base_clone(self):
  73         return self._clone(klass=models.query.QuerySet)
  74
  75     def get(self, *args, **kwargs):
  76         key = self.model.infer_cache_key(kwargs)
  77
  78         if key is not None:
  79             obj = cache.get(key)
  80
  81             if obj is None:
  82                 obj = self._base_clone().get(*args, **kwargs)
  83                 obj.cache()
  84             else:
  85                 obj = self.obj_from_datadict(obj)
  86
  87             obj.reset_original_state()
  88
  89             return obj
  90
  91         return self._base_clone().get(*args, **kwargs)
  92
  93     def _fetch_from_query_cache(self, key):
  94         invalidation_key = self.model._get_cache_query_invalidation_key()
  95         cached_result = cache.get_many([invalidation_key, key])
  96
  97         if not invalidation_key in cached_result:
  98             self.model._set_query_cache_invalidation_timestamp()
  99             return None
 100
 101         if (key in cached_result) and(cached_result[invalidation_key] < cached_result[key][0]):
 102             return cached_result[key][1]
 103
 104         return None
 105
 106     def count(self):
 107         cache_key = self.model._generate_cache_key("CNT:%s" % self._get_query_hash())
 108         result = self._fetch_from_query_cache(cache_key)
 109
 110         if result is not None:
 111             return result
 112
 113         result = super(CachedQuerySet, self).count()
 114         cache.set(cache_key, (datetime.datetime.now(), result), 60 * 60)
 115         return result
 116
 117     def iterator(self):
 118         cache_key = self.model._generate_cache_key("QUERY:%s" % self._get_query_hash())
 119         on_cache_query_attr = self.model.value_to_list_on_cache_query()
 120
 121         to_return = None
 122         to_cache = {}
 123
 124         with_aggregates = len(self.query.aggregates) > 0
 125         key_list = self._fetch_from_query_cache(cache_key)
 126
 127         if key_list is None:
 128             if not with_aggregates:
 129                 values_list = [on_cache_query_attr]
 130
 131                 if len(self.query.extra):
 132                     values_list += self.query.extra.keys()
 133
 134                 key_list = [v[0] for v in self.values_list(*values_list)]
 135                 to_cache[cache_key] = (datetime.datetime.now(), key_list)
 136             else:
 137                 to_return = list(super(CachedQuerySet, self).iterator())
 138                 to_cache[cache_key] = (datetime.datetime.now(), [
 139                     (row.__dict__[on_cache_query_attr], dict([(k, row.__dict__[k]) for k in self.query.aggregates.keys()]))
 140                     for row in to_return])
 141         elif with_aggregates:
 142             tmp = key_list
 143             key_list = [k[0] for k in tmp]
 144             with_aggregates = [k[1] for k in tmp]
 145             del tmp
 146
 147         if (not to_return) and key_list:
 148             row_keys = [self.model.infer_cache_key({on_cache_query_attr: attr}) for attr in key_list]
 149             cached = cache.get_many(row_keys)
 150
 151             to_return = [
 152                 (ck in cached) and self.obj_from_datadict(cached[ck]) or ToFetch(force_unicode(key_list[i])) for i, ck in enumerate(row_keys)
 153             ]
 154
 155             if len(cached) != len(row_keys):
 156                 to_fetch = [unicode(tr) for tr in to_return if isinstance(tr, ToFetch)]
 157
 158                 fetched = dict([(force_unicode(r.__dict__[on_cache_query_attr]), r) for r in
 159                               models.query.QuerySet(self.model).filter(**{"%s__in" % on_cache_query_attr: to_fetch})])
 160
 161                 to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)] or tr) for tr in to_return]
 162                 to_cache.update(dict([(self.model.infer_cache_key({on_cache_query_attr: attr}), r._as_dict()) for attr, r in fetched.items()]))
 163
 164             if with_aggregates:
 165                 for i, r in enumerate(to_return):
 166                     r.__dict__.update(with_aggregates[i])
 167
 168
 169         if len(to_cache):
 170             cache.set_many(to_cache, 60 * 60)
 171
 172         if to_return:
 173             for row in to_return:
 174                 if hasattr(row, 'leaf'):
 175                     row = row.leaf
 176
 177                 row.reset_original_state()
 178                 yield row
 179
 180     def _get_query_hash(self):
 181         try:
 182             return md5(unicode(self.query).encode("utf-8")).hexdigest()
 183         except:
 184             try:
 185                 return md5(self.query).hexdigest()
 186             except:
 187                 return md5(str(self.query)).hexdigest()
 188
 189
 190 class CachedManager(models.Manager):
 191     use_for_related_fields = True
 192
 193     def get_query_set(self):
 194         return CachedQuerySet(self.model)
 195
 196     def get_or_create(self, *args, **kwargs):
 197         try:
 198             return self.get(*args, **kwargs)
 199         except:
 200             return super(CachedManager, self).get_or_create(*args, **kwargs)
 201
 202
 203 class DenormalizedField(object):
 204     def __init__(self, manager, *args, **kwargs):
 205         self.manager = manager
 206         self.filter = (args, kwargs)
 207
 208     def setup_class(self, cls, name):
 209         dict_name = '_%s_dencache_' % name
 210
 211         def getter(inst):
 212             val = inst.__dict__.get(dict_name, None)
 213
 214             if val is None:
 215                 val = getattr(inst, self.manager).filter(*self.filter[0], **self.filter[1]).count()
 216                 inst.__dict__[dict_name] = val
 217                 inst.cache()
 218
 219             return val
 220
 221         def reset_cache(inst):
 222             inst.__dict__.pop(dict_name, None)
 223             inst.uncache()
 224
 225         cls.add_to_class(name, property(getter))
 226         cls.add_to_class("reset_%s_cache" % name, reset_cache)
 227
 228
 229 class BaseMetaClass(models.Model.__metaclass__):
 230     to_denormalize = []
 231
 232     def __new__(cls, *args, **kwargs):
 233         new_cls = super(BaseMetaClass, cls).__new__(cls, *args, **kwargs)
 234
 235         BaseMetaClass.to_denormalize.extend(
 236             [(new_cls, name, field) for name, field in new_cls.__dict__.items() if isinstance(field, DenormalizedField)]
 237         )
 238
 239         return new_cls
 240
 241     @classmethod
 242     def setup_denormalizes(cls):
 243         for new_cls, name, field in BaseMetaClass.to_denormalize:
 244             field.setup_class(new_cls, name)
 245
 246
 247 class BaseModel(models.Model):
 248     __metaclass__ = BaseMetaClass
 249
 250     objects = CachedManager()
 251
 252     class Meta:
 253         abstract = True
 254         app_label = 'forum'
 255
 256     def __init__(self, *args, **kwargs):
 257         super(BaseModel, self).__init__(*args, **kwargs)
 258         self.reset_original_state(kwargs.keys())
 259
 260     def reset_original_state(self, reset_fields=None):
 261         self._original_state = self._as_dict()
 262
 263         if reset_fields:
 264             self._original_state.update(dict([(f, None) for f in reset_fields]))
 265
 266     def get_dirty_fields(self):
 267         return [f.name for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]]
 268
 269     def _as_dict(self):
 270         return dict([(name, getattr(self, name)) for name in
 271                      ([f.attname for f in self._meta.fields] + [k for k in self.__dict__.keys() if k.endswith('_dencache_')])
 272         ])
 273
 274     def _get_update_kwargs(self):
 275         return dict([
 276             (f.name, getattr(self, f.name)) for f in self._meta.fields if self._original_state[f.attname] != self.__dict__[f.attname]
 277         ])
 278
 279     def save(self, full_save=False, *args, **kwargs):
 280         put_back = [k for k, v in self.__dict__.items() if isinstance(v, models.expressions.ExpressionNode)]
 281
 282         if hasattr(self, '_state'):
 283             self._state.db = 'default'
 284
 285         if self.id and not full_save:
 286             self.__class__.objects.filter(id=self.id).update(**self._get_update_kwargs())
 287         else:
 288             super(BaseModel, self).save()
 289
 290         if put_back:
 291             try:
 292                 self.__dict__.update(
 293                     self.__class__.objects.filter(id=self.id).values(*put_back)[0]
 294                 )
 295             except:
 296                 logging.error("Unable to read %s from %s" % (", ".join(put_back), self.__class__.__name__))
 297                 self.uncache()
 298
 299         self.reset_original_state()
 300         self._set_query_cache_invalidation_timestamp()
 301         self.cache()
 302
 303     @classmethod
 304     def _get_cache_query_invalidation_key(cls):
 305         return cls._generate_cache_key("INV_TS")
 306
 307     @classmethod
 308     def _set_query_cache_invalidation_timestamp(cls):
 309         cache.set(cls._get_cache_query_invalidation_key(), datetime.datetime.now(), 60 * 60 * 24)
 310
 311         for base in filter(lambda c: issubclass(c, BaseModel) and (not c is BaseModel), cls.__bases__):
 312             base._set_query_cache_invalidation_timestamp()
 313
 314     @classmethod
 315     def _generate_cache_key(cls, key, group=None):
 316         if group is None:
 317             group = cls.__name__
 318
 319         return '%s:%s:%s' % (settings.APP_URL, group, key)
 320
 321     def cache_key(self):
 322         return self._generate_cache_key(self.id)
 323
 324     @classmethod
 325     def value_to_list_on_cache_query(cls):
 326         return 'id'
 327
 328     @classmethod
 329     def infer_cache_key(cls, querydict):
 330         try:
 331             pk = [v for (k,v) in querydict.items() if k in ('pk', 'pk__exact', 'id', 'id__exact'
 332                             ) or k.endswith('_ptr__pk') or k.endswith('_ptr__id')][0]
 333
 334             cache_key = cls._generate_cache_key(pk)
 335
 336             if len(cache_key) > django_settings.CACHE_MAX_KEY_LENGTH:
 337                 cache_key = cache_key[:django_settings.CACHE_MAX_KEY_LENGTH]
 338
 339             return cache_key
 340         except:
 341             return None
 342
 343     def cache(self):
 344         cache.set(self.cache_key(), self._as_dict(), 60 * 60)
 345
 346     def uncache(self):
 347         cache.delete(self.cache_key())
 348
 349     def delete(self):
 350         self.uncache()
 351         self._set_query_cache_invalidation_timestamp()
 352         super(BaseModel, self).delete()
 353
 354
 355 from user import User
 356 from node import Node, NodeRevision, NodeManager
 357 from action import Action
 358
 359
 360
 361
 362