Package web2py :: Package gluon :: Module cache |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.cache

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  This file is part of the web2py Web Framework
      6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
      8
      9  Basic caching classes and methods
     10  =================================
     11
     12  - Cache - The generic caching object interfacing with the others
     13  - CacheInRam - providing caching in ram
     14  - CacheInDisk - provides caches on disk
     15
     16  Memcache is also available via a different module (see gluon.contrib.memcache)
     17
     18  When web2py is running on Google App Engine,
     19  caching will be provided by the GAE memcache
     20  (see gluon.contrib.gae_memcache)
     21  """
     22
     23  import time
     24  import portalocker
     25  import shelve
     26  import thread
     27  import os
     28  import logging
     29  import re
     30
     31  logger = logging.getLogger("web2py.cache")
     32
     33  __all__ = ['Cache']
     34
     35
     36  DEFAULT_TIME_EXPIRE = 300
     37
     38
     39 -class CacheAbstract(object):
     40      """
     41      Abstract class for cache implementations.
     42      Main function is now to provide referenced api documentation.
     43
     44      Use CacheInRam or CacheOnDisk instead which are derived from this class.
     45      """
     46
     47      cache_stats_name = 'web2py_cache_statistics'
     48
     49 -    def __init__(self, request=None):
     50          """
     51          Paremeters
     52          ----------
     53          request:
     54              the global request object
     55          """
     56          raise NotImplementedError
     57
     58 -    def __call__(self, key, f,
     59                  time_expire = DEFAULT_TIME_EXPIRE):
     60          """
     61          Tries retrieve the value corresponding to `key` from the cache of the
     62          object exists and if it did not expire, else it called the function `f`
     63          and stores the output in the cache corresponding to `key`. In the case
     64          the output of the function is returned.
     65
     66          :param key: the key of the object to be store or retrieved
     67          :param f: the function, whose output is to be cached
     68          :param time_expire: expiration of the cache in microseconds
     69
     70          - `time_expire` is used to compare the current time with the time when
     71              the requested object was last saved in cache. It does not affect
     72              future requests.
     73          - Setting `time_expire` to 0 or negative value forces the cache to
     74              refresh.
     75
     76          If the function `f` is `None` the cache is cleared.
     77          """
     78          raise NotImplementedError
     79
     80 -    def clear(self, regex=None):
     81          """
     82          Clears the cache of all keys that match the provided regular expression.
     83          If no regular expression is provided, it clears all entries in cache.
     84
     85          Parameters
     86          ----------
     87          regex:
     88              if provided, only keys matching the regex will be cleared.
     89              Otherwise all keys are cleared.
     90          """
     91
     92          raise NotImplementedError
     93
     94 -    def increment(self, key, value=1):
     95          """
     96          Increments the cached value for the given key by the amount in value
     97
     98          Parameters
     99          ----------
    100          key:
    101              key for the cached object to be incremeneted
    102          value:
    103              amount of the increment (defaults to 1, can be negative)
    104          """
    105          raise NotImplementedError
    106
    107 -    def _clear(self, storage, regex):
    108          """
    109          Auxiliary function called by `clear` to search and clear cache entries
    110          """
    111          r = re.compile(regex)
    112          for (key, value) in storage.items():
    113              if r.match(str(key)):
    114                  del storage[key]
    115
    116 -class CacheInRam(CacheAbstract):
    117      """
    118      Ram based caching
    119
    120      This is implemented as global (per process, shared by all threads)
    121      dictionary.
    122      A mutex-lock mechanism avoid conflicts.
    123      """
    124
    125      locker = thread.allocate_lock()
    126      meta_storage = {}
    127
    128 -    def __init__(self, request=None):
    129          self.locker.acquire()
    130          self.request = request
    131          if request:
    132              app = request.application
    133          else:
    134              app = ''
    135          if not app in self.meta_storage:
    136              self.storage = self.meta_storage[app] = {CacheAbstract.cache_stats_name: {
    137                  'hit_total': 0,
    138                  'misses': 0,
    139              }}
    140          else:
    141              self.storage = self.meta_storage[app]
    142          self.locker.release()
    143
    144 -    def clear(self, regex=None):
    145          self.locker.acquire()
    146          storage = self.storage
    147          if regex == None:
    148              storage.clear()
    149          else:
    150              self._clear(storage, regex)
    151
    152          if not CacheAbstract.cache_stats_name in storage.keys():
    153              storage[CacheAbstract.cache_stats_name] = {
    154                  'hit_total': 0,
    155                  'misses': 0,
    156              }
    157
    158          self.locker.release()
    159
    160 -    def __call__(self, key, f,
    161                  time_expire = DEFAULT_TIME_EXPIRE):
    162          """
    163          Attention! cache.ram does not copy the cached object. It just stores a reference to it.
    164          Turns out the deepcopying the object has some problems:
    165          1) would break backward compatibility
    166          2) would be limiting because people may want to cache live objects
    167          3) would work unless we deepcopy no storage and retrival which would make things slow.
    168          Anyway. You can deepcopy explicitly in the function generating the value to be cached.
    169          """
    170
    171          dt = time_expire
    172
    173          self.locker.acquire()
    174          item = self.storage.get(key, None)
    175          if item and f == None:
    176              del self.storage[key]
    177          self.storage[CacheAbstract.cache_stats_name]['hit_total'] += 1
    178          self.locker.release()
    179
    180          if f is None:
    181              return None
    182          if item and (dt == None or item[0] > time.time() - dt):
    183              return item[1]
    184          value = f()
    185
    186          self.locker.acquire()
    187          self.storage[key] = (time.time(), value)
    188          self.storage[CacheAbstract.cache_stats_name]['misses'] += 1
    189          self.locker.release()
    190          return value
    191
    192 -    def increment(self, key, value=1):
    193          self.locker.acquire()
    194          try:
    195              if key in self.storage:
    196                  value = self.storage[key][1] + value
    197              self.storage[key] = (time.time(), value)
    198          except BaseException, e:
    199              self.locker.release()
    200              raise e
    201          self.locker.release()
    202          return value
    203
    204
    205 -class CacheOnDisk(CacheAbstract):
    206      """
    207      Disk based cache
    208
    209      This is implemented as a shelve object and it is shared by multiple web2py
    210      processes (and threads) as long as they share the same filesystem.
    211      The file is locked wen accessed.
    212
    213      Disk cache provides persistance when web2py is started/stopped but it slower
    214      than `CacheInRam`
    215
    216      Values stored in disk cache must be pickable.
    217      """
    218
    219      speedup_checks = set()
    220
    221 -    def __init__(self, request, folder=None):
    222          self.request = request
    223
    224          # Lets test if the cache folder exists, if not
    225          # we are going to create it
    226          folder = folder or os.path.join(request.folder, 'cache')
    227
    228          if not os.path.exists(folder):
    229              os.mkdir(folder)
    230
    231          ### we need this because of a possible bug in shelve that may
    232          ### or may not lock
    233          self.locker_name = os.path.join(folder,'cache.lock')
    234          self.shelve_name = os.path.join(folder,'cache.shelve')
    235
    236          locker, locker_locked = None, False
    237          speedup_key = (folder,CacheAbstract.cache_stats_name)
    238          if not speedup_key in self.speedup_checks or \
    239                  not os.path.exists(self.shelve_name):
    240              try:
    241                  locker = open(self.locker_name, 'a')
    242                  portalocker.lock(locker, portalocker.LOCK_EX)
    243                  locker_locked = True
    244                  storage = shelve.open(self.shelve_name)
    245                  try:
    246                      if not storage.has_key(CacheAbstract.cache_stats_name):
    247                          storage[CacheAbstract.cache_stats_name] = {
    248                              'hit_total': 0,
    249                              'misses': 0,
    250                              }
    251                          storage.sync()
    252                  finally:
    253                      storage.close()
    254                  self.speedup_checks.add(speedup_key)
    255              except ImportError:
    256                  pass # no module _bsddb, ignoring exception now so it makes a ticket only if used
    257              except:
    258                  logger.error('corrupted file %s, will try delete it!' \
    259                                   % self.shelve_name)
    260                  try:
    261                      os.unlink(self.shelve_name)
    262                  except IOError:
    263                      logger.warn('unable to delete file %s' % self.shelve_name)
    264              if locker_locked:
    265                  portalocker.unlock(locker)
    266              if locker:
    267                  locker.close()
    268
    269 -    def clear(self, regex=None):
    270          locker = open(self.locker_name,'a')
    271          portalocker.lock(locker, portalocker.LOCK_EX)
    272          storage = shelve.open(self.shelve_name)
    273          try:
    274              if regex == None:
    275                  storage.clear()
    276              else:
    277                  self._clear(storage, regex)
    278              if not CacheAbstract.cache_stats_name in storage.keys():
    279                  storage[CacheAbstract.cache_stats_name] = {
    280                      'hit_total': 0,
    281                      'misses': 0,
    282                  }
    283              storage.sync()
    284          finally:
    285              storage.close()
    286          portalocker.unlock(locker)
    287          locker.close()
    288
    289 -    def __call__(self, key, f,
    290                  time_expire = DEFAULT_TIME_EXPIRE):
    291          dt = time_expire
    292
    293          locker = open(self.locker_name,'a')
    294          portalocker.lock(locker, portalocker.LOCK_EX)
    295          storage = shelve.open(self.shelve_name)
    296
    297          item = storage.get(key, None)
    298          if item and f == None:
    299              del storage[key]
    300
    301          storage[CacheAbstract.cache_stats_name] = {
    302              'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1,
    303              'misses': storage[CacheAbstract.cache_stats_name]['misses']
    304          }
    305
    306          storage.sync()
    307
    308          portalocker.unlock(locker)
    309          locker.close()
    310
    311          if f is None:
    312              return None
    313          if item and (dt == None or item[0] > time.time() - dt):
    314              return item[1]
    315          value = f()
    316
    317          locker = open(self.locker_name,'a')
    318          portalocker.lock(locker, portalocker.LOCK_EX)
    319          storage[key] = (time.time(), value)
    320
    321          storage[CacheAbstract.cache_stats_name] = {
    322              'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'],
    323              'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1
    324          }
    325
    326          storage.sync()
    327
    328          storage.close()
    329          portalocker.unlock(locker)
    330          locker.close()
    331
    332          return value
    333
    334 -    def increment(self, key, value=1):
    335          locker = open(self.locker_name,'a')
    336          portalocker.lock(locker, portalocker.LOCK_EX)
    337          storage = shelve.open(self.shelve_name)
    338          try:
    339              if key in storage:
    340                  value = storage[key][1] + value
    341              storage[key] = (time.time(), value)
    342              storage.sync()
    343          finally:
    344              storage.close()
    345              portalocker.unlock(locker)
    346              locker.close()
    347          return value
    348
    349
    350 -class Cache(object):
    351      """
    352      Sets up generic caching, creating an instance of both CacheInRam and
    353      CacheOnDisk.
    354      In case of GAE will make use of gluon.contrib.gae_memcache.
    355
    356      - self.ram is an instance of CacheInRam
    357      - self.disk is an instance of CacheOnDisk
    358      """
    359
    360 -    def __init__(self, request):
    361          """
    362          Parameters
    363          ----------
    364          request:
    365              the global request object
    366          """
    367          # GAE will have a special caching
    368          import settings
    369          if settings.global_settings.web2py_runtime_gae:
    370              from contrib.gae_memcache import MemcacheClient
    371              self.ram=self.disk=MemcacheClient(request)
    372          else:
    373              # Otherwise use ram (and try also disk)
    374              self.ram = CacheInRam(request)
    375              try:
    376                  self.disk = CacheOnDisk(request)
    377              except IOError:
    378                  logger.warning('no cache.disk (IOError)')
    379              except AttributeError:
    380                  # normally not expected anymore, as GAE has already
    381                  # been accounted for
    382                  logger.warning('no cache.disk (AttributeError)')
    383
    384 -    def __call__(self,
    385                  key = None,
    386                  time_expire = DEFAULT_TIME_EXPIRE,
    387                  cache_model = None):
    388          """
    389          Decorator function that can be used to cache any function/method.
    390
    391          Example::
    392
    393              @cache('key', 5000, cache.ram)
    394              def f():
    395                  return time.ctime()
    396
    397          When the function f is called, web2py tries to retrieve
    398          the value corresponding to `key` from the cache of the
    399          object exists and if it did not expire, else it calles the function `f`
    400          and stores the output in the cache corresponding to `key`. In the case
    401          the output of the function is returned.
    402
    403          :param key: the key of the object to be store or retrieved
    404          :param time_expire: expiration of the cache in microseconds
    405          :param cache_model: `cache.ram`, `cache.disk`, or other
    406              (like `cache.memcache` if defined). It defaults to `cache.ram`.
    407
    408          Notes
    409          -----
    410          `time_expire` is used to compare the curret time with the time when the
    411          requested object was last saved in cache. It does not affect future
    412          requests.
    413          Setting `time_expire` to 0 or negative value forces the cache to
    414          refresh.
    415
    416          If the function `f` is an action, we suggest using
    417          `request.env.path_info` as key.
    418          """
    419          if not cache_model:
    420              cache_model = self.ram
    421
    422          def tmp(func):
    423              def action():
    424                  return cache_model(key, func, time_expire)
    425              action.__name___ = func.__name__
    426              action.__doc__ = func.__doc__
    427              return action
    428
    429          return tmp
    430