1 "File-based cache backend"

     2 

     3 import os

     4 import time

     5 import shutil

     6 try:

     7     import cPickle as pickle

     8 except ImportError:

     9     import pickle

    10 

    11 from django.core.cache.backends.base import BaseCache

    12 from django.utils.hashcompat import md5_constructor

    13 

    14 class CacheClass(BaseCache):

    15     def __init__(self, dir, params):

    16         BaseCache.__init__(self, params)

    17 

    18         max_entries = params.get('max_entries', 300)

    19         try:

    20             self._max_entries = int(max_entries)

    21         except (ValueError, TypeError):

    22             self._max_entries = 300

    23 

    24         cull_frequency = params.get('cull_frequency', 3)

    25         try:

    26             self._cull_frequency = int(cull_frequency)

    27         except (ValueError, TypeError):

    28             self._cull_frequency = 3

    29 

    30         self._dir = dir

    31         if not os.path.exists(self._dir):

    32             self._createdir()

    33 

    34     def add(self, key, value, timeout=None):

    35         if self.has_key(key):

    36             return False

    37 

    38         self.set(key, value, timeout)

    39         return True

    40 

    41     def get(self, key, default=None):

    42         fname = self._key_to_file(key)

    43         try:

    44             f = open(fname, 'rb')

    45             try:

    46                 exp = pickle.load(f)

    47                 now = time.time()

    48                 if exp < now:

    49                     self._delete(fname)

    50                 else:

    51                     return pickle.load(f)

    52             finally:

    53                 f.close()

    54         except (IOError, OSError, EOFError, pickle.PickleError):

    55             pass

    56         return default

    57 

    58     def set(self, key, value, timeout=None):

    59         fname = self._key_to_file(key)

    60         dirname = os.path.dirname(fname)

    61 

    62         if timeout is None:

    63             timeout = self.default_timeout

    64 

    65         self._cull()

    66 

    67         try:

    68             if not os.path.exists(dirname):

    69                 os.makedirs(dirname)

    70 

    71             f = open(fname, 'wb')

    72             try:

    73                 now = time.time()

    74                 pickle.dump(now + timeout, f, pickle.HIGHEST_PROTOCOL)

    75                 pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)

    76             finally:

    77                 f.close()

    78         except (IOError, OSError):

    79             pass

    80 

    81     def delete(self, key):

    82         try:

    83             self._delete(self._key_to_file(key))

    84         except (IOError, OSError):

    85             pass

    86 

    87     def _delete(self, fname):

    88         os.remove(fname)

    89         try:

    90             # Remove the 2 subdirs if they're empty

    91             dirname = os.path.dirname(fname)

    92             os.rmdir(dirname)

    93             os.rmdir(os.path.dirname(dirname))

    94         except (IOError, OSError):

    95             pass

    96 

    97     def has_key(self, key):

    98         fname = self._key_to_file(key)

    99         try:

   100             f = open(fname, 'rb')

   101             try:

   102                 exp = pickle.load(f)

   103                 now = time.time()

   104                 if exp < now:

   105                     self._delete(fname)

   106                     return False

   107                 else:

   108                     return True

   109             finally:

   110                 f.close()

   111         except (IOError, OSError, EOFError, pickle.PickleError):

   112             return False

   113 

   114     def _cull(self):

   115         if int(self._num_entries) < self._max_entries:

   116             return

   117 

   118         try:

   119             filelist = os.listdir(self._dir)

   120         except (IOError, OSError):

   121             return

   122 

   123         if self._cull_frequency == 0:

   124             doomed = filelist

   125         else:

   126             doomed = [os.path.join(self._dir, k) for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0]

   127 

   128         for topdir in doomed:

   129             try:

   130                 for root, _, files in os.walk(topdir):

   131                     for f in files:

   132                         self._delete(os.path.join(root, f))

   133             except (IOError, OSError):

   134                 pass

   135 

   136     def _createdir(self):

   137         try:

   138             os.makedirs(self._dir)

   139         except OSError:

   140             raise EnvironmentError("Cache directory '%s' does not exist and could not be created'" % self._dir)

   141 

   142     def _key_to_file(self, key):

   143         """

   144         Convert the filename into an md5 string. We'll turn the first couple

   145         bits of the path into directory prefixes to be nice to filesystems

   146         that have problems with large numbers of files in a directory.

   147 

   148         Thus, a cache key of "foo" gets turnned into a file named

   149         ``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.

   150         """

   151         path = md5_constructor(key.encode('utf-8')).hexdigest()

   152         path = os.path.join(path[:2], path[2:4], path[4:])

   153         return os.path.join(self._dir, path)

   154 

   155     def _get_num_entries(self):

   156         count = 0

   157         for _,_,files in os.walk(self._dir):

   158             count += len(files)

   159         return count

   160     _num_entries = property(_get_num_entries)

   161 

   162     def clear(self):

   163         try:

   164             shutil.rmtree(self._dir)

   165         except (IOError, OSError):

   166             pass