[IMP] tools.cache now use a LRU to avoid memory overload

bzr revid: chs@openerp.com-20101109165739-1pm8jhckcyzkb2z1

bin/tools/func.py

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 ##############################################################################
-#    
+#
 #    OpenERP, Open Source Management Solution
 #    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+#    Copyright (C) 2010 OpenERP s.a. (<http://openerp.com>).
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as
@@ -15,17 +16,17 @@
 #    GNU Affero General Public License for more details.
 #
 #    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.     
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 ##############################################################################
 
-__all__ = ['partial', 'wraps', 'update_wrapper']
+__all__ = ['partial', 'wraps', 'update_wrapper', 'synchronized']
 
 try:
     from functools import partial, wraps, update_wrapper
 except ImportError:
     # The functools module doesn't exist in python < 2.5
-    # Code taken from python 2.5 
+    # Code taken from python 2.5
     # http://svn.python.org/view/python/tags/r254/Lib/functools.py?view=markup
 
     def partial(fun, *args, **kwargs):
@@ -76,3 +77,19 @@ except ImportError:
         return partial(update_wrapper, wrapped=wrapped,
                        assigned=assigned, updated=updated)
 
+
+
+def synchronized(lock_attr='_lock'):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            lock = getattr(self, lock_attr)
+            try:
+                lock.acquire()
+                return func(self, *args, **kwargs)
+            finally:
+                lock.release()
+        return wrapper
+    return decorator
+
+

bin/tools/lru.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+# taken from http://code.activestate.com/recipes/252524-length-limited-o1-lru-cache-implementation/
+import threading
+from func import synchronized
+
+__all__ = ['LRU']
+
+class LRUNode(object):
+    __slots__ = ['prev', 'next', 'me']
+    def __init__(self, prev, me):
+        self.prev = prev
+        self.me = me
+        self.next = None
+
+class LRU:
+    """
+    Implementation of a length-limited O(1) LRU queue.
+    Built for and used by PyPE:
+    http://pype.sourceforge.net
+    Copyright 2003 Josiah Carlson.
+    """
+    def __init__(self, count, pairs=[]):
+        self._lock = threading.RLock()
+        self.count = max(count, 1)
+        self.d = {}
+        self.first = None
+        self.last = None
+        for key, value in pairs:
+            self[key] = value
+
+    @synchronized()
+    def __contains__(self, obj):
+        return obj in self.d
+
+    @synchronized()
+    def __getitem__(self, obj):
+        a = self.d[obj].me
+        self[a[0]] = a[1]
+        return a[1]
+
+    @synchronized()
+    def __setitem__(self, obj, val):
+        if obj in self.d:
+            del self[obj]
+        nobj = LRUNode(self.last, (obj, val))
+        if self.first is None:
+            self.first = nobj
+        if self.last:
+            self.last.next = nobj
+        self.last = nobj
+        self.d[obj] = nobj
+        if len(self.d) > self.count:
+            if self.first == self.last:
+                self.first = None
+                self.last = None
+                return
+            a = self.first
+            a.next.prev = None
+            self.first = a.next
+            a.next = None
+            del self.d[a.me[0]]
+            del a
+
+    @synchronized()
+    def __delitem__(self, obj):
+        nobj = self.d[obj]
+        if nobj.prev:
+            nobj.prev.next = nobj.next
+        else:
+            self.first = nobj.next
+        if nobj.next:
+            nobj.next.prev = nobj.prev
+        else:
+            self.last = nobj.prev
+        del self.d[obj]
+
+    @synchronized()
+    def __iter__(self):
+        cur = self.first
+        while cur != None:
+            cur2 = cur.next
+            yield cur.me[1]
+            cur = cur2
+
+    @synchronized()
+    def __len__(self):
+        return len(self.d)
+
+    @synchronized()
+    def iteritems(self):
+        cur = self.first
+        while cur != None:
+            cur2 = cur.next
+            yield cur.me
+            cur = cur2
+
+    @synchronized()
+    def iterkeys(self):
+        return iter(self.d)
+
+    @synchronized()
+    def itervalues(self):
+        for i,j in self.iteritems():
+            yield j
+
+    @synchronized()
+    def keys(self):
+        return self.d.keys()
+
+    @synchronized()
+    def pop(self,key):
+        v=self[key]
+        del self[key]
+        return v
+

bin/tools/misc.py

@@ -50,6 +50,7 @@ else:
 
 import netsvc
 from config import config
+from lru import LRU
 
 _logger = logging.getLogger('tools')
 
@@ -696,7 +697,7 @@ class cache(object):
 
     __caches = []
 
-    def __init__(self, timeout=None, skiparg=2, multi=None):
+    def __init__(self, timeout=None, skiparg=2, multi=None, size=8192):
         assert skiparg >= 2 # at least self and cr
         if timeout is None:
             self.timeout = config['cache_timeout']
@@ -705,7 +706,7 @@ class cache(object):
         self.skiparg = skiparg
         self.multi = multi
         self.lasttime = time.time()
-        self.cache = {}
+        self.cache = LRU(size)      # TODO take size from config
         self.fun = None
         cache.__caches.append(self)