From ad4f3dfc0b4a23088bdfec7b1e2d458f86993fb7 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Tue, 9 Nov 2010 17:57:39 +0100 Subject: [PATCH] [IMP] tools.cache now use a LRU to avoid memory overload bzr revid: chs@openerp.com-20101109165739-1pm8jhckcyzkb2z1 --- bin/tools/func.py | 25 ++++++++-- bin/tools/lru.py | 115 ++++++++++++++++++++++++++++++++++++++++++++++ bin/tools/misc.py | 5 +- 3 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 bin/tools/lru.py diff --git a/bin/tools/func.py b/bin/tools/func.py index 3168f5e6935..8cedf6715fd 100644 --- a/bin/tools/func.py +++ b/bin/tools/func.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- ############################################################################## -# +# # OpenERP, Open Source Management Solution # Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010 OpenERP s.a. (). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -15,17 +16,17 @@ # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . +# along with this program. If not, see . # ############################################################################## -__all__ = ['partial', 'wraps', 'update_wrapper'] +__all__ = ['partial', 'wraps', 'update_wrapper', 'synchronized'] try: from functools import partial, wraps, update_wrapper except ImportError: # The functools module doesn't exist in python < 2.5 - # Code taken from python 2.5 + # Code taken from python 2.5 # http://svn.python.org/view/python/tags/r254/Lib/functools.py?view=markup def partial(fun, *args, **kwargs): @@ -76,3 +77,19 @@ except ImportError: return partial(update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated) + + +def synchronized(lock_attr='_lock'): + def decorator(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + lock = getattr(self, lock_attr) + try: + lock.acquire() + return func(self, *args, **kwargs) + finally: + lock.release() + return wrapper + return decorator + + diff --git a/bin/tools/lru.py b/bin/tools/lru.py new file mode 100644 index 00000000000..190fe5e4ff6 --- /dev/null +++ b/bin/tools/lru.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# taken from http://code.activestate.com/recipes/252524-length-limited-o1-lru-cache-implementation/ +import threading +from func import synchronized + +__all__ = ['LRU'] + +class LRUNode(object): + __slots__ = ['prev', 'next', 'me'] + def __init__(self, prev, me): + self.prev = prev + self.me = me + self.next = None + +class LRU: + """ + Implementation of a length-limited O(1) LRU queue. + Built for and used by PyPE: + http://pype.sourceforge.net + Copyright 2003 Josiah Carlson. + """ + def __init__(self, count, pairs=[]): + self._lock = threading.RLock() + self.count = max(count, 1) + self.d = {} + self.first = None + self.last = None + for key, value in pairs: + self[key] = value + + @synchronized() + def __contains__(self, obj): + return obj in self.d + + @synchronized() + def __getitem__(self, obj): + a = self.d[obj].me + self[a[0]] = a[1] + return a[1] + + @synchronized() + def __setitem__(self, obj, val): + if obj in self.d: + del self[obj] + nobj = LRUNode(self.last, (obj, val)) + if self.first is None: + self.first = nobj + if self.last: + self.last.next = nobj + self.last = nobj + self.d[obj] = nobj + if len(self.d) > self.count: + if self.first == self.last: + self.first = None + self.last = None + return + a = self.first + a.next.prev = None + self.first = a.next + a.next = None + del self.d[a.me[0]] + del a + + @synchronized() + def __delitem__(self, obj): + nobj = self.d[obj] + if nobj.prev: + nobj.prev.next = nobj.next + else: + self.first = nobj.next + if nobj.next: + nobj.next.prev = nobj.prev + else: + self.last = nobj.prev + del self.d[obj] + + @synchronized() + def __iter__(self): + cur = self.first + while cur != None: + cur2 = cur.next + yield cur.me[1] + cur = cur2 + + @synchronized() + def __len__(self): + return len(self.d) + + @synchronized() + def iteritems(self): + cur = self.first + while cur != None: + cur2 = cur.next + yield cur.me + cur = cur2 + + @synchronized() + def iterkeys(self): + return iter(self.d) + + @synchronized() + def itervalues(self): + for i,j in self.iteritems(): + yield j + + @synchronized() + def keys(self): + return self.d.keys() + + @synchronized() + def pop(self,key): + v=self[key] + del self[key] + return v + diff --git a/bin/tools/misc.py b/bin/tools/misc.py index 6d3aba7b561..90b4c135214 100644 --- a/bin/tools/misc.py +++ b/bin/tools/misc.py @@ -50,6 +50,7 @@ else: import netsvc from config import config +from lru import LRU _logger = logging.getLogger('tools') @@ -696,7 +697,7 @@ class cache(object): __caches = [] - def __init__(self, timeout=None, skiparg=2, multi=None): + def __init__(self, timeout=None, skiparg=2, multi=None, size=8192): assert skiparg >= 2 # at least self and cr if timeout is None: self.timeout = config['cache_timeout'] @@ -705,7 +706,7 @@ class cache(object): self.skiparg = skiparg self.multi = multi self.lasttime = time.time() - self.cache = {} + self.cache = LRU(size) # TODO take size from config self.fun = None cache.__caches.append(self)