From b5bf0e8c1c9bb0ad104c292d58d31d657e46aeb5 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 8 Jun 2011 11:25:15 +0200 Subject: [PATCH] [IMP] tools.cache cleanup, 15% speedup bzr revid: al@openerp.com-20110608092515-gum2y4snkrcrpjqb --- openerp/addons/base/ir/ir_rule.py | 30 +---- openerp/addons/base/ir/ir_translation.py | 12 +- openerp/tools/cache.py | 149 +++++++++++++++++++++++ openerp/tools/misc.py | 111 +---------------- 4 files changed, 161 insertions(+), 141 deletions(-) create mode 100644 openerp/tools/cache.py diff --git a/openerp/addons/base/ir/ir_rule.py b/openerp/addons/base/ir/ir_rule.py index b62da2ac4ff..86ded0387e0 100644 --- a/openerp/addons/base/ir/ir_rule.py +++ b/openerp/addons/base/ir/ir_rule.py @@ -134,22 +134,7 @@ class ir_rule(osv.osv): return [] def clear_cache(self, cr, uid): - cr.execute("""SELECT DISTINCT m.model - FROM ir_rule r - JOIN ir_model m - ON r.model_id = m.id - WHERE r.global - OR EXISTS (SELECT 1 - FROM rule_group_rel g_rel - JOIN res_groups_users_rel u_rel - ON g_rel.group_id = u_rel.gid - WHERE g_rel.rule_group_id = r.id - AND u_rel.uid = %s) - """, (uid,)) - models = map(itemgetter(0), cr.fetchall()) - clear = partial(self._compute_domain.clear_cache, self, uid) - [clear(model, mode) for model in models for mode in self._MODES] - + self._compute_domain.clear_cache(self) def domain_get(self, cr, uid, model_name, mode='read', context=None): dom = self._compute_domain(cr, uid, model_name, mode) @@ -164,20 +149,17 @@ class ir_rule(osv.osv): def unlink(self, cr, uid, ids, context=None): res = super(ir_rule, self).unlink(cr, uid, ids, context=context) - # Restart the cache on the _compute_domain method of ir.rule - self._compute_domain.clear_cache(self) + self.clear_cache() return res - def create(self, cr, user, vals, context=None): - res = super(ir_rule, self).create(cr, user, vals, context=context) - # Restart the cache on the _compute_domain method of ir.rule - self._compute_domain.clear_cache(self) + def create(self, cr, uid, vals, context=None): + res = super(ir_rule, self).create(cr, uid, vals, context=context) + self.clear_cache(cr, uid) return res def write(self, cr, uid, ids, vals, context=None): res = super(ir_rule, self).write(cr, uid, ids, vals, context=context) - # Restart the cache on the _compute_domain method - self._compute_domain.clear_cache(self) + self.clear_cache(cr,uid) return res ir_rule() diff --git a/openerp/addons/base/ir/ir_translation.py b/openerp/addons/base/ir/ir_translation.py index 0e6374b0143..02aa8f4a8fe 100644 --- a/openerp/addons/base/ir/ir_translation.py +++ b/openerp/addons/base/ir/ir_translation.py @@ -92,7 +92,7 @@ class ir_translation(osv.osv): cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)') cr.commit() - @tools.ormcache_multi(skiparg=3, multi=7) + @tools.ormcache_multi(skiparg=3, multi=6) def _get_ids(self, cr, uid, name, tt, lang, ids): translations = dict.fromkeys(ids, False) if ids: @@ -178,13 +178,11 @@ class ir_translation(osv.osv): return tools.ustr(source) return trad - def create(self, cursor, user, vals, context=None): + def create(self, cr, uid, vals, context=None): if not context: context = {} - ids = super(ir_translation, self).create(cursor, user, vals, context=context) - for trans_obj in self.read(cursor, user, [ids], ['name','type','res_id','src','lang'], context=context): - self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) - self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) + ids = super(ir_translation, self).create(cr, uid, vals, context=context) + self._get_source.clear_cache(self, vals.get('name',0), vals.get('type',0), vals.get('lang',0), vals.get('src',0)) return ids def write(self, cursor, user, ids, vals, context=None): @@ -193,8 +191,8 @@ class ir_translation(osv.osv): if isinstance(ids, (int, long)): ids = [ids] result = super(ir_translation, self).write(cursor, user, ids, vals, context=context) + self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context): - self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) return result diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py new file mode 100644 index 00000000000..db4115aef36 --- /dev/null +++ b/openerp/tools/cache.py @@ -0,0 +1,149 @@ +import lru + +class ormcache(object): + """ LRU cache decorator for orm methods + """ + + def __init__(self, skiparg=2, size=8192, multi=None, timeout=None): + self.skiparg = skiparg + self.size = size + self.method = None + self.stat_miss = 0 + self.stat_hit = 0 + self.stat_err = 0 + + def __call__(self,m): + self.method = m + def lookup(self2, cr, *args): + r = self.lookup(self2, cr, *args) + #self.stat() + return r + def clear(self2, *args): + return self.clear(self2, *args) + lookup.clear_cache = self.clear + #print "lookup-func",lookup + return lookup + + def stat(self): + print "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % (self.stat_hit,self.stat_miss,self.stat_err, (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit) ) + + def lru(self, self2): + try: + ormcache = getattr(self2, '_ormcache') + except AttributeError: + ormcache = self2._ormcache = {} + try: + d = ormcache[self.method] + except KeyError: + d = ormcache[self.method] = lru.LRU(self.size) + return d + + def lookup(self, self2, cr, *args): + d = self.lru(self2) + key = args[self.skiparg-2:] + try: + r = d[key] + #print "lookup-hit",self2,cr,key,r + self.stat_hit += 1 + return r + except KeyError: + self.stat_miss += 1 + #print "lookup-miss",self2,cr,key + value = d[args] = self.method(self2, cr, *args) + #print "lookup-miss-value",value + return value + except TypeError: + self.stat_err += 1 + #print "lookup-error",self2,cr,key + return self.method(self2, cr, *args) + + def clear(self, self2, *args): + """ Remove *args entry from the cache or all keys if *args is undefined + """ + d = self.lru(self2) + if args: + try: + key = args[self.skiparg-2:] + #print "del",key + del d[key] + except KeyError: + pass + else: + d.clear() + +class ormcache_multi(ormcache): + def __init__(self, skiparg=2, size=8192, multi=3): + super(ormcache_multi,self).__init__(skiparg,size) + self.multi = multi - 2 + + def lookup(self, self2, cr, *args): + d = self.lru(self2) + args = list(args) + multi = self.multi + #print args, multi + ids = args[multi] + r = {} + miss = [] + + for i in ids: + args[multi] = i + key = tuple(args[self.skiparg-2:]) + try: + r[i] = d[key] + #print "lookup-hit",self2,cr,key,r[i] + self.stat_hit += 1 + except Exception: + self.stat_miss += 1 + miss.append(i) + #print "lookup-miss",self2,cr,key + + if miss: + args[multi] = miss + r.update(self.method(self2, cr, *args)) + + for i in miss: + args[multi] = i + key = tuple(args[self.skiparg-2:]) + d[key] = r[i] + + return r + +class dummy_cache(object): + """ Cache decorator replacement to actually do no caching. + """ + def __init__(self, *l, **kw): + pass + def __call__(self, fn): + fn.clear_cache = self.clear + return fn + def clear(self, *l, **kw): + pass + +#ormcache = dummy_cache +cache = dummy_cache + +if __name__ == '__main__': + + class A(): + @ormcache() + def m(self,a,b): + print "A::m(", self,a,b + return 1 + + @ormcache_multi(multi=3) + def n(self,cr,uid,ids): + print "m", self,cr,uid,ids + return dict([(i,i) for i in ids]) + + a=A() + #r=a.m(1,2) + #r=a.m(1,2) + r=a.n("cr",1,[1,2,3,4]) + r=a.n("cr",1,[1,2]) + print r + for i in a._ormcache: + print a._ormcache[i].d + a.n.clear_cache(a,1,1) + r=a.n("cr",1,[1,2]) + print r + #r=a.n("cr",1,[1,2]) diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py index 307cf04ce68..48a9a936783 100644 --- a/openerp/tools/misc.py +++ b/openerp/tools/misc.py @@ -57,7 +57,7 @@ except ImportError: import openerp.loglevels as loglevels from config import config -from lru import LRU +from cache import * # get_encodings, ustr and exception_to_unicode were originally from tools.misc. # There are moved to loglevels until we refactor tools. @@ -622,7 +622,6 @@ class UpdateableDict(local): def __ne__(self, y): return self.dict.__ne__(y) - # Don't use ! Use res.currency.round() class currency(float): @@ -639,114 +638,6 @@ class currency(float): # display_value = int(self*(10**(-self.accuracy))/self.rounding)*self.rounding/(10**(-self.accuracy)) # return str(display_value) -class ormcache(object): - """ LRU cache decorator for orm methods - """ - - def __init__(self, skiparg=2, size=8192, multi=None, timeout=None): - self.skiparg = skiparg - self.size = size - self.method = None - self.stat_miss = 0 - self.stat_hit = 0 - self.stat_err = 0 - - def __call__(self,m): - self.method = m - def lookup(self2, cr, *args): - r = self.lookup(m, m, self2, cr, *args) -# print "lookup-stats hit miss err",self.stat_hit,self.stat_miss,self.stat_err - return r - def clear(self2, *args): - return self.clear(self2, *args) - lookup.clear_cache = self.clear - print "returned",lookup - return lookup - - def lookup(self, cacheid, method, self2, cr, *args): - try: - ormcache = getattr(self2, '_ormcache') - except AttributeError: - ormcache = self2._ormcache = {} - try: - d = ormcache[cacheid] - except KeyError: - d = ormcache[cacheid] = LRU(self.size) - key = args[self.skiparg-2:] - try: - r = d[key] -# print "lookup-hit",self2,cr,key,r - self.stat_hit += 1 - return r - except KeyError: - self.stat_miss += 1 -# print "lookup-miss",self2,cr,key - value = d[args] = method(self2, cr, *args) -# print "lookup-miss-value",value - return value - except TypeError: - self.stat_err += 1 -# print "lookup-typeerror",self2,cr,key - return method(self2, cr, *args) - - def clear(self, self2, *args, **kw): - """ Remove *args entry from the cache or all keys if *args is undefined - """ - try: - ormcache = getattr(self2, '_ormcache') - except AttributeError: - ormcache = self2._ormcache = {} - try: - d = ormcache[self.method] - except KeyError: - d = ormcache[self.method] = LRU(self.size) - d.clear() - -class ormcache_multi(ormcache): - def __init__(self, skiparg=2, size=8192, multi=4): - super(ormcache_multi,self).__init__(skiparg,size) - self.multi = multi - 3 - - def lookup(self, cacheid, method, self2, cr, *args): - superlookup = super(ormcache_multi,self).lookup - args = list(args) - multi = self.multi -# print args,multi - ids = args[multi] - r = {} - miss = [] - - def add_to_miss(_self2, _cr, *_args): - miss.append(_args[multi]) - - for i in ids: - args[multi] = i - r[i] = superlookup(method, add_to_miss, self2, cr, *args) - - args[multi] = miss - r.update(method(self2, cr, *args)) - - for i in miss: - args[multi] = i - key = tuple(args[self.skiparg-2:]) - self2._ormcache[cacheid][key] = r[i] - - return r - -class dummy_cache(object): - """ Cache decorator replacement to actually do no caching. - """ - def __init__(self, *l, **kw): - pass - def clear(self, *l, **kw): - pass - def __call__(self, fn): - fn.clear_cache = self.clear - return fn - -#ormcache = dummy_cache -cache = dummy_cache - def to_xml(s): return s.replace('&','&').replace('<','<').replace('>','>')