pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] tools.cache cleanup, 15% speedup

Browse Source 
bzr revid: al@openerp.com-20110608092515-gum2y4snkrcrpjqb
This commit is contained in:
Antony Lesuisse
2011-06-08 11:25:15 +02:00
parent 970b19b8b4
commit b5bf0e8c1c
4 changed files with 161 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openerp/addons/base/ir/ir_rule.py
+6 -24
View File
@@ -134,22 +134,7 @@ class ir_rule(osv.osv):
return []
def clear_cache(self, cr, uid):
cr.execute("""SELECT DISTINCT m.model
FROM ir_rule r
JOIN ir_model m
ON r.model_id = m.id
WHERE r.global
OR EXISTS (SELECT 1
FROM rule_group_rel g_rel
JOIN res_groups_users_rel u_rel
ON g_rel.group_id = u_rel.gid
WHERE g_rel.rule_group_id = r.id
AND u_rel.uid = %s)
""", (uid,))
models = map(itemgetter(0), cr.fetchall())
clear = partial(self._compute_domain.clear_cache, self, uid)
[clear(model, mode) for model in models for mode in self._MODES]
self._compute_domain.clear_cache(self)
def domain_get(self, cr, uid, model_name, mode='read', context=None):
dom = self._compute_domain(cr, uid, model_name, mode)
@@ -164,20 +149,17 @@ class ir_rule(osv.osv):
def unlink(self, cr, uid, ids, context=None):
res = super(ir_rule, self).unlink(cr, uid, ids, context=context)
# Restart the cache on the _compute_domain method of ir.rule
self._compute_domain.clear_cache(self)
self.clear_cache()
return res
def create(self, cr, user, vals, context=None):
res = super(ir_rule, self).create(cr, user, vals, context=context)
# Restart the cache on the _compute_domain method of ir.rule
self._compute_domain.clear_cache(self)
def create(self, cr, uid, vals, context=None):
res = super(ir_rule, self).create(cr, uid, vals, context=context)
self.clear_cache(cr, uid)
return res
def write(self, cr, uid, ids, vals, context=None):
res = super(ir_rule, self).write(cr, uid, ids, vals, context=context)
# Restart the cache on the _compute_domain method
self._compute_domain.clear_cache(self)
self.clear_cache(cr,uid)
return res
ir_rule()
openerp/addons/base/ir/ir_translation.py
+5 -7
View File
@@ -92,7 +92,7 @@ class ir_translation(osv.osv):
cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)')
cr.commit()
@tools.ormcache_multi(skiparg=3, multi=7)
@tools.ormcache_multi(skiparg=3, multi=6)
def _get_ids(self, cr, uid, name, tt, lang, ids):
translations = dict.fromkeys(ids, False)
if ids:
@@ -178,13 +178,11 @@ class ir_translation(osv.osv):
return tools.ustr(source)
return trad
def create(self, cursor, user, vals, context=None):
def create(self, cr, uid, vals, context=None):
if not context:
context = {}
ids = super(ir_translation, self).create(cursor, user, vals, context=context)
for trans_obj in self.read(cursor, user, [ids], ['name','type','res_id','src','lang'], context=context):
self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src'])
self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']])
ids = super(ir_translation, self).create(cr, uid, vals, context=context)
self._get_source.clear_cache(self, vals.get('name',0), vals.get('type',0), vals.get('lang',0), vals.get('src',0))
return ids
def write(self, cursor, user, ids, vals, context=None):
@@ -193,8 +191,8 @@ class ir_translation(osv.osv):
if isinstance(ids, (int, long)):
ids = [ids]
result = super(ir_translation, self).write(cursor, user, ids, vals, context=context)
self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src'])
for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context):
self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src'])
self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']])
return result
openerp/tools/cache.py
+149
View File
@@ -0,0 +1,149 @@
import lru
class ormcache(object):
""" LRU cache decorator for orm methods
"""
def __init__(self, skiparg=2, size=8192, multi=None, timeout=None):
self.skiparg = skiparg
self.size = size
self.method = None
self.stat_miss = 0
self.stat_hit = 0
self.stat_err = 0
def __call__(self,m):
self.method = m
def lookup(self2, cr, *args):
r = self.lookup(self2, cr, *args)
#self.stat()
return r
def clear(self2, *args):
return self.clear(self2, *args)
lookup.clear_cache = self.clear
#print "lookup-func",lookup
return lookup
def stat(self):
print "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % (self.stat_hit,self.stat_miss,self.stat_err, (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit) )
def lru(self, self2):
try:
ormcache = getattr(self2, '_ormcache')
except AttributeError:
ormcache = self2._ormcache = {}
try:
d = ormcache[self.method]
except KeyError:
d = ormcache[self.method] = lru.LRU(self.size)
return d
def lookup(self, self2, cr, *args):
d = self.lru(self2)
key = args[self.skiparg-2:]
try:
r = d[key]
#print "lookup-hit",self2,cr,key,r
self.stat_hit += 1
return r
except KeyError:
self.stat_miss += 1
#print "lookup-miss",self2,cr,key
value = d[args] = self.method(self2, cr, *args)
#print "lookup-miss-value",value
return value
except TypeError:
self.stat_err += 1
#print "lookup-error",self2,cr,key
return self.method(self2, cr, *args)
def clear(self, self2, *args):
""" Remove *args entry from the cache or all keys if *args is undefined
"""
d = self.lru(self2)
if args:
try:
key = args[self.skiparg-2:]
#print "del",key
del d[key]
except KeyError:
pass
else:
d.clear()
class ormcache_multi(ormcache):
def __init__(self, skiparg=2, size=8192, multi=3):
super(ormcache_multi,self).__init__(skiparg,size)
self.multi = multi - 2
def lookup(self, self2, cr, *args):
d = self.lru(self2)
args = list(args)
multi = self.multi
#print args, multi
ids = args[multi]
r = {}
miss = []
for i in ids:
args[multi] = i
key = tuple(args[self.skiparg-2:])
try:
r[i] = d[key]
#print "lookup-hit",self2,cr,key,r[i]
self.stat_hit += 1
except Exception:
self.stat_miss += 1
miss.append(i)
#print "lookup-miss",self2,cr,key
if miss:
args[multi] = miss
r.update(self.method(self2, cr, *args))
for i in miss:
args[multi] = i
key = tuple(args[self.skiparg-2:])
d[key] = r[i]
return r
class dummy_cache(object):
""" Cache decorator replacement to actually do no caching.
"""
def __init__(self, *l, **kw):
pass
def __call__(self, fn):
fn.clear_cache = self.clear
return fn
def clear(self, *l, **kw):
pass
#ormcache = dummy_cache
cache = dummy_cache
if __name__ == '__main__':
class A():
@ormcache()
def m(self,a,b):
print "A::m(", self,a,b
return 1
@ormcache_multi(multi=3)
def n(self,cr,uid,ids):
print "m", self,cr,uid,ids
return dict([(i,i) for i in ids])
a=A()
#r=a.m(1,2)
#r=a.m(1,2)
r=a.n("cr",1,[1,2,3,4])
r=a.n("cr",1,[1,2])
print r
for i in a._ormcache:
print a._ormcache[i].d
a.n.clear_cache(a,1,1)
r=a.n("cr",1,[1,2])
print r
#r=a.n("cr",1,[1,2])
openerp/tools/misc.py
+1 -110
View File
@@ -57,7 +57,7 @@ except ImportError:
import openerp.loglevels as loglevels
from config import config
from lru import LRU
from cache import *
# get_encodings, ustr and exception_to_unicode were originally from tools.misc.
# There are moved to loglevels until we refactor tools.
@@ -622,7 +622,6 @@ class UpdateableDict(local):
def __ne__(self, y):
return self.dict.__ne__(y)
# Don't use ! Use res.currency.round()
class currency(float):
@@ -639,114 +638,6 @@ class currency(float):
# display_value = int(self*(10**(-self.accuracy))/self.rounding)*self.rounding/(10**(-self.accuracy))
# return str(display_value)
class ormcache(object):
""" LRU cache decorator for orm methods
"""
def __init__(self, skiparg=2, size=8192, multi=None, timeout=None):
self.skiparg = skiparg
self.size = size
self.method = None
self.stat_miss = 0
self.stat_hit = 0
self.stat_err = 0
def __call__(self,m):
self.method = m
def lookup(self2, cr, *args):
r = self.lookup(m, m, self2, cr, *args)
# print "lookup-stats hit miss err",self.stat_hit,self.stat_miss,self.stat_err
return r
def clear(self2, *args):
return self.clear(self2, *args)
lookup.clear_cache = self.clear
print "returned",lookup
return lookup
def lookup(self, cacheid, method, self2, cr, *args):
try:
ormcache = getattr(self2, '_ormcache')
except AttributeError:
ormcache = self2._ormcache = {}
try:
d = ormcache[cacheid]
except KeyError:
d = ormcache[cacheid] = LRU(self.size)
key = args[self.skiparg-2:]
try:
r = d[key]
# print "lookup-hit",self2,cr,key,r
self.stat_hit += 1
return r
except KeyError:
self.stat_miss += 1
# print "lookup-miss",self2,cr,key
value = d[args] = method(self2, cr, *args)
# print "lookup-miss-value",value
return value
except TypeError:
self.stat_err += 1
# print "lookup-typeerror",self2,cr,key
return method(self2, cr, *args)
def clear(self, self2, *args, **kw):
""" Remove *args entry from the cache or all keys if *args is undefined
"""
try:
ormcache = getattr(self2, '_ormcache')
except AttributeError:
ormcache = self2._ormcache = {}
try:
d = ormcache[self.method]
except KeyError:
d = ormcache[self.method] = LRU(self.size)
d.clear()
class ormcache_multi(ormcache):
def __init__(self, skiparg=2, size=8192, multi=4):
super(ormcache_multi,self).__init__(skiparg,size)
self.multi = multi - 3
def lookup(self, cacheid, method, self2, cr, *args):
superlookup = super(ormcache_multi,self).lookup
args = list(args)
multi = self.multi
# print args,multi
ids = args[multi]
r = {}
miss = []
def add_to_miss(_self2, _cr, *_args):
miss.append(_args[multi])
for i in ids:
args[multi] = i
r[i] = superlookup(method, add_to_miss, self2, cr, *args)
args[multi] = miss
r.update(method(self2, cr, *args))
for i in miss:
args[multi] = i
key = tuple(args[self.skiparg-2:])
self2._ormcache[cacheid][key] = r[i]
return r
class dummy_cache(object):
""" Cache decorator replacement to actually do no caching.
"""
def __init__(self, *l, **kw):
pass
def clear(self, *l, **kw):
pass
def __call__(self, fn):
fn.clear_cache = self.clear
return fn
#ormcache = dummy_cache
cache = dummy_cache
def to_xml(s):
return s.replace('&','&amp;').replace('<','&lt;').replace('>','&gt;')