From 970b19b8b44632937baf3e3a8f80f2b20030ab7b Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 8 Jun 2011 05:03:30 +0200 Subject: [PATCH 01/16] [IMP] new tools.cache cleaner, 10% speedup and more to come... bzr revid: al@openerp.com-20110608030330-d0dsv2k6n0w3lyd5 --- openerp/addons/base/ir/ir_model.py | 13 +- openerp/addons/base/ir/ir_rule.py | 12 +- openerp/addons/base/ir/ir_translation.py | 22 +-- openerp/addons/base/res/res_company.py | 4 +- openerp/addons/base/res/res_lang.py | 4 +- openerp/addons/base/res/res_user.py | 6 +- openerp/osv/orm.py | 20 +- openerp/tools/lru.py | 7 +- openerp/tools/misc.py | 241 +++++++++-------------- 9 files changed, 142 insertions(+), 187 deletions(-) diff --git a/openerp/addons/base/ir/ir_model.py b/openerp/addons/base/ir/ir_model.py index ad0912a59ee..7121833156d 100644 --- a/openerp/addons/base/ir/ir_model.py +++ b/openerp/addons/base/ir/ir_model.py @@ -450,6 +450,7 @@ class ir_model_access(osv.osv): # pass no groups -> no access return False + @tools.ormcache() def check(self, cr, uid, model, mode='read', raise_exception=True, context=None): if uid==1: # User root have all accesses @@ -513,8 +514,6 @@ class ir_model_access(osv.osv): raise except_orm(_('AccessError'), msgs[mode] % (model_name, groups) ) return r - check = tools.cache()(check) - __cache_clearing_methods = [] def register_cache_clearing_method(self, model, method): @@ -528,7 +527,7 @@ class ir_model_access(osv.osv): pass def call_cache_clearing_methods(self, cr): - self.check.clear_cache(cr.dbname) # clear the cache of check function + self.check.clear_cache(self) # clear the cache of check function for model, method in self.__cache_clearing_methods: object_ = self.pool.get(model) if object_: @@ -593,7 +592,7 @@ class ir_model_data(osv.osv): if not cr.fetchone(): cr.execute('CREATE INDEX ir_model_data_module_name_index ON ir_model_data (module, name)') - @tools.cache() + @tools.ormcache() def _get_id(self, cr, uid, module, xml_id): """Returns the id of the ir.model.data record corresponding to a given module and xml_id (cached) or raise a ValueError if not found""" ids = self.search(cr, uid, [('module','=',module), ('name','=', xml_id)]) @@ -602,7 +601,7 @@ class ir_model_data(osv.osv): # the sql constraints ensure us we have only one result return ids[0] - @tools.cache() + @tools.ormcache() def get_object_reference(self, cr, uid, module, xml_id): """Returns (model, res_id) corresponding to a given module and xml_id (cached) or raise ValueError if not found""" data_id = self._get_id(cr, uid, module, xml_id) @@ -652,8 +651,8 @@ class ir_model_data(osv.osv): results = cr.fetchall() for imd_id2,res_id2,real_id2 in results: if not real_id2: - self._get_id.clear_cache(cr.dbname, uid, module, xml_id) - self.get_object_reference.clear_cache(cr.dbname, uid, module, xml_id) + self._get_id.clear_cache(self, uid, module, xml_id) + self.get_object_reference.clear_cache(self, uid, module, xml_id) cr.execute('delete from ir_model_data where id=%s', (imd_id2,)) res_id = False else: diff --git a/openerp/addons/base/ir/ir_rule.py b/openerp/addons/base/ir/ir_rule.py index befa47e25b0..b62da2ac4ff 100644 --- a/openerp/addons/base/ir/ir_rule.py +++ b/openerp/addons/base/ir/ir_rule.py @@ -94,7 +94,7 @@ class ir_rule(osv.osv): return ['&'] * (count-1) + dom return [] - @tools.cache() + @tools.ormcache() def _compute_domain(self, cr, uid, model_name, mode="read"): if mode not in self._MODES: raise ValueError('Invalid mode: %r' % (mode,)) @@ -147,12 +147,12 @@ class ir_rule(osv.osv): AND u_rel.uid = %s) """, (uid,)) models = map(itemgetter(0), cr.fetchall()) - clear = partial(self._compute_domain.clear_cache, cr.dbname, uid) + clear = partial(self._compute_domain.clear_cache, self, uid) [clear(model, mode) for model in models for mode in self._MODES] def domain_get(self, cr, uid, model_name, mode='read', context=None): - dom = self._compute_domain(cr, uid, model_name, mode=mode) + dom = self._compute_domain(cr, uid, model_name, mode) if dom: # _where_calc is called as superuser. This means that rules can # involve objects on which the real uid has no acces rights. @@ -165,19 +165,19 @@ class ir_rule(osv.osv): def unlink(self, cr, uid, ids, context=None): res = super(ir_rule, self).unlink(cr, uid, ids, context=context) # Restart the cache on the _compute_domain method of ir.rule - self._compute_domain.clear_cache(cr.dbname) + self._compute_domain.clear_cache(self) return res def create(self, cr, user, vals, context=None): res = super(ir_rule, self).create(cr, user, vals, context=context) # Restart the cache on the _compute_domain method of ir.rule - self._compute_domain.clear_cache(cr.dbname) + self._compute_domain.clear_cache(self) return res def write(self, cr, uid, ids, vals, context=None): res = super(ir_rule, self).write(cr, uid, ids, vals, context=context) # Restart the cache on the _compute_domain method - self._compute_domain.clear_cache(cr.dbname) + self._compute_domain.clear_cache(self) return res ir_rule() diff --git a/openerp/addons/base/ir/ir_translation.py b/openerp/addons/base/ir/ir_translation.py index eb900363669..0e6374b0143 100644 --- a/openerp/addons/base/ir/ir_translation.py +++ b/openerp/addons/base/ir/ir_translation.py @@ -92,7 +92,7 @@ class ir_translation(osv.osv): cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)') cr.commit() - @tools.cache(skiparg=3, multi='ids') + @tools.ormcache_multi(skiparg=3, multi=7) def _get_ids(self, cr, uid, name, tt, lang, ids): translations = dict.fromkeys(ids, False) if ids: @@ -112,9 +112,9 @@ class ir_translation(osv.osv): tr = self._get_ids(cr, uid, name, tt, lang, ids) for res_id in tr: if tr[res_id]: - self._get_source.clear_cache(cr.dbname, uid, name, tt, lang, tr[res_id]) - self._get_source.clear_cache(cr.dbname, uid, name, tt, lang) - self._get_ids.clear_cache(cr.dbname, uid, name, tt, lang, ids) + self._get_source.clear_cache(self, uid, name, tt, lang, tr[res_id]) + self._get_source.clear_cache(self, uid, name, tt, lang) + self._get_ids.clear_cache(self, uid, name, tt, lang, ids) cr.execute('delete from ir_translation ' \ 'where lang=%s ' \ @@ -133,7 +133,7 @@ class ir_translation(osv.osv): }) return len(ids) - @tools.cache(skiparg=3) + @tools.ormcache(skiparg=3) def _get_source(self, cr, uid, name, types, lang, source=None): """ Returns the translation for the given combination of name, type, language @@ -183,8 +183,8 @@ class ir_translation(osv.osv): context = {} ids = super(ir_translation, self).create(cursor, user, vals, context=context) for trans_obj in self.read(cursor, user, [ids], ['name','type','res_id','src','lang'], context=context): - self._get_source.clear_cache(cursor.dbname, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) - self._get_ids.clear_cache(cursor.dbname, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) + self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) + self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) return ids def write(self, cursor, user, ids, vals, context=None): @@ -194,8 +194,8 @@ class ir_translation(osv.osv): ids = [ids] result = super(ir_translation, self).write(cursor, user, ids, vals, context=context) for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context): - self._get_source.clear_cache(cursor.dbname, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) - self._get_ids.clear_cache(cursor.dbname, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) + self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) + self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) return result def unlink(self, cursor, user, ids, context=None): @@ -204,8 +204,8 @@ class ir_translation(osv.osv): if isinstance(ids, (int, long)): ids = [ids] for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context): - self._get_source.clear_cache(cursor.dbname, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) - self._get_ids.clear_cache(cursor.dbname, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) + self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) + self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) result = super(ir_translation, self).unlink(cursor, user, ids, context=context) return result diff --git a/openerp/addons/base/res/res_company.py b/openerp/addons/base/res/res_company.py index 0143b03fd2a..bb281b9c0ea 100644 --- a/openerp/addons/base/res/res_company.py +++ b/openerp/addons/base/res/res_company.py @@ -132,7 +132,7 @@ class res_company(osv.osv): ids = self._get_company_children(cr, uid, company) return ids - @tools.cache() + @tools.ormcache() def _get_company_children(self, cr, uid=None, company=None): if not company: return [] @@ -159,7 +159,7 @@ class res_company(osv.osv): # This function restart the cache on the _get_company_children method # def cache_restart(self, cr): - self._get_company_children.clear_cache(cr.dbname) + self._get_company_children.clear_cache(self) def create(self, cr, uid, vals, context=None): if not vals.get('name', False) or vals.get('partner_id', False): diff --git a/openerp/addons/base/res/res_lang.py b/openerp/addons/base/res/res_lang.py index 600f31aead2..f9a7bd00dc5 100644 --- a/openerp/addons/base/res/res_lang.py +++ b/openerp/addons/base/res/res_lang.py @@ -160,7 +160,7 @@ class lang(osv.osv): (_check_format, 'Invalid date/time format directive specified. Please refer to the list of allowed directives, displayed when you edit a language.', ['time_format', 'date_format']) ] - @tools.cache(skiparg=3) + @tools.ormcache(skiparg=3) def _lang_data_get(self, cr, uid, lang_id, monetary=False): conv = localeconv() lang_obj = self.browse(cr, uid, lang_id) @@ -171,7 +171,7 @@ class lang(osv.osv): def write(self, cr, uid, ids, vals, context=None): for lang_id in ids : - self._lang_data_get.clear_cache(cr.dbname,lang_id= lang_id) + self._lang_data_get.clear_cache(self,lang_id= lang_id) return super(lang, self).write(cr, uid, ids, vals, context) def unlink(self, cr, uid, ids, context=None): diff --git a/openerp/addons/base/res/res_user.py b/openerp/addons/base/res/res_user.py index 8a76ba9a6f4..c9eb1da381c 100644 --- a/openerp/addons/base/res/res_user.py +++ b/openerp/addons/base/res/res_user.py @@ -267,7 +267,7 @@ class users(osv.osv): o['password'] = '********' return o result = super(users, self).read(cr, uid, ids, fields, context, load) - canwrite = self.pool.get('ir.model.access').check(cr, uid, 'res.users', 'write', raise_exception=False) + canwrite = self.pool.get('ir.model.access').check(cr, uid, 'res.users', 'write', False) if not canwrite: if isinstance(ids, (int, float)): result = override_password(result) @@ -351,7 +351,7 @@ class users(osv.osv): 'menu_tips':True } - @tools.cache() + @tools.ormcache() def company_get(self, cr, uid, uid2, context=None): return self._get_company(cr, uid, context=context, uid2=uid2) @@ -374,7 +374,7 @@ class users(osv.osv): res = super(users, self).write(cr, uid, ids, values, context=context) # clear caches linked to the users - self.company_get.clear_cache(cr.dbname) + self.company_get.clear_cache(self) self.pool.get('ir.model.access').call_cache_clearing_methods(cr) clear = partial(self.pool.get('ir.rule').clear_cache, cr) map(clear, ids) diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 5197f92f3d0..1041e7ac73c 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -1981,7 +1981,7 @@ class orm_template(object): def read_string(self, cr, uid, id, langs, fields=None, context=None): res = {} res2 = {} - self.pool.get('ir.model.access').check(cr, uid, 'ir.translation', 'read', context=context) + self.pool.get('ir.model.access').check(cr, uid, 'ir.translation', 'read') if not fields: fields = self._columns.keys() + self._inherit_fields.keys() #FIXME: collect all calls to _get_source into one SQL call. @@ -2005,7 +2005,7 @@ class orm_template(object): return res def write_string(self, cr, uid, id, langs, vals, context=None): - self.pool.get('ir.model.access').check(cr, uid, 'ir.translation', 'write', context=context) + self.pool.get('ir.model.access').check(cr, uid, 'ir.translation', 'write') #FIXME: try to only call the translation in one SQL for lang in langs: for field in vals: @@ -2312,7 +2312,7 @@ class orm(orm_template): """ context = context or {} - self.pool.get('ir.model.access').check(cr, uid, self._name, 'read', context=context) + self.pool.get('ir.model.access').check(cr, uid, self._name, 'read') if not fields: fields = self._columns.keys() @@ -3122,14 +3122,14 @@ class orm(orm_template): """ ira = self.pool.get('ir.model.access') - write_access = ira.check(cr, user, self._name, 'write', raise_exception=False, context=context) or \ - ira.check(cr, user, self._name, 'create', raise_exception=False, context=context) + write_access = ira.check(cr, user, self._name, 'write', False) or \ + ira.check(cr, user, self._name, 'create', False) return super(orm, self).fields_get(cr, user, fields, context, write_access) def read(self, cr, user, ids, fields=None, context=None, load='_classic_read'): if not context: context = {} - self.pool.get('ir.model.access').check(cr, user, self._name, 'read', context=context) + self.pool.get('ir.model.access').check(cr, user, self._name, 'read') if not fields: fields = list(set(self._columns.keys() + self._inherit_fields.keys())) if isinstance(ids, (int, long)): @@ -3415,7 +3415,7 @@ class orm(orm_template): self._check_concurrency(cr, ids, context) - self.pool.get('ir.model.access').check(cr, uid, self._name, 'unlink', context=context) + self.pool.get('ir.model.access').check(cr, uid, self._name, 'unlink') properties = self.pool.get('ir.property') domain = [('res_id', '=', False), @@ -3549,7 +3549,7 @@ class orm(orm_template): ids = [ids] self._check_concurrency(cr, ids, context) - self.pool.get('ir.model.access').check(cr, user, self._name, 'write', context=context) + self.pool.get('ir.model.access').check(cr, user, self._name, 'write') result = self._store_get_values(cr, user, ids, vals.keys(), context) or [] @@ -3758,7 +3758,7 @@ class orm(orm_template): """ if not context: context = {} - self.pool.get('ir.model.access').check(cr, user, self._name, 'create', context=context) + self.pool.get('ir.model.access').check(cr, user, self._name, 'create') vals = self._add_missing_default_values(cr, user, vals, context) @@ -4214,7 +4214,7 @@ class orm(orm_template): """ if context is None: context = {} - self.pool.get('ir.model.access').check(cr, access_rights_uid or user, self._name, 'read', context=context) + self.pool.get('ir.model.access').check(cr, access_rights_uid or user, self._name, 'read') query = self._where_calc(cr, user, args, context=context) self._apply_ir_rules(cr, user, query, 'read', context=context) diff --git a/openerp/tools/lru.py b/openerp/tools/lru.py index 190fe5e4ff6..aa7b14a745c 100644 --- a/openerp/tools/lru.py +++ b/openerp/tools/lru.py @@ -12,7 +12,7 @@ class LRUNode(object): self.me = me self.next = None -class LRU: +class LRU(object): """ Implementation of a length-limited O(1) LRU queue. Built for and used by PyPE: @@ -113,3 +113,8 @@ class LRU: del self[key] return v + @synchronized() + def clear(self): + self.d = {} + self.first = None + self.last = None diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py index 5aa01d4f267..307cf04ce68 100644 --- a/openerp/tools/misc.py +++ b/openerp/tools/misc.py @@ -639,162 +639,113 @@ class currency(float): # display_value = int(self*(10**(-self.accuracy))/self.rounding)*self.rounding/(10**(-self.accuracy)) # return str(display_value) +class ormcache(object): + """ LRU cache decorator for orm methods + """ -def is_hashable(h): - try: - hash(h) - return True - except TypeError: - return False + def __init__(self, skiparg=2, size=8192, multi=None, timeout=None): + self.skiparg = skiparg + self.size = size + self.method = None + self.stat_miss = 0 + self.stat_hit = 0 + self.stat_err = 0 + + def __call__(self,m): + self.method = m + def lookup(self2, cr, *args): + r = self.lookup(m, m, self2, cr, *args) +# print "lookup-stats hit miss err",self.stat_hit,self.stat_miss,self.stat_err + return r + def clear(self2, *args): + return self.clear(self2, *args) + lookup.clear_cache = self.clear + print "returned",lookup + return lookup + + def lookup(self, cacheid, method, self2, cr, *args): + try: + ormcache = getattr(self2, '_ormcache') + except AttributeError: + ormcache = self2._ormcache = {} + try: + d = ormcache[cacheid] + except KeyError: + d = ormcache[cacheid] = LRU(self.size) + key = args[self.skiparg-2:] + try: + r = d[key] +# print "lookup-hit",self2,cr,key,r + self.stat_hit += 1 + return r + except KeyError: + self.stat_miss += 1 +# print "lookup-miss",self2,cr,key + value = d[args] = method(self2, cr, *args) +# print "lookup-miss-value",value + return value + except TypeError: + self.stat_err += 1 +# print "lookup-typeerror",self2,cr,key + return method(self2, cr, *args) + + def clear(self, self2, *args, **kw): + """ Remove *args entry from the cache or all keys if *args is undefined + """ + try: + ormcache = getattr(self2, '_ormcache') + except AttributeError: + ormcache = self2._ormcache = {} + try: + d = ormcache[self.method] + except KeyError: + d = ormcache[self.method] = LRU(self.size) + d.clear() + +class ormcache_multi(ormcache): + def __init__(self, skiparg=2, size=8192, multi=4): + super(ormcache_multi,self).__init__(skiparg,size) + self.multi = multi - 3 + + def lookup(self, cacheid, method, self2, cr, *args): + superlookup = super(ormcache_multi,self).lookup + args = list(args) + multi = self.multi +# print args,multi + ids = args[multi] + r = {} + miss = [] + + def add_to_miss(_self2, _cr, *_args): + miss.append(_args[multi]) + + for i in ids: + args[multi] = i + r[i] = superlookup(method, add_to_miss, self2, cr, *args) + + args[multi] = miss + r.update(method(self2, cr, *args)) + + for i in miss: + args[multi] = i + key = tuple(args[self.skiparg-2:]) + self2._ormcache[cacheid][key] = r[i] + + return r class dummy_cache(object): """ Cache decorator replacement to actually do no caching. - - This can be useful to benchmark and/or track memory leak. - """ - - def __init__(self, timeout=None, skiparg=2, multi=None, size=8192): + def __init__(self, *l, **kw): pass - - def clear(self, dbname, *args, **kwargs): + def clear(self, *l, **kw): pass - - @classmethod - def clean_caches_for_db(cls, dbname): - pass - def __call__(self, fn): fn.clear_cache = self.clear return fn -class real_cache(object): - """ - Use it as a decorator of the function you plan to cache - Timeout: 0 = no timeout, otherwise in seconds - """ - - __caches = [] - - def __init__(self, timeout=None, skiparg=2, multi=None, size=8192): - assert skiparg >= 2 # at least self and cr - if timeout is None: - self.timeout = config['cache_timeout'] - else: - self.timeout = timeout - self.skiparg = skiparg - self.multi = multi - self.lasttime = time.time() - self.cache = LRU(size) # TODO take size from config - self.fun = None - cache.__caches.append(self) - - - def _generate_keys(self, dbname, kwargs2): - """ - Generate keys depending of the arguments and the self.mutli value - """ - - def to_tuple(d): - pairs = d.items() - pairs.sort(key=lambda (k,v): k) - for i, (k, v) in enumerate(pairs): - if isinstance(v, dict): - pairs[i] = (k, to_tuple(v)) - if isinstance(v, (list, set)): - pairs[i] = (k, tuple(v)) - elif not is_hashable(v): - pairs[i] = (k, repr(v)) - return tuple(pairs) - - if not self.multi: - key = (('dbname', dbname),) + to_tuple(kwargs2) - yield key, None - else: - multis = kwargs2[self.multi][:] - for id in multis: - kwargs2[self.multi] = (id,) - key = (('dbname', dbname),) + to_tuple(kwargs2) - yield key, id - - def _unify_args(self, *args, **kwargs): - # Update named arguments with positional argument values (without self and cr) - kwargs2 = self.fun_default_values.copy() - kwargs2.update(kwargs) - kwargs2.update(dict(zip(self.fun_arg_names, args[self.skiparg-2:]))) - return kwargs2 - - def clear(self, dbname, *args, **kwargs): - """clear the cache for database dbname - if *args and **kwargs are both empty, clear all the keys related to this database - """ - if not args and not kwargs: - keys_to_del = [key for key in self.cache.keys() if key[0][1] == dbname] - else: - kwargs2 = self._unify_args(*args, **kwargs) - keys_to_del = [key for key, _ in self._generate_keys(dbname, kwargs2) if key in self.cache.keys()] - - for key in keys_to_del: - self.cache.pop(key) - - @classmethod - def clean_caches_for_db(cls, dbname): - for c in cls.__caches: - c.clear(dbname) - - def __call__(self, fn): - if self.fun is not None: - raise Exception("Can not use a cache instance on more than one function") - self.fun = fn - - argspec = inspect.getargspec(fn) - self.fun_arg_names = argspec[0][self.skiparg:] - self.fun_default_values = {} - if argspec[3]: - self.fun_default_values = dict(zip(self.fun_arg_names[-len(argspec[3]):], argspec[3])) - - def cached_result(self2, cr, *args, **kwargs): - if time.time()-int(self.timeout) > self.lasttime: - self.lasttime = time.time() - t = time.time()-int(self.timeout) - old_keys = [key for key in self.cache.keys() if self.cache[key][1] < t] - for key in old_keys: - self.cache.pop(key) - - kwargs2 = self._unify_args(*args, **kwargs) - - result = {} - notincache = {} - for key, id in self._generate_keys(cr.dbname, kwargs2): - if key in self.cache: - result[id] = self.cache[key][0] - else: - notincache[id] = key - - if notincache: - if self.multi: - kwargs2[self.multi] = notincache.keys() - - result2 = fn(self2, cr, *args[:self.skiparg-2], **kwargs2) - if not self.multi: - key = notincache[None] - self.cache[key] = (result2, time.time()) - result[None] = result2 - else: - for id in result2: - key = notincache[id] - self.cache[key] = (result2[id], time.time()) - result.update(result2) - - if not self.multi: - return result[None] - return result - - cached_result.clear_cache = self.clear - return cached_result - -# TODO make it an option -cache = real_cache +#ormcache = dummy_cache +cache = dummy_cache def to_xml(s): return s.replace('&','&').replace('<','<').replace('>','>') From b5bf0e8c1c9bb0ad104c292d58d31d657e46aeb5 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 8 Jun 2011 11:25:15 +0200 Subject: [PATCH 02/16] [IMP] tools.cache cleanup, 15% speedup bzr revid: al@openerp.com-20110608092515-gum2y4snkrcrpjqb --- openerp/addons/base/ir/ir_rule.py | 30 +---- openerp/addons/base/ir/ir_translation.py | 12 +- openerp/tools/cache.py | 149 +++++++++++++++++++++++ openerp/tools/misc.py | 111 +---------------- 4 files changed, 161 insertions(+), 141 deletions(-) create mode 100644 openerp/tools/cache.py diff --git a/openerp/addons/base/ir/ir_rule.py b/openerp/addons/base/ir/ir_rule.py index b62da2ac4ff..86ded0387e0 100644 --- a/openerp/addons/base/ir/ir_rule.py +++ b/openerp/addons/base/ir/ir_rule.py @@ -134,22 +134,7 @@ class ir_rule(osv.osv): return [] def clear_cache(self, cr, uid): - cr.execute("""SELECT DISTINCT m.model - FROM ir_rule r - JOIN ir_model m - ON r.model_id = m.id - WHERE r.global - OR EXISTS (SELECT 1 - FROM rule_group_rel g_rel - JOIN res_groups_users_rel u_rel - ON g_rel.group_id = u_rel.gid - WHERE g_rel.rule_group_id = r.id - AND u_rel.uid = %s) - """, (uid,)) - models = map(itemgetter(0), cr.fetchall()) - clear = partial(self._compute_domain.clear_cache, self, uid) - [clear(model, mode) for model in models for mode in self._MODES] - + self._compute_domain.clear_cache(self) def domain_get(self, cr, uid, model_name, mode='read', context=None): dom = self._compute_domain(cr, uid, model_name, mode) @@ -164,20 +149,17 @@ class ir_rule(osv.osv): def unlink(self, cr, uid, ids, context=None): res = super(ir_rule, self).unlink(cr, uid, ids, context=context) - # Restart the cache on the _compute_domain method of ir.rule - self._compute_domain.clear_cache(self) + self.clear_cache() return res - def create(self, cr, user, vals, context=None): - res = super(ir_rule, self).create(cr, user, vals, context=context) - # Restart the cache on the _compute_domain method of ir.rule - self._compute_domain.clear_cache(self) + def create(self, cr, uid, vals, context=None): + res = super(ir_rule, self).create(cr, uid, vals, context=context) + self.clear_cache(cr, uid) return res def write(self, cr, uid, ids, vals, context=None): res = super(ir_rule, self).write(cr, uid, ids, vals, context=context) - # Restart the cache on the _compute_domain method - self._compute_domain.clear_cache(self) + self.clear_cache(cr,uid) return res ir_rule() diff --git a/openerp/addons/base/ir/ir_translation.py b/openerp/addons/base/ir/ir_translation.py index 0e6374b0143..02aa8f4a8fe 100644 --- a/openerp/addons/base/ir/ir_translation.py +++ b/openerp/addons/base/ir/ir_translation.py @@ -92,7 +92,7 @@ class ir_translation(osv.osv): cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)') cr.commit() - @tools.ormcache_multi(skiparg=3, multi=7) + @tools.ormcache_multi(skiparg=3, multi=6) def _get_ids(self, cr, uid, name, tt, lang, ids): translations = dict.fromkeys(ids, False) if ids: @@ -178,13 +178,11 @@ class ir_translation(osv.osv): return tools.ustr(source) return trad - def create(self, cursor, user, vals, context=None): + def create(self, cr, uid, vals, context=None): if not context: context = {} - ids = super(ir_translation, self).create(cursor, user, vals, context=context) - for trans_obj in self.read(cursor, user, [ids], ['name','type','res_id','src','lang'], context=context): - self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) - self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) + ids = super(ir_translation, self).create(cr, uid, vals, context=context) + self._get_source.clear_cache(self, vals.get('name',0), vals.get('type',0), vals.get('lang',0), vals.get('src',0)) return ids def write(self, cursor, user, ids, vals, context=None): @@ -193,8 +191,8 @@ class ir_translation(osv.osv): if isinstance(ids, (int, long)): ids = [ids] result = super(ir_translation, self).write(cursor, user, ids, vals, context=context) + self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context): - self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src']) self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']]) return result diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py new file mode 100644 index 00000000000..db4115aef36 --- /dev/null +++ b/openerp/tools/cache.py @@ -0,0 +1,149 @@ +import lru + +class ormcache(object): + """ LRU cache decorator for orm methods + """ + + def __init__(self, skiparg=2, size=8192, multi=None, timeout=None): + self.skiparg = skiparg + self.size = size + self.method = None + self.stat_miss = 0 + self.stat_hit = 0 + self.stat_err = 0 + + def __call__(self,m): + self.method = m + def lookup(self2, cr, *args): + r = self.lookup(self2, cr, *args) + #self.stat() + return r + def clear(self2, *args): + return self.clear(self2, *args) + lookup.clear_cache = self.clear + #print "lookup-func",lookup + return lookup + + def stat(self): + print "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % (self.stat_hit,self.stat_miss,self.stat_err, (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit) ) + + def lru(self, self2): + try: + ormcache = getattr(self2, '_ormcache') + except AttributeError: + ormcache = self2._ormcache = {} + try: + d = ormcache[self.method] + except KeyError: + d = ormcache[self.method] = lru.LRU(self.size) + return d + + def lookup(self, self2, cr, *args): + d = self.lru(self2) + key = args[self.skiparg-2:] + try: + r = d[key] + #print "lookup-hit",self2,cr,key,r + self.stat_hit += 1 + return r + except KeyError: + self.stat_miss += 1 + #print "lookup-miss",self2,cr,key + value = d[args] = self.method(self2, cr, *args) + #print "lookup-miss-value",value + return value + except TypeError: + self.stat_err += 1 + #print "lookup-error",self2,cr,key + return self.method(self2, cr, *args) + + def clear(self, self2, *args): + """ Remove *args entry from the cache or all keys if *args is undefined + """ + d = self.lru(self2) + if args: + try: + key = args[self.skiparg-2:] + #print "del",key + del d[key] + except KeyError: + pass + else: + d.clear() + +class ormcache_multi(ormcache): + def __init__(self, skiparg=2, size=8192, multi=3): + super(ormcache_multi,self).__init__(skiparg,size) + self.multi = multi - 2 + + def lookup(self, self2, cr, *args): + d = self.lru(self2) + args = list(args) + multi = self.multi + #print args, multi + ids = args[multi] + r = {} + miss = [] + + for i in ids: + args[multi] = i + key = tuple(args[self.skiparg-2:]) + try: + r[i] = d[key] + #print "lookup-hit",self2,cr,key,r[i] + self.stat_hit += 1 + except Exception: + self.stat_miss += 1 + miss.append(i) + #print "lookup-miss",self2,cr,key + + if miss: + args[multi] = miss + r.update(self.method(self2, cr, *args)) + + for i in miss: + args[multi] = i + key = tuple(args[self.skiparg-2:]) + d[key] = r[i] + + return r + +class dummy_cache(object): + """ Cache decorator replacement to actually do no caching. + """ + def __init__(self, *l, **kw): + pass + def __call__(self, fn): + fn.clear_cache = self.clear + return fn + def clear(self, *l, **kw): + pass + +#ormcache = dummy_cache +cache = dummy_cache + +if __name__ == '__main__': + + class A(): + @ormcache() + def m(self,a,b): + print "A::m(", self,a,b + return 1 + + @ormcache_multi(multi=3) + def n(self,cr,uid,ids): + print "m", self,cr,uid,ids + return dict([(i,i) for i in ids]) + + a=A() + #r=a.m(1,2) + #r=a.m(1,2) + r=a.n("cr",1,[1,2,3,4]) + r=a.n("cr",1,[1,2]) + print r + for i in a._ormcache: + print a._ormcache[i].d + a.n.clear_cache(a,1,1) + r=a.n("cr",1,[1,2]) + print r + #r=a.n("cr",1,[1,2]) diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py index 307cf04ce68..48a9a936783 100644 --- a/openerp/tools/misc.py +++ b/openerp/tools/misc.py @@ -57,7 +57,7 @@ except ImportError: import openerp.loglevels as loglevels from config import config -from lru import LRU +from cache import * # get_encodings, ustr and exception_to_unicode were originally from tools.misc. # There are moved to loglevels until we refactor tools. @@ -622,7 +622,6 @@ class UpdateableDict(local): def __ne__(self, y): return self.dict.__ne__(y) - # Don't use ! Use res.currency.round() class currency(float): @@ -639,114 +638,6 @@ class currency(float): # display_value = int(self*(10**(-self.accuracy))/self.rounding)*self.rounding/(10**(-self.accuracy)) # return str(display_value) -class ormcache(object): - """ LRU cache decorator for orm methods - """ - - def __init__(self, skiparg=2, size=8192, multi=None, timeout=None): - self.skiparg = skiparg - self.size = size - self.method = None - self.stat_miss = 0 - self.stat_hit = 0 - self.stat_err = 0 - - def __call__(self,m): - self.method = m - def lookup(self2, cr, *args): - r = self.lookup(m, m, self2, cr, *args) -# print "lookup-stats hit miss err",self.stat_hit,self.stat_miss,self.stat_err - return r - def clear(self2, *args): - return self.clear(self2, *args) - lookup.clear_cache = self.clear - print "returned",lookup - return lookup - - def lookup(self, cacheid, method, self2, cr, *args): - try: - ormcache = getattr(self2, '_ormcache') - except AttributeError: - ormcache = self2._ormcache = {} - try: - d = ormcache[cacheid] - except KeyError: - d = ormcache[cacheid] = LRU(self.size) - key = args[self.skiparg-2:] - try: - r = d[key] -# print "lookup-hit",self2,cr,key,r - self.stat_hit += 1 - return r - except KeyError: - self.stat_miss += 1 -# print "lookup-miss",self2,cr,key - value = d[args] = method(self2, cr, *args) -# print "lookup-miss-value",value - return value - except TypeError: - self.stat_err += 1 -# print "lookup-typeerror",self2,cr,key - return method(self2, cr, *args) - - def clear(self, self2, *args, **kw): - """ Remove *args entry from the cache or all keys if *args is undefined - """ - try: - ormcache = getattr(self2, '_ormcache') - except AttributeError: - ormcache = self2._ormcache = {} - try: - d = ormcache[self.method] - except KeyError: - d = ormcache[self.method] = LRU(self.size) - d.clear() - -class ormcache_multi(ormcache): - def __init__(self, skiparg=2, size=8192, multi=4): - super(ormcache_multi,self).__init__(skiparg,size) - self.multi = multi - 3 - - def lookup(self, cacheid, method, self2, cr, *args): - superlookup = super(ormcache_multi,self).lookup - args = list(args) - multi = self.multi -# print args,multi - ids = args[multi] - r = {} - miss = [] - - def add_to_miss(_self2, _cr, *_args): - miss.append(_args[multi]) - - for i in ids: - args[multi] = i - r[i] = superlookup(method, add_to_miss, self2, cr, *args) - - args[multi] = miss - r.update(method(self2, cr, *args)) - - for i in miss: - args[multi] = i - key = tuple(args[self.skiparg-2:]) - self2._ormcache[cacheid][key] = r[i] - - return r - -class dummy_cache(object): - """ Cache decorator replacement to actually do no caching. - """ - def __init__(self, *l, **kw): - pass - def clear(self, *l, **kw): - pass - def __call__(self, fn): - fn.clear_cache = self.clear - return fn - -#ormcache = dummy_cache -cache = dummy_cache - def to_xml(s): return s.replace('&','&').replace('<','<').replace('>','>') From b687cc3bb4c035cce3975d49313e96b76a768da2 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 17:01:22 +0200 Subject: [PATCH 03/16] [FIX] remove unused clear function bzr revid: al@openerp.com-20110824150122-ejryb7ni1l92d1hp --- openerp/tools/cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index db4115aef36..b9323fa0d02 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -18,8 +18,6 @@ class ormcache(object): r = self.lookup(self2, cr, *args) #self.stat() return r - def clear(self2, *args): - return self.clear(self2, *args) lookup.clear_cache = self.clear #print "lookup-func",lookup return lookup From 362399c6ac4c71ea5ac1fa68b81d47e75e77219b Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 17:29:05 +0200 Subject: [PATCH 04/16] [FIX] remove cache on used function bzr revid: al@openerp.com-20110824152905-8en0wclyumjz5105 --- openerp/addons/base/res/res_user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openerp/addons/base/res/res_user.py b/openerp/addons/base/res/res_user.py index c9eb1da381c..a7e28730657 100644 --- a/openerp/addons/base/res/res_user.py +++ b/openerp/addons/base/res/res_user.py @@ -351,7 +351,6 @@ class users(osv.osv): 'menu_tips':True } - @tools.ormcache() def company_get(self, cr, uid, uid2, context=None): return self._get_company(cr, uid, context=context, uid2=uid2) From 8d08cbf5610bd2df8b9186f2cc3b05e33b191d02 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 17:30:42 +0200 Subject: [PATCH 05/16] [FIX] remove cache on used function bzr revid: al@openerp.com-20110824153042-w6wgegxjruu06ln1 --- openerp/addons/base/res/res_user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openerp/addons/base/res/res_user.py b/openerp/addons/base/res/res_user.py index a7e28730657..dc672fc5146 100644 --- a/openerp/addons/base/res/res_user.py +++ b/openerp/addons/base/res/res_user.py @@ -373,7 +373,6 @@ class users(osv.osv): res = super(users, self).write(cr, uid, ids, values, context=context) # clear caches linked to the users - self.company_get.clear_cache(self) self.pool.get('ir.model.access').call_cache_clearing_methods(cr) clear = partial(self.pool.get('ir.rule').clear_cache, cr) map(clear, ids) From 93f9a0b0cd51595e8706f4c5faec87ee487e5070 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 18:19:18 +0200 Subject: [PATCH 06/16] [IMP] ir_rule distribute clear_cache bzr revid: al@openerp.com-20110824161918-dkexqhuc1mzpx9fs --- openerp/addons/base/ir/ir_rule.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/openerp/addons/base/ir/ir_rule.py b/openerp/addons/base/ir/ir_rule.py index 1c43227a76e..d7144cfa01b 100644 --- a/openerp/addons/base/ir/ir_rule.py +++ b/openerp/addons/base/ir/ir_rule.py @@ -138,9 +138,6 @@ class ir_rule(osv.osv): return domain return [] - def clear_cache(self, cr, uid): - self._compute_domain.clear_cache(self) - def domain_get(self, cr, uid, model_name, mode='read', context=None): dom = self._compute_domain(cr, uid, model_name, mode) if dom: @@ -154,17 +151,17 @@ class ir_rule(osv.osv): def unlink(self, cr, uid, ids, context=None): res = super(ir_rule, self).unlink(cr, uid, ids, context=context) - self.clear_cache() + self._compute_domain.clear_cache(self) return res def create(self, cr, uid, vals, context=None): res = super(ir_rule, self).create(cr, uid, vals, context=context) - self.clear_cache(cr, uid) + self._compute_domain.clear_cache(self) return res def write(self, cr, uid, ids, vals, context=None): res = super(ir_rule, self).write(cr, uid, ids, vals, context=context) - self.clear_cache(cr,uid) + self._compute_domain.clear_cache(self) return res ir_rule() From 52a84f0758fefa2174e7a6bf27508551ef9ef521 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 19:22:37 +0200 Subject: [PATCH 07/16] [FIX] revert previous commit ir_rule clear_cache bzr revid: al@openerp.com-20110824172237-arzved9kk54u3hwu --- openerp/addons/base/ir/ir_rule.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/openerp/addons/base/ir/ir_rule.py b/openerp/addons/base/ir/ir_rule.py index d7144cfa01b..734d00eb9a2 100644 --- a/openerp/addons/base/ir/ir_rule.py +++ b/openerp/addons/base/ir/ir_rule.py @@ -138,6 +138,9 @@ class ir_rule(osv.osv): return domain return [] + def clear_cache(self, cr, uid): + self._compute_domain.clear_cache(self) + def domain_get(self, cr, uid, model_name, mode='read', context=None): dom = self._compute_domain(cr, uid, model_name, mode) if dom: @@ -151,17 +154,17 @@ class ir_rule(osv.osv): def unlink(self, cr, uid, ids, context=None): res = super(ir_rule, self).unlink(cr, uid, ids, context=context) - self._compute_domain.clear_cache(self) + self.clear_cache(cr, uid) return res def create(self, cr, uid, vals, context=None): res = super(ir_rule, self).create(cr, uid, vals, context=context) - self._compute_domain.clear_cache(self) + self.clear_cache(cr, uid) return res def write(self, cr, uid, ids, vals, context=None): res = super(ir_rule, self).write(cr, uid, ids, vals, context=context) - self._compute_domain.clear_cache(self) + self.clear_cache(cr,uid) return res ir_rule() From 8d31082408b8d650c724ead0a53da93416a6c599 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 23:19:48 +0200 Subject: [PATCH 08/16] [FIX] ir.model.access check use positional args bzr revid: al@openerp.com-20110824211948-gvtc7uv9e3oeeo3e --- openerp/addons/base/ir/ir_attachment.py | 4 ++-- openerp/addons/base/ir/ir_ui_menu.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index 2c3d4e61e84..ecdd2ceca37 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -33,7 +33,6 @@ class ir_attachment(osv.osv): """ if not ids: return - ima = self.pool.get('ir.model.access') res_ids = {} if ids: if isinstance(ids, (int, long)): @@ -47,12 +46,13 @@ class ir_attachment(osv.osv): if 'res_model' in values and 'res_id' in values: res_ids.setdefault(values['res_model'],set()).add(values['res_id']) + ima = self.pool.get('ir.model.access') for model, mids in res_ids.items(): # ignore attachments that are not attached to a resource anymore when checking access rights # (resource was deleted but attachment was not) cr.execute('select id from '+self.pool.get(model)._table+' where id in %s', (tuple(mids),)) mids = [x[0] for x in cr.fetchall()] - ima.check(cr, uid, model, mode, context=context) + ima.check(cr, uid, model, mode) self.pool.get(model).check_access_rule(cr, uid, mids, mode, context=context) def search(self, cr, uid, args, offset=0, limit=None, order=None, diff --git a/openerp/addons/base/ir/ir_ui_menu.py b/openerp/addons/base/ir/ir_ui_menu.py index 9ed4d7806bc..83660cfe753 100644 --- a/openerp/addons/base/ir/ir_ui_menu.py +++ b/openerp/addons/base/ir/ir_ui_menu.py @@ -88,7 +88,7 @@ class ir_ui_menu(osv.osv): field = model_field.get(menu.action._name) if field and data[field]: - if not modelaccess.check(cr, uid, data[field], raise_exception=False): + if not modelaccess.check(cr, uid, data[field], False): continue else: # if there is no action, it's a 'folder' menu From 8c0fb375b45804cc037b9d2976427f4f6e1b774c Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 23:44:16 +0200 Subject: [PATCH 09/16] [FIX] ir.model clear_cache use self bzr revid: al@openerp.com-20110824214416-9eby8h7mnqmlr8a3 --- openerp/addons/base/ir/ir_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openerp/addons/base/ir/ir_model.py b/openerp/addons/base/ir/ir_model.py index 4dba1d57dad..0c4b52ac1ac 100644 --- a/openerp/addons/base/ir/ir_model.py +++ b/openerp/addons/base/ir/ir_model.py @@ -638,8 +638,8 @@ class ir_model_data(osv.osv): """ Regular unlink method, but make sure to clear the caches. """ ref_ids = self.browse(cr, uid, ids, context=context) for ref_id in ref_ids: - self._get_id.clear_cache(cr.dbname, uid, ref_id.module, ref_id.name) - self.get_object_reference.clear_cache(cr.dbname, uid, ref_id.module, ref_id.name) + self._get_id.clear_cache(self, uid, ref_id.module, ref_id.name) + self.get_object_reference.clear_cache(self, uid, ref_id.module, ref_id.name) return super(ir_model_data,self).unlink(cr, uid, ids, context=context) def _update(self,cr, uid, model, module, values, xml_id=False, store=True, noupdate=False, mode='init', res_id=False, context=None): From 7a52778235730d039cfcd11ce11ff15264364f16 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Wed, 24 Aug 2011 23:48:43 +0200 Subject: [PATCH 10/16] [IMP] ir.model.data unlink clear full cache bzr revid: al@openerp.com-20110824214843-c75b3jl1fad0mv5a --- openerp/addons/base/ir/ir_model.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openerp/addons/base/ir/ir_model.py b/openerp/addons/base/ir/ir_model.py index 0c4b52ac1ac..a9d2a21b7a5 100644 --- a/openerp/addons/base/ir/ir_model.py +++ b/openerp/addons/base/ir/ir_model.py @@ -636,10 +636,8 @@ class ir_model_data(osv.osv): def unlink(self, cr, uid, ids, context=None): """ Regular unlink method, but make sure to clear the caches. """ - ref_ids = self.browse(cr, uid, ids, context=context) - for ref_id in ref_ids: - self._get_id.clear_cache(self, uid, ref_id.module, ref_id.name) - self.get_object_reference.clear_cache(self, uid, ref_id.module, ref_id.name) + self._get_id.clear_cache(self) + self.get_object_reference.clear_cache(self) return super(ir_model_data,self).unlink(cr, uid, ids, context=context) def _update(self,cr, uid, model, module, values, xml_id=False, store=True, noupdate=False, mode='init', res_id=False, context=None): From c7076adb06e7b77ad77bb801b1e98e111a448614 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Thu, 25 Aug 2011 00:03:24 +0200 Subject: [PATCH 11/16] [FIX] ir_attachement check use positional args bzr revid: al@openerp.com-20110824220324-xfo7swm45stcooun --- openerp/addons/base/ir/ir_attachment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index ecdd2ceca37..95a4cd8cab0 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -80,7 +80,7 @@ class ir_attachment(osv.osv): # performed in batch as much as possible. ima = self.pool.get('ir.model.access') for model, targets in model_attachments.iteritems(): - if not ima.check(cr, uid, model, 'read', raise_exception=False, context=context): + if not ima.check(cr, uid, model, 'read', False): # remove all corresponding attachment ids for attach_id in itertools.chain(*targets.values()): ids.remove(attach_id) From 4be82ffd5223b9dc1ea23d32a57e2789590883ef Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Thu, 25 Aug 2011 00:23:30 +0200 Subject: [PATCH 12/16] [FIX] remove debugging bzr revid: al@openerp.com-20110824222330-zrfnr4qme71r9oqi --- openerp/tools/cache.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index b9323fa0d02..fd28a659377 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -1,7 +1,7 @@ import lru class ormcache(object): - """ LRU cache decorator for orm methods + """ LRU cache decorator for orm methods, """ def __init__(self, skiparg=2, size=8192, multi=None, timeout=None): @@ -16,14 +16,12 @@ class ormcache(object): self.method = m def lookup(self2, cr, *args): r = self.lookup(self2, cr, *args) - #self.stat() return r lookup.clear_cache = self.clear - #print "lookup-func",lookup return lookup def stat(self): - print "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % (self.stat_hit,self.stat_miss,self.stat_err, (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit) ) + return "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % (self.stat_hit,self.stat_miss,self.stat_err, (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit) ) def lru(self, self2): try: @@ -41,18 +39,14 @@ class ormcache(object): key = args[self.skiparg-2:] try: r = d[key] - #print "lookup-hit",self2,cr,key,r self.stat_hit += 1 return r except KeyError: self.stat_miss += 1 - #print "lookup-miss",self2,cr,key value = d[args] = self.method(self2, cr, *args) - #print "lookup-miss-value",value return value except TypeError: self.stat_err += 1 - #print "lookup-error",self2,cr,key return self.method(self2, cr, *args) def clear(self, self2, *args): @@ -62,7 +56,6 @@ class ormcache(object): if args: try: key = args[self.skiparg-2:] - #print "del",key del d[key] except KeyError: pass @@ -78,7 +71,6 @@ class ormcache_multi(ormcache): d = self.lru(self2) args = list(args) multi = self.multi - #print args, multi ids = args[multi] r = {} miss = [] @@ -88,12 +80,10 @@ class ormcache_multi(ormcache): key = tuple(args[self.skiparg-2:]) try: r[i] = d[key] - #print "lookup-hit",self2,cr,key,r[i] self.stat_hit += 1 except Exception: self.stat_miss += 1 miss.append(i) - #print "lookup-miss",self2,cr,key if miss: args[multi] = miss @@ -134,8 +124,8 @@ if __name__ == '__main__': return dict([(i,i) for i in ids]) a=A() - #r=a.m(1,2) - #r=a.m(1,2) + r=a.m(1,2) + r=a.m(1,2) r=a.n("cr",1,[1,2,3,4]) r=a.n("cr",1,[1,2]) print r @@ -144,4 +134,4 @@ if __name__ == '__main__': a.n.clear_cache(a,1,1) r=a.n("cr",1,[1,2]) print r - #r=a.n("cr",1,[1,2]) + r=a.n("cr",1,[1,2]) From 86965e38848baa706d5948d3837485e63b1b8e0f Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Thu, 25 Aug 2011 01:11:31 +0200 Subject: [PATCH 13/16] [FIX] orm ir.translation get_source to use positional args bzr revid: al@openerp.com-20110824231131-y33ng5x4d6j2vr6w --- openerp/osv/orm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index b721de5accc..9c099355427 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -1264,7 +1264,7 @@ class orm_template(object): else: translated_msg = tmp_msg else: - translated_msg = trans._get_source(cr, uid, self._name, 'constraint', lng, source=msg) or msg + translated_msg = trans._get_source(cr, uid, self._name, 'constraint', lng, msg) or msg error_msgs.append( _("Error occurred while validating the field(s) %s: %s") % (','.join(fields), translated_msg) ) From 6fdadb04f51c468c772a9dcc9e5f344d77d7be66 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 25 Aug 2011 14:47:11 +0200 Subject: [PATCH 14/16] [IMP] tools.cache: added missing clean_caches_for_db replacement. bzr revid: vmt@openerp.com-20110825124711-y2uckmvcoa0arz4h --- openerp/modules/registry.py | 24 ++++++++++++++++++++++++ openerp/osv/orm.py | 12 ++++++++++++ openerp/service/web_services.py | 4 +++- openerp/sql_db.py | 1 - 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 11f484a71a0..19b6227d621 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -82,6 +82,15 @@ class Registry(object): return res + def clear_caches + """ Clear the caches + + This clears the caches associated to methods decorated with + ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models. + """ + for model in self.models.itervalues(): + model.clear_caches() + class RegistryManager(object): """ Model registries manager. @@ -155,4 +164,19 @@ class RegistryManager(object): del cls.registries[db_name] + @classmethod + def clear_caches(db_name) + """ Clear the caches + + This clears the caches associated to methods decorated with + ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models + of the given database name. + + This method is given to spare you a ``RegistryManager.get(db_name)`` + that would loads the given database if it was not already loaded. + """ + if db_name in cls.registries: + del cls.registries[db_name].clear_caches() + + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 9c099355427..d2e1cb95487 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -2251,6 +2251,18 @@ class orm_template(object): values = defaults return values + def clear_caches(self): + """ Clear the caches + + This clears the caches associated to methods decorated with + ``tools.ormcache`` or ``tools.ormcache_multi``. + """ + try: + getattr(self, '_ormcache') + self._ormcache = {} + except AttributeError: + pass + class orm_memory(orm_template): _protected = ['read', 'write', 'create', 'default_get', 'perm_read', 'unlink', 'fields_get', 'fields_view_get', 'search', 'name_get', 'distinct_field_get', 'name_search', 'copy', 'import_data', 'search_count', 'exists'] diff --git a/openerp/service/web_services.py b/openerp/service/web_services.py index c3b23a4dbb7..c768a3cb877 100644 --- a/openerp/service/web_services.py +++ b/openerp/service/web_services.py @@ -155,6 +155,7 @@ class db(netsvc.ExportService): def exp_drop(self, db_name): sql_db.close_db(db_name) + openerp.modules.registry.RegistryManager.clear_caches(db_name) openerp.netsvc.Agent.cancel(db_name) logger = netsvc.Logger() @@ -258,7 +259,8 @@ class db(netsvc.ExportService): def exp_rename(self, old_name, new_name): sql_db.close_db(old_name) - openerp.netsvc.Agent.cancel(db_name) + openerp.modules.registry.RegistryManager.clear_caches(old_name) + openerp.netsvc.Agent.cancel(old_name) logger = netsvc.Logger() db = sql_db.db_connect('template1') diff --git a/openerp/sql_db.py b/openerp/sql_db.py index 18efb62eb18..f4a899c8db2 100644 --- a/openerp/sql_db.py +++ b/openerp/sql_db.py @@ -500,7 +500,6 @@ def db_connect(db_name): def close_db(db_name): """ You might want to call openerp.netsvc.Agent.cancel(db_name) along this function.""" _Pool.close_all(dsn(db_name)) - tools.cache.clean_caches_for_db(db_name) ct = currentThread() if hasattr(ct, 'dbname'): delattr(ct, 'dbname') From dad3528ab7191bd4097d0fac275b89bccbfd6119 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 26 Aug 2011 14:10:26 +0200 Subject: [PATCH 15/16] [FIX] registry.py: typos (invalid syntax). I wonder how come I made the previous commit. bzr revid: vmt@openerp.com-20110826121026-da90yx3iapj6y1v1 --- openerp/modules/registry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 19b6227d621..e651c5d7994 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -82,7 +82,7 @@ class Registry(object): return res - def clear_caches + def clear_caches(): """ Clear the caches This clears the caches associated to methods decorated with @@ -165,7 +165,7 @@ class RegistryManager(object): @classmethod - def clear_caches(db_name) + def clear_caches(db_name): """ Clear the caches This clears the caches associated to methods decorated with @@ -176,7 +176,7 @@ class RegistryManager(object): that would loads the given database if it was not already loaded. """ if db_name in cls.registries: - del cls.registries[db_name].clear_caches() + cls.registries[db_name].clear_caches() # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: From 174e25e66fcb69fe44080b644b42fa3eb33279e9 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 26 Aug 2011 14:16:39 +0200 Subject: [PATCH 16/16] [IMP] tools.cache: removed unused variable. bzr revid: vmt@openerp.com-20110826121639-irm0g0eh5yec1c88 --- openerp/tools/cache.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index fd28a659377..090651cf6a4 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -107,9 +107,6 @@ class dummy_cache(object): def clear(self, *l, **kw): pass -#ormcache = dummy_cache -cache = dummy_cache - if __name__ == '__main__': class A():