Xavier Morel
083c70bbb6
Before this, invalidations to the UID cache is not synchronised between workers because it's an ad-hoc solution (so a user changing their password or an admin disabling a user would only lock out an attacker currently using the API of one of possibly several workers). Shift the entire thing to ormcache which already has proper support for synchronising cache invalidation between workers. Also simplify the cache invalidation mess in Users.write because the caches have been unified into a single registry-level LRU, so the half-dozen cache clears on specific ormcached methods & models is pretty much the same as repeatedly calling clear_caches on the current model. **However** registry.cache is trivially accessible from server actions and safe_eval as long as they provide access to a model (through `model.pool.cache`). Which is common, and an issue given we're very much putting sensible data in there. Fix this by renaming `Registry.cache` to `Registry.__cache`, this requires few editions and mangled names are not accessible from safe_eval contexts. The alternative would have been to add more bespoke handling of the uid cache to hook it into the cache invalidation propagation machinery. After discussion with (@)odony, fixing LRU access and using that seems cleaner and less error-prone. Note on lazy_property ===================== Make Registry.cache / Registry.__cache into a regular attribute: the overhead of the LRU is not that high (compared to that of the registry itself), it's rare that we *don't* need it, and it's assumed to be a persisted attribute (it's not just a cache) so making it a normal attribute seems fine; and lazy_property doesn't work for mangled names: the name of the property is mangled using the name of the definition class, but the name of the symbol (fget) is not mangled so lazy_property would set the __cache attribute but then Python would lookup _Registry__cache, creating a new cache every access. And we can't (always) mangle things correctly on `__get__(obj, owner)`: `owner` is just `type(obj)`, meaning in the case of inheritance the type we get is the type through which the property is accessed rather than the one it's defined on. So it would work in the cases where no inheritance is involved (such as Registry.__cache) but not in general (lest we want to play around walking the MRO ourselves to find the definition source, which doesn't seem worth it). lazy_property *could* be made to work properly on Python 3.6+: the descriptor protocol gains `__set_name__(name, owner)`, which is called with the properly mangled name — and with the definition class to boot (though there might still be issues when overriding lazy properties as the override will be mangled & named differently... or maybe that's a feature?). However we're still supporting 3.5 at this point, AFAIK, so that's not an option. Plus it feels unnecessary / not very useful. However add an assertion to `lazy_property` so it signals when we try to use it on a mangled method (as otherwise it kinda sorta work in the sense that the property / object is accessible but is in effect a slower way to write a regular property).
233 lines
8.1 KiB
Python
233 lines
8.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
|
|
|
# decorator makes wrappers that have the same API as their wrapped function
|
|
from collections import Counter, defaultdict
|
|
from decorator import decorator
|
|
from inspect import signature
|
|
import logging
|
|
|
|
unsafe_eval = eval
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ormcache_counter(object):
|
|
""" Statistic counters for cache entries. """
|
|
__slots__ = ['hit', 'miss', 'err']
|
|
|
|
def __init__(self):
|
|
self.hit = 0
|
|
self.miss = 0
|
|
self.err = 0
|
|
|
|
@property
|
|
def ratio(self):
|
|
return 100.0 * self.hit / (self.hit + self.miss or 1)
|
|
|
|
# statistic counters dictionary, maps (dbname, modelname, method) to counter
|
|
STAT = defaultdict(ormcache_counter)
|
|
|
|
|
|
class ormcache(object):
|
|
""" LRU cache decorator for model methods.
|
|
The parameters are strings that represent expressions referring to the
|
|
signature of the decorated method, and are used to compute a cache key::
|
|
|
|
@ormcache('model_name', 'mode')
|
|
def _compute_domain(self, model_name, mode="read"):
|
|
...
|
|
|
|
For the sake of backward compatibility, the decorator supports the named
|
|
parameter `skiparg`::
|
|
|
|
@ormcache(skiparg=1)
|
|
def _compute_domain(self, model_name, mode="read"):
|
|
...
|
|
|
|
Methods implementing this decorator should never return a Recordset,
|
|
because the underlying cursor will eventually be closed and raise a
|
|
`psycopg2.OperationalError`.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
self.args = args
|
|
self.skiparg = kwargs.get('skiparg')
|
|
|
|
def __call__(self, method):
|
|
self.method = method
|
|
self.determine_key()
|
|
lookup = decorator(self.lookup, method)
|
|
lookup.clear_cache = self.clear
|
|
return lookup
|
|
|
|
def determine_key(self):
|
|
""" Determine the function that computes a cache key from arguments. """
|
|
if self.skiparg is None:
|
|
# build a string that represents function code and evaluate it
|
|
args = str(signature(self.method))[1:-1]
|
|
if self.args:
|
|
code = "lambda %s: (%s,)" % (args, ", ".join(self.args))
|
|
else:
|
|
code = "lambda %s: ()" % (args,)
|
|
self.key = unsafe_eval(code)
|
|
else:
|
|
# backward-compatible function that uses self.skiparg
|
|
self.key = lambda *args, **kwargs: args[self.skiparg:]
|
|
|
|
def lru(self, model):
|
|
counter = STAT[(model.pool.db_name, model._name, self.method)]
|
|
return model.pool._Registry__cache, (model._name, self.method), counter
|
|
|
|
def lookup(self, method, *args, **kwargs):
|
|
d, key0, counter = self.lru(args[0])
|
|
key = key0 + self.key(*args, **kwargs)
|
|
try:
|
|
r = d[key]
|
|
counter.hit += 1
|
|
return r
|
|
except KeyError:
|
|
counter.miss += 1
|
|
value = d[key] = self.method(*args, **kwargs)
|
|
return value
|
|
except TypeError:
|
|
_logger.warning("cache lookup error on %r", key, exc_info=True)
|
|
counter.err += 1
|
|
return self.method(*args, **kwargs)
|
|
|
|
def clear(self, model, *args):
|
|
""" Clear the registry cache """
|
|
model.pool._clear_cache()
|
|
|
|
|
|
class ormcache_context(ormcache):
|
|
""" This LRU cache decorator is a variant of :class:`ormcache`, with an
|
|
extra parameter ``keys`` that defines a sequence of dictionary keys. Those
|
|
keys are looked up in the ``context`` parameter and combined to the cache
|
|
key made by :class:`ormcache`.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
super(ormcache_context, self).__init__(*args, **kwargs)
|
|
self.keys = kwargs['keys']
|
|
|
|
def determine_key(self):
|
|
""" Determine the function that computes a cache key from arguments. """
|
|
assert self.skiparg is None, "ormcache_context() no longer supports skiparg"
|
|
# build a string that represents function code and evaluate it
|
|
sign = signature(self.method)
|
|
args = str(sign)[1:-1]
|
|
cont_expr = "(context or {})" if 'context' in sign.parameters else "self._context"
|
|
keys_expr = "tuple(%s.get(k) for k in %r)" % (cont_expr, self.keys)
|
|
if self.args:
|
|
code = "lambda %s: (%s, %s)" % (args, ", ".join(self.args), keys_expr)
|
|
else:
|
|
code = "lambda %s: (%s,)" % (args, keys_expr)
|
|
self.key = unsafe_eval(code)
|
|
|
|
|
|
class ormcache_multi(ormcache):
|
|
""" This LRU cache decorator is a variant of :class:`ormcache`, with an
|
|
extra parameter ``multi`` that gives the name of a parameter. Upon call, the
|
|
corresponding argument is iterated on, and every value leads to a cache
|
|
entry under its own key.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
super(ormcache_multi, self).__init__(*args, **kwargs)
|
|
self.multi = kwargs['multi']
|
|
|
|
def determine_key(self):
|
|
""" Determine the function that computes a cache key from arguments. """
|
|
assert self.skiparg is None, "ormcache_multi() no longer supports skiparg"
|
|
assert isinstance(self.multi, str), "ormcache_multi() parameter multi must be an argument name"
|
|
|
|
super(ormcache_multi, self).determine_key()
|
|
|
|
# key_multi computes the extra element added to the key
|
|
sign = signature(self.method)
|
|
args = str(sign)[1:-1]
|
|
code_multi = "lambda %s: %s" % (args, self.multi)
|
|
self.key_multi = unsafe_eval(code_multi)
|
|
|
|
# self.multi_pos is the position of self.multi in args
|
|
self.multi_pos = list(sign.parameters).index(self.multi)
|
|
|
|
def lookup(self, method, *args, **kwargs):
|
|
d, key0, counter = self.lru(args[0])
|
|
base_key = key0 + self.key(*args, **kwargs)
|
|
ids = self.key_multi(*args, **kwargs)
|
|
result = {}
|
|
missed = []
|
|
|
|
# first take what is available in the cache
|
|
for i in ids:
|
|
key = base_key + (i,)
|
|
try:
|
|
result[i] = d[key]
|
|
counter.hit += 1
|
|
except Exception:
|
|
counter.miss += 1
|
|
missed.append(i)
|
|
|
|
if missed:
|
|
# call the method for the ids that were not in the cache; note that
|
|
# thanks to decorator(), the multi argument will be bound and passed
|
|
# positionally in args.
|
|
args = list(args)
|
|
args[self.multi_pos] = missed
|
|
result.update(method(*args, **kwargs))
|
|
|
|
# store those new results back in the cache
|
|
for i in missed:
|
|
key = base_key + (i,)
|
|
d[key] = result[i]
|
|
|
|
return result
|
|
|
|
|
|
class dummy_cache(object):
|
|
""" Cache decorator replacement to actually do no caching. """
|
|
def __init__(self, *l, **kw):
|
|
pass
|
|
|
|
def __call__(self, fn):
|
|
fn.clear_cache = self.clear
|
|
return fn
|
|
|
|
def clear(self, *l, **kw):
|
|
pass
|
|
|
|
|
|
def log_ormcache_stats(sig=None, frame=None):
|
|
""" Log statistics of ormcache usage by database, model, and method. """
|
|
from odoo.modules.registry import Registry
|
|
import threading
|
|
|
|
me = threading.currentThread()
|
|
me_dbname = getattr(me, 'dbname', 'n/a')
|
|
|
|
for dbname, reg in sorted(Registry.registries.d.items()):
|
|
# set logger prefix to dbname
|
|
me.dbname = dbname
|
|
entries = Counter(k[:2] for k in reg._Registry__cache.d)
|
|
# show entries sorted by model name, method name
|
|
for key in sorted(entries, key=lambda key: (key[0], key[1].__name__)):
|
|
model, method = key
|
|
stat = STAT[(dbname, model, method)]
|
|
_logger.info(
|
|
"%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s",
|
|
entries[key], stat.hit, stat.miss, stat.err, stat.ratio, model, method.__name__,
|
|
)
|
|
|
|
me.dbname = me_dbname
|
|
|
|
|
|
def get_cache_key_counter(bound_method, *args, **kwargs):
|
|
""" Return the cache, key and stat counter for the given call. """
|
|
model = bound_method.__self__
|
|
ormcache = bound_method.clear_cache.__self__
|
|
cache, key0, counter = ormcache.lru(model)
|
|
key = key0 + ormcache.key(model, *args, **kwargs)
|
|
return cache, key, counter
|
|
|
|
# For backward compatibility
|
|
cache = ormcache
|