From d539be329e4469660270651ee1530eae8c1b9604 Mon Sep 17 00:00:00 2001 From: Nicolas Lempereur Date: Wed, 18 Mar 2015 15:32:54 +0100 Subject: [PATCH 01/25] [FIX] view_form: add context to search_count Currently, the view_form search_count doesn't propagate the context, so when searching on a translated field, the count can be different than the one we would expect and get with a search in a current language different than en_US. opw-628792 opw-630212 closes #5825 --- addons/web/static/src/js/view_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/web/static/src/js/view_form.js b/addons/web/static/src/js/view_form.js index a1241288450..5b58594147a 100644 --- a/addons/web/static/src/js/view_form.js +++ b/addons/web/static/src/js/view_form.js @@ -2574,7 +2574,7 @@ instance.web.form.FieldCharDomain = instance.web.form.AbstractField.extend(insta var model = this.options.model || this.field_manager.get_field_value(this.options.model_field); var domain = instance.web.pyeval.eval('domain', this.get('value')); var ds = new instance.web.DataSetStatic(self, model, self.build_context()); - ds.call('search_count', [domain]).then(function (results) { + ds.call('search_count', [domain, self.build_context()]).then(function (results) { $('.oe_domain_count', self.$el).text(results + ' records selected'); if (self.get('effective_readonly')) { $('button span', self.$el).text(' See selection'); From accd3b472a1b0c01e62d47cb571a68dd88238d21 Mon Sep 17 00:00:00 2001 From: Nicolas Piganeau Date: Sun, 11 Jan 2015 16:02:07 +0100 Subject: [PATCH 02/25] Fix unwanted opening of stock.picking when scrapping from production --- addons/stock/wizard/stock_move.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/addons/stock/wizard/stock_move.py b/addons/stock/wizard/stock_move.py index 7016494572d..4e034be0510 100644 --- a/addons/stock/wizard/stock_move.py +++ b/addons/stock/wizard/stock_move.py @@ -88,16 +88,16 @@ class stock_move_scrap(osv.osv_memory): context=context) if context.get('active_id'): move = self.pool.get('stock.move').browse(cr, uid, context['active_id'], context=context) - return { - 'view_type': 'form', - 'view_mode': 'form', - 'res_model': 'stock.picking', - 'type': 'ir.actions.act_window', - 'res_id': move.picking_id.id, - 'context': context - } - else: - return {'type': 'ir.actions.act_window_close'} + if move.picking_id: + return { + 'view_type': 'form', + 'view_mode': 'form', + 'res_model': 'stock.picking', + 'type': 'ir.actions.act_window', + 'res_id': move.picking_id.id, + 'context': context + } + return {'type': 'ir.actions.act_window_close'} From efe8bfe5f17c0d60636d76e31852d64e8366e1ea Mon Sep 17 00:00:00 2001 From: Nicolas Lempereur Date: Fri, 20 Mar 2015 16:09:29 +0100 Subject: [PATCH 03/25] [FIX] report_invoice: minor css fix to right align A field was not rightly aligned in a report, this fix add a class which fixes this. opw-630789 (fix by gab) --- addons/sale_layout/views/report_invoice_layouted.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/sale_layout/views/report_invoice_layouted.xml b/addons/sale_layout/views/report_invoice_layouted.xml index 7dec3a0098c..592fe7de047 100644 --- a/addons/sale_layout/views/report_invoice_layouted.xml +++ b/addons/sale_layout/views/report_invoice_layouted.xml @@ -16,7 +16,7 @@ - + From 77a75d4e1f34832650c03f2cc46a8cd82d940478 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Sat, 21 Mar 2015 15:59:05 +0100 Subject: [PATCH 04/25] [FIX] web: remove leftover `future_display_name`. see 694fe1f --- addons/web/static/src/js/view_form.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/addons/web/static/src/js/view_form.js b/addons/web/static/src/js/view_form.js index 5b58594147a..ca401a34179 100644 --- a/addons/web/static/src/js/view_form.js +++ b/addons/web/static/src/js/view_form.js @@ -961,10 +961,7 @@ instance.web.FormView = instance.web.View.extend(instance.web.form.FieldManagerM fields.push('display_name'); return self.dataset.read_index(fields, { - context: { - 'bin_size': true, - 'future_display_name': true - }, + context: { 'bin_size': true }, check_access_rule: true }).then(function(r) { self.trigger('load_record', r); From 23873da6b62e090afb9e4f53f1d9b7284f1122bc Mon Sep 17 00:00:00 2001 From: Josse Colpaert Date: Sat, 21 Mar 2015 14:35:19 +0100 Subject: [PATCH 05/25] [FIX] stock: quant_ids relations should not be copied when copying moves fixes #5848 The same holds for the inverse history_ids relationship. When an object is copied, the many2many fields their links are copied. When we copy a done move as is done in the return wizard to create the reverse, it copied also the relationship with the quants. This is problematic as this field indicates the quants that were transferred by the move and the new move will think it will have returned all the quants even before it is done. --- addons/stock/stock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/stock/stock.py b/addons/stock/stock.py index 61391bb2315..c3401862747 100644 --- a/addons/stock/stock.py +++ b/addons/stock/stock.py @@ -305,7 +305,7 @@ class stock_quant(osv.osv): 'create_date': fields.datetime('Creation Date', readonly=True), 'in_date': fields.datetime('Incoming Date', readonly=True, select=True), - 'history_ids': fields.many2many('stock.move', 'stock_quant_move_rel', 'quant_id', 'move_id', 'Moves', help='Moves that operate(d) on this quant'), + 'history_ids': fields.many2many('stock.move', 'stock_quant_move_rel', 'quant_id', 'move_id', 'Moves', help='Moves that operate(d) on this quant', copy=False), 'company_id': fields.many2one('res.company', 'Company', help="The company to which the quants belong", required=True, readonly=True, select=True), 'inventory_value': fields.function(_calc_inventory_value, string="Inventory Value", type='float', readonly=True), @@ -1777,7 +1777,7 @@ class stock_move(osv.osv): # used for colors in tree views: 'scrapped': fields.related('location_dest_id', 'scrap_location', type='boolean', relation='stock.location', string='Scrapped', readonly=True), - 'quant_ids': fields.many2many('stock.quant', 'stock_quant_move_rel', 'move_id', 'quant_id', 'Moved Quants'), + 'quant_ids': fields.many2many('stock.quant', 'stock_quant_move_rel', 'move_id', 'quant_id', 'Moved Quants', copy=False), 'reserved_quant_ids': fields.one2many('stock.quant', 'reservation_id', 'Reserved quants'), 'linked_move_operation_ids': fields.one2many('stock.move.operation.link', 'move_id', string='Linked Operations', readonly=True, help='Operations that impact this move for the computation of the remaining quantities'), 'remaining_qty': fields.function(_get_remaining_qty, type='float', string='Remaining Quantity', digits=0, From e3a80afbaf305b3419aafab1ffd07c241326b9ca Mon Sep 17 00:00:00 2001 From: Denis Ledoux Date: Mon, 23 Mar 2015 13:44:57 +0100 Subject: [PATCH 06/25] [FIX] im_livechat: prevent translation of JS snippet QWeb templates can be translated, javascript code included. Nothing prevents you to translate the javascript reserved word "undefined", for instance. Preventing the livechat JS snippet to be translated prevent translations mistakes, like translate ```undefined``` by ```Niet gedefinieerd``` opw-630335 --- addons/im_livechat/views/im_livechat.xml | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/addons/im_livechat/views/im_livechat.xml b/addons/im_livechat/views/im_livechat.xml index b680990ebfc..33ee2e0b906 100644 --- a/addons/im_livechat/views/im_livechat.xml +++ b/addons/im_livechat/views/im_livechat.xml @@ -123,19 +123,21 @@ From e2ea691cef35941e9c0a3e807d1c303ba030663d Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 3 Mar 2015 11:26:10 +0100 Subject: [PATCH 07/25] [IMP] ormcache: turn it into a global LRU cache shared among registries The ormcache is now shared among registries. The cached methods use keys like (DBNAME, MODELNAME, METHOD, args...). This allows registries with high load to use more cache than other registries. --- openerp/addons/base/tests/test_api.py | 4 +--- openerp/models.py | 5 +---- openerp/modules/registry.py | 17 +++++++++++++++++ openerp/tools/cache.py | 23 +++++++++-------------- openerp/tools/lru.py | 8 ++++++++ 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/openerp/addons/base/tests/test_api.py b/openerp/addons/base/tests/test_api.py index 1f5203260e6..7deeafa946a 100644 --- a/openerp/addons/base/tests/test_api.py +++ b/openerp/addons/base/tests/test_api.py @@ -31,12 +31,10 @@ class TestAPI(common.TransactionCase): self.assertTrue(ids) self.assertTrue(partners) - # partners and its contents are instance of the model, and share its ormcache + # partners and its contents are instance of the model self.assertIsRecordset(partners, 'res.partner') - self.assertIs(partners._ormcache, self.env['res.partner']._ormcache) for p in partners: self.assertIsRecord(p, 'res.partner') - self.assertIs(p._ormcache, self.env['res.partner']._ormcache) self.assertEqual([p.id for p in partners], ids) self.assertEqual(self.env['res.partner'].browse(ids), partners) diff --git a/openerp/models.py b/openerp/models.py index 73782dea625..2f820cec4e2 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -827,9 +827,6 @@ class BaseModel(object): "TransientModels must have log_access turned on, " \ "in order to implement their access rights policy" - # prepare ormcache, which must be shared by all instances of the model - cls._ormcache = {} - @api.model @ormcache() def _is_an_ordinary_table(self): @@ -1827,7 +1824,7 @@ class BaseModel(object): ``tools.ormcache`` or ``tools.ormcache_multi``. """ try: - self._ormcache.clear() + self.pool.cache.clear_prefix((self.pool.db_name, self._name)) self.pool._any_cache_cleared = True except AttributeError: pass diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index c6e3e2ed1d2..5c9870a488a 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -104,6 +104,10 @@ class Registry(Mapping): """ Same as ``self[model_name]``. """ return self.models[model_name] + @lazy_property + def cache(self): + return RegistryManager.cache + @lazy_property def pure_function_fields(self): """ Return the list of pure function fields (field objects) """ @@ -287,6 +291,7 @@ class RegistryManager(object): """ _registries = None + _cache = None _lock = threading.RLock() _saved_lock = None @@ -307,6 +312,18 @@ class RegistryManager(object): cls._registries = LRU(size) return cls._registries + @classproperty + def cache(cls): + """ Return the global LRU ormcache. Its keys are tuples with the + following structure: (db_name, model_name, method, args...). + """ + with cls.lock(): + if cls._cache is None: + # we allocate one cache entry per 32KB of memory + size = max(8192, int(config['limit_memory_soft'] / 32768)) + cls._cache = LRU(size) + return cls._cache + @classmethod def lock(cls): """ Return the current registry lock. """ diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index 57dea0888c1..ab387de361a 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -52,16 +52,11 @@ class ormcache(object): (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit)) def lru(self, model): - ormcache = model._ormcache - try: - d = ormcache[self.method] - except KeyError: - d = ormcache[self.method] = lru.LRU(self.size) - return d + return model.pool.cache, (model.pool.db_name, model._name, self.method) def lookup(self, method, *args, **kwargs): - d = self.lru(args[0]) - key = args[self.skiparg:] + d, key0 = self.lru(args[0]) + key = key0 + args[self.skiparg:] try: r = d[key] self.stat_hit += 1 @@ -76,12 +71,12 @@ class ormcache(object): def clear(self, model, *args): """ Remove *args entry from the cache or all keys if *args is undefined """ - d = self.lru(model) + d, key0 = self.lru(model) if args: logger.warn("ormcache.clear arguments are deprecated and ignored " "(while clearing caches on (%s).%s)", model._name, self.method.__name__) - d.clear() + d.clear_prefix(key0) model.pool._any_cache_cleared = True @@ -97,7 +92,7 @@ class ormcache_context(ormcache): return super(ormcache_context, self).__call__(method) def lookup(self, method, *args, **kwargs): - d = self.lru(args[0]) + d, key0 = self.lru(args[0]) # Note. The decorator() wrapper (used in __call__ above) will resolve # arguments, and pass them positionally to lookup(). This is why context @@ -109,7 +104,7 @@ class ormcache_context(ormcache): ckey = [(k, context[k]) for k in self.accepted_keys if k in context] # Beware: do not take the context from args! - key = args[self.skiparg:self.context_pos] + tuple(ckey) + key = key0 + args[self.skiparg:self.context_pos] + tuple(ckey) try: r = d[key] self.stat_hit += 1 @@ -130,8 +125,8 @@ class ormcache_multi(ormcache): self.multi = multi def lookup(self, method, *args, **kwargs): - d = self.lru(args[0]) - base_key = args[self.skiparg:self.multi] + args[self.multi+1:] + d, key0 = self.lru(args[0]) + base_key = key0 + args[self.skiparg:self.multi] + args[self.multi+1:] ids = args[self.multi] result = {} missed = [] diff --git a/openerp/tools/lru.py b/openerp/tools/lru.py index 13b76f387b7..c9de62130ce 100644 --- a/openerp/tools/lru.py +++ b/openerp/tools/lru.py @@ -119,4 +119,12 @@ class LRU(object): self.first = None self.last = None + @synchronized() + def clear_prefix(self, prefix): + """ Remove from `self` all the items with the given `prefix`. """ + n = len(prefix) + for key in self.keys(): + if key[:n] == prefix: + del self[key] + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: From 0f9b452c33edd012824bf2b7c82cbc529502f73e Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 4 Mar 2015 10:25:55 +0100 Subject: [PATCH 08/25] [IMP] models: convert deprecated model._all_columns into a dynamic property The mappings model._all_columns takes quite some memory (around 2MB per registry) because of the numerous dictionaries (one per model) and inefficient memory storage of column_info. Since it is deprecated and almost no longer used, it can be computed on demand. --- .../addons/test_new_api/tests/test_related.py | 3 --- openerp/models.py | 25 ++++++++----------- openerp/osv/fields.py | 2 ++ 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/openerp/addons/test_new_api/tests/test_related.py b/openerp/addons/test_new_api/tests/test_related.py index b7bb773c056..32869da69ca 100644 --- a/openerp/addons/test_new_api/tests/test_related.py +++ b/openerp/addons/test_new_api/tests/test_related.py @@ -111,9 +111,6 @@ class TestPropertyField(common.TransactionCase): self.partner._columns.update({ 'property_country': fields.property(type='many2one', relation="res.country", string="Country by company"), }) - self.partner._all_columns.update({ - 'property_country': fields.column_info('property_country', self.partner._columns['property_country'], None, None, None), - }) self.partner._field_create(cr) partner_id = self.partner.create(cr, alice, { diff --git a/openerp/models.py b/openerp/models.py index 2f820cec4e2..3b93d2c4158 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -341,7 +341,7 @@ class BaseModel(object): # 1. includes self fields, # 2. uses column_info instead of a triple. # Warning: _all_columns is deprecated, use _fields instead - _all_columns = {} + # _all_columns = {} _table = None _log_create = False @@ -491,7 +491,6 @@ class BaseModel(object): """ field = cls._fields.pop(name) cls._columns.pop(name, None) - cls._all_columns.pop(name, None) if hasattr(cls, name): delattr(cls, name) return field @@ -2898,7 +2897,7 @@ class BaseModel(object): @classmethod def _inherits_reload(cls): - """ Recompute the _inherit_fields and _all_columns mappings. """ + """ Recompute the _inherit_fields mapping. """ cls._inherit_fields = struct = {} for parent_model, parent_field in cls._inherits.iteritems(): parent = cls.pool[parent_model] @@ -2908,19 +2907,17 @@ class BaseModel(object): for name, source in parent._inherit_fields.iteritems(): struct[name] = (parent_model, parent_field, source[2], source[3]) - # old-api stuff - cls._all_columns = cls._get_column_infos() - - @classmethod - def _get_column_infos(cls): - """Returns a dict mapping all fields names (direct fields and - inherited field via _inherits) to a ``column_info`` struct - giving detailed columns """ + @property + def _all_columns(self): + """ Returns a dict mapping all fields names (direct fields and inherited + field via _inherits) to a ``column_info`` struct giving detailed columns. + Deprecated. + """ result = {} # do not inverse for loops, since local fields may hide inherited ones! - for k, (parent, m2o, col, original_parent) in cls._inherit_fields.iteritems(): + for k, (parent, m2o, col, original_parent) in self._inherit_fields.iteritems(): result[k] = fields.column_info(k, col, parent, m2o, original_parent) - for k, col in cls._columns.iteritems(): + for k, col in self._columns.iteritems(): result[k] = fields.column_info(k, col) return result @@ -3024,7 +3021,7 @@ class BaseModel(object): field = model._fields[field_name] field._triggers.update(triggers) - # determine old-api cls._inherit_fields and cls._all_columns + # determine old-api structures about inherited fields cls._inherits_reload() # register stuff about low-level function fields diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index 53ac8d9e942..cc11669ef33 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -1681,6 +1681,8 @@ class column_info(object): contains it i.e in case of multilevel inheritance, ``None`` for local columns. """ + __slots__ = ['name', 'column', 'parent_model', 'parent_column', 'original_parent'] + def __init__(self, name, column, parent_model=None, parent_column=None, original_parent=None): self.name = name self.column = column From 9bce04de7966346a75226d1ee6b8217deb266858 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 4 Mar 2015 14:44:14 +0100 Subject: [PATCH 09/25] [IMP] models: build a flat hierarchy of classes for models Instead of composing classes in a binary tree hierarchy, we make one class that inherits from all its base classes. This avoids keeping intermediate data structures for columns, inherits, constraints, etc. This saves about 600Kb per registry. --- openerp/models.py | 86 ++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/openerp/models.py b/openerp/models.py index 3b93d2c4158..eb45259c6ba 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -569,11 +569,22 @@ class BaseModel(object): """ - # IMPORTANT: the registry contains an instance for each model. The class - # of each model carries inferred metadata that is shared among the - # model's instances for this registry, but not among registries. Hence - # we cannot use that "registry class" for combining model classes by - # inheritance, since it confuses the metadata inference process. + # The model's class inherits from cls and the classes of the inherited + # models. All those classes are combined in a flat hierarchy: + # + # Model the base class of all models + # / | \ + # cls c2 c1 the classes defined in modules + # \ | / + # ModelClass the final class of the model + # / | \ + # model recordset ... the class' instances + # + # The registry contains the instance `model`. Its class, `ModelClass`, + # carries inferred metadata that is shared between all the model's + # instances for this registry only. When we '_inherit' from another + # model, we do not inherit its `ModelClass`, but this class' parents. + # This is a limitation of the inheritance mechanism. # Keep links to non-inherited constraints in cls; this is useful for # instance when exporting translations @@ -590,65 +601,48 @@ class BaseModel(object): # determine the module that introduced the model original_module = pool[name]._original_module if name in parents else cls._module - # build the class hierarchy for the model + # determine all the classes the model should inherit from + bases = [cls] for parent in parents: if parent not in pool: raise TypeError('The model "%s" specifies an unexisting parent class "%s"\n' 'You may need to add a dependency on the parent class\' module.' % (name, parent)) - parent_model = pool[parent] + bases += type(pool[parent]).__bases__ - # do no use the class of parent_model, since that class contains - # inferred metadata; use its ancestor instead - parent_class = type(parent_model).__base__ + # determine the attributes of the model's class + inherits = {} + depends = {} + constraints = {} + sql_constraints = [] - inherits = dict(parent_class._inherits) - inherits.update(cls._inherits) + for base in reversed(bases): + inherits.update(base._inherits) - depends = dict(parent_class._depends) - for m, fs in cls._depends.iteritems(): - depends[m] = depends.get(m, []) + fs + for mname, fnames in base._depends.iteritems(): + depends[mname] = depends.get(mname, []) + fnames - old_constraints = parent_class._constraints - new_constraints = cls._constraints - # filter out from old_constraints the ones overridden by a - # constraint with the same function name in new_constraints - constraints = new_constraints + [oldc - for oldc in old_constraints - if not any(newc[2] == oldc[2] and same_name(newc[0], oldc[0]) - for newc in new_constraints) - ] + for cons in base._constraints: + # cons may override a constraint with the same function name + constraints[getattr(cons[0], '__name__', id(cons[0]))] = cons - sql_constraints = cls._sql_constraints + \ - parent_class._sql_constraints + sql_constraints += base._sql_constraints - attrs = { - '_name': name, - '_register': False, - '_inherits': inherits, - '_depends': depends, - '_constraints': constraints, - '_sql_constraints': sql_constraints, - } - cls = type(name, (cls, parent_class), attrs) - - # introduce the "registry class" of the model; - # duplicate some attributes so that the ORM can modify them - attrs = { + # build the actual class of the model + ModelClass = type(name, tuple(bases), { '_name': name, '_register': False, '_columns': None, # recomputed in _setup_fields() '_defaults': None, # recomputed in _setup_base() '_fields': frozendict(), # idem - '_inherits': dict(cls._inherits), - '_depends': dict(cls._depends), - '_constraints': list(cls._constraints), - '_sql_constraints': list(cls._sql_constraints), + '_inherits': inherits, + '_depends': depends, + '_constraints': constraints.values(), + '_sql_constraints': sql_constraints, '_original_module': original_module, - } - cls = type(cls._name, (cls,), attrs) + }) # instantiate the model, and initialize it - model = object.__new__(cls) + model = object.__new__(ModelClass) model.__init__(pool, cr) return model From bf703fd9a3951f9bd9bce1c6c62dc21ac2a4ac23 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 5 Mar 2015 09:42:38 +0100 Subject: [PATCH 10/25] [FIX] models: reorder the base classes of models following an equivalent class hierarchy Sometimes, the expected mro of the model is not the same as the one built with a binary class hierarchy. So we reorder the base classes in order to match the equivalent binary class hierarchy. This also fixes the cases where duplicates appear in base classes. --- openerp/models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openerp/models.py b/openerp/models.py index eb45259c6ba..76ad1e042bf 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -603,11 +603,17 @@ class BaseModel(object): # determine all the classes the model should inherit from bases = [cls] + hierarchy = cls for parent in parents: if parent not in pool: raise TypeError('The model "%s" specifies an unexisting parent class "%s"\n' 'You may need to add a dependency on the parent class\' module.' % (name, parent)) - bases += type(pool[parent]).__bases__ + parent_class = type(pool[parent]) + bases += parent_class.__bases__ + hierarchy = type(name, (hierarchy, parent_class), {'_register': False}) + + # order bases following the mro of class hierarchy + bases = [base for base in hierarchy.mro() if base in bases] # determine the attributes of the model's class inherits = {} From 9aad3d873b7e0e6fae3d87ac931f213ac58894bb Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 3 Mar 2015 14:25:10 +0100 Subject: [PATCH 11/25] [IMP] fields: turn field.digits and column.digits into dynamic properties The computed value of parameter digits is no longer stored into fields and columns; instead the value is recomputed everytime it is needed. Note that performance is not an issue, since the method `get_precision` of model 'decimal.precision' is cached by the orm. This simplifies the management of digits on fields and saves about 300Kb per registry. --- addons/decimal_precision/decimal_precision.py | 5 -- openerp/__init__.py | 10 ++- openerp/fields.py | 27 ++++---- openerp/models.py | 2 - openerp/osv/fields.py | 69 +++++++++++-------- 5 files changed, 62 insertions(+), 51 deletions(-) diff --git a/addons/decimal_precision/decimal_precision.py b/addons/decimal_precision/decimal_precision.py index f169b698ebd..728fa9cc99b 100644 --- a/addons/decimal_precision/decimal_precision.py +++ b/addons/decimal_precision/decimal_precision.py @@ -48,11 +48,6 @@ class decimal_precision(orm.Model): def clear_cache(self, cr): """clear cache and update models. Notify other workers to restart their registry.""" self.precision_get.clear_cache(self) - env = openerp.api.Environment(cr, SUPERUSER_ID, {}) - for model in self.pool.values(): - for field in model._fields.values(): - if field.type == 'float': - field._setup_digits(env) RegistryManager.signal_registry_change(cr.dbname) def create(self, cr, uid, data, context=None): diff --git a/openerp/__init__.py b/openerp/__init__.py index 24a193bf759..691a7d27c1c 100644 --- a/openerp/__init__.py +++ b/openerp/__init__.py @@ -56,11 +56,15 @@ del time # The hard-coded super-user id (a.k.a. administrator, or root user). SUPERUSER_ID = 1 -def registry(database_name): +def registry(database_name=None): """ - Return the model registry for the given database. If the registry does not - exist yet, it is created on the fly. + Return the model registry for the given database, or the database mentioned + on the current thread. If the registry does not exist yet, it is created on + the fly. """ + if database_name is None: + import threading + database_name = threading.currentThread().dbname return modules.registry.RegistryManager.get(database_name) #---------------------------------------------------------- diff --git a/openerp/fields.py b/openerp/fields.py index da0ba4dc374..9aadca4b3cb 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -991,26 +991,28 @@ class Float(Field): """ type = 'float' _digits = None # digits argument passed to class initializer - digits = None # digits as computed by setup() group_operator = None # operator for aggregating values def __init__(self, string=None, digits=None, **kwargs): super(Float, self).__init__(string=string, _digits=digits, **kwargs) + @property + def digits(self): + if callable(self._digits): + with registry().cursor() as cr: + return self._digits(cr) + else: + return self._digits + def _setup_digits(self, env): """ Setup the digits for `self` and its corresponding column """ - self.digits = self._digits(env.cr) if callable(self._digits) else self._digits - if self.digits: - assert isinstance(self.digits, (tuple, list)) and len(self.digits) >= 2, \ - "Float field %s with digits %r, expecting (total, decimal)" % (self, self.digits) - if self.column: - self.column.digits_change(env.cr) + pass def _setup_regular(self, env): super(Float, self)._setup_regular(env) self._setup_digits(env) - _related_digits = property(attrgetter('digits')) + _related__digits = property(attrgetter('_digits')) _related_group_operator = property(attrgetter('group_operator')) _description_digits = property(attrgetter('digits')) @@ -1021,10 +1023,9 @@ class Float(Field): def convert_to_cache(self, value, record, validate=True): # apply rounding here, otherwise value in cache may be wrong! - if self.digits: - return float_round(float(value or 0.0), precision_digits=self.digits[1]) - else: - return float(value or 0.0) + value = float(value or 0.0) + digits = self.digits + return float_round(value, precision_digits=digits[1]) if digits else value class _String(Field): @@ -1790,7 +1791,7 @@ class Id(Field): raise TypeError("field 'id' cannot be assigned") # imported here to avoid dependency cycle issues -from openerp import SUPERUSER_ID +from openerp import SUPERUSER_ID, registry from .exceptions import Warning, AccessError, MissingError from .models import BaseModel, MAGIC_COLUMNS from .osv import fields diff --git a/openerp/models.py b/openerp/models.py index 76ad1e042bf..0846521888d 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -659,8 +659,6 @@ class BaseModel(object): # process store of low-level function fields for fname, column in cls._columns.iteritems(): - if hasattr(column, 'digits_change'): - column.digits_change(cr) # filter out existing store about this field pool._store_function[cls._name] = [ stored diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index cc11669ef33..239a55e394f 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -51,7 +51,7 @@ from openerp.tools.translate import _ from openerp.tools import float_round, float_repr from openerp.tools import html_sanitize import simplejson -from openerp import SUPERUSER_ID +from openerp import SUPERUSER_ID, registry _logger = logging.getLogger(__name__) @@ -334,36 +334,42 @@ class html(text): import __builtin__ +def _symbol_set_float(self, x): + result = __builtin__.float(x or 0.0) + digits = self.digits + if digits: + precision, scale = digits + result = float_repr(float_round(result, precision_digits=scale), precision_digits=scale) + return result + class float(_column): _type = 'float' _symbol_c = '%s' - _symbol_f = lambda x: __builtin__.float(x or 0.0) - _symbol_set = (_symbol_c, _symbol_f) _symbol_get = lambda self,x: x or 0.0 + @property + def digits(self): + if self._digits_compute: + with registry().cursor() as cr: + return self._digits_compute(cr) + else: + return self._digits + def __init__(self, string='unknown', digits=None, digits_compute=None, required=False, **args): _column.__init__(self, string=string, required=required, **args) - self.digits = digits # synopsis: digits_compute(cr) -> (precision, scale) - self.digits_compute = digits_compute - - def new(self, _computed_field=False, **args): - # float columns are database-dependent, so always recreate them - return type(self)(**args) + self._digits = digits + self._digits_compute = digits_compute + self._symbol_f = lambda x: _symbol_set_float(self, x) + self._symbol_set = (self._symbol_c, self._symbol_f) def to_field_args(self): args = super(float, self).to_field_args() - args['digits'] = self.digits_compute or self.digits + args['digits'] = self._digits_compute or self._digits return args def digits_change(self, cr): - if self.digits_compute: - self.digits = self.digits_compute(cr) - if self.digits: - precision, scale = self.digits - self._symbol_set = ('%s', lambda x: float_repr(float_round(__builtin__.float(x or 0.0), - precision_digits=scale), - precision_digits=scale)) + pass class date(_column): _type = 'date' @@ -1240,10 +1246,22 @@ class function(_column): # function fields are not copied by default copy = False + @property + def digits(self): + if self._digits_compute: + with registry().cursor() as cr: + return self._digits_compute(cr) + else: + return self._digits + # # multi: compute several fields in one call # def __init__(self, fnct, arg=None, fnct_inv=None, fnct_inv_arg=None, type='float', fnct_search=None, obj=None, store=False, multi=False, **args): + # pop attributes that should not be assigned to self + self._digits = args.pop('digits', (16,2)) + self._digits_compute = args.pop('digits_compute', None) + _column.__init__(self, **args) self._obj = obj self._fnct = fnct @@ -1253,8 +1271,6 @@ class function(_column): if 'relation' in args: self._obj = args['relation'] - self.digits = args.get('digits', (16,2)) - self.digits_compute = args.get('digits_compute', None) if callable(args.get('selection')): from openerp import api self.selection = api.expected(api.cr_uid_context, args['selection']) @@ -1283,6 +1299,10 @@ class function(_column): self._symbol_c = char._symbol_c self._symbol_f = lambda x: _symbol_set_char(self, x) self._symbol_set = (self._symbol_c, self._symbol_f) + elif type == 'float': + self._symbol_c = float._symbol_c + self._symbol_f = lambda x: _symbol_set_float(self, x) + self._symbol_set = (self._symbol_c, self._symbol_f) else: type_class = globals().get(type) if type_class is not None: @@ -1304,7 +1324,7 @@ class function(_column): args = super(function, self).to_field_args() args['store'] = bool(self.store) if self._type in ('float',): - args['digits'] = self.digits_compute or self.digits + args['digits'] = self._digits_compute or self._digits elif self._type in ('selection', 'reference'): args['selection'] = self.selection elif self._type in ('many2one', 'one2many', 'many2many'): @@ -1312,14 +1332,7 @@ class function(_column): return args def digits_change(self, cr): - if self._type == 'float': - if self.digits_compute: - self.digits = self.digits_compute(cr) - if self.digits: - precision, scale = self.digits - self._symbol_set = ('%s', lambda x: float_repr(float_round(__builtin__.float(x or 0.0), - precision_digits=scale), - precision_digits=scale)) + pass def search(self, cr, uid, obj, name, args, context=None): if not self._fnct_search: From 4d232969a3158f424db9510b3d57be3ab021bcf0 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 10 Mar 2015 11:58:12 +0100 Subject: [PATCH 12/25] [IMP] fields: remove attribute `_free_attrs` and use `_attrs` instead This saves about 400Kb per registry by not allocating those lists, which are in most cases empty. The removal of the attribute will also simplify a bit the management of free attributes. --- openerp/fields.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index 9aadca4b3cb..fda36696b7a 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -253,8 +253,9 @@ class Field(object): """ __metaclass__ = MetaField - _attrs = None # dictionary with all field attributes - _free_attrs = None # list of semantic-free attribute names + _attrs = None # dictionary of field attributes; it contains: + # - all attributes after __init__() + # - free attributes only after set_class_name() automatic = False # whether the field is automatically created ("magic" field) inherited = False # whether the field is inherited (_inherits) @@ -295,7 +296,6 @@ class Field(object): def __init__(self, string=None, **kwargs): kwargs['string'] = string self._attrs = {key: val for key, val in kwargs.iteritems() if val is not None} - self._free_attrs = [] # self._triggers is a set of pairs (field, path) that represents the # computed fields that depend on `self`. When `self` is modified, it @@ -337,9 +337,10 @@ class Field(object): if not isinstance(attrs.get('column'), (NoneType, fields.function)): attrs.pop('store', None) + self._attrs = {} for attr, value in attrs.iteritems(): if not hasattr(self, attr): - self._free_attrs.append(attr) + self._attrs[attr] = value setattr(self, attr, value) if not self.string and not self.related: @@ -478,10 +479,10 @@ class Field(object): if not getattr(self, attr): setattr(self, attr, getattr(field, prop)) - for attr in field._free_attrs: - if attr not in self._free_attrs: - self._free_attrs.append(attr) - setattr(self, attr, getattr(field, attr)) + for attr, value in field._attrs.iteritems(): + if attr not in self._attrs: + self._attrs[attr] = value + setattr(self, attr, value) # special case for states: copy it only for inherited fields if not self.states and self.inherited: @@ -648,8 +649,8 @@ class Field(object): args = {} for attr, prop in self.column_attrs: args[attr] = getattr(self, prop) - for attr in self._free_attrs: - args[attr] = getattr(self, attr) + for attr, value in self._attrs.iteritems(): + args[attr] = value if self.company_dependent: # company-dependent fields are mapped to former property fields From 3df77540870c7b021ab056fda0f5d8b316c503e9 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 11 Mar 2015 10:16:34 +0100 Subject: [PATCH 13/25] [IMP] fields: reduce memory footprint of `field._attrs` and `column._args` The optimization consists in sharing the dictionary when it is empty, which is the common case. This saves around 1.5Mb per registry. --- openerp/fields.py | 24 ++++++++++++++++-------- openerp/osv/fields.py | 7 ++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index fda36696b7a..98e5e3f4a1f 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -30,12 +30,13 @@ import logging import pytz import xmlrpclib -from openerp.tools import float_round, ustr, html_sanitize +from openerp.tools import float_round, frozendict, html_sanitize, ustr from openerp.tools import DEFAULT_SERVER_DATE_FORMAT as DATE_FORMAT from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT as DATETIME_FORMAT DATE_LENGTH = len(date.today().strftime(DATE_FORMAT)) DATETIME_LENGTH = len(datetime.now().strftime(DATETIME_FORMAT)) +EMPTY_DICT = frozendict() _logger = logging.getLogger(__name__) @@ -295,7 +296,8 @@ class Field(object): def __init__(self, string=None, **kwargs): kwargs['string'] = string - self._attrs = {key: val for key, val in kwargs.iteritems() if val is not None} + attrs = {key: val for key, val in kwargs.iteritems() if val is not None} + self._attrs = attrs or EMPTY_DICT # self._triggers is a set of pairs (field, path) that represents the # computed fields that depend on `self`. When `self` is modified, it @@ -304,6 +306,15 @@ class Field(object): self._triggers = set() self.inverse_fields = [] + def _set_attr(self, name, value): + """ Set the given field attribute, and add it to `_attrs` if necessary. """ + object.__setattr__(self, name, value) + if not hasattr(type(self), name): + if self._attrs: + self._attrs[name] = value + else: + self._attrs = {name: value} # replace EMPTY_DICT + def new(self, **kwargs): """ Return a field of the same type as `self`, with its own parameters. """ return type(self)(**kwargs) @@ -337,11 +348,9 @@ class Field(object): if not isinstance(attrs.get('column'), (NoneType, fields.function)): attrs.pop('store', None) - self._attrs = {} + self._attrs = EMPTY_DICT for attr, value in attrs.iteritems(): - if not hasattr(self, attr): - self._attrs[attr] = value - setattr(self, attr, value) + self._set_attr(attr, value) if not self.string and not self.related: # related fields get their string from their parent field @@ -481,8 +490,7 @@ class Field(object): for attr, value in field._attrs.iteritems(): if attr not in self._attrs: - self._attrs[attr] = value - setattr(self, attr, value) + self._set_attr(attr, value) # special case for states: copy it only for inherited fields if not self.states and self.inherited: diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index 239a55e394f..bf324af15cb 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -48,11 +48,12 @@ from psycopg2 import Binary import openerp import openerp.tools as tools from openerp.tools.translate import _ -from openerp.tools import float_round, float_repr -from openerp.tools import html_sanitize +from openerp.tools import float_repr, float_round, frozendict, html_sanitize import simplejson from openerp import SUPERUSER_ID, registry +EMPTY_DICT = frozendict() + _logger = logging.getLogger(__name__) def _symbol_set(symb): @@ -136,7 +137,7 @@ class _column(object): if val: setattr(self, key, val) - self._args = args + self._args = args or EMPTY_DICT for key, val in args.iteritems(): setattr(self, key, val) From ffa7f28d34a449d65ef353403cc2d2a12e256d93 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 11 Mar 2015 10:36:17 +0100 Subject: [PATCH 14/25] [IMP] fields: reduce memory footprint of list/set field attributes The optimization consists in using tuples for attributes `inverse_fields`, `computed_fields` and `_triggers`, and to let them share their value when it is empty, which is common. This saves around 1.8Mb per registry. --- openerp/fields.py | 34 ++++++++++++++++++++-------------- openerp/models.py | 15 ++++++++------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index 98e5e3f4a1f..3167819534a 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -268,7 +268,6 @@ class Field(object): relational = False # whether the field is a relational one model_name = None # name of the model of this field comodel_name = None # name of the model of values (if relational) - inverse_fields = None # list of inverse fields (objects) store = True # whether the field is stored in database index = False # whether the field is indexed in database @@ -294,18 +293,15 @@ class Field(object): change_default = None # whether the field may trigger a "user-onchange" deprecated = None # whether the field is ... deprecated + inverse_fields = () # collection of inverse fields (objects) + computed_fields = () # fields computed with the same method as self + _triggers = () # invalidation and recomputation triggers + def __init__(self, string=None, **kwargs): kwargs['string'] = string attrs = {key: val for key, val in kwargs.iteritems() if val is not None} self._attrs = attrs or EMPTY_DICT - # self._triggers is a set of pairs (field, path) that represents the - # computed fields that depend on `self`. When `self` is modified, it - # invalidates the cache of each `field`, and registers the records to - # recompute based on `path`. See method `modified` below for details. - self._triggers = set() - self.inverse_fields = [] - def _set_attr(self, name, value): """ Set the given field attribute, and add it to `_attrs` if necessary. """ object.__setattr__(self, name, value) @@ -542,6 +538,16 @@ class Field(object): # # Setup of field triggers # + # The triggers is a collection of pairs (field, path) of computed fields + # that depend on `self`. When `self` is modified, it invalidates the cache + # of each `field`, and registers the records to recompute based on `path`. + # See method `modified` below for details. + # + + def add_trigger(self, trigger): + """ Add a recomputation trigger on `self`. """ + if trigger not in self._triggers: + self._triggers += (trigger,) def setup_triggers(self, env): """ Add the necessary triggers to invalidate/recompute `self`. """ @@ -570,12 +576,12 @@ class Field(object): continue #_logger.debug("Add trigger on %s to recompute %s", field, self) - field._triggers.add((self, '.'.join(path0 or ['id']))) + field.add_trigger((self, '.'.join(path0 or ['id']))) # add trigger on inverse fields, too for invf in field.inverse_fields: #_logger.debug("Add trigger on %s to recompute %s", invf, self) - invf._triggers.add((self, '.'.join(path0 + [head]))) + invf.add_trigger((self, '.'.join(path0 + [head]))) # recursively traverse the dependency if tail: @@ -1684,8 +1690,8 @@ class One2many(_RelationalMulti): # (res_model/res_id pattern). Only inverse the field if this is # a `Many2one` field. if isinstance(invf, Many2one): - self.inverse_fields.append(invf) - invf.inverse_fields.append(self) + self.inverse_fields += (invf,) + invf.inverse_fields += (self,) _description_relation_field = property(attrgetter('inverse_name')) @@ -1756,8 +1762,8 @@ class Many2many(_RelationalMulti): # if inverse field has already been setup, it is present in m2m invf = m2m.get((self.relation, self.column2, self.column1)) if invf: - self.inverse_fields.append(invf) - invf.inverse_fields.append(self) + self.inverse_fields += (invf,) + invf.inverse_fields += (self,) else: # add self in m2m, so that its inverse field can find it m2m[(self.relation, self.column1, self.column2)] = self diff --git a/openerp/models.py b/openerp/models.py index 0846521888d..d9f565cb781 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -2992,14 +2992,15 @@ class BaseModel(object): if column: cls._columns[name] = column - # group fields by compute to determine field.computed_fields - fields_by_compute = defaultdict(list) + # determine field.computed_fields + computed_fields = defaultdict(list) for field in cls._fields.itervalues(): if field.compute: - field.computed_fields = fields_by_compute[field.compute] - field.computed_fields.append(field) - else: - field.computed_fields = [] + computed_fields[field.compute].append(field) + + for fields in computed_fields.itervalues(): + for field in fields: + field.computed_fields = fields @api.model def _setup_complete(self): @@ -3017,7 +3018,7 @@ class BaseModel(object): model = self.env[model_name] for field_name in field_names: field = model._fields[field_name] - field._triggers.update(triggers) + map(field.add_trigger, triggers) # determine old-api structures about inherited fields cls._inherits_reload() From 915af86a915f44c4405509c27a0526bed64fc943 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 11 Mar 2015 12:26:02 +0100 Subject: [PATCH 15/25] [IMP] fields: simplify definition of ID field --- openerp/fields.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index 3167819534a..aaf6ba20f9c 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -75,8 +75,8 @@ class MetaField(type): def __init__(cls, name, bases, attrs): super(MetaField, cls).__init__(name, bases, attrs) - if cls.type: - cls.by_type[cls.type] = cls + if cls.type and cls.type not in MetaField.by_type: + MetaField.by_type[cls.type] = cls # compute class attributes to avoid calling dir() on fields cls.column_attrs = [] @@ -1784,15 +1784,15 @@ class Serialized(Field): class Id(Field): """ Special case for field 'id'. """ + type = 'integer' + + string = 'ID' store = True #: Can't write this! readonly = True - def __init__(self, string=None, **kwargs): - super(Id, self).__init__(type='integer', string=string, **kwargs) - def to_column(self): - self.column = fields.integer('ID') + self.column = fields.integer(self.string) return self.column def __get__(self, record, owner): From e7928e1265b1d3bb0d937c0e3adcaf21185560cf Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 11 Mar 2015 12:37:10 +0100 Subject: [PATCH 16/25] [IMP] fields: add `__slots__` on all fields Introduce slots on all field classes for common attributes; slots take much less memory than a `__dict__`. The other attributes are stored in a dictionary `_attrs`; all fields with an empty value for `_attrs` (common case) share the same empty dictionary. This saves quite some memory (around 4.5Mb per registry), given the number of field instances created for a registry. Another mechanism is used for the default values of attributes, since slots cannot be assigned on classes. --- openerp/fields.py | 287 +++++++++++++++++++++++++++++++++------------- 1 file changed, 207 insertions(+), 80 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index aaf6ba20f9c..b70d95709b0 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -79,11 +79,14 @@ class MetaField(type): MetaField.by_type[cls.type] = cls # compute class attributes to avoid calling dir() on fields + cls.default_attrs = {} cls.column_attrs = [] cls.related_attrs = [] cls.description_attrs = [] for attr in dir(cls): - if attr.startswith('_column_'): + if attr.startswith('_default_'): + cls.default_attrs[attr[9:]] = getattr(cls, attr) + elif attr.startswith('_column_'): cls.column_attrs.append((attr[8:], attr)) elif attr.startswith('_related_'): cls.related_attrs.append((attr[9:], attr)) @@ -254,69 +257,132 @@ class Field(object): """ __metaclass__ = MetaField - _attrs = None # dictionary of field attributes; it contains: + type = None # type of the field (string) + relational = False # whether the field is a relational one + + __slots__ = [ + '_attrs', # dictionary of field attributes; it contains: # - all attributes after __init__() # - free attributes only after set_class_name() - automatic = False # whether the field is automatically created ("magic" field) - inherited = False # whether the field is inherited (_inherits) - column = None # the column corresponding to the field - setup_done = False # whether the field has been set up + 'automatic', # whether the field is automatically created ("magic" field) + 'inherited', # whether the field is inherited (_inherits) + 'column', # the column corresponding to the field + 'setup_done', # whether the field has been set up - name = None # name of the field - type = None # type of the field (string) - relational = False # whether the field is a relational one - model_name = None # name of the model of this field - comodel_name = None # name of the model of values (if relational) + 'name', # name of the field + 'model_name', # name of the model of this field + 'comodel_name', # name of the model of values (if relational) - store = True # whether the field is stored in database - index = False # whether the field is indexed in database - manual = False # whether the field is a custom field - copy = True # whether the field is copied over by BaseModel.copy() - depends = () # collection of field dependencies - recursive = False # whether self depends on itself - compute = None # compute(recs) computes field on recs - compute_sudo = False # whether field should be recomputed as admin - inverse = None # inverse(recs) inverses field on recs - search = None # search(recs, operator, value) searches on self - related = None # sequence of field names, for related fields - related_sudo = True # whether related fields should be read as admin - company_dependent = False # whether `self` is company-dependent (property field) - default = None # default(recs) returns the default value + 'store', # whether the field is stored in database + 'index', # whether the field is indexed in database + 'manual', # whether the field is a custom field + 'copy', # whether the field is copied over by BaseModel.copy() + 'depends', # collection of field dependencies + 'recursive', # whether self depends on itself + 'compute', # compute(recs) computes field on recs + 'compute_sudo', # whether field should be recomputed as admin + 'inverse', # inverse(recs) inverses field on recs + 'search', # search(recs, operator, value) searches on self + 'related', # sequence of field names, for related fields + 'related_sudo', # whether related fields should be read as admin + 'company_dependent', # whether `self` is company-dependent (property field) + 'default', # default(recs) returns the default value - string = None # field label - help = None # field tooltip - readonly = False - required = False - states = None - groups = False # csv list of group xml ids - change_default = None # whether the field may trigger a "user-onchange" - deprecated = None # whether the field is ... deprecated + 'string', # field label + 'help', # field tooltip + 'readonly', # whether the field is readonly + 'required', # whether the field is required + 'states', # set readonly and required depending on state + 'groups', # csv list of group xml ids + 'change_default', # whether the field may trigger a "user-onchange" + 'deprecated', # whether the field is deprecated - inverse_fields = () # collection of inverse fields (objects) - computed_fields = () # fields computed with the same method as self - _triggers = () # invalidation and recomputation triggers + 'inverse_fields', # collection of inverse fields (objects) + 'computed_fields', # fields computed with the same method as self + 'related_field', # corresponding related field + '_triggers', # invalidation and recomputation triggers + ] + + # default values for slots + _default_automatic = False + _default_inherited = False + _default_column = None + _default_setup_done = False + + _default_name = None + _default_model_name = None + _default_comodel_name = None + + _default_store = True + _default_index = False + _default_manual = False + _default_copy = True + _default_depends = () + _default_recursive = False + _default_compute = None + _default_compute_sudo = False + _default_inverse = None + _default_search = None + _default_related = None + _default_related_sudo = True + _default_company_dependent = False + _default_default = None + + _default_string = None + _default_help = None + _default_readonly = False + _default_required = False + _default_states = None + _default_groups = False + _default_change_default = None + _default_deprecated = None + + _default_inverse_fields = () + _default_computed_fields = () + _default_related_field = None + _default__triggers = () def __init__(self, string=None, **kwargs): kwargs['string'] = string attrs = {key: val for key, val in kwargs.iteritems() if val is not None} self._attrs = attrs or EMPTY_DICT - def _set_attr(self, name, value): - """ Set the given field attribute, and add it to `_attrs` if necessary. """ - object.__setattr__(self, name, value) - if not hasattr(type(self), name): + def __getattr__(self, name): + """ Access non-slot field attribute. """ + if name == '_attrs': + raise AttributeError(name) + try: + return self._attrs[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + """ Set slot or non-slot field attribute. """ + try: + object.__setattr__(self, name, value) + except AttributeError: if self._attrs: self._attrs[name] = value else: self._attrs = {name: value} # replace EMPTY_DICT + def __delattr__(self, name): + """ Remove non-slot field attribute. """ + try: + del self._attrs[name] + except KeyError: + raise AttributeError(name) + def new(self, **kwargs): """ Return a field of the same type as `self`, with its own parameters. """ return type(self)(**kwargs) def set_class_name(self, cls, name): """ Assign the model class and field name of `self`. """ + for attr, value in self.default_attrs.iteritems(): + setattr(self, attr, value) + self.model_name = cls._name self.name = name @@ -346,7 +412,7 @@ class Field(object): self._attrs = EMPTY_DICT for attr, value in attrs.iteritems(): - self._set_attr(attr, value) + setattr(self, attr, value) if not self.string and not self.related: # related fields get their string from their parent field @@ -486,7 +552,7 @@ class Field(object): for attr, value in field._attrs.iteritems(): if attr not in self._attrs: - self._set_attr(attr, value) + setattr(self, attr, value) # special case for states: copy it only for inherited fields if not self.states and self.inherited: @@ -963,6 +1029,8 @@ class Field(object): class Boolean(Field): type = 'boolean' + __slots__ = [] + def convert_to_cache(self, value, record, validate=True): return bool(value) @@ -974,10 +1042,13 @@ class Boolean(Field): class Integer(Field): type = 'integer' - group_operator = None # operator for aggregating values + __slots__ = [ + 'group_operator', # operator for aggregating values + ] + + _default_group_operator = None _related_group_operator = property(attrgetter('group_operator')) - _column_group_operator = property(attrgetter('group_operator')) def convert_to_cache(self, value, record, validate=True): @@ -1005,8 +1076,14 @@ class Float(Field): cursor and returning a pair (total, decimal) """ type = 'float' - _digits = None # digits argument passed to class initializer - group_operator = None # operator for aggregating values + + __slots__ = [ + '_digits', # digits argument passed to class initializer + 'group_operator', # operator for aggregating values + ] + + _default__digits = None + _default_group_operator = None def __init__(self, string=None, digits=None, **kwargs): super(Float, self).__init__(string=string, _digits=digits, **kwargs) @@ -1045,8 +1122,11 @@ class Float(Field): class _String(Field): """ Abstract class for string fields. """ - translate = False + __slots__ = [ + 'translate', # whether the field is translated + ] + _default_translate = False _column_translate = property(attrgetter('translate')) _related_translate = property(attrgetter('translate')) _description_translate = property(attrgetter('translate')) @@ -1060,17 +1140,21 @@ class Char(_String): :param bool translate: whether the values of this field can be translated """ type = 'char' - size = None + + __slots__ = [ + 'size', # maximum size of values (deprecated) + ] + + _default_size = None + _column_size = property(attrgetter('size')) + _related_size = property(attrgetter('size')) + _description_size = property(attrgetter('size')) def _setup_regular(self, env): super(Char, self)._setup_regular(env) assert isinstance(self.size, (NoneType, int)), \ "Char field %s with non-integer size %r" % (self, self.size) - _column_size = property(attrgetter('size')) - _related_size = property(attrgetter('size')) - _description_size = property(attrgetter('size')) - def convert_to_cache(self, value, record, validate=True): if value is None or value is False: return False @@ -1083,6 +1167,7 @@ class Text(_String): :param translate: whether the value of this field can be translated """ type = 'text' + __slots__ = [] def convert_to_cache(self, value, record, validate=True): if value is None or value is False: @@ -1091,13 +1176,18 @@ class Text(_String): class Html(_String): type = 'html' - sanitize = True # whether value must be sanitized - strip_style = False # whether to strip style attributes + __slots__ = [ + 'sanitize', # whether value must be sanitized + 'strip_style', # whether to strip style attributes + ] + + _default_sanitize = True _column_sanitize = property(attrgetter('sanitize')) _related_sanitize = property(attrgetter('sanitize')) _description_sanitize = property(attrgetter('sanitize')) + _default_strip_style = False _column_strip_style = property(attrgetter('strip_style')) _related_strip_style = property(attrgetter('strip_style')) _description_strip_style = property(attrgetter('strip_style')) @@ -1112,6 +1202,7 @@ class Html(_String): class Date(Field): type = 'date' + __slots__ = [] @staticmethod def today(*args): @@ -1172,6 +1263,7 @@ class Date(Field): class Datetime(Field): type = 'datetime' + __slots__ = [] @staticmethod def now(*args): @@ -1246,6 +1338,7 @@ class Datetime(Field): class Binary(Field): type = 'binary' + __slots__ = [] class Selection(Field): @@ -1261,8 +1354,12 @@ class Selection(Field): `. """ type = 'selection' - selection = None # [(value, string), ...], function or method name - selection_add = None # [(value, string), ...] + + __slots__ = [ + 'selection', # [(value, string), ...], function or method name + ] + + _default_selection = None def __init__(self, selection=None, string=None, **kwargs): if callable(selection): @@ -1353,20 +1450,20 @@ class Selection(Field): class Reference(Selection): type = 'reference' - size = None - def __init__(self, selection=None, string=None, **kwargs): - super(Reference, self).__init__(selection=selection, string=string, **kwargs) + __slots__ = [ + 'size', # maximum size of values (deprecated) + ] + + _default_size = None + _related_size = property(attrgetter('size')) + _column_size = property(attrgetter('size')) def _setup_regular(self, env): super(Reference, self)._setup_regular(env) assert isinstance(self.size, (NoneType, int)), \ "Reference field %s with non-integer size %r" % (self, self.size) - _related_size = property(attrgetter('size')) - - _column_size = property(attrgetter('size')) - def convert_to_cache(self, value, record, validate=True): if isinstance(value, BaseModel): if ((not validate or value._name in self.get_values(record.env)) @@ -1392,8 +1489,14 @@ class Reference(Selection): class _Relational(Field): """ Abstract class for relational fields. """ relational = True - domain = None # domain for searching values - context = None # context for searching values + + __slots__ = [ + 'domain', # domain for searching values + 'context', # context for searching values + ] + + _default_domain = [] + _default_context = {} def _setup_regular(self, env): super(_Relational, self)._setup_regular(env) @@ -1461,9 +1564,16 @@ class Many2one(_Relational): fields or field extensions. """ type = 'many2one' - ondelete = 'set null' # what to do when value is deleted - auto_join = False # whether joins are generated upon search - delegate = False # whether self implements delegation + + __slots__ = [ + 'ondelete', # what to do when value is deleted + 'auto_join', # whether joins are generated upon search + 'delegate', # whether self implements delegation + ] + + _default_ondelete = 'set null' + _default_auto_join = False + _default_delegate = False def __init__(self, comodel_name=None, string=None, **kwargs): super(Many2one, self).__init__(comodel_name=comodel_name, string=string, **kwargs) @@ -1543,6 +1653,7 @@ class UnionUpdate(SpecialValue): class _RelationalMulti(_Relational): """ Abstract class for relational fields *2many. """ + __slots__ = [] def _update(self, records, value): """ Update the cached value of `self` for `records` with `value`. """ @@ -1666,10 +1777,17 @@ class One2many(_RelationalMulti): the case of related fields or field extensions. """ type = 'one2many' - inverse_name = None # name of the inverse field - auto_join = False # whether joins are generated upon search - limit = None # optional limit to use upon read - copy = False # o2m are not copied by default + + __slots__ = [ + 'inverse_name', # name of the inverse field + 'auto_join', # whether joins are generated upon search + 'limit', # optional limit to use upon read + ] + + _default_inverse_name = None + _default_auto_join = False + _default_limit = None + _default_copy = False # o2m are not copied by default def __init__(self, comodel_name=None, inverse_name=None, string=None, **kwargs): super(One2many, self).__init__( @@ -1731,10 +1849,18 @@ class Many2many(_RelationalMulti): """ type = 'many2many' - relation = None # name of table - column1 = None # column of table referring to model - column2 = None # column of table referring to comodel - limit = None # optional limit to use upon read + + __slots__ = [ + 'relation', # name of table + 'column1', # column of table referring to model + 'column2', # column of table referring to comodel + 'limit', # optional limit to use upon read + ] + + _default_relation = None + _default_column1 = None + _default_column2 = None + _default_limit = None def __init__(self, comodel_name=None, relation=None, column1=None, column2=None, string=None, **kwargs): @@ -1777,6 +1903,7 @@ class Many2many(_RelationalMulti): class Serialized(Field): """ Minimal support for existing sparse and serialized fields. """ type = 'serialized' + __slots__ = [] def convert_to_cache(self, value, record, validate=True): return value or {} @@ -1785,11 +1912,11 @@ class Serialized(Field): class Id(Field): """ Special case for field 'id'. """ type = 'integer' + __slots__ = [] - string = 'ID' - store = True - #: Can't write this! - readonly = True + _default_string = 'ID' + _default_store = True + _default_readonly = True def to_column(self): self.column = fields.integer(self.string) From 4259fe007233be0507b6c483ddb6446f5cce65b5 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 11 Mar 2015 14:05:39 +0100 Subject: [PATCH 17/25] [IMP] openerp/osv/fields: add `__slots__` on all column classes Use the same technique as for field classes. This saves about 1.6Mb of memory per registry. --- openerp/osv/fields.py | 247 +++++++++++++++++++++++++++--------------- 1 file changed, 161 insertions(+), 86 deletions(-) diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index bf324af15cb..ddf34e4da63 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -74,7 +74,6 @@ class _column(object): _classic_read = True _classic_write = True _auto_join = False - _prefetch = True _properties = False _type = 'unknown' _obj = None @@ -85,59 +84,64 @@ class _column(object): _symbol_get = None _deprecated = False - copy = True # whether value is copied by BaseModel.copy() - string = None - help = "" - required = False - readonly = False - _domain = [] - _context = {} - states = None - priority = 0 - change_default = False - size = None - ondelete = None - translate = False - select = False - manual = False - write = False - read = False - selectable = True - group_operator = False - groups = False # CSV list of ext IDs of groups - deprecated = False # Optional deprecation warning + __slots__ = [ + 'copy', # whether value is copied by BaseModel.copy() + 'string', + 'help', + 'required', + 'readonly', + '_domain', + '_context', + 'states', + 'priority', + 'change_default', + 'size', + 'ondelete', + 'translate', + 'select', + 'manual', + 'write', + 'read', + 'selectable', + 'group_operator', + 'groups', # CSV list of ext IDs of groups + 'deprecated', # Optional deprecation warning + '_args', + '_prefetch', + ] - def __init__(self, string='unknown', required=False, readonly=False, domain=None, context=None, states=None, priority=0, change_default=False, size=None, ondelete=None, translate=False, select=False, manual=False, **args): + def __init__(self, string='unknown', required=False, readonly=False, domain=[], context={}, states=None, priority=0, change_default=False, size=None, ondelete=None, translate=False, select=False, manual=False, **args): """ The 'manual' keyword argument specifies if the field is a custom one. It corresponds to the 'state' column in ir_model_fields. """ - args0 = { - 'string': string, - 'help': args.pop('help', None), - 'required': required, - 'readonly': readonly, - '_domain': domain, - '_context': context, - 'states': states, - 'priority': priority, - 'change_default': change_default, - 'size': size, - 'ondelete': ondelete.lower() if ondelete else None, - 'translate': translate, - 'select': select, - 'manual': manual, - 'group_operator': args.pop('group_operator', None), - 'groups': args.pop('groups', None), - 'deprecated': args.pop('deprecated', None), - } - for key, val in args0.iteritems(): - if val: - setattr(self, key, val) + # add parameters and default values + args['copy'] = args.get('copy', True) + args['string'] = string + args['help'] = args.get('help', '') + args['required'] = required + args['readonly'] = readonly + args['_domain'] = domain + args['_context'] = context + args['states'] = states + args['priority'] = priority + args['change_default'] = change_default + args['size'] = size + args['ondelete'] = ondelete.lower() if ondelete else None + args['translate'] = translate + args['select'] = select + args['manual'] = manual + args['write'] = args.get('write', False) + args['read'] = args.get('read', False) + args['selectable'] = args.get('selectable', True) + args['group_operator'] = args.get('group_operator', None) + args['groups'] = args.get('groups', None) + args['deprecated'] = args.get('deprecated', None) + args['_prefetch'] = args.get('_prefetch', True) - self._args = args or EMPTY_DICT + self._args = EMPTY_DICT for key, val in args.iteritems(): setattr(self, key, val) @@ -145,6 +149,32 @@ class _column(object): if not self._classic_write or self.deprecated or self.manual: self._prefetch = False + def __getattr__(self, name): + """ Access a non-slot attribute. """ + if name == '_args': + raise AttributeError(name) + try: + return self._args[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + """ Set a slot or non-slot attribute. """ + try: + object.__setattr__(self, name, value) + except AttributeError: + if self._args: + self._args[name] = value + else: + self._args = {name: value} # replace EMPTY_DICT + + def __delattr__(self, name): + """ Remove a non-slot attribute. """ + try: + del self._args[name] + except KeyError: + raise AttributeError(name) + def new(self, _computed_field=False, **args): """ Return a column like `self` with the given parameters; the parameter `_computed_field` tells whether the corresponding field is computed. @@ -224,6 +254,7 @@ class boolean(_column): _symbol_c = '%s' _symbol_f = bool _symbol_set = (_symbol_c, _symbol_f) + __slots__ = [] def __init__(self, string='unknown', required=False, **args): super(boolean, self).__init__(string=string, required=required, **args) @@ -239,6 +270,7 @@ class integer(_column): _symbol_f = lambda x: int(x or 0) _symbol_set = (_symbol_c, _symbol_f) _symbol_get = lambda self,x: x or 0 + __slots__ = [] def __init__(self, string='unknown', required=False, **args): super(integer, self).__init__(string=string, required=required, **args) @@ -246,6 +278,7 @@ class integer(_column): class reference(_column): _type = 'reference' _classic_read = False # post-process to handle missing target + __slots__ = ['selection'] def __init__(self, string, selection, size=None, **args): if callable(selection): @@ -298,6 +331,7 @@ def _symbol_set_char(self, symb): class char(_column): _type = 'char' + __slots__ = ['_symbol_f', '_symbol_set', '_symbol_set_char'] def __init__(self, string="unknown", size=None, **args): _column.__init__(self, string=string, size=size or None, **args) @@ -307,11 +341,13 @@ class char(_column): class text(_column): _type = 'text' + __slots__ = [] class html(text): _type = 'html' _symbol_c = '%s' + __slots__ = ['_sanitize', '_strip_style', '_symbol_f', '_symbol_set'] def _symbol_set_html(self, value): if value is None or value is False: @@ -347,6 +383,7 @@ class float(_column): _type = 'float' _symbol_c = '%s' _symbol_get = lambda self,x: x or 0.0 + __slots__ = ['_digits', '_digits_compute', '_symbol_f', '_symbol_set'] @property def digits(self): @@ -374,6 +411,7 @@ class float(_column): class date(_column): _type = 'date' + __slots__ = [] MONTHS = [ ('01', 'January'), @@ -464,6 +502,7 @@ class date(_column): class datetime(_column): _type = 'datetime' + __slots__ = [] MONTHS = [ ('01', 'January'), @@ -533,7 +572,7 @@ class datetime(_column): class binary(_column): _type = 'binary' - _symbol_c = '%s' + _classic_read = False # Binary values may be byte strings (python 2.6 byte array), but # the legacy OpenERP convention is to transfer and store binaries @@ -541,17 +580,16 @@ class binary(_column): # unicode in some circumstances, hence the str() cast in symbol_f. # This str coercion will only work for pure ASCII unicode strings, # on purpose - non base64 data must be passed as a 8bit byte strings. + _symbol_c = '%s' _symbol_f = lambda symb: symb and Binary(str(symb)) or None - _symbol_set = (_symbol_c, _symbol_f) _symbol_get = lambda self, x: x and str(x) - _classic_read = False - _prefetch = False + __slots__ = ['filters'] def __init__(self, string='unknown', filters=None, **args): - _column.__init__(self, string=string, **args) - self.filters = filters + args['_prefetch'] = args.get('_prefetch', False) + _column.__init__(self, string=string, filters=filters, **args) def get(self, cr, obj, ids, name, user=None, context=None, values=None): if not context: @@ -579,13 +617,13 @@ class binary(_column): class selection(_column): _type = 'selection' + __slots__ = ['selection'] def __init__(self, selection, string='unknown', **args): if callable(selection): from openerp import api selection = api.expected(api.cr_uid_context, selection) - _column.__init__(self, string=string, **args) - self.selection = selection + _column.__init__(self, string=string, selection=selection, **args) def to_field_args(self): args = super(selection, self).to_field_args() @@ -646,9 +684,10 @@ class many2one(_column): _symbol_f = lambda x: x or None _symbol_set = (_symbol_c, _symbol_f) - ondelete = 'set null' + __slots__ = ['ondelete', '_obj', '_auto_join'] def __init__(self, obj, string='unknown', auto_join=False, **args): + args['ondelete'] = args.get('ondelete', 'set null') _column.__init__(self, string=string, **args) self._obj = obj self._auto_join = auto_join @@ -694,13 +733,14 @@ class many2one(_column): class one2many(_column): _classic_read = False _classic_write = False - _prefetch = False _type = 'one2many' - # one2many columns are not copied by default - copy = False + __slots__ = ['_obj', '_fields_id', '_limit', '_auto_join'] def __init__(self, obj, fields_id, string='unknown', limit=None, auto_join=False, **args): + # one2many columns are not copied by default + args['copy'] = args.get('copy', False) + args['_prefetch'] = args.get('_prefetch', False) _column.__init__(self, string=string, **args) self._obj = obj self._fields_id = fields_id @@ -841,12 +881,14 @@ class many2many(_column): """ _classic_read = False _classic_write = False - _prefetch = False _type = 'many2many' + __slots__ = ['_obj', '_rel', '_id1', '_id2', '_limit', '_auto_join'] + def __init__(self, obj, rel=None, id1=None, id2=None, string='unknown', limit=None, **args): """ """ + args['_prefetch'] = args.get('_prefetch', False) _column.__init__(self, string=string, **args) self._obj = obj if rel and '.' in rel: @@ -856,6 +898,7 @@ class many2many(_column): self._id1 = id1 self._id2 = id2 self._limit = limit + self._auto_join = False def to_field_args(self): args = super(many2many, self).to_field_args() @@ -1238,14 +1281,30 @@ class function(_column): } """ - _classic_read = False - _classic_write = False - _prefetch = False - _type = 'function' _properties = True - # function fields are not copied by default - copy = False + __slots__ = [ + '_type', + '_classic_read', + '_classic_write', + '_symbol_c', + '_symbol_f', + '_symbol_set', + '_symbol_get', + + '_fnct', + '_arg', + '_fnct_inv', + '_fnct_inv_arg', + '_fnct_search', + '_multi', + 'store', + + '_digits', + '_digits_compute', + 'selection', + '_obj', + ] @property def digits(self): @@ -1259,33 +1318,43 @@ class function(_column): # multi: compute several fields in one call # def __init__(self, fnct, arg=None, fnct_inv=None, fnct_inv_arg=None, type='float', fnct_search=None, obj=None, store=False, multi=False, **args): + self._classic_read = False + self._classic_write = False + self._prefetch = False + self._symbol_c = '%s' + self._symbol_f = _symbol_set + self._symbol_set = (self._symbol_c, self._symbol_f) + self._symbol_get = None + # pop attributes that should not be assigned to self self._digits = args.pop('digits', (16,2)) self._digits_compute = args.pop('digits_compute', None) + self._obj = args.pop('relation', obj) + + # function fields are not copied by default + args['copy'] = args.get('copy', False) _column.__init__(self, **args) - self._obj = obj + + self._type = type self._fnct = fnct - self._fnct_inv = fnct_inv self._arg = arg + self._fnct_inv = fnct_inv + self._fnct_inv_arg = fnct_inv_arg + self._fnct_search = fnct_search + self.store = store self._multi = multi - if 'relation' in args: - self._obj = args['relation'] + + if not fnct_inv: + self.readonly = 1 + + if not fnct_search and not store: + self.selectable = False if callable(args.get('selection')): from openerp import api self.selection = api.expected(api.cr_uid_context, args['selection']) - self._fnct_inv_arg = fnct_inv_arg - if not fnct_inv: - self.readonly = 1 - self._type = type - self._fnct_search = fnct_search - self.store = store - - if not fnct_search and not store: - self.selectable = False - if store: if self._type != 'many2one': # m2o fields need to return tuples with name_get, not just foreign keys @@ -1421,6 +1490,7 @@ class related(function): 'bar': fields.related('foo_id', 'frol', type='char', string='Frol of Foo'), } """ + __slots__ = ['arg', '_relations'] def _fnct_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None): # assume self._arg = ('foo', 'bar', 'baz') @@ -1472,7 +1542,8 @@ class related(function): pass -class sparse(function): +class sparse(function): + __slots__ = ['serialization_field'] def convert_value(self, obj, cr, uid, record, value, read_value, context=None): """ @@ -1521,7 +1592,6 @@ class sparse(function): return read_value return value - def _fnct_write(self,obj,cr, uid, ids, field_name, value, args, context=None): if not type(ids) == list: ids = [ids] @@ -1564,7 +1634,6 @@ class sparse(function): def __init__(self, serialization_field, **kwargs): self.serialization_field = serialization_field super(sparse, self).__init__(self._fnct_read, fnct_inv=self._fnct_write, multi='__sparse_multi', **kwargs) - # --------------------------------------------------------- @@ -1572,6 +1641,8 @@ class sparse(function): # --------------------------------------------------------- class dummy(function): + __slots__ = ['arg', '_relations'] + def _fnct_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None): return [] @@ -1595,23 +1666,27 @@ class serialized(_column): Note: only plain components allowed. """ - + _type = 'serialized' + __slots__ = [] + def _symbol_set_struct(val): return simplejson.dumps(val) def _symbol_get_struct(self, val): return simplejson.loads(val or '{}') - - _prefetch = False - _type = 'serialized' _symbol_c = '%s' _symbol_f = _symbol_set_struct _symbol_set = (_symbol_c, _symbol_f) _symbol_get = _symbol_get_struct + def __init__(self, *args, **kwargs): + kwargs['_prefetch'] = kwargs.get('_prefetch', False) + super(serialized, self).__init__(*args, **kwargs) + # TODO: review completly this class for speed improvement class property(function): + __slots__ = [] def to_field_args(self): args = super(property, self).to_field_args() From 1df3467888ee5af9d24f5d33081bd695327e1ca9 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 12 Mar 2015 10:53:01 +0100 Subject: [PATCH 18/25] [IMP] fields: remove unnecessary code In `__getattr__`, remove the test on accessing `_attrs`. This situation should never happen, since the attribute `_attrs` is set in method `__init__()`. --- openerp/fields.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index b70d95709b0..6dd8d6a77d2 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -350,8 +350,6 @@ class Field(object): def __getattr__(self, name): """ Access non-slot field attribute. """ - if name == '_attrs': - raise AttributeError(name) try: return self._attrs[name] except KeyError: From f93a14e3b2158e1398456d76373303e608d40a03 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 12 Mar 2015 10:57:20 +0100 Subject: [PATCH 19/25] [IMP] fields: remove comments about `_all_columns`, and improve its docstring --- openerp/models.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/openerp/models.py b/openerp/models.py index d9f565cb781..c626ee92e23 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -336,13 +336,6 @@ class BaseModel(object): # field_column_obj, origina_parent_model), ... } _inherit_fields = {} - # Mapping field name/column_info object - # This is similar to _inherit_fields but: - # 1. includes self fields, - # 2. uses column_info instead of a triple. - # Warning: _all_columns is deprecated, use _fields instead - # _all_columns = {} - _table = None _log_create = False _sql_constraints = [] @@ -2907,9 +2900,9 @@ class BaseModel(object): @property def _all_columns(self): - """ Returns a dict mapping all fields names (direct fields and inherited - field via _inherits) to a ``column_info`` struct giving detailed columns. - Deprecated. + """ Returns a dict mapping all fields names (self fields and inherited + field via _inherits) to a ``column_info`` object giving detailed column + information. This property is deprecated, use ``_fields`` instead. """ result = {} # do not inverse for loops, since local fields may hide inherited ones! From 93c341a52cab8070e94af5078c99cc9ac1efc8a7 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 12 Mar 2015 11:00:16 +0100 Subject: [PATCH 20/25] [IMP] models: avoid using `map` instead of a `for` loop (thanks chs@odoo.com) --- openerp/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openerp/models.py b/openerp/models.py index c626ee92e23..b536d19476a 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -3011,7 +3011,8 @@ class BaseModel(object): model = self.env[model_name] for field_name in field_names: field = model._fields[field_name] - map(field.add_trigger, triggers) + for trigger in triggers: + field.add_trigger(trigger) # determine old-api structures about inherited fields cls._inherits_reload() From b156c2e27ef598ba22910c811621e2085dfbf822 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 12 Mar 2015 15:46:47 +0100 Subject: [PATCH 21/25] [IMP] registry: adapt LRU sizes for registries and ormcache The registry size is now assumed to be around 10Mb, and ormcache size is proportional to the maximum number of registries. Statistics about ormcache usage is shown to the log when receiving signal SIGUSR1. --- openerp/modules/registry.py | 9 +++++---- openerp/service/server.py | 8 +++++++- openerp/tools/cache.py | 31 +++++++++++++++++++++++++------ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 5c9870a488a..352c6f156af 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -305,8 +305,9 @@ class RegistryManager(object): # cannot specify the memory limit soft on windows... size = 42 else: - # On average, a clean registry take 25MB of memory + cache - avgsz = 30 * 1024 * 1024 + # A registry takes 10MB of memory on average, so we reserve + # 10Mb (registry) + 5Mb (working memory) per registry + avgsz = 15 * 1024 * 1024 size = int(config['limit_memory_soft'] / avgsz) cls._registries = LRU(size) @@ -319,8 +320,8 @@ class RegistryManager(object): """ with cls.lock(): if cls._cache is None: - # we allocate one cache entry per 32KB of memory - size = max(8192, int(config['limit_memory_soft'] / 32768)) + # we allocate 8192 cache entries per registry + size = 8192 * cls.registries.count cls._cache = LRU(size) return cls._cache diff --git a/openerp/service/server.py b/openerp/service/server.py index 0ec039e83ad..b6c57fa7325 100644 --- a/openerp/service/server.py +++ b/openerp/service/server.py @@ -38,7 +38,7 @@ import openerp from openerp.modules.registry import RegistryManager from openerp.release import nt_service_name import openerp.tools.config as config -from openerp.tools.misc import stripped_sys_argv, dumpstacks +from openerp.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats _logger = logging.getLogger(__name__) @@ -296,6 +296,7 @@ class ThreadedServer(CommonServer): signal.signal(signal.SIGCHLD, self.signal_handler) signal.signal(signal.SIGHUP, self.signal_handler) signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) elif os.name == 'nt': import win32api win32api.SetConsoleCtrlHandler(lambda sig: self.signal_handler(sig, None), 1) @@ -389,6 +390,7 @@ class GeventServer(CommonServer): if os.name == 'posix': signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) gevent.spawn(self.watch_parent) self.httpd = WSGIServer((self.interface, self.port), self.app) @@ -510,6 +512,9 @@ class PreforkServer(CommonServer): elif sig == signal.SIGQUIT: # dump stacks on kill -3 self.dumpstacks() + elif sig == signal.SIGUSR1: + # log ormcache stats on kill -SIGUSR1 + log_ormcache_stats() elif sig == signal.SIGTTIN: # increase number of workers self.population += 1 @@ -586,6 +591,7 @@ class PreforkServer(CommonServer): signal.signal(signal.SIGTTIN, self.signal_handler) signal.signal(signal.SIGTTOU, self.signal_handler) signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) # listen to socket self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index ab387de361a..99364f8da98 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -27,7 +27,7 @@ from inspect import getargspec import lru import logging -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) class ormcache(object): @@ -47,9 +47,9 @@ class ormcache(object): return lookup def stat(self): - return "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % \ + return "lookup-stats %6d hit %6d miss %6d err %4.1f ratio" % \ (self.stat_hit, self.stat_miss, self.stat_err, - (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit)) + (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit or 1)) def lru(self, model): return model.pool.cache, (model.pool.db_name, model._name, self.method) @@ -73,9 +73,9 @@ class ormcache(object): """ Remove *args entry from the cache or all keys if *args is undefined """ d, key0 = self.lru(model) if args: - logger.warn("ormcache.clear arguments are deprecated and ignored " - "(while clearing caches on (%s).%s)", - model._name, self.method.__name__) + _logger.warn("ormcache.clear arguments are deprecated and ignored " + "(while clearing caches on (%s).%s)", + model._name, self.method.__name__) d.clear_prefix(key0) model.pool._any_cache_cleared = True @@ -168,6 +168,25 @@ class dummy_cache(object): pass +def log_ormcache_stats(sig=None, frame=None): + """ Log statistics of ormcache usage by database, model, and method. """ + from openerp.modules.registry import RegistryManager + from collections import defaultdict + import threading + + me = threading.currentThread() + entries = defaultdict(int) + for key in RegistryManager.cache.iterkeys(): + entries[key[:3]] += 1 + for (dbname, model_name, method), count in sorted(entries.items()): + me.dbname = dbname + model = RegistryManager.get(dbname)[model_name] + func = getattr(model, method.__name__).im_func + ormcache = func.clear_cache.im_self + _logger.info("%6d entries, %s, for %s.%s", + count, ormcache.stat(), model_name, method.__name__) + + # For backward compatibility cache = ormcache From 524bb0a52027ec81245750bb5f0a9e5320faf922 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 12 Mar 2015 16:39:12 +0100 Subject: [PATCH 22/25] [IMP] fields: simplify the definition of `__slots__` via the `Field` metaclass --- openerp/fields.py | 288 +++++++++++++++++----------------------------- 1 file changed, 108 insertions(+), 180 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index 6dd8d6a77d2..14deaa8d601 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -73,20 +73,32 @@ class MetaField(type): """ Metaclass for field classes. """ by_type = {} + def __new__(meta, name, bases, attrs): + """ Combine the ``_slots`` dict from parent classes, and determine + `__slots__` for them on the new class. + """ + base_slots = {} + for base in reversed(bases): + base_slots.update(getattr(base, '_slots', ())) + + slots = dict(base_slots) + slots.update(attrs.get('_slots', ())) + + attrs['__slots__'] = set(slots) - set(base_slots) + attrs['_slots'] = slots + return type.__new__(meta, name, bases, attrs) + def __init__(cls, name, bases, attrs): super(MetaField, cls).__init__(name, bases, attrs) if cls.type and cls.type not in MetaField.by_type: MetaField.by_type[cls.type] = cls # compute class attributes to avoid calling dir() on fields - cls.default_attrs = {} cls.column_attrs = [] cls.related_attrs = [] cls.description_attrs = [] for attr in dir(cls): - if attr.startswith('_default_'): - cls.default_attrs[attr[9:]] = getattr(cls, attr) - elif attr.startswith('_column_'): + if attr.startswith('_column_'): cls.column_attrs.append((attr[8:], attr)) elif attr.startswith('_related_'): cls.related_attrs.append((attr[9:], attr)) @@ -257,91 +269,52 @@ class Field(object): """ __metaclass__ = MetaField - type = None # type of the field (string) - relational = False # whether the field is a relational one + type = None # type of the field (string) + relational = False # whether the field is a relational one - __slots__ = [ - '_attrs', # dictionary of field attributes; it contains: - # - all attributes after __init__() - # - free attributes only after set_class_name() + _slots = { + '_attrs': EMPTY_DICT, # dictionary of field attributes; it contains: + # - all attributes after __init__() + # - free attributes only after set_class_name() - 'automatic', # whether the field is automatically created ("magic" field) - 'inherited', # whether the field is inherited (_inherits) - 'column', # the column corresponding to the field - 'setup_done', # whether the field has been set up + 'automatic': False, # whether the field is automatically created ("magic" field) + 'inherited': False, # whether the field is inherited (_inherits) + 'column': None, # the column corresponding to the field + 'setup_done': False, # whether the field has been set up - 'name', # name of the field - 'model_name', # name of the model of this field - 'comodel_name', # name of the model of values (if relational) + 'name': None, # name of the field + 'model_name': None, # name of the model of this field + 'comodel_name': None, # name of the model of values (if relational) - 'store', # whether the field is stored in database - 'index', # whether the field is indexed in database - 'manual', # whether the field is a custom field - 'copy', # whether the field is copied over by BaseModel.copy() - 'depends', # collection of field dependencies - 'recursive', # whether self depends on itself - 'compute', # compute(recs) computes field on recs - 'compute_sudo', # whether field should be recomputed as admin - 'inverse', # inverse(recs) inverses field on recs - 'search', # search(recs, operator, value) searches on self - 'related', # sequence of field names, for related fields - 'related_sudo', # whether related fields should be read as admin - 'company_dependent', # whether `self` is company-dependent (property field) - 'default', # default(recs) returns the default value + 'store': True, # whether the field is stored in database + 'index': False, # whether the field is indexed in database + 'manual': False, # whether the field is a custom field + 'copy': True, # whether the field is copied over by BaseModel.copy() + 'depends': (), # collection of field dependencies + 'recursive': False, # whether self depends on itself + 'compute': None, # compute(recs) computes field on recs + 'compute_sudo': False, # whether field should be recomputed as admin + 'inverse': None, # inverse(recs) inverses field on recs + 'search': None, # search(recs, operator, value) searches on self + 'related': None, # sequence of field names, for related fields + 'related_sudo': True, # whether related fields should be read as admin + 'company_dependent': False, # whether `self` is company-dependent (property field) + 'default': None, # default(recs) returns the default value - 'string', # field label - 'help', # field tooltip - 'readonly', # whether the field is readonly - 'required', # whether the field is required - 'states', # set readonly and required depending on state - 'groups', # csv list of group xml ids - 'change_default', # whether the field may trigger a "user-onchange" - 'deprecated', # whether the field is deprecated + 'string': None, # field label + 'help': None, # field tooltip + 'readonly': False, # whether the field is readonly + 'required': False, # whether the field is required + 'states': None, # set readonly and required depending on state + 'groups': None, # csv list of group xml ids + 'change_default': False, # whether the field may trigger a "user-onchange" + 'deprecated': None, # whether the field is deprecated - 'inverse_fields', # collection of inverse fields (objects) - 'computed_fields', # fields computed with the same method as self - 'related_field', # corresponding related field - '_triggers', # invalidation and recomputation triggers - ] - - # default values for slots - _default_automatic = False - _default_inherited = False - _default_column = None - _default_setup_done = False - - _default_name = None - _default_model_name = None - _default_comodel_name = None - - _default_store = True - _default_index = False - _default_manual = False - _default_copy = True - _default_depends = () - _default_recursive = False - _default_compute = None - _default_compute_sudo = False - _default_inverse = None - _default_search = None - _default_related = None - _default_related_sudo = True - _default_company_dependent = False - _default_default = None - - _default_string = None - _default_help = None - _default_readonly = False - _default_required = False - _default_states = None - _default_groups = False - _default_change_default = None - _default_deprecated = None - - _default_inverse_fields = () - _default_computed_fields = () - _default_related_field = None - _default__triggers = () + 'inverse_fields': (), # collection of inverse fields (objects) + 'computed_fields': (), # fields computed with the same method as self + 'related_field': None, # corresponding related field + '_triggers': (), # invalidation and recomputation triggers + } def __init__(self, string=None, **kwargs): kwargs['string'] = string @@ -378,7 +351,8 @@ class Field(object): def set_class_name(self, cls, name): """ Assign the model class and field name of `self`. """ - for attr, value in self.default_attrs.iteritems(): + self_attrs = self._attrs + for attr, value in self._slots.iteritems(): setattr(self, attr, value) self.model_name = cls._name @@ -391,7 +365,7 @@ class Field(object): attrs.update(field._attrs) else: attrs.clear() - attrs.update(self._attrs) # necessary in case self is not in cls + attrs.update(self_attrs) # necessary in case self is not in cls # initialize `self` with `attrs` if attrs.get('compute'): @@ -408,7 +382,6 @@ class Field(object): if not isinstance(attrs.get('column'), (NoneType, fields.function)): attrs.pop('store', None) - self._attrs = EMPTY_DICT for attr, value in attrs.iteritems(): setattr(self, attr, value) @@ -1027,8 +1000,6 @@ class Field(object): class Boolean(Field): type = 'boolean' - __slots__ = [] - def convert_to_cache(self, value, record, validate=True): return bool(value) @@ -1040,12 +1011,10 @@ class Boolean(Field): class Integer(Field): type = 'integer' + _slots = { + 'group_operator': None, # operator for aggregating values + } - __slots__ = [ - 'group_operator', # operator for aggregating values - ] - - _default_group_operator = None _related_group_operator = property(attrgetter('group_operator')) _column_group_operator = property(attrgetter('group_operator')) @@ -1074,14 +1043,10 @@ class Float(Field): cursor and returning a pair (total, decimal) """ type = 'float' - - __slots__ = [ - '_digits', # digits argument passed to class initializer - 'group_operator', # operator for aggregating values - ] - - _default__digits = None - _default_group_operator = None + _slots = { + '_digits': None, # digits argument passed to class initializer + 'group_operator': None, # operator for aggregating values + } def __init__(self, string=None, digits=None, **kwargs): super(Float, self).__init__(string=string, _digits=digits, **kwargs) @@ -1120,11 +1085,10 @@ class Float(Field): class _String(Field): """ Abstract class for string fields. """ - __slots__ = [ - 'translate', # whether the field is translated - ] + _slots = { + 'translate': False, # whether the field is translated + } - _default_translate = False _column_translate = property(attrgetter('translate')) _related_translate = property(attrgetter('translate')) _description_translate = property(attrgetter('translate')) @@ -1138,12 +1102,10 @@ class Char(_String): :param bool translate: whether the values of this field can be translated """ type = 'char' + _slots = { + 'size': None, # maximum size of values (deprecated) + } - __slots__ = [ - 'size', # maximum size of values (deprecated) - ] - - _default_size = None _column_size = property(attrgetter('size')) _related_size = property(attrgetter('size')) _description_size = property(attrgetter('size')) @@ -1165,7 +1127,6 @@ class Text(_String): :param translate: whether the value of this field can be translated """ type = 'text' - __slots__ = [] def convert_to_cache(self, value, record, validate=True): if value is None or value is False: @@ -1174,18 +1135,15 @@ class Text(_String): class Html(_String): type = 'html' + _slots = { + 'sanitize': True, # whether value must be sanitized + 'strip_style': False, # whether to strip style attributes + } - __slots__ = [ - 'sanitize', # whether value must be sanitized - 'strip_style', # whether to strip style attributes - ] - - _default_sanitize = True _column_sanitize = property(attrgetter('sanitize')) _related_sanitize = property(attrgetter('sanitize')) _description_sanitize = property(attrgetter('sanitize')) - _default_strip_style = False _column_strip_style = property(attrgetter('strip_style')) _related_strip_style = property(attrgetter('strip_style')) _description_strip_style = property(attrgetter('strip_style')) @@ -1200,7 +1158,6 @@ class Html(_String): class Date(Field): type = 'date' - __slots__ = [] @staticmethod def today(*args): @@ -1261,7 +1218,6 @@ class Date(Field): class Datetime(Field): type = 'datetime' - __slots__ = [] @staticmethod def now(*args): @@ -1336,7 +1292,6 @@ class Datetime(Field): class Binary(Field): type = 'binary' - __slots__ = [] class Selection(Field): @@ -1352,12 +1307,9 @@ class Selection(Field): `. """ type = 'selection' - - __slots__ = [ - 'selection', # [(value, string), ...], function or method name - ] - - _default_selection = None + _slots = { + 'selection': None, # [(value, string), ...], function or method name + } def __init__(self, selection=None, string=None, **kwargs): if callable(selection): @@ -1448,12 +1400,10 @@ class Selection(Field): class Reference(Selection): type = 'reference' + _slots = { + 'size': None, # maximum size of values (deprecated) + } - __slots__ = [ - 'size', # maximum size of values (deprecated) - ] - - _default_size = None _related_size = property(attrgetter('size')) _column_size = property(attrgetter('size')) @@ -1487,14 +1437,10 @@ class Reference(Selection): class _Relational(Field): """ Abstract class for relational fields. """ relational = True - - __slots__ = [ - 'domain', # domain for searching values - 'context', # context for searching values - ] - - _default_domain = [] - _default_context = {} + _slots = { + 'domain': [], # domain for searching values + 'context': {}, # context for searching values + } def _setup_regular(self, env): super(_Relational, self)._setup_regular(env) @@ -1562,16 +1508,11 @@ class Many2one(_Relational): fields or field extensions. """ type = 'many2one' - - __slots__ = [ - 'ondelete', # what to do when value is deleted - 'auto_join', # whether joins are generated upon search - 'delegate', # whether self implements delegation - ] - - _default_ondelete = 'set null' - _default_auto_join = False - _default_delegate = False + _slots = { + 'ondelete': 'set null', # what to do when value is deleted + 'auto_join': False, # whether joins are generated upon search + 'delegate': False, # whether self implements delegation + } def __init__(self, comodel_name=None, string=None, **kwargs): super(Many2one, self).__init__(comodel_name=comodel_name, string=string, **kwargs) @@ -1651,7 +1592,6 @@ class UnionUpdate(SpecialValue): class _RelationalMulti(_Relational): """ Abstract class for relational fields *2many. """ - __slots__ = [] def _update(self, records, value): """ Update the cached value of `self` for `records` with `value`. """ @@ -1775,17 +1715,12 @@ class One2many(_RelationalMulti): the case of related fields or field extensions. """ type = 'one2many' - - __slots__ = [ - 'inverse_name', # name of the inverse field - 'auto_join', # whether joins are generated upon search - 'limit', # optional limit to use upon read - ] - - _default_inverse_name = None - _default_auto_join = False - _default_limit = None - _default_copy = False # o2m are not copied by default + _slots = { + 'inverse_name': None, # name of the inverse field + 'auto_join': False, # whether joins are generated upon search + 'limit': None, # optional limit to use upon read + 'copy': False, # o2m are not copied by default + } def __init__(self, comodel_name=None, inverse_name=None, string=None, **kwargs): super(One2many, self).__init__( @@ -1847,18 +1782,12 @@ class Many2many(_RelationalMulti): """ type = 'many2many' - - __slots__ = [ - 'relation', # name of table - 'column1', # column of table referring to model - 'column2', # column of table referring to comodel - 'limit', # optional limit to use upon read - ] - - _default_relation = None - _default_column1 = None - _default_column2 = None - _default_limit = None + _slots = { + 'relation': None, # name of table + 'column1': None, # column of table referring to model + 'column2': None, # column of table referring to comodel + 'limit': None, # optional limit to use upon read + } def __init__(self, comodel_name=None, relation=None, column1=None, column2=None, string=None, **kwargs): @@ -1901,7 +1830,6 @@ class Many2many(_RelationalMulti): class Serialized(Field): """ Minimal support for existing sparse and serialized fields. """ type = 'serialized' - __slots__ = [] def convert_to_cache(self, value, record, validate=True): return value or {} @@ -1910,11 +1838,11 @@ class Serialized(Field): class Id(Field): """ Special case for field 'id'. """ type = 'integer' - __slots__ = [] - - _default_string = 'ID' - _default_store = True - _default_readonly = True + _slots = { + 'string': 'ID', + 'store': True, + 'readonly': True, + } def to_column(self): self.column = fields.integer(self.string) From c0d79a78f02ddc1bdb90d9a19c36d6a4c64b6a8e Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Mon, 16 Mar 2015 10:31:13 +0100 Subject: [PATCH 23/25] [IMP] fields: rename methods to avoid collisions with `__slots__` names --- openerp/osv/fields.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index ddf34e4da63..9c4dcc8258a 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -1492,13 +1492,13 @@ class related(function): """ __slots__ = ['arg', '_relations'] - def _fnct_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None): + def _related_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None): # assume self._arg = ('foo', 'bar', 'baz') # domain = [(name, op, val)] => search [('foo.bar.baz', op, val)] field = '.'.join(self._arg) return map(lambda x: (field, x[1], x[2]), domain) - def _fnct_write(self, obj, cr, uid, ids, field_name, values, args, context=None): + def _related_write(self, obj, cr, uid, ids, field_name, values, args, context=None): if isinstance(ids, (int, long)): ids = [ids] for instance in obj.browse(cr, uid, ids, context=context): @@ -1509,7 +1509,7 @@ class related(function): # write on the last field of the target record instance.write({self.arg[-1]: values}) - def _fnct_read(self, obj, cr, uid, ids, field_name, args, context=None): + def _related_read(self, obj, cr, uid, ids, field_name, args, context=None): res = {} for record in obj.browse(cr, SUPERUSER_ID, ids, context=context): value = record @@ -1536,7 +1536,7 @@ class related(function): def __init__(self, *arg, **args): self.arg = arg self._relations = [] - super(related, self).__init__(self._fnct_read, arg, self._fnct_write, fnct_inv_arg=arg, fnct_search=self._fnct_search, **args) + super(related, self).__init__(self._related_read, arg, self._related_write, fnct_inv_arg=arg, fnct_search=self._related_search, **args) if self.store is True: # TODO: improve here to change self.store = {...} according to related objects pass @@ -1592,7 +1592,7 @@ class sparse(function): return read_value return value - def _fnct_write(self,obj,cr, uid, ids, field_name, value, args, context=None): + def _sparse_write(self,obj,cr, uid, ids, field_name, value, args, context=None): if not type(ids) == list: ids = [ids] records = obj.browse(cr, uid, ids, context=context) @@ -1607,7 +1607,7 @@ class sparse(function): obj.write(cr, uid, ids, {self.serialization_field: serialized}, context=context) return True - def _fnct_read(self, obj, cr, uid, ids, field_names, args, context=None): + def _sparse_read(self, obj, cr, uid, ids, field_names, args, context=None): results = {} records = obj.browse(cr, uid, ids, context=context) for record in records: @@ -1633,7 +1633,7 @@ class sparse(function): def __init__(self, serialization_field, **kwargs): self.serialization_field = serialization_field - super(sparse, self).__init__(self._fnct_read, fnct_inv=self._fnct_write, multi='__sparse_multi', **kwargs) + super(sparse, self).__init__(self._sparse_read, fnct_inv=self._sparse_write, multi='__sparse_multi', **kwargs) # --------------------------------------------------------- @@ -1643,19 +1643,19 @@ class sparse(function): class dummy(function): __slots__ = ['arg', '_relations'] - def _fnct_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None): + def _dummy_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None): return [] - def _fnct_write(self, obj, cr, uid, ids, field_name, values, args, context=None): + def _dummy_write(self, obj, cr, uid, ids, field_name, values, args, context=None): return False - def _fnct_read(self, obj, cr, uid, ids, field_name, args, context=None): + def _dummy_read(self, obj, cr, uid, ids, field_name, args, context=None): return {} def __init__(self, *arg, **args): self.arg = arg self._relations = [] - super(dummy, self).__init__(self._fnct_read, arg, self._fnct_write, fnct_inv_arg=arg, fnct_search=self._fnct_search, **args) + super(dummy, self).__init__(self._dummy_read, arg, self._dummy_write, fnct_inv_arg=arg, fnct_search=self._dummy_search, **args) # --------------------------------------------------------- # Serialized fields @@ -1693,19 +1693,19 @@ class property(function): args['company_dependent'] = True return args - def _fnct_search(self, tobj, cr, uid, obj, name, domain, context=None): + def _property_search(self, tobj, cr, uid, obj, name, domain, context=None): ir_property = obj.pool['ir.property'] result = [] for field, operator, value in domain: result += ir_property.search_multi(cr, uid, name, tobj._name, operator, value, context=context) return result - def _fnct_write(self, obj, cr, uid, id, prop_name, value, obj_dest, context=None): + def _property_write(self, obj, cr, uid, id, prop_name, value, obj_dest, context=None): ir_property = obj.pool['ir.property'] ir_property.set_multi(cr, uid, prop_name, obj._name, {id: value}, context=context) return True - def _fnct_read(self, obj, cr, uid, ids, prop_names, obj_dest, context=None): + def _property_read(self, obj, cr, uid, ids, prop_names, obj_dest, context=None): ir_property = obj.pool['ir.property'] res = {id: {} for id in ids} @@ -1734,9 +1734,9 @@ class property(function): args = dict(args) args['obj'] = args.pop('relation', '') or args.get('obj', '') super(property, self).__init__( - fnct=self._fnct_read, - fnct_inv=self._fnct_write, - fnct_search=self._fnct_search, + fnct=self._property_read, + fnct_inv=self._property_write, + fnct_search=self._property_search, multi='properties', **args ) From 68969c97160addfcb152cc35c103530493c34a1c Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Mon, 16 Mar 2015 11:35:05 +0100 Subject: [PATCH 24/25] [IMP] fields: avoid redundant slot in class many2one --- openerp/osv/fields.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index 9c4dcc8258a..0ca684253b5 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -151,8 +151,6 @@ class _column(object): def __getattr__(self, name): """ Access a non-slot attribute. """ - if name == '_args': - raise AttributeError(name) try: return self._args[name] except KeyError: @@ -684,7 +682,7 @@ class many2one(_column): _symbol_f = lambda x: x or None _symbol_set = (_symbol_c, _symbol_f) - __slots__ = ['ondelete', '_obj', '_auto_join'] + __slots__ = ['_obj', '_auto_join'] def __init__(self, obj, string='unknown', auto_join=False, **args): args['ondelete'] = args.get('ondelete', 'set null') From 5aa23f828053d30a5f67b88272775919542ebf0d Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 18 Mar 2015 15:31:16 +0100 Subject: [PATCH 25/25] [IMP] ormcache: compute hit/miss/err statistics per database In ormcache, those statistic numbers were counted per cached method for all registries. We introduce separate statistic counters, and create a counter per (database, model, method). The existing attributes that are no longer used have been removed. --- openerp/tools/cache.py | 57 +++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index 99364f8da98..7f40421e13a 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -21,24 +21,36 @@ # decorator makes wrappers that have the same API as their wrapped function; # this is important for the openerp.api.guess() that relies on signatures +from collections import defaultdict from decorator import decorator from inspect import getargspec - -import lru import logging _logger = logging.getLogger(__name__) +class ormcache_counter(object): + """ Statistic counters for cache entries. """ + __slots__ = ['hit', 'miss', 'err'] + + def __init__(self): + self.hit = 0 + self.miss = 0 + self.err = 0 + + @property + def ratio(self): + return 100.0 * self.hit / (self.hit + self.miss or 1) + +# statistic counters dictionary, maps (dbname, modelname, method) to counter +STAT = defaultdict(ormcache_counter) + + class ormcache(object): """ LRU cache decorator for orm methods. """ def __init__(self, skiparg=2, size=8192, multi=None, timeout=None): self.skiparg = skiparg - self.size = size - self.stat_miss = 0 - self.stat_hit = 0 - self.stat_err = 0 def __call__(self, method): self.method = method @@ -46,11 +58,6 @@ class ormcache(object): lookup.clear_cache = self.clear return lookup - def stat(self): - return "lookup-stats %6d hit %6d miss %6d err %4.1f ratio" % \ - (self.stat_hit, self.stat_miss, self.stat_err, - (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit or 1)) - def lru(self, model): return model.pool.cache, (model.pool.db_name, model._name, self.method) @@ -59,14 +66,14 @@ class ormcache(object): key = key0 + args[self.skiparg:] try: r = d[key] - self.stat_hit += 1 + STAT[key0].hit += 1 return r except KeyError: - self.stat_miss += 1 + STAT[key0].miss += 1 value = d[key] = self.method(*args, **kwargs) return value except TypeError: - self.stat_err += 1 + STAT[key0].err += 1 return self.method(*args, **kwargs) def clear(self, model, *args): @@ -107,14 +114,14 @@ class ormcache_context(ormcache): key = key0 + args[self.skiparg:self.context_pos] + tuple(ckey) try: r = d[key] - self.stat_hit += 1 + STAT[key0].hit += 1 return r except KeyError: - self.stat_miss += 1 + STAT[key0].miss += 1 value = d[key] = self.method(*args, **kwargs) return value except TypeError: - self.stat_err += 1 + STAT[key0].err += 1 return self.method(*args, **kwargs) @@ -136,9 +143,9 @@ class ormcache_multi(ormcache): key = base_key + (i,) try: result[i] = d[key] - self.stat_hit += 1 + STAT[key0].hit += 1 except Exception: - self.stat_miss += 1 + STAT[key0].miss += 1 missed.append(i) if missed: @@ -171,20 +178,18 @@ class dummy_cache(object): def log_ormcache_stats(sig=None, frame=None): """ Log statistics of ormcache usage by database, model, and method. """ from openerp.modules.registry import RegistryManager - from collections import defaultdict import threading me = threading.currentThread() entries = defaultdict(int) for key in RegistryManager.cache.iterkeys(): entries[key[:3]] += 1 - for (dbname, model_name, method), count in sorted(entries.items()): + for key, count in sorted(entries.items()): + dbname, model_name, method = key me.dbname = dbname - model = RegistryManager.get(dbname)[model_name] - func = getattr(model, method.__name__).im_func - ormcache = func.clear_cache.im_self - _logger.info("%6d entries, %s, for %s.%s", - count, ormcache.stat(), model_name, method.__name__) + stat = STAT[key] + _logger.info("%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s", + count, stat.hit, stat.miss, stat.err, stat.ratio, model_name, method.__name__) # For backward compatibility