From f668a123d94e3e27ea7c1e7343ab8fafd5ceb69f Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Tue, 11 Dec 2012 11:59:54 +0100 Subject: [PATCH 1/4] [IMP] Reduce considerably the loading time of a new registry. Loading time was mesured on the loading of a second database (identical to the first one), i.e. by passing -d xx,yy on the command line, using cProfile. The databases were installed with sale, mrp, and the crm. The cProfile code is commited as part of this patch and should be removed (or maybe guarded by some command-line flag) (just as the commenting-out of the cron startup). The patch was also applied on top of the trunk-simple-table-stats-vmt branch which provides SQL queries counters. Results indicate that the number of SQL queries are reduced from about 2100 to 27. Loading time is reduced from 1.3s to 0.26s (i.e. improved by 5). Changes: All calls to ir_model_fields to fetch manual (custom) fields are done in a single call (prior to instanciate all models). Checks for empty module descriptions are not done unless we are in init or update mode. (The behavior was the opposite, which was probably a mistake). Some calls to ir_translation, passing en_US because there was no lang in the context, are not done anymore. The improved time is also a result of a change in the decimal_precision module where precision_get fetches all digits/applications instead of one at a time (and thus implements its own caching instead of relying on openerp.tools.ormache). bzr revid: vmt@openerp.com-20121211105954-lwgs5js7yw3tzghs --- openerp/cli/server.py | 10 +++++++--- openerp/modules/loading.py | 19 ++++++++++++++----- openerp/osv/orm.py | 31 ++++++++++++++++++------------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/openerp/cli/server.py b/openerp/cli/server.py index f030f801967..67ce518155c 100644 --- a/openerp/cli/server.py +++ b/openerp/cli/server.py @@ -94,10 +94,11 @@ def setup_pid_file(): def preload_registry(dbname): """ Preload a registry, and start the cron.""" try: - db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=openerp.tools.config['init'] or openerp.tools.config['update'], pooljobs=False) + update_module = True if openerp.tools.config['init'] or openerp.tools.config['update'] else False + db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=update_module, pooljobs=False) # jobs will start to be processed later, when openerp.cron.start_master_thread() is called by openerp.service.start_services() - registry.schedule_cron_jobs() + #registry.schedule_cron_jobs() except Exception: _logger.exception('Failed to initialize database `%s`.', dbname) @@ -258,9 +259,12 @@ def main(args): else: openerp.service.start_services() + import cProfile if config['db_name']: for dbname in config['db_name'].split(','): - preload_registry(dbname) + prof = cProfile.Profile() + prof.runcall(preload_registry, dbname) + prof.dump_stats('preload_registry_%s.profile' % dbname) if config["stop_after_init"]: sys.exit(0) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index e7a6c50286a..48f816035ff 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -146,6 +146,11 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= cr.execute("select (now() at time zone 'UTC')::timestamp") dt_before_load = cr.fetchone()[0] + pool.fields_by_model = {'whatever': {}} + cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',)) + for field in cr.dictfetchall(): + pool.fields_by_model.setdefault(field['model'], []).append(field) + # register, instantiate and initialize models for each modules for index, package in enumerate(graph): module_name = package.name @@ -159,6 +164,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= load_openerp_module(package.name) models = pool.load(cr, package) + loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) @@ -220,6 +226,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= delattr(package, kind) cr.commit() + + pool.fields_by_model = {} cr.commit() @@ -239,7 +247,7 @@ def _check_module_names(cr, module_names): incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) -def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules): +def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks): """Loads modules marked with ``states``, adding them to ``graph`` and ``loaded_modules`` and returns a list of installed/upgraded modules.""" processed_modules = [] @@ -248,7 +256,7 @@ def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_m module_list = [name for (name,) in cr.fetchall() if name not in graph] graph.add_modules(cr, module_list, force) _logger.debug('Updating graph with %d more modules', len(module_list)) - loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules) + loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks) processed_modules.extend(processed) loaded_modules.extend(loaded) if not processed: break @@ -293,7 +301,8 @@ def load_modules(db, force_demo=False, status=None, update_module=False): # processed_modules: for cleanup step after install # loaded_modules: to avoid double loading report = pool._assertion_report - loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report) + print update_module + loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report) if tools.config['load_language']: for lang in tools.config['load_language'].split(','): @@ -333,11 +342,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False): # be dropped in STEP 6 later, before restarting the loading # process. states_to_load = ['installed', 'to upgrade', 'to remove'] - processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules) + processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module) processed_modules.extend(processed) if update_module: states_to_load = ['to install'] - processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules) + processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module) processed_modules.extend(processed) # load custom models diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 060d6da0277..4ba3d24b4c4 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -1018,10 +1018,14 @@ class BaseModel(object): # Load manual fields - cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields')) - if cr.fetchone(): - cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual')) - for field in cr.dictfetchall(): + #cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields')) + if True: #cr.fetchone(): + if self.pool.fields_by_model: + fields__ = self.pool.fields_by_model.get(self._name, []) + else: + cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual')) + fields__ = cr.dictfetchall() + for field in fields__: if field['name'] in self._columns: continue attrs = { @@ -3628,15 +3632,16 @@ class BaseModel(object): else: res = map(lambda x: {'id': x}, ids) - for f in fields_pre: - if f == self.CONCURRENCY_CHECK_FIELD: - continue - if self._columns[f].translate: - ids = [x['id'] for x in res] - #TODO: optimize out of this loop - res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context.get('lang', False) or 'en_US', ids) - for r in res: - r[f] = res_trans.get(r['id'], False) or r[f] + if context.get('lang'): + for f in fields_pre: + if f == self.CONCURRENCY_CHECK_FIELD: + continue + if self._columns[f].translate: + ids = [x['id'] for x in res] + #TODO: optimize out of this loop + res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context['lang'], ids) + for r in res: + r[f] = res_trans.get(r['id'], False) or r[f] for table in self._inherits: col = self._inherits[table] From 3667e3c6190d8a22e358bfd2eac48e0cdca9a8a3 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 14 Dec 2012 11:58:20 +0100 Subject: [PATCH 2/4] [IMP] module loading: removed unnecessary indentation, added comments. bzr revid: vmt@openerp.com-20121214105820-9nlgzu9pm7cvh1pz --- openerp/modules/loading.py | 7 +++- openerp/osv/orm.py | 85 +++++++++++++++++++------------------- 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index 48f816035ff..b4ad08af7a1 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -146,7 +146,10 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= cr.execute("select (now() at time zone 'UTC')::timestamp") dt_before_load = cr.fetchone()[0] - pool.fields_by_model = {'whatever': {}} + # Query manual fields for all models at once and save them on the registry + # so the initialization code for each model does not have to do it + # one model at a time. + pool.fields_by_model = {'so_this_dict_is_not_empty': {}} cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',)) for field in cr.dictfetchall(): pool.fields_by_model.setdefault(field['model'], []).append(field) @@ -227,6 +230,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= cr.commit() + # The query won't be valid for models created later (i.e. custom model + # created after the registry has been loaded), so empty its result. pool.fields_by_model = {} cr.commit() diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 4ba3d24b4c4..7d89139b2d5 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -1018,49 +1018,50 @@ class BaseModel(object): # Load manual fields - #cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields')) - if True: #cr.fetchone(): - if self.pool.fields_by_model: - fields__ = self.pool.fields_by_model.get(self._name, []) - else: - cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual')) - fields__ = cr.dictfetchall() - for field in fields__: - if field['name'] in self._columns: - continue - attrs = { - 'string': field['field_description'], - 'required': bool(field['required']), - 'readonly': bool(field['readonly']), - 'domain': eval(field['domain']) if field['domain'] else None, - 'size': field['size'], - 'ondelete': field['on_delete'], - 'translate': (field['translate']), - 'manual': True, - #'select': int(field['select_level']) - } + # Check the query is already done for all modules of if we need to + # do it ourselves. + if self.pool.fields_by_model: + manual_fields = self.pool.fields_by_model.get(self._name, []) + else: + cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual')) + manual_fields = cr.dictfetchall() + for field in manual_fields: + if field['name'] in self._columns: + continue + attrs = { + 'string': field['field_description'], + 'required': bool(field['required']), + 'readonly': bool(field['readonly']), + 'domain': eval(field['domain']) if field['domain'] else None, + 'size': field['size'], + 'ondelete': field['on_delete'], + 'translate': (field['translate']), + 'manual': True, + #'select': int(field['select_level']) + } + + if field['serialization_field_id']: + cr.execute('SELECT name FROM ir_model_fields WHERE id=%s', (field['serialization_field_id'],)) + attrs.update({'serialization_field': cr.fetchone()[0], 'type': field['ttype']}) + if field['ttype'] in ['many2one', 'one2many', 'many2many']: + attrs.update({'relation': field['relation']}) + self._columns[field['name']] = fields.sparse(**attrs) + elif field['ttype'] == 'selection': + self._columns[field['name']] = fields.selection(eval(field['selection']), **attrs) + elif field['ttype'] == 'reference': + self._columns[field['name']] = fields.reference(selection=eval(field['selection']), **attrs) + elif field['ttype'] == 'many2one': + self._columns[field['name']] = fields.many2one(field['relation'], **attrs) + elif field['ttype'] == 'one2many': + self._columns[field['name']] = fields.one2many(field['relation'], field['relation_field'], **attrs) + elif field['ttype'] == 'many2many': + _rel1 = field['relation'].replace('.', '_') + _rel2 = field['model'].replace('.', '_') + _rel_name = 'x_%s_%s_%s_rel' % (_rel1, _rel2, field['name']) + self._columns[field['name']] = fields.many2many(field['relation'], _rel_name, 'id1', 'id2', **attrs) + else: + self._columns[field['name']] = getattr(fields, field['ttype'])(**attrs) - if field['serialization_field_id']: - cr.execute('SELECT name FROM ir_model_fields WHERE id=%s', (field['serialization_field_id'],)) - attrs.update({'serialization_field': cr.fetchone()[0], 'type': field['ttype']}) - if field['ttype'] in ['many2one', 'one2many', 'many2many']: - attrs.update({'relation': field['relation']}) - self._columns[field['name']] = fields.sparse(**attrs) - elif field['ttype'] == 'selection': - self._columns[field['name']] = fields.selection(eval(field['selection']), **attrs) - elif field['ttype'] == 'reference': - self._columns[field['name']] = fields.reference(selection=eval(field['selection']), **attrs) - elif field['ttype'] == 'many2one': - self._columns[field['name']] = fields.many2one(field['relation'], **attrs) - elif field['ttype'] == 'one2many': - self._columns[field['name']] = fields.one2many(field['relation'], field['relation_field'], **attrs) - elif field['ttype'] == 'many2many': - _rel1 = field['relation'].replace('.', '_') - _rel2 = field['model'].replace('.', '_') - _rel_name = 'x_%s_%s_%s_rel' % (_rel1, _rel2, field['name']) - self._columns[field['name']] = fields.many2many(field['relation'], _rel_name, 'id1', 'id2', **attrs) - else: - self._columns[field['name']] = getattr(fields, field['ttype'])(**attrs) self._inherits_check() self._inherits_reload() if not self._sequence: From 08a082f63fb9b2d86ae62932d83ca535d1684b59 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 14 Dec 2012 15:11:14 +0100 Subject: [PATCH 3/4] [FIX] registry: Set the fields_by_model attribute in __init__(), use None to flag non-existing fields-per-model cache. bzr revid: vmt@openerp.com-20121214141114-em9r66e3sfy21t2r --- openerp/modules/loading.py | 4 ++-- openerp/modules/registry.py | 1 + openerp/osv/orm.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index b4ad08af7a1..935ea8a2436 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -149,7 +149,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= # Query manual fields for all models at once and save them on the registry # so the initialization code for each model does not have to do it # one model at a time. - pool.fields_by_model = {'so_this_dict_is_not_empty': {}} + pool.fields_by_model = {} cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',)) for field in cr.dictfetchall(): pool.fields_by_model.setdefault(field['model'], []).append(field) @@ -232,7 +232,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= # The query won't be valid for models created later (i.e. custom model # created after the registry has been loaded), so empty its result. - pool.fields_by_model = {} + pool.fields_by_model = None cr.commit() diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index d898cc7e8db..afa8351bce9 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -50,6 +50,7 @@ class Registry(object): self._init = True self._init_parent = {} self._assertion_report = assertion_report.assertion_report() + self.fields_by_model = None # modules fully loaded (maintained during init phase by `loading` module) self._init_modules = set() diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 7d89139b2d5..69aee5adcdd 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -1020,7 +1020,7 @@ class BaseModel(object): # Check the query is already done for all modules of if we need to # do it ourselves. - if self.pool.fields_by_model: + if self.pool.fields_by_model is not None: manual_fields = self.pool.fields_by_model.get(self._name, []) else: cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual')) From 14ad37779c39595f551bfbf2c78055d2e64250f0 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Sun, 16 Dec 2012 02:52:14 +0100 Subject: [PATCH 4/4] cleanups bzr revid: al@openerp.com-20121216015214-kfo6zlarh4gdccrx --- openerp/cli/server.py | 8 +------- openerp/modules/loading.py | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/openerp/cli/server.py b/openerp/cli/server.py index 67ce518155c..d97d4c81571 100644 --- a/openerp/cli/server.py +++ b/openerp/cli/server.py @@ -96,9 +96,6 @@ def preload_registry(dbname): try: update_module = True if openerp.tools.config['init'] or openerp.tools.config['update'] else False db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=update_module, pooljobs=False) - - # jobs will start to be processed later, when openerp.cron.start_master_thread() is called by openerp.service.start_services() - #registry.schedule_cron_jobs() except Exception: _logger.exception('Failed to initialize database `%s`.', dbname) @@ -259,12 +256,9 @@ def main(args): else: openerp.service.start_services() - import cProfile if config['db_name']: for dbname in config['db_name'].split(','): - prof = cProfile.Profile() - prof.runcall(preload_registry, dbname) - prof.dump_stats('preload_registry_%s.profile' % dbname) + preload_registry(dbname) if config["stop_after_init"]: sys.exit(0) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index 935ea8a2436..44486fe52dc 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -306,7 +306,6 @@ def load_modules(db, force_demo=False, status=None, update_module=False): # processed_modules: for cleanup step after install # loaded_modules: to avoid double loading report = pool._assertion_report - print update_module loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report) if tools.config['load_language']: