[MERGE] Registry loading speedup by thu

All calls to ir_model_fields to fetch manual (aka custom or x_) fields are done
in a single call (prior to instanciate all models).

Checks for empty module descriptions are not done unless we are in init or
update mode. (The behavior was the opposite, which was probably a mistake).

Some calls to ir_translation, passing en_US because there was no lang in the
context, are not done anymore.

bzr revid: al@openerp.com-20121216030722-thxk3ddff8lkw05y

openerp/cli/server.py

@@ -94,10 +94,8 @@ def setup_pid_file():
 def preload_registry(dbname):
     """ Preload a registry, and start the cron."""
     try:
-        db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=openerp.tools.config['init'] or openerp.tools.config['update'], pooljobs=False)
-
-        # jobs will start to be processed later, when openerp.cron.start_master_thread() is called by openerp.service.start_services()
-        registry.schedule_cron_jobs()
+        update_module = True if openerp.tools.config['init'] or openerp.tools.config['update'] else False
+        db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=update_module,  pooljobs=False)
     except Exception:
         _logger.exception('Failed to initialize database `%s`.', dbname)
 

openerp/modules/loading.py

@@ -146,6 +146,14 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
     cr.execute("select (now() at time zone 'UTC')::timestamp")
     dt_before_load = cr.fetchone()[0]
 
+    # Query manual fields for all models at once and save them on the registry
+    # so the initialization code for each model does not have to do it
+    # one model at a time.
+    pool.fields_by_model = {}
+    cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
+    for field in cr.dictfetchall():
+        pool.fields_by_model.setdefault(field['model'], []).append(field)
+
     # register, instantiate and initialize models for each modules
     for index, package in enumerate(graph):
         module_name = package.name
@@ -159,6 +167,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
         load_openerp_module(package.name)
 
         models = pool.load(cr, package)
+
         loaded_modules.append(package.name)
         if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
             init_module_models(cr, package.name, models)
@@ -220,6 +229,10 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
                     delattr(package, kind)
 
         cr.commit()
+
+    # The query won't be valid for models created later (i.e. custom model
+    # created after the registry has been loaded), so empty its result.
+    pool.fields_by_model = None
     
     cr.commit()
 
@@ -239,7 +252,7 @@ def _check_module_names(cr, module_names):
             incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
             _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
 
-def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
+def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks):
     """Loads modules marked with ``states``, adding them to ``graph`` and
        ``loaded_modules`` and returns a list of installed/upgraded modules."""
     processed_modules = []
@@ -248,7 +261,7 @@ def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_m
         module_list = [name for (name,) in cr.fetchall() if name not in graph]
         graph.add_modules(cr, module_list, force)
         _logger.debug('Updating graph with %d more modules', len(module_list))
-        loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
+        loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks)
         processed_modules.extend(processed)
         loaded_modules.extend(loaded)
         if not processed: break
@@ -293,7 +306,7 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
         # processed_modules: for cleanup step after install
         # loaded_modules: to avoid double loading
         report = pool._assertion_report
-        loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)
+        loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
 
         if tools.config['load_language']:
             for lang in tools.config['load_language'].split(','):
@@ -333,11 +346,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
         #            be dropped in STEP 6 later, before restarting the loading
         #            process.
         states_to_load = ['installed', 'to upgrade', 'to remove']
-        processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
+        processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
         processed_modules.extend(processed)
         if update_module:
             states_to_load = ['to install']
-            processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
+            processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
             processed_modules.extend(processed)
 
         # load custom models

openerp/modules/registry.py

@@ -50,6 +50,7 @@ class Registry(object):
         self._init = True
         self._init_parent = {}
         self._assertion_report = assertion_report.assertion_report()
+        self.fields_by_model = None
 
         # modules fully loaded (maintained during init phase by `loading` module)
         self._init_modules = set()

openerp/osv/orm.py

@@ -1014,45 +1014,50 @@ class BaseModel(object):
 
         # Load manual fields
 
-        cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields'))
-        if cr.fetchone():
+        # Check the query is already done for all modules of if we need to
+        # do it ourselves.
+        if self.pool.fields_by_model is not None:
+            manual_fields = self.pool.fields_by_model.get(self._name, [])
+        else:
             cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual'))
-            for field in cr.dictfetchall():
-                if field['name'] in self._columns:
-                    continue
-                attrs = {
-                    'string': field['field_description'],
-                    'required': bool(field['required']),
-                    'readonly': bool(field['readonly']),
-                    'domain': eval(field['domain']) if field['domain'] else None,
-                    'size': field['size'],
-                    'ondelete': field['on_delete'],
-                    'translate': (field['translate']),
-                    'manual': True,
-                    #'select': int(field['select_level'])
-                }
+            manual_fields = cr.dictfetchall()
+        for field in manual_fields:
+            if field['name'] in self._columns:
+                continue
+            attrs = {
+                'string': field['field_description'],
+                'required': bool(field['required']),
+                'readonly': bool(field['readonly']),
+                'domain': eval(field['domain']) if field['domain'] else None,
+                'size': field['size'],
+                'ondelete': field['on_delete'],
+                'translate': (field['translate']),
+                'manual': True,
+                #'select': int(field['select_level'])
+            }
+
+            if field['serialization_field_id']:
+                cr.execute('SELECT name FROM ir_model_fields WHERE id=%s', (field['serialization_field_id'],))
+                attrs.update({'serialization_field': cr.fetchone()[0], 'type': field['ttype']})
+                if field['ttype'] in ['many2one', 'one2many', 'many2many']:
+                    attrs.update({'relation': field['relation']})
+                self._columns[field['name']] = fields.sparse(**attrs)
+            elif field['ttype'] == 'selection':
+                self._columns[field['name']] = fields.selection(eval(field['selection']), **attrs)
+            elif field['ttype'] == 'reference':
+                self._columns[field['name']] = fields.reference(selection=eval(field['selection']), **attrs)
+            elif field['ttype'] == 'many2one':
+                self._columns[field['name']] = fields.many2one(field['relation'], **attrs)
+            elif field['ttype'] == 'one2many':
+                self._columns[field['name']] = fields.one2many(field['relation'], field['relation_field'], **attrs)
+            elif field['ttype'] == 'many2many':
+                _rel1 = field['relation'].replace('.', '_')
+                _rel2 = field['model'].replace('.', '_')
+                _rel_name = 'x_%s_%s_%s_rel' % (_rel1, _rel2, field['name'])
+                self._columns[field['name']] = fields.many2many(field['relation'], _rel_name, 'id1', 'id2', **attrs)
+            else:
+                self._columns[field['name']] = getattr(fields, field['ttype'])(**attrs)
 
-                if field['serialization_field_id']:
-                    cr.execute('SELECT name FROM ir_model_fields WHERE id=%s', (field['serialization_field_id'],))
-                    attrs.update({'serialization_field': cr.fetchone()[0], 'type': field['ttype']})
-                    if field['ttype'] in ['many2one', 'one2many', 'many2many']:
-                        attrs.update({'relation': field['relation']})
-                    self._columns[field['name']] = fields.sparse(**attrs)
-                elif field['ttype'] == 'selection':
-                    self._columns[field['name']] = fields.selection(eval(field['selection']), **attrs)
-                elif field['ttype'] == 'reference':
-                    self._columns[field['name']] = fields.reference(selection=eval(field['selection']), **attrs)
-                elif field['ttype'] == 'many2one':
-                    self._columns[field['name']] = fields.many2one(field['relation'], **attrs)
-                elif field['ttype'] == 'one2many':
-                    self._columns[field['name']] = fields.one2many(field['relation'], field['relation_field'], **attrs)
-                elif field['ttype'] == 'many2many':
-                    _rel1 = field['relation'].replace('.', '_')
-                    _rel2 = field['model'].replace('.', '_')
-                    _rel_name = 'x_%s_%s_%s_rel' % (_rel1, _rel2, field['name'])
-                    self._columns[field['name']] = fields.many2many(field['relation'], _rel_name, 'id1', 'id2', **attrs)
-                else:
-                    self._columns[field['name']] = getattr(fields, field['ttype'])(**attrs)
         self._inherits_check()
         self._inherits_reload()
         if not self._sequence:
@@ -3624,15 +3629,16 @@ class BaseModel(object):
         else:
             res = map(lambda x: {'id': x}, ids)
 
-        for f in fields_pre:
-            if f == self.CONCURRENCY_CHECK_FIELD:
-                continue
-            if self._columns[f].translate:
-                ids = [x['id'] for x in res]
-                #TODO: optimize out of this loop
-                res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context.get('lang', False) or 'en_US', ids)
-                for r in res:
-                    r[f] = res_trans.get(r['id'], False) or r[f]
+        if context.get('lang'):
+            for f in fields_pre:
+                if f == self.CONCURRENCY_CHECK_FIELD:
+                    continue
+                if self._columns[f].translate:
+                    ids = [x['id'] for x in res]
+                    #TODO: optimize out of this loop
+                    res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context['lang'], ids)
+                    for r in res:
+                        r[f] = res_trans.get(r['id'], False) or r[f]
 
         for table in self._inherits:
             col = self._inherits[table]