[IMP] base, tools, web_edeitor: improve ir_translation insertion

The ir_translations were pretty slow to load, this was mainly caused by the condition in variable 'find_expr' and the import of po values in the temporary table. To improve the load performance, we made two improvement: 1. Insert the row in the temporary table by batch instead of making an insertion for each record. 2. Replacing the condition in the variable 'find_expr' by partial unique index. To perform such a thing, we had to separate the type model in two types, 'model' and 'model_terms' because it wasn't possible to create a partial unique index on type 'model' because the type was used for two different use cases. The type model is now used for fields that have have a value not callable for the attribute translate, these field can only have one record for the same field, model, res_id and language. And the type 'model_terms' is used for the fields callable, because they can have multiple sources for the same res_id. We also removed the deprecated types 'report', 'help', 'view', 'field'. Thanks to @rco-odoo for the improvement of the first point.
2018-09-18 14:30:03 +02:00
parent 83e460b8da
commit 7288b4779a
5 changed files with 93 additions and 78 deletions
@@ -27,7 +27,7 @@ class IrTranslation(models.Model):
        """ Convert the HTML fragment ``value`` to XML if necessary, and write
        it as the value of translation ``self``.
        """
-        assert len(self) == 1 and self.type == 'model'
+        assert len(self) == 1 and self.type == 'model_terms'
        mname, fname = self.name.split(',')
        field = self.env[mname]._fields[fname]
        if field.translate == xml_translate:
@@ -13,12 +13,9 @@ from odoo.tools import pycompat
 _logger = logging.getLogger(__name__)

 TRANSLATION_TYPE = [
-    ('field', 'Field'),                         # deprecated
-    ('model', 'Object'),
-    ('report', 'Report/Template'),
+    ('model', 'Model Field'),
+    ('model_terms', 'Structured Model Field'),
    ('selection', 'Selection'),
-    ('view', 'View'),                           # deprecated
-    ('help', 'Help'),                           # deprecated
    ('code', 'Code'),
    ('constraint', 'Constraint'),
    ('sql_constraint', 'SQL Constraint')
@@ -43,6 +40,7 @@ class IrTranslationImport(object):
        self._model_table = model._table
        self._overwrite = model._context.get('overwrite', False)
        self._debug = False
+        self._rows = []

        # Note that Postgres will NOT inherit the constraints or indexes
        # of ir_translation, so this copy will be much faster.
@@ -88,18 +86,23 @@ class IrTranslationImport(object):
            params['name'] = 'ir.ui.view,arch_db'
            params['imd_model'] = "ir.ui.view"

-        query = """ INSERT INTO %s (name, lang, res_id, src, type, imd_model, module, imd_name, value, state, comments)
-                    VALUES (%%(name)s, %%(lang)s, %%(res_id)s, %%(src)s, %%(type)s, %%(imd_model)s, %%(module)s,
-                            %%(imd_name)s, %%(value)s, %%(state)s, %%(comments)s) """ % self._table
-        self._cr.execute(query, params)
+        self._rows.append((params['name'], params['lang'], params['res_id'],
+                           params['src'], params['type'], params['imd_model'],
+                           params['module'], params['imd_name'], params['value'],
+                           params['state'], params['comments']))

    def finish(self):
        """ Transfer the data from the temp table to ir.translation """
        cr = self._cr
-        if self._debug:
-            cr.execute("SELECT count(*) FROM %s" % self._table)
-            count = cr.fetchone()[0]
-            _logger.debug("ir.translation.cursor: We have %d entries to process", count)
+
+        # Step 0: insert rows in batch
+        query = """ INSERT INTO %s (name, lang, res_id, src, type, imd_model,
+                                    module, imd_name, value, state, comments)
+                    VALUES """ % self._table
+        for rows in cr.split_for_in_conditions(self._rows):
+            cr.execute(query + ", ".join(["%s"] * len(rows)), rows)
+
+        _logger.debug("ir.translation.cursor: We have %d entries to process", len(self._rows))

        # Step 1: resolve ir.model.data references to res_ids
        cr.execute(""" UPDATE %s AS ti
@@ -129,54 +132,56 @@ class IrTranslationImport(object):
                if hasattr(field, 'translate') and callable(field.translate):
                    src_relevant_fields.append("%s,%s" % (model, field_name))

-        find_expr = """
-                irt.lang = ti.lang
-            AND irt.type = ti.type
-            AND irt.name = ti.name
-            AND (
-                    (ti.type = 'model' AND ti.res_id = irt.res_id AND ti.name IN %s AND irt.src = ti.src)
-                 OR (ti.type = 'model' AND ti.res_id = irt.res_id AND ti.name NOT IN %s)
-                 OR (ti.type = 'view' AND (irt.res_id IS NULL OR ti.res_id = irt.res_id) AND irt.src = ti.src)
-                 OR (ti.type = 'field')
-                 OR (ti.type = 'help')
-                 OR (ti.type NOT IN ('model', 'view', 'field', 'help') AND irt.src = ti.src)
-            )
-        """
-
-        # Step 2: update existing (matching) translations
+        # Step 2: insert new or upsert translations
        if self._overwrite:
-            cr.execute(""" UPDATE ONLY %s AS irt
-                           SET value = ti.value,
-                               src = ti.src,
-                               state = 'translated'
-                           FROM %s AS ti
-                          WHERE %s
-                            AND ti.value IS NOT NULL
-                            AND ti.value != ''
-                            AND noupdate IS NOT TRUE
-                       """ % (self._model_table, self._table, find_expr),
-                       (tuple(src_relevant_fields), tuple(src_relevant_fields)))
-
-        # Step 3: insert new translations
-        cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
-                       SELECT name, lang, res_id, src, type, value, module, state, comments
-                       FROM %s AS ti
-                       WHERE NOT EXISTS(SELECT 1 FROM ONLY %s AS irt WHERE %s)
-                       ON CONFLICT DO NOTHING;
-                   """ % (self._model_table, self._table, self._model_table, find_expr),
-                   (tuple(src_relevant_fields), tuple(src_relevant_fields)))
+            cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
+                           SELECT name, lang, res_id, src, type, value, module, state, comments
+                           FROM %s
+                           WHERE type = 'code'
+                           ON CONFLICT (type, lang, md5(src)) WHERE type = 'code'
+                            DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
+                       """ % (self._model_table, self._table))
+            count = cr.rowcount
+            cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
+                           SELECT name, lang, res_id, src, type, value, module, state, comments
+                           FROM %s
+                           WHERE type = 'model'
+                           ON CONFLICT (type, lang, name, res_id) WHERE type = 'model'
+                            DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
+                       """ % (self._model_table, self._table))
+            count += cr.rowcount
+            cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
+                           SELECT name, lang, res_id, src, type, value, module, state, comments
+                           FROM %s
+                           WHERE type IN ('selection', 'constraint', 'sql_constraint')
+                           ON CONFLICT (type, lang, name, md5(src)) WHERE type IN ('selection', 'constraint', 'sql_constraint')
+                            DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
+                       """ % (self._model_table, self._table))
+            count += cr.rowcount
+            cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
+                           SELECT name, lang, res_id, src, type, value, module, state, comments
+                           FROM %s
+                           WHERE type = 'model_terms'
+                           ON CONFLICT (type, name, lang, res_id, md5(src))
+                            DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
+                       """ % (self._model_table, self._table))
+            count += cr.rowcount
+        else:
+            cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
+                           SELECT name, lang, res_id, src, type, value, module, state, comments
+                           FROM %s
+                           ON CONFLICT DO NOTHING;
+                       """ % (self._model_table, self._table))
+            count = cr.rowcount

        if self._debug:
            cr.execute("SELECT COUNT(*) FROM ONLY %s" % self._model_table)
            total = cr.fetchone()[0]
-            cr.execute("SELECT COUNT(*) FROM ONLY %s AS irt, %s AS ti WHERE %s" % \
-                       (self._model_table, self._table, find_expr),
-                       (tuple(src_relevant_fields), tuple(src_relevant_fields)))
-            count = cr.fetchone()[0]
            _logger.debug("ir.translation.cursor: %d entries now in ir.translation, %d common entries with tmp", total, count)

-        # Step 4: cleanup
+        # Step 3: cleanup
        cr.execute("DROP TABLE %s" % self._table)
+        self._rows.clear()
        return True


@@ -262,8 +267,15 @@ class IrTranslation(models.Model):
        res = super(IrTranslation, self)._auto_init()
        # Add separate md5 index on src (no size limit on values, and good performance).
        tools.create_index(self._cr, 'ir_translation_src_md5', self._table, ['md5(src)'])
+        # Cover 'model_terms' type
        tools.create_unique_index(self._cr, 'ir_translation_unique', self._table,
                                  ['type', 'name', 'lang', 'res_id', 'md5(src)'])
+        if not tools.index_exists(self._cr, 'ir_translation_code_unique'):
+            self._cr.execute("CREATE UNIQUE INDEX ir_translation_code_unique ON ir_translation (type, lang, md5(src)) WHERE type = 'code'")
+        if not tools.index_exists(self._cr, 'ir_translation_model_unique'):
+            self._cr.execute("CREATE UNIQUE INDEX ir_translation_model_unique ON ir_translation (type, lang, name, res_id) WHERE type = 'model'")
+        if not tools.index_exists(self._cr, 'ir_translation_selection_unique'):
+            self._cr.execute("CREATE UNIQUE INDEX ir_translation_selection_unique ON ir_translation (type, lang, name, md5(src)) WHERE type IN ('selection', 'constraint', 'sql_constraint')")
        return res

    @api.model
@@ -403,7 +415,7 @@ class IrTranslation(models.Model):
        query = """ SELECT * FROM ir_translation
                    WHERE lang=%s AND type=%s AND name=%s AND res_id IN %s """
        name = "%s,%s" % (field.model_name, field.name)
-        params = (records.env.lang, 'model', name, tuple(records.ids))
+        params = (records.env.lang, 'model_terms', name, tuple(records.ids))
        return query, params

    @api.model
@@ -448,7 +460,7 @@ class IrTranslation(models.Model):
            value = record[field.name]
            terms = set(field.get_trans_terms(value))
            record_trans = trans.search([
-                ('type', '=', 'model'),
+                ('type', '=', 'model_terms'),
                ('name', '=', "%s,%s" % (field.model_name, field.name)),
                ('res_id', '=', record.id),
            ])
@@ -591,12 +603,13 @@ class IrTranslation(models.Model):
        if callable(field.translate):
            # insert missing translations for each term in src
            query = """ INSERT INTO ir_translation (lang, type, name, res_id, src, value, module)
-                        SELECT l.code, 'model', %(name)s, %(res_id)s, %(src)s, %(src)s, %(module)s
+                        SELECT l.code, 'model_terms', %(name)s, %(res_id)s, %(src)s, %(src)s, %(module)s
                        FROM res_lang l
                        WHERE l.active AND l.translatable AND NOT EXISTS (
                            SELECT 1 FROM ir_translation
                            WHERE lang=l.code AND type='model' AND name=%(name)s AND res_id=%(res_id)s AND src=%(src)s
-                        );
+                        )
+                        ON CONFLICT DO NOTHING;
                    """
            for record in records:
                module = external_ids[record.id].split('.')[0]
@@ -4,8 +4,10 @@
 import unittest

 from odoo.tools import pycompat
+from odoo.tools import mute_logger
 from odoo.tools.translate import quote, unquote, xml_translate, html_translate
 from odoo.tests.common import TransactionCase, tagged
+from psycopg2 import IntegrityError


@tagged('standard', 'at_install')
@@ -286,21 +288,23 @@ class TestTranslation(TransactionCase):
        })
        # change name and insert a duplicate manually
        padawans.write({'name': 'Padawans'})
-        self.env['ir.translation'].create({
-            'type': 'model',
-            'name': 'res.partner.category,name',
-            'module':'base',
-            'lang': 'fr_FR',
-            'res_id': padawans.id,
-            'value': 'Apprentis',
-            'state': 'translated',
-        })
+        with self.assertRaises(IntegrityError), mute_logger('odoo.sql_db'):
+            with self.env.cr.savepoint():
+                self.env['ir.translation'].create({
+                    'type': 'model',
+                    'name': 'res.partner.category,name',
+                    'module':'base',
+                    'lang': 'fr_FR',
+                    'res_id': padawans.id,
+                    'value': 'Apprentis',
+                    'state': 'translated',
+                })
        self.env['ir.translation'].translate_fields('res.partner.category', padawans.id, 'name')
        translations = self.env['ir.translation'].search([
            ('res_id', '=', padawans.id), ('name', '=', 'res.partner.category,name')
        ])
-        self.assertEqual(len(translations), 1, "Translations were not merged after `translate_fields` call")
-        self.assertEqual(translations.value, "Apprentis", "The most recent translation must stay")
+        self.assertEqual(len(translations), 1, "Translations were not duplicated after `translate_fields` call")
+        self.assertEqual(translations.value, "Apprenti", "The first translation must stay")


 class TestXMLTranslation(TransactionCase):
@@ -323,7 +327,7 @@ class TestXMLTranslation(TransactionCase):
        })
        for src, value in list(pycompat.izip(terms_en, terms_fr)):
            self.env['ir.translation'].create({
-                'type': 'model',
+                'type': 'model_terms',
                'name': 'ir.ui.view,arch_db',
                'lang': 'fr_FR',
                'res_id': view0.id,
@@ -363,7 +367,7 @@ class TestXMLTranslation(TransactionCase):
        })
        for src, value in list(pycompat.izip(terms_en, terms_fr)):
            self.env['ir.translation'].create({
-                'type': 'model',
+                'type': 'model_terms',
                'name': 'ir.ui.view,arch_db',
                'lang': 'fr_FR',
                'res_id': view0.id,
@@ -618,7 +618,7 @@ class TestNoModel(ViewCase):
            'type': 'qweb',
        })
        self.env['ir.translation'].create({
-            'type': 'model',
+            'type': 'model_terms',
            'name': 'ir.ui.view,arch_db',
            'res_id': view.id,
            'lang': 'fr_FR',
@@ -830,7 +830,7 @@ def trans_generate(lang, modules, cr):
        tnx = (module, source, name, id, type, tuple(comments or ()))
        to_translate.add(tnx)

-    query = 'SELECT name, model, res_id, module FROM ir_model_data'
+    query = 'SELECT min(name), model, res_id, module FROM ir_model_data'
    query_models = """SELECT m.id, m.model, imd.module
                      FROM ir_model AS m, ir_model_data AS imd
                      WHERE m.id = imd.res_id AND imd.model = 'ir.model'"""
@@ -848,7 +848,7 @@ def trans_generate(lang, modules, cr):
        query_models += ' AND imd.module != %s'
        query_param = ('__export__',)

-    query += ' ORDER BY module, model, name'
+    query += ' GROUP BY model, res_id, module ORDER BY module, model, min(name)'
    query_models += ' ORDER BY module, model'

    cr.execute(query, query_param)
@@ -894,7 +894,8 @@ def trans_generate(lang, modules, cr):
                except Exception:
                    continue
                for term in set(field.get_trans_terms(value)):
-                    push_translation(module, 'model', name, xml_name, term)
+                    trans_type = 'model_terms' if callable(field.translate) else 'model'
+                    push_translation(module, trans_type, name, xml_name, term)

        # End of data for ir.model.data query results

@@ -982,9 +983,6 @@ def trans_generate(lang, modules, cr):
        for root, dummy, files in walksymlinks(path):
            for fname in fnmatch.filter(files, '*.py'):
                babel_extract_terms(fname, path, root)
-            # mako provides a babel extractor: http://docs.makotemplates.org/en/latest/usage.html#babel
-            for fname in fnmatch.filter(files, '*.mako'):
-                babel_extract_terms(fname, path, root, 'mako', trans_type='report')
            # Javascript source files in the static/src/js directory, rest is ignored (libs)
            if fnmatch.fnmatch(root, '*/static/src/js*'):
                for fname in fnmatch.filter(files, '*.js'):