pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] base, tools, web_edeitor: improve ir_translation insertion

Browse Source 
The ir_translations were pretty slow to load, this was mainly caused by
the condition in variable 'find_expr' and the import of po values in the
temporary table.

To improve the load performance, we made two improvement:

1. Insert the row in the temporary table by batch instead of making an
   insertion for each record.
2. Replacing the condition in the variable 'find_expr' by partial unique
   index. To perform such a thing, we had to separate the type model in
   two types, 'model' and 'model_terms' because it wasn't possible to
   create a partial unique index on type 'model' because the type was
   used for two different use cases. The type model is now used for
   fields that have have a value not callable for the attribute
   translate, these field can only have one record for the same field,
   model, res_id and language. And the type 'model_terms' is used for
   the fields callable, because they can have multiple sources for the
   same res_id.

We also removed the deprecated types 'report', 'help', 'view', 'field'.

Thanks to @rco-odoo for the improvement of the first point.
This commit is contained in:
Pierre Masereel
2018-09-18 14:30:03 +02:00
committed by Martin Trigaux
parent 83e460b8da
commit 7288b4779a
5 changed files with 93 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/web_editor/models/ir_translation.py
+1 -1
View File
@@ -27,7 +27,7 @@ class IrTranslation(models.Model):
""" Convert the HTML fragment ``value`` to XML if necessary, and write """ Convert the HTML fragment ``value`` to XML if necessary, and write
it as the value of translation ``self``. it as the value of translation ``self``.
""" """
assert len(self) == 1 and self.type == 'model' assert len(self) == 1 and self.type == 'model_terms'
mname, fname = self.name.split(',') mname, fname = self.name.split(',')
field = self.env[mname]._fields[fname] field = self.env[mname]._fields[fname]
if field.translate == xml_translate: if field.translate == xml_translate:
odoo/addons/base/models/ir_translation.py
+70 -57
View File
@@ -13,12 +13,9 @@ from odoo.tools import pycompat
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
TRANSLATION_TYPE = [ TRANSLATION_TYPE = [
('field', 'Field'), # deprecated ('model', 'Model Field'),
('model', 'Object'), ('model_terms', 'Structured Model Field'),
('report', 'Report/Template'),
('selection', 'Selection'), ('selection', 'Selection'),
('view', 'View'), # deprecated
('help', 'Help'), # deprecated
('code', 'Code'), ('code', 'Code'),
('constraint', 'Constraint'), ('constraint', 'Constraint'),
('sql_constraint', 'SQL Constraint') ('sql_constraint', 'SQL Constraint')
@@ -43,6 +40,7 @@ class IrTranslationImport(object):
self._model_table = model._table self._model_table = model._table
self._overwrite = model._context.get('overwrite', False) self._overwrite = model._context.get('overwrite', False)
self._debug = False self._debug = False
self._rows = []
# Note that Postgres will NOT inherit the constraints or indexes # Note that Postgres will NOT inherit the constraints or indexes
# of ir_translation, so this copy will be much faster. # of ir_translation, so this copy will be much faster.
@@ -88,18 +86,23 @@ class IrTranslationImport(object):
params['name'] = 'ir.ui.view,arch_db' params['name'] = 'ir.ui.view,arch_db'
params['imd_model'] = "ir.ui.view" params['imd_model'] = "ir.ui.view"
query = """ INSERT INTO %s (name, lang, res_id, src, type, imd_model, module, imd_name, value, state, comments) self._rows.append((params['name'], params['lang'], params['res_id'],
VALUES (%%(name)s, %%(lang)s, %%(res_id)s, %%(src)s, %%(type)s, %%(imd_model)s, %%(module)s, params['src'], params['type'], params['imd_model'],
%%(imd_name)s, %%(value)s, %%(state)s, %%(comments)s) """ % self._table params['module'], params['imd_name'], params['value'],
self._cr.execute(query, params) params['state'], params['comments']))
def finish(self): def finish(self):
""" Transfer the data from the temp table to ir.translation """ """ Transfer the data from the temp table to ir.translation """
cr = self._cr cr = self._cr
if self._debug:
cr.execute("SELECT count(*) FROM %s" % self._table) # Step 0: insert rows in batch
count = cr.fetchone()[0] query = """ INSERT INTO %s (name, lang, res_id, src, type, imd_model,
_logger.debug("ir.translation.cursor: We have %d entries to process", count) module, imd_name, value, state, comments)
VALUES """ % self._table
for rows in cr.split_for_in_conditions(self._rows):
cr.execute(query + ", ".join(["%s"] * len(rows)), rows)
_logger.debug("ir.translation.cursor: We have %d entries to process", len(self._rows))
# Step 1: resolve ir.model.data references to res_ids # Step 1: resolve ir.model.data references to res_ids
cr.execute(""" UPDATE %s AS ti cr.execute(""" UPDATE %s AS ti
@@ -129,54 +132,56 @@ class IrTranslationImport(object):
if hasattr(field, 'translate') and callable(field.translate): if hasattr(field, 'translate') and callable(field.translate):
src_relevant_fields.append("%s,%s" % (model, field_name)) src_relevant_fields.append("%s,%s" % (model, field_name))
find_expr = """ # Step 2: insert new or upsert translations
irt.lang = ti.lang
AND irt.type = ti.type
AND irt.name = ti.name
AND (
(ti.type = 'model' AND ti.res_id = irt.res_id AND ti.name IN %s AND irt.src = ti.src)
OR (ti.type = 'model' AND ti.res_id = irt.res_id AND ti.name NOT IN %s)
OR (ti.type = 'view' AND (irt.res_id IS NULL OR ti.res_id = irt.res_id) AND irt.src = ti.src)
OR (ti.type = 'field')
OR (ti.type = 'help')
OR (ti.type NOT IN ('model', 'view', 'field', 'help') AND irt.src = ti.src)
)
"""
# Step 2: update existing (matching) translations
if self._overwrite: if self._overwrite:
cr.execute(""" UPDATE ONLY %s AS irt cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
SET value = ti.value, SELECT name, lang, res_id, src, type, value, module, state, comments
src = ti.src, FROM %s
state = 'translated' WHERE type = 'code'
FROM %s AS ti ON CONFLICT (type, lang, md5(src)) WHERE type = 'code'
WHERE %s DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
AND ti.value IS NOT NULL """ % (self._model_table, self._table))
AND ti.value != '' count = cr.rowcount
AND noupdate IS NOT TRUE cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
""" % (self._model_table, self._table, find_expr), SELECT name, lang, res_id, src, type, value, module, state, comments
(tuple(src_relevant_fields), tuple(src_relevant_fields))) FROM %s
WHERE type = 'model'
# Step 3: insert new translations ON CONFLICT (type, lang, name, res_id) WHERE type = 'model'
cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments) DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
SELECT name, lang, res_id, src, type, value, module, state, comments """ % (self._model_table, self._table))
FROM %s AS ti count += cr.rowcount
WHERE NOT EXISTS(SELECT 1 FROM ONLY %s AS irt WHERE %s) cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
ON CONFLICT DO NOTHING; SELECT name, lang, res_id, src, type, value, module, state, comments
""" % (self._model_table, self._table, self._model_table, find_expr), FROM %s
(tuple(src_relevant_fields), tuple(src_relevant_fields))) WHERE type IN ('selection', 'constraint', 'sql_constraint')
ON CONFLICT (type, lang, name, md5(src)) WHERE type IN ('selection', 'constraint', 'sql_constraint')
DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
""" % (self._model_table, self._table))
count += cr.rowcount
cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
SELECT name, lang, res_id, src, type, value, module, state, comments
FROM %s
WHERE type = 'model_terms'
ON CONFLICT (type, name, lang, res_id, md5(src))
DO UPDATE SET (name, lang, res_id, src, type, value, module, state, comments) = (EXCLUDED.name, EXCLUDED.lang, EXCLUDED.res_id, EXCLUDED.src, EXCLUDED.type, EXCLUDED.value, EXCLUDED.module, EXCLUDED.state, EXCLUDED.comments);
""" % (self._model_table, self._table))
count += cr.rowcount
else:
cr.execute(""" INSERT INTO %s(name, lang, res_id, src, type, value, module, state, comments)
SELECT name, lang, res_id, src, type, value, module, state, comments
FROM %s
ON CONFLICT DO NOTHING;
""" % (self._model_table, self._table))
count = cr.rowcount
if self._debug: if self._debug:
cr.execute("SELECT COUNT(*) FROM ONLY %s" % self._model_table) cr.execute("SELECT COUNT(*) FROM ONLY %s" % self._model_table)
total = cr.fetchone()[0] total = cr.fetchone()[0]
cr.execute("SELECT COUNT(*) FROM ONLY %s AS irt, %s AS ti WHERE %s" % \
(self._model_table, self._table, find_expr),
(tuple(src_relevant_fields), tuple(src_relevant_fields)))
count = cr.fetchone()[0]
_logger.debug("ir.translation.cursor: %d entries now in ir.translation, %d common entries with tmp", total, count) _logger.debug("ir.translation.cursor: %d entries now in ir.translation, %d common entries with tmp", total, count)
# Step 4: cleanup # Step 3: cleanup
cr.execute("DROP TABLE %s" % self._table) cr.execute("DROP TABLE %s" % self._table)
self._rows.clear()
return True return True
@@ -262,8 +267,15 @@ class IrTranslation(models.Model):
res = super(IrTranslation, self)._auto_init() res = super(IrTranslation, self)._auto_init()
# Add separate md5 index on src (no size limit on values, and good performance). # Add separate md5 index on src (no size limit on values, and good performance).
tools.create_index(self._cr, 'ir_translation_src_md5', self._table, ['md5(src)']) tools.create_index(self._cr, 'ir_translation_src_md5', self._table, ['md5(src)'])
# Cover 'model_terms' type
tools.create_unique_index(self._cr, 'ir_translation_unique', self._table, tools.create_unique_index(self._cr, 'ir_translation_unique', self._table,
['type', 'name', 'lang', 'res_id', 'md5(src)']) ['type', 'name', 'lang', 'res_id', 'md5(src)'])
if not tools.index_exists(self._cr, 'ir_translation_code_unique'):
self._cr.execute("CREATE UNIQUE INDEX ir_translation_code_unique ON ir_translation (type, lang, md5(src)) WHERE type = 'code'")
if not tools.index_exists(self._cr, 'ir_translation_model_unique'):
self._cr.execute("CREATE UNIQUE INDEX ir_translation_model_unique ON ir_translation (type, lang, name, res_id) WHERE type = 'model'")
if not tools.index_exists(self._cr, 'ir_translation_selection_unique'):
self._cr.execute("CREATE UNIQUE INDEX ir_translation_selection_unique ON ir_translation (type, lang, name, md5(src)) WHERE type IN ('selection', 'constraint', 'sql_constraint')")
return res return res
@api.model @api.model
@@ -403,7 +415,7 @@ class IrTranslation(models.Model):
query = """ SELECT * FROM ir_translation query = """ SELECT * FROM ir_translation
WHERE lang=%s AND type=%s AND name=%s AND res_id IN %s """ WHERE lang=%s AND type=%s AND name=%s AND res_id IN %s """
name = "%s,%s" % (field.model_name, field.name) name = "%s,%s" % (field.model_name, field.name)
params = (records.env.lang, 'model', name, tuple(records.ids)) params = (records.env.lang, 'model_terms', name, tuple(records.ids))
return query, params return query, params
@api.model @api.model
@@ -448,7 +460,7 @@ class IrTranslation(models.Model):
value = record[field.name] value = record[field.name]
terms = set(field.get_trans_terms(value)) terms = set(field.get_trans_terms(value))
record_trans = trans.search([ record_trans = trans.search([
('type', '=', 'model'), ('type', '=', 'model_terms'),
('name', '=', "%s,%s" % (field.model_name, field.name)), ('name', '=', "%s,%s" % (field.model_name, field.name)),
('res_id', '=', record.id), ('res_id', '=', record.id),
]) ])
@@ -591,12 +603,13 @@ class IrTranslation(models.Model):
if callable(field.translate): if callable(field.translate):
# insert missing translations for each term in src # insert missing translations for each term in src
query = """ INSERT INTO ir_translation (lang, type, name, res_id, src, value, module) query = """ INSERT INTO ir_translation (lang, type, name, res_id, src, value, module)
SELECT l.code, 'model', %(name)s, %(res_id)s, %(src)s, %(src)s, %(module)s SELECT l.code, 'model_terms', %(name)s, %(res_id)s, %(src)s, %(src)s, %(module)s
FROM res_lang l FROM res_lang l
WHERE l.active AND l.translatable AND NOT EXISTS ( WHERE l.active AND l.translatable AND NOT EXISTS (
SELECT 1 FROM ir_translation SELECT 1 FROM ir_translation
WHERE lang=l.code AND type='model' AND name=%(name)s AND res_id=%(res_id)s AND src=%(src)s WHERE lang=l.code AND type='model' AND name=%(name)s AND res_id=%(res_id)s AND src=%(src)s
); )
ON CONFLICT DO NOTHING;
""" """
for record in records: for record in records:
module = external_ids[record.id].split('.')[0] module = external_ids[record.id].split('.')[0]
odoo/addons/base/tests/test_translate.py
+17 -13
View File
@@ -4,8 +4,10 @@
import unittest import unittest
from odoo.tools import pycompat from odoo.tools import pycompat
from odoo.tools import mute_logger
from odoo.tools.translate import quote, unquote, xml_translate, html_translate from odoo.tools.translate import quote, unquote, xml_translate, html_translate
from odoo.tests.common import TransactionCase, tagged from odoo.tests.common import TransactionCase, tagged
from psycopg2 import IntegrityError
@tagged('standard', 'at_install') @tagged('standard', 'at_install')
@@ -286,21 +288,23 @@ class TestTranslation(TransactionCase):
}) })
# change name and insert a duplicate manually # change name and insert a duplicate manually
padawans.write({'name': 'Padawans'}) padawans.write({'name': 'Padawans'})
self.env['ir.translation'].create({ with self.assertRaises(IntegrityError), mute_logger('odoo.sql_db'):
'type': 'model', with self.env.cr.savepoint():
'name': 'res.partner.category,name', self.env['ir.translation'].create({
'module':'base', 'type': 'model',
'lang': 'fr_FR', 'name': 'res.partner.category,name',
'res_id': padawans.id, 'module':'base',
'value': 'Apprentis', 'lang': 'fr_FR',
'state': 'translated', 'res_id': padawans.id,
}) 'value': 'Apprentis',
'state': 'translated',
})
self.env['ir.translation'].translate_fields('res.partner.category', padawans.id, 'name') self.env['ir.translation'].translate_fields('res.partner.category', padawans.id, 'name')
translations = self.env['ir.translation'].search([ translations = self.env['ir.translation'].search([
('res_id', '=', padawans.id), ('name', '=', 'res.partner.category,name') ('res_id', '=', padawans.id), ('name', '=', 'res.partner.category,name')
]) ])
self.assertEqual(len(translations), 1, "Translations were not merged after `translate_fields` call") self.assertEqual(len(translations), 1, "Translations were not duplicated after `translate_fields` call")
self.assertEqual(translations.value, "Apprentis", "The most recent translation must stay") self.assertEqual(translations.value, "Apprenti", "The first translation must stay")
class TestXMLTranslation(TransactionCase): class TestXMLTranslation(TransactionCase):
@@ -323,7 +327,7 @@ class TestXMLTranslation(TransactionCase):
}) })
for src, value in list(pycompat.izip(terms_en, terms_fr)): for src, value in list(pycompat.izip(terms_en, terms_fr)):
self.env['ir.translation'].create({ self.env['ir.translation'].create({
'type': 'model', 'type': 'model_terms',
'name': 'ir.ui.view,arch_db', 'name': 'ir.ui.view,arch_db',
'lang': 'fr_FR', 'lang': 'fr_FR',
'res_id': view0.id, 'res_id': view0.id,
@@ -363,7 +367,7 @@ class TestXMLTranslation(TransactionCase):
}) })
for src, value in list(pycompat.izip(terms_en, terms_fr)): for src, value in list(pycompat.izip(terms_en, terms_fr)):
self.env['ir.translation'].create({ self.env['ir.translation'].create({
'type': 'model', 'type': 'model_terms',
'name': 'ir.ui.view,arch_db', 'name': 'ir.ui.view,arch_db',
'lang': 'fr_FR', 'lang': 'fr_FR',
'res_id': view0.id, 'res_id': view0.id,
odoo/addons/base/tests/test_views.py
+1 -1
View File
@@ -618,7 +618,7 @@ class TestNoModel(ViewCase):
'type': 'qweb', 'type': 'qweb',
}) })
self.env['ir.translation'].create({ self.env['ir.translation'].create({
'type': 'model', 'type': 'model_terms',
'name': 'ir.ui.view,arch_db', 'name': 'ir.ui.view,arch_db',
'res_id': view.id, 'res_id': view.id,
'lang': 'fr_FR', 'lang': 'fr_FR',
odoo/tools/translate.py
+4 -6
View File
@@ -830,7 +830,7 @@ def trans_generate(lang, modules, cr):
tnx = (module, source, name, id, type, tuple(comments or ())) tnx = (module, source, name, id, type, tuple(comments or ()))
to_translate.add(tnx) to_translate.add(tnx)
query = 'SELECT name, model, res_id, module FROM ir_model_data' query = 'SELECT min(name), model, res_id, module FROM ir_model_data'
query_models = """SELECT m.id, m.model, imd.module query_models = """SELECT m.id, m.model, imd.module
FROM ir_model AS m, ir_model_data AS imd FROM ir_model AS m, ir_model_data AS imd
WHERE m.id = imd.res_id AND imd.model = 'ir.model'""" WHERE m.id = imd.res_id AND imd.model = 'ir.model'"""
@@ -848,7 +848,7 @@ def trans_generate(lang, modules, cr):
query_models += ' AND imd.module != %s' query_models += ' AND imd.module != %s'
query_param = ('__export__',) query_param = ('__export__',)
query += ' ORDER BY module, model, name' query += ' GROUP BY model, res_id, module ORDER BY module, model, min(name)'
query_models += ' ORDER BY module, model' query_models += ' ORDER BY module, model'
cr.execute(query, query_param) cr.execute(query, query_param)
@@ -894,7 +894,8 @@ def trans_generate(lang, modules, cr):
except Exception: except Exception:
continue continue
for term in set(field.get_trans_terms(value)): for term in set(field.get_trans_terms(value)):
push_translation(module, 'model', name, xml_name, term) trans_type = 'model_terms' if callable(field.translate) else 'model'
push_translation(module, trans_type, name, xml_name, term)
# End of data for ir.model.data query results # End of data for ir.model.data query results
@@ -982,9 +983,6 @@ def trans_generate(lang, modules, cr):
for root, dummy, files in walksymlinks(path): for root, dummy, files in walksymlinks(path):
for fname in fnmatch.filter(files, '*.py'): for fname in fnmatch.filter(files, '*.py'):
babel_extract_terms(fname, path, root) babel_extract_terms(fname, path, root)
# mako provides a babel extractor: http://docs.makotemplates.org/en/latest/usage.html#babel
for fname in fnmatch.filter(files, '*.mako'):
babel_extract_terms(fname, path, root, 'mako', trans_type='report')
# Javascript source files in the static/src/js directory, rest is ignored (libs) # Javascript source files in the static/src/js directory, rest is ignored (libs)
if fnmatch.fnmatch(root, '*/static/src/js*'): if fnmatch.fnmatch(root, '*/static/src/js*'):
for fname in fnmatch.filter(files, '*.js'): for fname in fnmatch.filter(files, '*.js'):