pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] core: improve translation import speed

Browse Source 
this commit improves the performance of importing translations by
1. groupup udpating data for different languages and different modules
2. read model_terms translations data directly with xmlid to get rid of fetching
   data from ir_model_data

for modules:sale_management, point_of_sale, industry_fsm, helpdesk, mrp, mrp_plm
(62 installed modules)
time for test_language_install is 45.14% less

for all modules:
time for test_language_install is 68.03% less

closes odoo/odoo#107332

X-original-commit: 727ef9e0a2ca93089b472eb1313117fee8dc876d
Signed-off-by: Raphael Collet <rco@odoo.com>
This commit is contained in:
Chong Wang (cwg)
2022-12-06 17:12:21 +01:00
committed by Raphael Collet
parent 0f69da6ebf
commit a0bd7d0d72
8 changed files with 245 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/mail/models/template_reset_mixin.py
+6 -20
View File
@@ -11,7 +11,7 @@ from odoo.modules import get_module_resource
from odoo.modules.module import get_resource_from_path, get_resource_path
from odoo.tools.convert import xml_import
from odoo.tools.misc import file_open
from odoo.tools.translate import TranslationFileReader, _trans_load_data
from odoo.tools.translate import TranslationImporter
class TemplateResetMixin(models.AbstractModel):
@@ -61,7 +61,8 @@ class TemplateResetMixin(models.AbstractModel):
# -------------------------------------------------------------------------
def _override_translation_term(self, module_name, xml_ids):
processed_base_langs = []
translation_importer = TranslationImporter(self.env.cr)
for code, _ in self.env['res.lang'].get_installed():
lang_code = tools.get_iso_codes(code)
# In case of sub languages (e.g fr_BE), load the base language first, (e.g fr.po) and
@@ -72,30 +73,15 @@ class TemplateResetMixin(models.AbstractModel):
base_lang_code = lang_code.split('_')[0]
base_trans_file = get_module_resource(module_name, 'i18n', base_lang_code + '.po')
if base_trans_file:
if base_lang_code not in processed_base_langs:
processed_base_langs.append(base_lang_code)
self._process_translation_data(base_trans_file, code, xml_ids)
translation_importer.load_file(base_trans_file, code, xmlids=xml_ids)
# Step 2: reset translation file with main language file (can possibly override the
# terms coming from the base language)
trans_file = get_module_resource(module_name, 'i18n', lang_code + '.po')
if trans_file:
self._process_translation_data(trans_file, code, xml_ids)
translation_importer.load_file(trans_file, code, xmlids=xml_ids)
def _process_translation_data(self, trans_file, lang, xml_ids):
"""Load translations.
:param trans_file: path to a translation file to open, e.g. {addon_path}/mail/i18n/es.po
:param lang: language code of the translations contained in `trans_file`
language must be present and activated in the database
:return Boolean: True if translation data is available and processed, False otherwise
"""
with file_open(trans_file, mode='rb', filter_ext=(".po",)) as fileobj:
reader = TranslationFileReader(fileobj)
# Process a single PO entry
_trans_load_data(self.env.cr, reader, lang, overwrite=True, force_overwrite=True, xml_ids=xml_ids)
translation_importer.save(overwrite=True, force_overwrite=True)
def reset_template(self):
"""Resets the Template with values given in source file. We ignore the case of
odoo/addons/base/models/ir_module.py
+11 -10
View File
@@ -31,6 +31,7 @@ from odoo.exceptions import AccessDenied, UserError
from odoo.osv import expression
from odoo.tools.parse_version import parse_version
from odoo.tools.misc import topological_sort
from odoo.tools.translate import TranslationImporter
from odoo.http import request
from odoo.modules import get_module_path, get_module_resource
@@ -932,13 +933,14 @@ class Module(models.Model):
def _load_module_terms(self, modules, langs, overwrite=False):
""" Load PO files of the given modules for the given languages. """
# load i18n files
translation_importer = TranslationImporter(self.env.cr, verbose=False)
for module_name in modules:
modpath = get_module_path(module_name)
if not modpath:
continue
for lang in langs:
lang_code = tools.get_iso_codes(lang)
lang_overwrite = overwrite
base_lang_code = None
if '_' in lang_code:
base_lang_code = lang_code.split('_')[0]
@@ -948,29 +950,28 @@ class Module(models.Model):
base_trans_file = get_module_resource(module_name, 'i18n', base_lang_code + '.po')
if base_trans_file:
_logger.info('module %s: loading base translation file %s for language %s', module_name, base_lang_code, lang)
tools.trans_load(self._cr, base_trans_file, lang, verbose=False, overwrite=lang_overwrite)
lang_overwrite = True # make sure the requested translation will override the base terms later
translation_importer.load_file(base_trans_file, lang)
# i18n_extra folder is for additional translations handle manually (eg: for l10n_be)
base_trans_extra_file = get_module_resource(module_name, 'i18n_extra', base_lang_code + '.po')
if base_trans_extra_file:
_logger.info('module %s: loading extra base translation file %s for language %s', module_name, base_lang_code, lang)
tools.trans_load(self._cr, base_trans_extra_file, lang, verbose=False, overwrite=lang_overwrite)
lang_overwrite = True # make sure the requested translation will override the base terms later
translation_importer.load_file(base_trans_extra_file, lang)
# Step 2: then load the main translation file, possibly overriding the terms coming from the base language
trans_file = get_module_resource(module_name, 'i18n', lang_code + '.po')
if trans_file:
_logger.info('module %s: loading translation file (%s) for language %s', module_name, lang_code, lang)
tools.trans_load(self._cr, trans_file, lang, verbose=False, overwrite=lang_overwrite)
_logger.info('module %s: loading translation file %s for language %s', module_name, lang_code, lang)
translation_importer.load_file(trans_file, lang)
elif lang_code != 'en_US':
_logger.info('module %s: no translation for language %s', module_name, lang_code)
trans_extra_file = get_module_resource(module_name, 'i18n_extra', lang_code + '.po')
if trans_extra_file:
_logger.info('module %s: loading extra translation file (%s) for language %s', module_name, lang_code, lang)
tools.trans_load(self._cr, trans_extra_file, lang, verbose=False, overwrite=lang_overwrite)
return True
_logger.info('module %s: loading extra translation file %s for language %s', module_name, lang_code, lang)
translation_importer.load_file(trans_extra_file, lang)
translation_importer.save(overwrite=overwrite)
DEP_STATES = STATES + [('unknown', 'Unknown')]
odoo/addons/base/tests/test_translate.py
+11 -5
View File
@@ -9,8 +9,8 @@ from psycopg2.extras import Json
import io
from odoo.exceptions import AccessError, ValidationError
from odoo.tools import trans_load_data, sql
from odoo.tools.translate import quote, unquote, xml_translate, html_translate
from odoo.tools import sql
from odoo.tools.translate import quote, unquote, xml_translate, html_translate, TranslationImporter
from odoo.tests.common import TransactionCase, BaseCase, new_test_user, tagged
_stats_logger = logging.getLogger('odoo.tests.stats')
@@ -295,7 +295,9 @@ class TestTranslation(TransactionCase):
''' % cls.customers_xml_id
with io.BytesIO(bytes(po_string, encoding='utf-8')) as f:
f.name = 'dummy'
trans_load_data(cls.env.cr, f, 'po', 'fr_FR', verbose=True, overwrite=True)
translation_importer = TranslationImporter(cls.env.cr, verbose=True)
translation_importer.load(f, 'po', 'fr_FR')
translation_importer.save(overwrite=True)
def test_101_create_translated_record(self):
category = self.customers.with_context({})
@@ -465,7 +467,9 @@ class TestTranslationWrite(TransactionCase):
''' % self.category_xml_id
with io.BytesIO(bytes(po_string, encoding='utf-8')) as f:
f.name = 'dummy'
trans_load_data(self.env.cr, f, 'po', 'en_US', verbose=True, overwrite=True)
translation_importer = TranslationImporter(self.env.cr, verbose=True)
translation_importer.load(f, 'po', 'fr_FR')
translation_importer.save(overwrite=True)
self.category.with_context(lang='fr_FR').write({'name': 'French Name'})
self.category.with_context(lang='en_US').write({'name': 'English Name'})
@@ -651,7 +655,9 @@ class TestTranslationWrite(TransactionCase):
''' % (ir_model_field_xml_id, LABEL)
with io.BytesIO(bytes(po_string, encoding='utf-8')) as f:
f.name = 'dummy'
trans_load_data(self.env.cr, f, 'po', 'fr_FR', verbose=True, overwrite=True)
translation_importer = TranslationImporter(self.env.cr, verbose=True)
translation_importer.load(f, 'po', 'fr_FR')
translation_importer.save(overwrite=True)
# check that fields_get() returns the expected label
model = self.env['ir.model'].with_context(lang='fr_FR')
odoo/addons/base/wizard/base_import_language.py
+20 -36
View File
@@ -3,13 +3,12 @@
import base64
import logging
import os
from tempfile import TemporaryFile
from psycopg2 import ProgrammingError
from contextlib import closing
from os.path import splitext
from odoo import api, fields, models, tools, sql_db, _
from odoo.exceptions import UserError
from odoo.tools.translate import TranslationImporter
_logger = logging.getLogger(__name__)
@@ -29,36 +28,21 @@ class BaseLanguageImport(models.TransientModel):
"will be overwritten and replaced by those in this file")
def import_lang(self):
this = self[0]
with TemporaryFile('wb+') as buf:
try:
buf.write(base64.decodebytes(this.data))
# now we determine the file format
buf.seek(0)
fileformat = os.path.splitext(this.filename)[-1][1:].lower()
Lang = self.env["res.lang"]
lang = Lang._activate_lang(self.code) or Lang._create_lang(
self.code, lang_name=self.name
)
tools.trans_load_data(
this._cr, buf, fileformat, this.code, overwrite=self.overwrite
)
except ProgrammingError as e:
_logger.exception('File unsuccessfully imported, due to a malformed file.')
with closing(sql_db.db_connect(self._cr.dbname).cursor()) as cr:
raise UserError(_('File %r not imported due to a malformed file.\n\n'
'This issue can be caused by duplicates entries who are referring to the same field. '
'Please check the content of the file you are trying to import.\n\n'
'Technical Details:\n%s') % (self.filename, tools.ustr(e)))
except Exception as e:
_logger.exception('File unsuccessfully imported, due to format mismatch.')
raise UserError(
_('File %r not imported due to format mismatch or a malformed file.'
' (Valid formats are .csv, .po, .pot)\n\nTechnical Details:\n%s') % \
(this.filename, tools.ustr(e))
)
return True
self.ensure_one()
Lang = self.env["res.lang"]
try:
if not Lang._activate_lang(self.code):
Lang._create_lang(self.code, lang_name=self.name)
with TemporaryFile('wb+') as buf:
buf.write(base64.decodebytes(self.data))
fileformat = splitext(self.filename)[-1][1:].lower()
translation_importer = TranslationImporter(self.env.cr)
translation_importer.load(buf, fileformat, self.code)
translation_importer.save(overwrite=self.overwrite)
except Exception as e:
_logger.exception('File unsuccessfully imported, due to format mismatch.')
raise UserError(
_('File %r not imported due to format mismatch or a malformed file.'
' (Valid formats are .csv, .po, .pot)\n\nTechnical Details:\n%s') % \
(self.filename, tools.ustr(e))
)
odoo/addons/test_new_api/tests/test_related_translation.py
+4 -2
View File
@@ -2,7 +2,7 @@
import io
import odoo.tests
from odoo.tools import trans_load_data
from odoo.tools.translate import TranslationImporter
@odoo.tests.tagged('post_install', '-at_install')
@@ -126,7 +126,9 @@ class TestRelatedTranslation(odoo.tests.TransactionCase):
''' % test1_xml_id
with io.BytesIO(bytes(po_string, encoding='utf-8')) as f:
f.name = 'dummy'
trans_load_data(self.test1.env.cr, f, 'po', 'fr_FR', verbose=True, overwrite=True)
translation_importer = TranslationImporter(self.env.cr, verbose=True)
translation_importer.load(f, 'po', 'fr_FR')
translation_importer.save(overwrite=True)
self.assertEqual(self.test2.with_context(lang='fr_FR').name, 'Nouveau couteau')
def test_translate_from_ori_term(self):
odoo/addons/test_translation_import/tests/test_term_count.py
+5 -12
View File
@@ -1,10 +1,8 @@
# -*- coding: utf-8 -*-
from contextlib import closing
import base64
import io
import odoo
from odoo.tests import common, tagged
from odoo.tools.misc import file_open, mute_logger
from odoo.tools.translate import TranslationModuleReader, code_translations, CodeTranslations, PYTHON_TRANSLATION_COMMENT, JAVASCRIPT_TRANSLATION_COMMENT, WEB_TRANSLATION_COMMENT
@@ -18,8 +16,6 @@ class TestImport(common.TransactionCase):
self.env['res.lang']._activate_lang('fr_FR')
# Tip: code translations don't need to be imported explicitly
# odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False)
model = self.env['test.translation.import.model1']
self.assertEqual(
model.with_context(lang='fr_FR').get_code_translation(),
@@ -28,7 +24,7 @@ class TestImport(common.TransactionCase):
def test_import_model_translation(self):
self.env['res.lang']._activate_lang('fr_FR')
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False)
self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR'])
record = self.env.ref('test_translation_import.test_translation_import_model1_record1')
self.assertEqual(
@@ -38,7 +34,7 @@ class TestImport(common.TransactionCase):
def test_import_model_term_translation(self):
self.env['res.lang']._activate_lang('fr_FR')
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False)
self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR'])
record = self.env.ref('test_translation_import.test_translation_import_model1_record1')
self.assertEqual(
@@ -54,11 +50,11 @@ class TestImport(common.TransactionCase):
self.assertEqual(menu.name, 'Test translation model1')
# install french and change translation content
self.env['res.lang']._activate_lang('fr_FR')
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False)
self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR'])
self.assertEqual(menu.with_context(lang='fr_FR').name, "Test translation import in french")
menu.with_context(lang='fr_FR').name = "Nouveau nom"
# reload with overwrite
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False, overwrite=True)
self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR'], overwrite=True)
self.assertEqual(menu.name, "Test translation model1")
self.assertEqual(menu.with_context(lang='fr_FR').name, "Nouveau nom")
@@ -66,10 +62,7 @@ class TestImport(common.TransactionCase):
def test_lang_with_base(self):
self.env['res.lang']._activate_lang('fr_BE')
self.env['res.lang']._activate_lang('fr_CA')
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_BE', verbose=False)
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr_BE.po', 'fr_BE', verbose=False, overwrite=True)
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_CA', verbose=False)
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr_CA.po', 'fr_CA', verbose=False, overwrite=True)
self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_BE', 'fr_CA'], overwrite=True)
# language override base language
record = self.env.ref('test_translation_import.test_translation_import_model1_record1')
odoo/cli/server.py
+3 -3
View File
@@ -118,9 +118,9 @@ def import_translation():
registry = odoo.modules.registry.Registry.new(dbname)
with registry.cursor() as cr:
odoo.tools.trans_load(
cr, config["translate_in"], config["language"], overwrite=overwrite,
)
translation_importer = odoo.tools.translate.TranslationImporter(cr)
translation_importer.load_file(config["translate_in"], config["language"])
translation_importer.save(overwrite=overwrite)
def main(args):
check_root_user()
odoo/tools/translate.py
+185 -152
View File
@@ -14,8 +14,8 @@ from tokenize import generate_tokens, STRING, NEWLINE, INDENT, DEDENT
import polib
import re
import tarfile
import tempfile
import threading
import warnings
from collections import defaultdict, namedtuple
from datetime import datetime
from os.path import join
@@ -1094,7 +1094,7 @@ class TranslationModuleReader:
# From our business perspective, the parent column is no need to be translated,
# but it is need to be set to jsonb column, since the child columns need to be translated
# And export the parent field may make one value to be translated twice in transifex
if field.translate and not (field.model_name == 'ir.actions.actions' and field.name == 'name'):
if field.translate and field.store and str(field) != 'ir.actions.actions.name':
name = model + "," + field_name
try:
value_en = record[field_name] or ''
@@ -1198,176 +1198,209 @@ class TranslationModuleReader:
break
def trans_load(cr, filename, lang, verbose=True, overwrite=False):
try:
with file_open(filename, mode='rb') as fileobj:
_logger.info("loading %s", filename)
fileformat = os.path.splitext(filename)[-1][1:].lower()
return trans_load_data(cr, fileobj, fileformat, lang,
verbose=verbose,
overwrite=overwrite)
except IOError:
if verbose:
_logger.error("couldn't read translation file %s", filename)
return None
def DeepDefaultDict():
return defaultdict(DeepDefaultDict)
def trans_load_data(cr, fileobj, fileformat, lang,
verbose=True, overwrite=False):
"""Load translations.
:param cr:
:param fileobj: buffer open to a translation file
:param fileformat: format of the `fielobj` file, one of 'po' or 'csv'
:param lang: language code of the translations contained in `fileobj`
language must be present and activated in the database
:param verbose: increase log output
:param overwrite: if a translation already exists for a term, replace it with
the one in `fileobj`
class TranslationImporter:
""" Helper object for importing translation files to a database.
This class provides a convenient API to load the translations from many
files and import them all at once, which helps speeding up the whole import.
"""
if verbose:
_logger.info('loading translation file for language %s', lang)
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {'lang': lang})
if not env['res.lang']._lang_get(lang):
_logger.error("Couldn't read translation for lang '%s', language not found", lang)
return None
try:
fileobj.seek(0)
reader = TranslationFileReader(fileobj, fileformat=fileformat)
_trans_load_data(cr, reader, lang, overwrite)
if verbose:
_logger.info("translation file loaded successfully")
def __init__(self, cr, verbose=True):
self.cr = cr
self.verbose = verbose
self.env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
except IOError:
iso_lang = get_iso_codes(lang)
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
_logger.exception("couldn't read translation file %s", filename)
# {model_name: {field_name: {xmlid: {lang: value}}}}
self.model_translations = DeepDefaultDict()
# {model_name: {field_name: {xmlid: {src: {lang: value}}}}}
self.model_terms_translations = DeepDefaultDict()
def load_file(self, filepath, lang, xmlids=None):
""" Load translations from the given file path.
def _trans_load_data(cr, reader, lang, overwrite=False, force_overwrite=False, xml_ids=None):
# logical overwrite: if a translation should be updated when it has already been in the DB before
# logical overwrite = force_overwrite or (not noudpate and overwrite)
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {'lang': lang})
if force_overwrite:
overwrite = True
:param filepath: file path to open
:param lang: language code of the translations contained in the file;
the language must be present and activated in the database
:param xmlids: if given, only translations for records with xmlid in xmlids will be loaded
"""
with file_open(filepath, mode='rb') as fileobj:
fileformat = os.path.splitext(filepath)[-1][1:].lower()
self.load(fileobj, fileformat, lang, xmlids=xmlids)
# load all translation in to dict
deep_defaultdict = lambda: defaultdict(deep_defaultdict)
# {model_name: {xmlid: {field_name, {src: value}}}}
translations = deep_defaultdict()
# [(module_name, imd_name), (module_name, imd_name), ...] only for model_terms translation
module_imd_names = []
# {model_name: {field_name: [xmlid, xmlid, ...]}}
model_name_to_field_name_to_xmlids = defaultdict(lambda: defaultdict(list))
def load(self, fileobj, fileformat, lang, xmlids=None):
"""Load translations from the given file object.
for row in reader:
if not row.get('value') or not row.get('src'): # ignore empty translations
continue
if row.get('type') == 'code': # ignore code translations
continue
# TODO: CWG if the po file should not be trusted, we need to check each model term
model_name = row.get('imd_model')
module_name = row['module']
imd_name = row['imd_name']
if model_name in env:
:param fileobj: buffer open to a translation file
:param fileformat: format of the `fielobj` file, one of 'po' or 'csv'
:param lang: language code of the translations contained in `fileobj`;
the language must be present and activated in the database
:param xmlids: if given, only translations for records with xmlid in xmlids will be loaded
"""
if self.verbose:
_logger.info('loading translation file for language %s', lang)
if not self.env['res.lang']._lang_get(lang):
_logger.error("Couldn't read translation for lang '%s', language not found", lang)
return None
try:
fileobj.seek(0)
reader = TranslationFileReader(fileobj, fileformat=fileformat)
self._load(reader, lang, xmlids)
except IOError:
iso_lang = get_iso_codes(self.lang)
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
_logger.exception("couldn't read translation file %s", filename)
def _load(self, reader, lang, xmlids=None):
if xmlids and not isinstance(xmlids, set):
xmlids = set(xmlids)
for row in reader:
if not row.get('value') or not row.get('src'): # ignore empty translations
continue
if row.get('type') == 'code': # ignore code translations
continue
# TODO: CWG if the po file should not be trusted, we need to check each model term
model_name = row.get('imd_model')
module_name = row['module']
if model_name not in self.env:
continue
field_name = row['name'].split(',')[1]
field = env[model_name]._fields.get(field_name)
field = self.env[model_name]._fields.get(field_name)
if not field or not field.translate or not field.store:
continue
xmlid = module_name + '.' + row['imd_name']
if xml_ids and xmlid not in xml_ids:
if xmlids and xmlid not in xmlids:
continue
if row['type'] == 'model_terms':
module_imd_names.append((module_name, imd_name))
model_name_to_field_name_to_xmlids[model_name][field_name].append(xmlid)
translations[model_name][xmlid][field_name][row['src']] = row['value']
if row.get('type') == 'model':
self.model_translations[model_name][field_name][xmlid][lang] = row['value']
elif row.get('type') == 'model_terms':
self.model_terms_translations[model_name][field_name][xmlid][row['src']][lang] = row['value']
if not translations:
return
env.flush_all()
def save(self, overwrite=False, force_overwrite=False):
""" Save translations to the database.
# get mapping xmlid_to_id, id_to_xmlid
xmlid_to_id = {}
for sub_module_imd_names in cr.split_for_in_conditions(module_imd_names):
query = "SELECT module || '.' || name, res_id, noupdate, model FROM ir_model_data WHERE "
query += " OR ".join(["module = %s AND name = %s"] * len(sub_module_imd_names))
cr.execute(query, [param for params in sub_module_imd_names for param in params])
# {xmlid: (id, noupdate, model_name)}
xmlid_to_id.update((
(xmlid, (id_, not force_overwrite and noupdate, model_name))
for xmlid, id_, noupdate, model_name in cr.fetchall()
))
id_to_xmlid = {
(model_name, id_): (xmlid, noupdate)
for xmlid, (id_, noupdate, model_name) in xmlid_to_id.items()
}
For a record with 'noupdate' in ``ir_model_data``, its existing translations
will be overwritten if ``force_overwrite or (not noupdate and overwrite)``.
# translate model terms
for model_name, model_dictionary in translations.items():
if model_name not in env:
continue
Model = env[model_name]
model_table = Model._table
field_name_to_xmlids = model_name_to_field_name_to_xmlids[model_name]
fields = Model._fields
field_type = defaultdict(list)
for field_name in field_name_to_xmlids.keys():
translate = fields[field_name].translate
field_type['no_translate' if not translate else 'model_terms' if callable(translate) else 'model'].append(field_name)
if field_type['no_translate']:
pass # ignore illegal translations
for field_name in field_type['model_terms']:
record_ids = {
it[0]
for it in map(xmlid_to_id.get, field_name_to_xmlids[field_name])
if it
}
field = fields[field_name]
for sub_ids in cr.split_for_in_conditions(record_ids):
cr.execute(f'SELECT id, "{field_name}" FROM "{model_table}" WHERE id IN %s', (sub_ids,))
for id_, values in cr.fetchall():
if not values:
continue
value_en = values.get('en_US')
if not value_en:
continue
value_lang = values.get(lang, value_en)
xmlid, noupdate = id_to_xmlid[(model_name, id_)]
field_dictionary = {k: v for k, v in model_dictionary[xmlid][field_name].items()} # len(field.get_trans_terms(v)) == 1
translation_dictionary = field.get_translation_dictionary(value_en, {lang: value_lang})
An existing translation means:
* model translation: the ``jsonb`` value in database has the language code as key;
* model terms translation: the term value in the language is different from the term value in ``en_US``.
"""
if not self.model_translations and not self.model_terms_translations:
return
# update translation_dictionary using new translations
for term_en, term in translation_dictionary.items():
if not overwrite and noupdate and term[lang] != term_en:
cr = self.cr
env = self.env
env.flush_all()
for model_name, model_dictionary in self.model_terms_translations.items():
Model = env[model_name]
model_table = Model._table
fields = Model._fields
# field_name, {xmlid: {src: {lang: value}}}
for field_name, field_dictionary in model_dictionary.items():
field = fields.get(field_name)
for sub_xmlids in cr.split_for_in_conditions(field_dictionary.keys()):
# [module_name, imd_name, module_name, imd_name, ...]
params = []
for xmlid in sub_xmlids:
params.extend(xmlid.split('.'))
cr.execute(f'''
SELECT m.id, imd.module || '.' || imd.name, m."{field_name}", imd.noupdate
FROM "{model_table}" m, "ir_model_data" imd
WHERE m.id = imd.res_id
AND ({" OR ".join(["(imd.module = %s AND imd.name = %s)"] * (len(params) // 2))})
''', params)
# [id, translations, id, translations, ...]
params = []
for id_, xmlid, values, noupdate in cr.fetchall():
if not values:
continue
value_en = values.get('en_US')
if not value_en:
continue
term[lang] = field_dictionary.get(term_en, term[lang])
new_value_lang = field.translate(lambda term: translation_dictionary.get(term, {}).get(lang), value_en)
values[lang] = new_value_lang
env.cache.update_raw(Model.browse(id_), field, [values], dirty=True)
for field_name in field_type['model']:
field_translations = []
for xmlid in field_name_to_xmlids[field_name]:
src, value = list(model_dictionary[xmlid][field_name].items())[0]
field_translations.extend([xmlid.split('.')[-1], src, Json({lang: value})])
if field_translations:
if not force_overwrite:
value_query = f"""CASE WHEN {overwrite} is True AND imd.noupdate is False
THEN m."{field_name}" || t.value
ELSE t.value || m."{field_name}"END"""
else:
value_query = f'm."{field_name}" || t.value'
env.cr.execute(f"""
UPDATE "{model_table}" AS m
SET "{field_name}" = {value_query}
FROM (
VALUES {', '.join(['(%s, %s, %s::jsonb)'] * int(len(field_translations) / 3))}
) AS t(imd_name, src, value)
JOIN "ir_model_data" AS imd
ON imd."model" = '{model_name}' AND imd.name = t.imd_name
WHERE imd."res_id" = m."id" AND m."{field_name}"->>'en_US' = t.src
""", field_translations)
env.invalidate_all()
# {src: {lang: value}}
record_dictionary = field_dictionary[xmlid]
langs = {lang for translations in record_dictionary.values() for lang in translations.keys()}
translation_dictionary = field.get_translation_dictionary(
value_en,
{k: v for k, v in values.items() if k in langs}
)
if force_overwrite or (not noupdate and overwrite):
# overwrite existing translations
for term_en, translations in record_dictionary.items():
translation_dictionary[term_en].update(translations)
else:
# keep existing translations
for term_en, translations in record_dictionary.items():
translations.update({k: v for k, v in translation_dictionary[term_en].items() if v != term_en})
translation_dictionary[term_en] = translations
for lang in langs:
values[lang] = field.translate(lambda term: translation_dictionary.get(term, {}).get(lang), value_en)
params.extend((id_, Json(values)))
if params:
env.cr.execute(f"""
UPDATE "{model_table}" AS m
SET "{field_name}" = t.value
FROM (
VALUES {', '.join(['(%s, %s::jsonb)'] * (len(params) // 2))}
) AS t(id, value)
WHERE m.id = t.id
""", params)
self.model_terms_translations.clear()
for model_name, model_dictionary in self.model_translations.items():
Model = env[model_name]
model_table = Model._table
for field_name, field_dictionary in model_dictionary.items():
for sub_field_dictionary in cr.split_for_in_conditions(field_dictionary.items()):
# [xmlid, translations, xmlid, translations, ...]
params = []
for xmlid, translations in sub_field_dictionary:
params.extend([xmlid.split('.')[-1], Json(translations)])
if not force_overwrite:
value_query = f"""CASE WHEN {overwrite} IS TRUE AND imd.noupdate IS FALSE
THEN m."{field_name}" || t.value
ELSE t.value || m."{field_name}"END"""
else:
value_query = f'm."{field_name}" || t.value'
env.cr.execute(f"""
UPDATE "{model_table}" AS m
SET "{field_name}" = {value_query}
FROM (
VALUES {', '.join(['(%s, %s::jsonb)'] * (len(params) // 2))}
) AS t(imd_name, value)
JOIN "ir_model_data" AS imd
ON imd."model" = '{model_name}' AND imd.name = t.imd_name
WHERE imd."res_id" = m."id"
""", params)
self.model_translations.clear()
env.invalidate_all()
if self.verbose:
_logger.info("translations are loaded successfully")
def trans_load(cr, filepath, lang, verbose=True, overwrite=False):
warnings.warn('The function trans_load is deprecated in favor of TranslationImporter', DeprecationWarning)
translation_importer = TranslationImporter(cr, verbose=verbose)
translation_importer.load_file(filepath, lang)
translation_importer.save(overwrite=overwrite)
def trans_load_data(cr, fileobj, fileformat, lang, verbose=True, overwrite=False):
warnings.warn('The function trans_load_data is deprecated in favor of TranslationImporter', DeprecationWarning)
translation_importer = TranslationImporter(cr, verbose=verbose)
translation_importer.load(fileobj, fileformat, lang)
translation_importer.save(overwrite=overwrite)
def get_locales(lang=None):