From a0bd7d0d72e314bbcaf3a5a48524bc1434a009c4 Mon Sep 17 00:00:00 2001 From: "Chong Wang (cwg)" Date: Wed, 16 Nov 2022 18:41:31 +0000 Subject: [PATCH] [IMP] core: improve translation import speed this commit improves the performance of importing translations by 1. groupup udpating data for different languages and different modules 2. read model_terms translations data directly with xmlid to get rid of fetching data from ir_model_data for modules:sale_management, point_of_sale, industry_fsm, helpdesk, mrp, mrp_plm (62 installed modules) time for test_language_install is 45.14% less for all modules: time for test_language_install is 68.03% less closes odoo/odoo#107332 X-original-commit: 727ef9e0a2ca93089b472eb1313117fee8dc876d Signed-off-by: Raphael Collet --- addons/mail/models/template_reset_mixin.py | 26 +- odoo/addons/base/models/ir_module.py | 21 +- odoo/addons/base/tests/test_translate.py | 16 +- .../base/wizard/base_import_language.py | 56 ++- .../tests/test_related_translation.py | 6 +- .../tests/test_term_count.py | 17 +- odoo/cli/server.py | 6 +- odoo/tools/translate.py | 337 ++++++++++-------- 8 files changed, 245 insertions(+), 240 deletions(-) diff --git a/addons/mail/models/template_reset_mixin.py b/addons/mail/models/template_reset_mixin.py index 9b7b59fce3d..073271c31f6 100644 --- a/addons/mail/models/template_reset_mixin.py +++ b/addons/mail/models/template_reset_mixin.py @@ -11,7 +11,7 @@ from odoo.modules import get_module_resource from odoo.modules.module import get_resource_from_path, get_resource_path from odoo.tools.convert import xml_import from odoo.tools.misc import file_open -from odoo.tools.translate import TranslationFileReader, _trans_load_data +from odoo.tools.translate import TranslationImporter class TemplateResetMixin(models.AbstractModel): @@ -61,7 +61,8 @@ class TemplateResetMixin(models.AbstractModel): # ------------------------------------------------------------------------- def _override_translation_term(self, module_name, xml_ids): - processed_base_langs = [] + translation_importer = TranslationImporter(self.env.cr) + for code, _ in self.env['res.lang'].get_installed(): lang_code = tools.get_iso_codes(code) # In case of sub languages (e.g fr_BE), load the base language first, (e.g fr.po) and @@ -72,30 +73,15 @@ class TemplateResetMixin(models.AbstractModel): base_lang_code = lang_code.split('_')[0] base_trans_file = get_module_resource(module_name, 'i18n', base_lang_code + '.po') if base_trans_file: - if base_lang_code not in processed_base_langs: - processed_base_langs.append(base_lang_code) - self._process_translation_data(base_trans_file, code, xml_ids) + translation_importer.load_file(base_trans_file, code, xmlids=xml_ids) # Step 2: reset translation file with main language file (can possibly override the # terms coming from the base language) trans_file = get_module_resource(module_name, 'i18n', lang_code + '.po') if trans_file: - self._process_translation_data(trans_file, code, xml_ids) + translation_importer.load_file(trans_file, code, xmlids=xml_ids) - - def _process_translation_data(self, trans_file, lang, xml_ids): - """Load translations. - - :param trans_file: path to a translation file to open, e.g. {addon_path}/mail/i18n/es.po - :param lang: language code of the translations contained in `trans_file` - language must be present and activated in the database - - :return Boolean: True if translation data is available and processed, False otherwise - """ - with file_open(trans_file, mode='rb', filter_ext=(".po",)) as fileobj: - reader = TranslationFileReader(fileobj) - # Process a single PO entry - _trans_load_data(self.env.cr, reader, lang, overwrite=True, force_overwrite=True, xml_ids=xml_ids) + translation_importer.save(overwrite=True, force_overwrite=True) def reset_template(self): """Resets the Template with values given in source file. We ignore the case of diff --git a/odoo/addons/base/models/ir_module.py b/odoo/addons/base/models/ir_module.py index b3827571531..f7520b669cd 100644 --- a/odoo/addons/base/models/ir_module.py +++ b/odoo/addons/base/models/ir_module.py @@ -31,6 +31,7 @@ from odoo.exceptions import AccessDenied, UserError from odoo.osv import expression from odoo.tools.parse_version import parse_version from odoo.tools.misc import topological_sort +from odoo.tools.translate import TranslationImporter from odoo.http import request from odoo.modules import get_module_path, get_module_resource @@ -932,13 +933,14 @@ class Module(models.Model): def _load_module_terms(self, modules, langs, overwrite=False): """ Load PO files of the given modules for the given languages. """ # load i18n files + translation_importer = TranslationImporter(self.env.cr, verbose=False) + for module_name in modules: modpath = get_module_path(module_name) if not modpath: continue for lang in langs: lang_code = tools.get_iso_codes(lang) - lang_overwrite = overwrite base_lang_code = None if '_' in lang_code: base_lang_code = lang_code.split('_')[0] @@ -948,29 +950,28 @@ class Module(models.Model): base_trans_file = get_module_resource(module_name, 'i18n', base_lang_code + '.po') if base_trans_file: _logger.info('module %s: loading base translation file %s for language %s', module_name, base_lang_code, lang) - tools.trans_load(self._cr, base_trans_file, lang, verbose=False, overwrite=lang_overwrite) - lang_overwrite = True # make sure the requested translation will override the base terms later + translation_importer.load_file(base_trans_file, lang) # i18n_extra folder is for additional translations handle manually (eg: for l10n_be) base_trans_extra_file = get_module_resource(module_name, 'i18n_extra', base_lang_code + '.po') if base_trans_extra_file: _logger.info('module %s: loading extra base translation file %s for language %s', module_name, base_lang_code, lang) - tools.trans_load(self._cr, base_trans_extra_file, lang, verbose=False, overwrite=lang_overwrite) - lang_overwrite = True # make sure the requested translation will override the base terms later + translation_importer.load_file(base_trans_extra_file, lang) # Step 2: then load the main translation file, possibly overriding the terms coming from the base language trans_file = get_module_resource(module_name, 'i18n', lang_code + '.po') if trans_file: - _logger.info('module %s: loading translation file (%s) for language %s', module_name, lang_code, lang) - tools.trans_load(self._cr, trans_file, lang, verbose=False, overwrite=lang_overwrite) + _logger.info('module %s: loading translation file %s for language %s', module_name, lang_code, lang) + translation_importer.load_file(trans_file, lang) elif lang_code != 'en_US': _logger.info('module %s: no translation for language %s', module_name, lang_code) trans_extra_file = get_module_resource(module_name, 'i18n_extra', lang_code + '.po') if trans_extra_file: - _logger.info('module %s: loading extra translation file (%s) for language %s', module_name, lang_code, lang) - tools.trans_load(self._cr, trans_extra_file, lang, verbose=False, overwrite=lang_overwrite) - return True + _logger.info('module %s: loading extra translation file %s for language %s', module_name, lang_code, lang) + translation_importer.load_file(trans_extra_file, lang) + + translation_importer.save(overwrite=overwrite) DEP_STATES = STATES + [('unknown', 'Unknown')] diff --git a/odoo/addons/base/tests/test_translate.py b/odoo/addons/base/tests/test_translate.py index 4ceeb8f7efd..1bd1c96092a 100644 --- a/odoo/addons/base/tests/test_translate.py +++ b/odoo/addons/base/tests/test_translate.py @@ -9,8 +9,8 @@ from psycopg2.extras import Json import io from odoo.exceptions import AccessError, ValidationError -from odoo.tools import trans_load_data, sql -from odoo.tools.translate import quote, unquote, xml_translate, html_translate +from odoo.tools import sql +from odoo.tools.translate import quote, unquote, xml_translate, html_translate, TranslationImporter from odoo.tests.common import TransactionCase, BaseCase, new_test_user, tagged _stats_logger = logging.getLogger('odoo.tests.stats') @@ -295,7 +295,9 @@ class TestTranslation(TransactionCase): ''' % cls.customers_xml_id with io.BytesIO(bytes(po_string, encoding='utf-8')) as f: f.name = 'dummy' - trans_load_data(cls.env.cr, f, 'po', 'fr_FR', verbose=True, overwrite=True) + translation_importer = TranslationImporter(cls.env.cr, verbose=True) + translation_importer.load(f, 'po', 'fr_FR') + translation_importer.save(overwrite=True) def test_101_create_translated_record(self): category = self.customers.with_context({}) @@ -465,7 +467,9 @@ class TestTranslationWrite(TransactionCase): ''' % self.category_xml_id with io.BytesIO(bytes(po_string, encoding='utf-8')) as f: f.name = 'dummy' - trans_load_data(self.env.cr, f, 'po', 'en_US', verbose=True, overwrite=True) + translation_importer = TranslationImporter(self.env.cr, verbose=True) + translation_importer.load(f, 'po', 'fr_FR') + translation_importer.save(overwrite=True) self.category.with_context(lang='fr_FR').write({'name': 'French Name'}) self.category.with_context(lang='en_US').write({'name': 'English Name'}) @@ -651,7 +655,9 @@ class TestTranslationWrite(TransactionCase): ''' % (ir_model_field_xml_id, LABEL) with io.BytesIO(bytes(po_string, encoding='utf-8')) as f: f.name = 'dummy' - trans_load_data(self.env.cr, f, 'po', 'fr_FR', verbose=True, overwrite=True) + translation_importer = TranslationImporter(self.env.cr, verbose=True) + translation_importer.load(f, 'po', 'fr_FR') + translation_importer.save(overwrite=True) # check that fields_get() returns the expected label model = self.env['ir.model'].with_context(lang='fr_FR') diff --git a/odoo/addons/base/wizard/base_import_language.py b/odoo/addons/base/wizard/base_import_language.py index ae31b34447b..cb98d60f0c9 100644 --- a/odoo/addons/base/wizard/base_import_language.py +++ b/odoo/addons/base/wizard/base_import_language.py @@ -3,13 +3,12 @@ import base64 import logging -import os from tempfile import TemporaryFile -from psycopg2 import ProgrammingError -from contextlib import closing +from os.path import splitext from odoo import api, fields, models, tools, sql_db, _ from odoo.exceptions import UserError +from odoo.tools.translate import TranslationImporter _logger = logging.getLogger(__name__) @@ -29,36 +28,21 @@ class BaseLanguageImport(models.TransientModel): "will be overwritten and replaced by those in this file") def import_lang(self): - this = self[0] - with TemporaryFile('wb+') as buf: - try: - buf.write(base64.decodebytes(this.data)) - - # now we determine the file format - buf.seek(0) - fileformat = os.path.splitext(this.filename)[-1][1:].lower() - - Lang = self.env["res.lang"] - lang = Lang._activate_lang(self.code) or Lang._create_lang( - self.code, lang_name=self.name - ) - - tools.trans_load_data( - this._cr, buf, fileformat, this.code, overwrite=self.overwrite - ) - except ProgrammingError as e: - _logger.exception('File unsuccessfully imported, due to a malformed file.') - - with closing(sql_db.db_connect(self._cr.dbname).cursor()) as cr: - raise UserError(_('File %r not imported due to a malformed file.\n\n' - 'This issue can be caused by duplicates entries who are referring to the same field. ' - 'Please check the content of the file you are trying to import.\n\n' - 'Technical Details:\n%s') % (self.filename, tools.ustr(e))) - except Exception as e: - _logger.exception('File unsuccessfully imported, due to format mismatch.') - raise UserError( - _('File %r not imported due to format mismatch or a malformed file.' - ' (Valid formats are .csv, .po, .pot)\n\nTechnical Details:\n%s') % \ - (this.filename, tools.ustr(e)) - ) - return True + self.ensure_one() + Lang = self.env["res.lang"] + try: + if not Lang._activate_lang(self.code): + Lang._create_lang(self.code, lang_name=self.name) + with TemporaryFile('wb+') as buf: + buf.write(base64.decodebytes(self.data)) + fileformat = splitext(self.filename)[-1][1:].lower() + translation_importer = TranslationImporter(self.env.cr) + translation_importer.load(buf, fileformat, self.code) + translation_importer.save(overwrite=self.overwrite) + except Exception as e: + _logger.exception('File unsuccessfully imported, due to format mismatch.') + raise UserError( + _('File %r not imported due to format mismatch or a malformed file.' + ' (Valid formats are .csv, .po, .pot)\n\nTechnical Details:\n%s') % \ + (self.filename, tools.ustr(e)) + ) diff --git a/odoo/addons/test_new_api/tests/test_related_translation.py b/odoo/addons/test_new_api/tests/test_related_translation.py index 64c67e875c8..2dd2c91432a 100644 --- a/odoo/addons/test_new_api/tests/test_related_translation.py +++ b/odoo/addons/test_new_api/tests/test_related_translation.py @@ -2,7 +2,7 @@ import io import odoo.tests -from odoo.tools import trans_load_data +from odoo.tools.translate import TranslationImporter @odoo.tests.tagged('post_install', '-at_install') @@ -126,7 +126,9 @@ class TestRelatedTranslation(odoo.tests.TransactionCase): ''' % test1_xml_id with io.BytesIO(bytes(po_string, encoding='utf-8')) as f: f.name = 'dummy' - trans_load_data(self.test1.env.cr, f, 'po', 'fr_FR', verbose=True, overwrite=True) + translation_importer = TranslationImporter(self.env.cr, verbose=True) + translation_importer.load(f, 'po', 'fr_FR') + translation_importer.save(overwrite=True) self.assertEqual(self.test2.with_context(lang='fr_FR').name, 'Nouveau couteau') def test_translate_from_ori_term(self): diff --git a/odoo/addons/test_translation_import/tests/test_term_count.py b/odoo/addons/test_translation_import/tests/test_term_count.py index 200ff02528e..cbfa00e68ea 100644 --- a/odoo/addons/test_translation_import/tests/test_term_count.py +++ b/odoo/addons/test_translation_import/tests/test_term_count.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- -from contextlib import closing import base64 import io -import odoo from odoo.tests import common, tagged from odoo.tools.misc import file_open, mute_logger from odoo.tools.translate import TranslationModuleReader, code_translations, CodeTranslations, PYTHON_TRANSLATION_COMMENT, JAVASCRIPT_TRANSLATION_COMMENT, WEB_TRANSLATION_COMMENT @@ -18,8 +16,6 @@ class TestImport(common.TransactionCase): self.env['res.lang']._activate_lang('fr_FR') # Tip: code translations don't need to be imported explicitly - # odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False) - model = self.env['test.translation.import.model1'] self.assertEqual( model.with_context(lang='fr_FR').get_code_translation(), @@ -28,7 +24,7 @@ class TestImport(common.TransactionCase): def test_import_model_translation(self): self.env['res.lang']._activate_lang('fr_FR') - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False) + self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR']) record = self.env.ref('test_translation_import.test_translation_import_model1_record1') self.assertEqual( @@ -38,7 +34,7 @@ class TestImport(common.TransactionCase): def test_import_model_term_translation(self): self.env['res.lang']._activate_lang('fr_FR') - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False) + self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR']) record = self.env.ref('test_translation_import.test_translation_import_model1_record1') self.assertEqual( @@ -54,11 +50,11 @@ class TestImport(common.TransactionCase): self.assertEqual(menu.name, 'Test translation model1') # install french and change translation content self.env['res.lang']._activate_lang('fr_FR') - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False) + self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR']) self.assertEqual(menu.with_context(lang='fr_FR').name, "Test translation import in french") menu.with_context(lang='fr_FR').name = "Nouveau nom" # reload with overwrite - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False, overwrite=True) + self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_FR'], overwrite=True) self.assertEqual(menu.name, "Test translation model1") self.assertEqual(menu.with_context(lang='fr_FR').name, "Nouveau nom") @@ -66,10 +62,7 @@ class TestImport(common.TransactionCase): def test_lang_with_base(self): self.env['res.lang']._activate_lang('fr_BE') self.env['res.lang']._activate_lang('fr_CA') - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_BE', verbose=False) - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr_BE.po', 'fr_BE', verbose=False, overwrite=True) - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_CA', verbose=False) - odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr_CA.po', 'fr_CA', verbose=False, overwrite=True) + self.env['ir.module.module']._load_module_terms(['test_translation_import'], ['fr_BE', 'fr_CA'], overwrite=True) # language override base language record = self.env.ref('test_translation_import.test_translation_import_model1_record1') diff --git a/odoo/cli/server.py b/odoo/cli/server.py index 8b80505cb6c..e3871f52044 100644 --- a/odoo/cli/server.py +++ b/odoo/cli/server.py @@ -118,9 +118,9 @@ def import_translation(): registry = odoo.modules.registry.Registry.new(dbname) with registry.cursor() as cr: - odoo.tools.trans_load( - cr, config["translate_in"], config["language"], overwrite=overwrite, - ) + translation_importer = odoo.tools.translate.TranslationImporter(cr) + translation_importer.load_file(config["translate_in"], config["language"]) + translation_importer.save(overwrite=overwrite) def main(args): check_root_user() diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py index 1162e42ebd6..e4dff4d836a 100644 --- a/odoo/tools/translate.py +++ b/odoo/tools/translate.py @@ -14,8 +14,8 @@ from tokenize import generate_tokens, STRING, NEWLINE, INDENT, DEDENT import polib import re import tarfile -import tempfile import threading +import warnings from collections import defaultdict, namedtuple from datetime import datetime from os.path import join @@ -1094,7 +1094,7 @@ class TranslationModuleReader: # From our business perspective, the parent column is no need to be translated, # but it is need to be set to jsonb column, since the child columns need to be translated # And export the parent field may make one value to be translated twice in transifex - if field.translate and not (field.model_name == 'ir.actions.actions' and field.name == 'name'): + if field.translate and field.store and str(field) != 'ir.actions.actions.name': name = model + "," + field_name try: value_en = record[field_name] or '' @@ -1198,176 +1198,209 @@ class TranslationModuleReader: break -def trans_load(cr, filename, lang, verbose=True, overwrite=False): - try: - with file_open(filename, mode='rb') as fileobj: - _logger.info("loading %s", filename) - fileformat = os.path.splitext(filename)[-1][1:].lower() - return trans_load_data(cr, fileobj, fileformat, lang, - verbose=verbose, - overwrite=overwrite) - except IOError: - if verbose: - _logger.error("couldn't read translation file %s", filename) - return None +def DeepDefaultDict(): + return defaultdict(DeepDefaultDict) -def trans_load_data(cr, fileobj, fileformat, lang, - verbose=True, overwrite=False): - """Load translations. - :param cr: - :param fileobj: buffer open to a translation file - :param fileformat: format of the `fielobj` file, one of 'po' or 'csv' - :param lang: language code of the translations contained in `fileobj` - language must be present and activated in the database - :param verbose: increase log output - :param overwrite: if a translation already exists for a term, replace it with - the one in `fileobj` +class TranslationImporter: + """ Helper object for importing translation files to a database. + This class provides a convenient API to load the translations from many + files and import them all at once, which helps speeding up the whole import. """ - if verbose: - _logger.info('loading translation file for language %s', lang) - env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {'lang': lang}) - if not env['res.lang']._lang_get(lang): - _logger.error("Couldn't read translation for lang '%s', language not found", lang) - return None - try: - fileobj.seek(0) - reader = TranslationFileReader(fileobj, fileformat=fileformat) - _trans_load_data(cr, reader, lang, overwrite) - if verbose: - _logger.info("translation file loaded successfully") + def __init__(self, cr, verbose=True): + self.cr = cr + self.verbose = verbose + self.env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {}) - except IOError: - iso_lang = get_iso_codes(lang) - filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat) - _logger.exception("couldn't read translation file %s", filename) + # {model_name: {field_name: {xmlid: {lang: value}}}} + self.model_translations = DeepDefaultDict() + # {model_name: {field_name: {xmlid: {src: {lang: value}}}}} + self.model_terms_translations = DeepDefaultDict() + def load_file(self, filepath, lang, xmlids=None): + """ Load translations from the given file path. -def _trans_load_data(cr, reader, lang, overwrite=False, force_overwrite=False, xml_ids=None): - # logical overwrite: if a translation should be updated when it has already been in the DB before - # logical overwrite = force_overwrite or (not noudpate and overwrite) - env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {'lang': lang}) - if force_overwrite: - overwrite = True + :param filepath: file path to open + :param lang: language code of the translations contained in the file; + the language must be present and activated in the database + :param xmlids: if given, only translations for records with xmlid in xmlids will be loaded + """ + with file_open(filepath, mode='rb') as fileobj: + fileformat = os.path.splitext(filepath)[-1][1:].lower() + self.load(fileobj, fileformat, lang, xmlids=xmlids) - # load all translation in to dict - deep_defaultdict = lambda: defaultdict(deep_defaultdict) - # {model_name: {xmlid: {field_name, {src: value}}}} - translations = deep_defaultdict() - # [(module_name, imd_name), (module_name, imd_name), ...] only for model_terms translation - module_imd_names = [] - # {model_name: {field_name: [xmlid, xmlid, ...]}} - model_name_to_field_name_to_xmlids = defaultdict(lambda: defaultdict(list)) + def load(self, fileobj, fileformat, lang, xmlids=None): + """Load translations from the given file object. - for row in reader: - if not row.get('value') or not row.get('src'): # ignore empty translations - continue - if row.get('type') == 'code': # ignore code translations - continue - # TODO: CWG if the po file should not be trusted, we need to check each model term - model_name = row.get('imd_model') - module_name = row['module'] - imd_name = row['imd_name'] - if model_name in env: + :param fileobj: buffer open to a translation file + :param fileformat: format of the `fielobj` file, one of 'po' or 'csv' + :param lang: language code of the translations contained in `fileobj`; + the language must be present and activated in the database + :param xmlids: if given, only translations for records with xmlid in xmlids will be loaded + """ + if self.verbose: + _logger.info('loading translation file for language %s', lang) + if not self.env['res.lang']._lang_get(lang): + _logger.error("Couldn't read translation for lang '%s', language not found", lang) + return None + try: + fileobj.seek(0) + reader = TranslationFileReader(fileobj, fileformat=fileformat) + self._load(reader, lang, xmlids) + except IOError: + iso_lang = get_iso_codes(self.lang) + filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat) + _logger.exception("couldn't read translation file %s", filename) + + def _load(self, reader, lang, xmlids=None): + if xmlids and not isinstance(xmlids, set): + xmlids = set(xmlids) + for row in reader: + if not row.get('value') or not row.get('src'): # ignore empty translations + continue + if row.get('type') == 'code': # ignore code translations + continue + # TODO: CWG if the po file should not be trusted, we need to check each model term + model_name = row.get('imd_model') + module_name = row['module'] + if model_name not in self.env: + continue field_name = row['name'].split(',')[1] - field = env[model_name]._fields.get(field_name) + field = self.env[model_name]._fields.get(field_name) if not field or not field.translate or not field.store: continue xmlid = module_name + '.' + row['imd_name'] - if xml_ids and xmlid not in xml_ids: + if xmlids and xmlid not in xmlids: continue - if row['type'] == 'model_terms': - module_imd_names.append((module_name, imd_name)) - model_name_to_field_name_to_xmlids[model_name][field_name].append(xmlid) - translations[model_name][xmlid][field_name][row['src']] = row['value'] + if row.get('type') == 'model': + self.model_translations[model_name][field_name][xmlid][lang] = row['value'] + elif row.get('type') == 'model_terms': + self.model_terms_translations[model_name][field_name][xmlid][row['src']][lang] = row['value'] - if not translations: - return - env.flush_all() + def save(self, overwrite=False, force_overwrite=False): + """ Save translations to the database. - # get mapping xmlid_to_id, id_to_xmlid - xmlid_to_id = {} - for sub_module_imd_names in cr.split_for_in_conditions(module_imd_names): - query = "SELECT module || '.' || name, res_id, noupdate, model FROM ir_model_data WHERE " - query += " OR ".join(["module = %s AND name = %s"] * len(sub_module_imd_names)) - cr.execute(query, [param for params in sub_module_imd_names for param in params]) - # {xmlid: (id, noupdate, model_name)} - xmlid_to_id.update(( - (xmlid, (id_, not force_overwrite and noupdate, model_name)) - for xmlid, id_, noupdate, model_name in cr.fetchall() - )) - id_to_xmlid = { - (model_name, id_): (xmlid, noupdate) - for xmlid, (id_, noupdate, model_name) in xmlid_to_id.items() - } + For a record with 'noupdate' in ``ir_model_data``, its existing translations + will be overwritten if ``force_overwrite or (not noupdate and overwrite)``. - # translate model terms - for model_name, model_dictionary in translations.items(): - if model_name not in env: - continue - Model = env[model_name] - model_table = Model._table - field_name_to_xmlids = model_name_to_field_name_to_xmlids[model_name] - fields = Model._fields - field_type = defaultdict(list) - for field_name in field_name_to_xmlids.keys(): - translate = fields[field_name].translate - field_type['no_translate' if not translate else 'model_terms' if callable(translate) else 'model'].append(field_name) - if field_type['no_translate']: - pass # ignore illegal translations - for field_name in field_type['model_terms']: - record_ids = { - it[0] - for it in map(xmlid_to_id.get, field_name_to_xmlids[field_name]) - if it - } - field = fields[field_name] - for sub_ids in cr.split_for_in_conditions(record_ids): - cr.execute(f'SELECT id, "{field_name}" FROM "{model_table}" WHERE id IN %s', (sub_ids,)) - for id_, values in cr.fetchall(): - if not values: - continue - value_en = values.get('en_US') - if not value_en: - continue - value_lang = values.get(lang, value_en) - xmlid, noupdate = id_to_xmlid[(model_name, id_)] - field_dictionary = {k: v for k, v in model_dictionary[xmlid][field_name].items()} # len(field.get_trans_terms(v)) == 1 - translation_dictionary = field.get_translation_dictionary(value_en, {lang: value_lang}) + An existing translation means: + * model translation: the ``jsonb`` value in database has the language code as key; + * model terms translation: the term value in the language is different from the term value in ``en_US``. + """ + if not self.model_translations and not self.model_terms_translations: + return - # update translation_dictionary using new translations - for term_en, term in translation_dictionary.items(): - if not overwrite and noupdate and term[lang] != term_en: + cr = self.cr + env = self.env + env.flush_all() + + for model_name, model_dictionary in self.model_terms_translations.items(): + Model = env[model_name] + model_table = Model._table + fields = Model._fields + # field_name, {xmlid: {src: {lang: value}}} + for field_name, field_dictionary in model_dictionary.items(): + field = fields.get(field_name) + for sub_xmlids in cr.split_for_in_conditions(field_dictionary.keys()): + # [module_name, imd_name, module_name, imd_name, ...] + params = [] + for xmlid in sub_xmlids: + params.extend(xmlid.split('.')) + cr.execute(f''' + SELECT m.id, imd.module || '.' || imd.name, m."{field_name}", imd.noupdate + FROM "{model_table}" m, "ir_model_data" imd + WHERE m.id = imd.res_id + AND ({" OR ".join(["(imd.module = %s AND imd.name = %s)"] * (len(params) // 2))}) + ''', params) + + # [id, translations, id, translations, ...] + params = [] + for id_, xmlid, values, noupdate in cr.fetchall(): + if not values: + continue + value_en = values.get('en_US') + if not value_en: continue - term[lang] = field_dictionary.get(term_en, term[lang]) - new_value_lang = field.translate(lambda term: translation_dictionary.get(term, {}).get(lang), value_en) - values[lang] = new_value_lang - env.cache.update_raw(Model.browse(id_), field, [values], dirty=True) - for field_name in field_type['model']: - field_translations = [] - for xmlid in field_name_to_xmlids[field_name]: - src, value = list(model_dictionary[xmlid][field_name].items())[0] - field_translations.extend([xmlid.split('.')[-1], src, Json({lang: value})]) - if field_translations: - if not force_overwrite: - value_query = f"""CASE WHEN {overwrite} is True AND imd.noupdate is False - THEN m."{field_name}" || t.value - ELSE t.value || m."{field_name}"END""" - else: - value_query = f'm."{field_name}" || t.value' - env.cr.execute(f""" - UPDATE "{model_table}" AS m - SET "{field_name}" = {value_query} - FROM ( - VALUES {', '.join(['(%s, %s, %s::jsonb)'] * int(len(field_translations) / 3))} - ) AS t(imd_name, src, value) - JOIN "ir_model_data" AS imd - ON imd."model" = '{model_name}' AND imd.name = t.imd_name - WHERE imd."res_id" = m."id" AND m."{field_name}"->>'en_US' = t.src - """, field_translations) - env.invalidate_all() + # {src: {lang: value}} + record_dictionary = field_dictionary[xmlid] + langs = {lang for translations in record_dictionary.values() for lang in translations.keys()} + translation_dictionary = field.get_translation_dictionary( + value_en, + {k: v for k, v in values.items() if k in langs} + ) + + if force_overwrite or (not noupdate and overwrite): + # overwrite existing translations + for term_en, translations in record_dictionary.items(): + translation_dictionary[term_en].update(translations) + else: + # keep existing translations + for term_en, translations in record_dictionary.items(): + translations.update({k: v for k, v in translation_dictionary[term_en].items() if v != term_en}) + translation_dictionary[term_en] = translations + + for lang in langs: + values[lang] = field.translate(lambda term: translation_dictionary.get(term, {}).get(lang), value_en) + params.extend((id_, Json(values))) + if params: + env.cr.execute(f""" + UPDATE "{model_table}" AS m + SET "{field_name}" = t.value + FROM ( + VALUES {', '.join(['(%s, %s::jsonb)'] * (len(params) // 2))} + ) AS t(id, value) + WHERE m.id = t.id + """, params) + + self.model_terms_translations.clear() + + for model_name, model_dictionary in self.model_translations.items(): + Model = env[model_name] + model_table = Model._table + for field_name, field_dictionary in model_dictionary.items(): + for sub_field_dictionary in cr.split_for_in_conditions(field_dictionary.items()): + # [xmlid, translations, xmlid, translations, ...] + params = [] + for xmlid, translations in sub_field_dictionary: + params.extend([xmlid.split('.')[-1], Json(translations)]) + if not force_overwrite: + value_query = f"""CASE WHEN {overwrite} IS TRUE AND imd.noupdate IS FALSE + THEN m."{field_name}" || t.value + ELSE t.value || m."{field_name}"END""" + else: + value_query = f'm."{field_name}" || t.value' + env.cr.execute(f""" + UPDATE "{model_table}" AS m + SET "{field_name}" = {value_query} + FROM ( + VALUES {', '.join(['(%s, %s::jsonb)'] * (len(params) // 2))} + ) AS t(imd_name, value) + JOIN "ir_model_data" AS imd + ON imd."model" = '{model_name}' AND imd.name = t.imd_name + WHERE imd."res_id" = m."id" + """, params) + + self.model_translations.clear() + + env.invalidate_all() + if self.verbose: + _logger.info("translations are loaded successfully") + + +def trans_load(cr, filepath, lang, verbose=True, overwrite=False): + warnings.warn('The function trans_load is deprecated in favor of TranslationImporter', DeprecationWarning) + translation_importer = TranslationImporter(cr, verbose=verbose) + translation_importer.load_file(filepath, lang) + translation_importer.save(overwrite=overwrite) + + +def trans_load_data(cr, fileobj, fileformat, lang, verbose=True, overwrite=False): + warnings.warn('The function trans_load_data is deprecated in favor of TranslationImporter', DeprecationWarning) + translation_importer = TranslationImporter(cr, verbose=verbose) + translation_importer.load(fileobj, fileformat, lang) + translation_importer.save(overwrite=overwrite) def get_locales(lang=None):