From e15b75efb4454102b19a33d4f6a6845784bc9be1 Mon Sep 17 00:00:00 2001 From: "Chong Wang (cwg)" Date: Mon, 9 Oct 2023 12:38:22 +0200 Subject: [PATCH] [IMP] core: export record translation Currently, bulk-importing translations for non-module loaded data is hard 1. PO file import works fine, but exporting a PO template for non-module loaded data is near impossible (since PO exports will only export entire modules) 2. Import of translated values during csv/excel file import is not supported This commit fix the issue by improve 1 which reuses the translation export wizard for modules to export translations for non-module records. So that user can export translations for selected records with a domain and import the po file after translating [DEBUG MODE] Settings -> Translations -> Export translations -> Export Type ("model") -> Select `Model to Export` and `Model Domain` -> Export The framework will 1. create external ids for records without external ids 2. export translations for stored translated and inherited translated fields closes odoo/odoo#138531 Task: 3463505 Signed-off-by: Raphael Collet --- .../base/wizard/base_export_language.py | 16 +- .../wizard/base_export_language_views.xml | 6 +- .../data/test_translation_import_data.xml | 3 + .../test_translation_import/models/models.py | 8 + .../security/ir.model.access.csv | 3 +- .../tests/test_term_count.py | 34 +++- odoo/tools/translate.py | 164 +++++++++++++----- 7 files changed, 182 insertions(+), 52 deletions(-) diff --git a/odoo/addons/base/wizard/base_export_language.py b/odoo/addons/base/wizard/base_export_language.py index d53a3a284e2..31fcd733638 100644 --- a/odoo/addons/base/wizard/base_export_language.py +++ b/odoo/addons/base/wizard/base_export_language.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import ast import base64 import contextlib import io @@ -23,8 +24,13 @@ class BaseLanguageExport(models.TransientModel): lang = fields.Selection(_get_languages, string='Language', required=True, default=NEW_LANG_KEY) format = fields.Selection([('csv','CSV File'), ('po','PO File'), ('tgz', 'TGZ Archive')], string='File Format', required=True, default='po') + export_type = fields.Selection([('module', 'Module'), ('model', 'Model')], + string='Export Type', required=True, default='module') modules = fields.Many2many('ir.module.module', 'rel_modules_langexport', 'wiz_id', 'module_id', string='Apps To Export', domain=[('state','=','installed')]) + model_id = fields.Many2one('ir.model', string='Model to Export', domain=[('transient', '=', False)]) + model_name = fields.Char(string="Model Name", related="model_id.model") + domain = fields.Char(string="Model Domain", default='[]') data = fields.Binary('File', readonly=True, attachment=False) state = fields.Selection([('choose', 'choose'), ('get', 'get')], # choose language or get the file default='choose') @@ -32,15 +38,21 @@ class BaseLanguageExport(models.TransientModel): def act_getfile(self): this = self[0] lang = this.lang if this.lang != NEW_LANG_KEY else False - mods = sorted(this.mapped('modules.name')) or ['all'] with contextlib.closing(io.BytesIO()) as buf: - tools.trans_export(lang, mods, buf, this.format, self._cr) + if this.export_type == 'model': + ids = self.env[this.model_name].search(ast.literal_eval(this.domain)).ids + tools.trans_export_records(lang, this.model_name, ids, buf, this.format, self._cr) + else: + mods = sorted(this.mapped('modules.name')) or ['all'] + tools.trans_export(lang, mods, buf, this.format, self._cr) out = base64.encodebytes(buf.getvalue()) filename = 'new' if lang: filename = tools.get_iso_codes(lang) + elif this.export_type == 'model': + filename = this.model_name.replace('.', '_') elif len(mods) == 1: filename = mods[0] extension = this.format diff --git a/odoo/addons/base/wizard/base_export_language_views.xml b/odoo/addons/base/wizard/base_export_language_views.xml index 98ef89dcf69..42f405bd6a9 100644 --- a/odoo/addons/base/wizard/base_export_language_views.xml +++ b/odoo/addons/base/wizard/base_export_language_views.xml @@ -11,7 +11,11 @@ - + + + + +

Export Complete

diff --git a/odoo/addons/test_translation_import/data/test_translation_import_data.xml b/odoo/addons/test_translation_import/data/test_translation_import_data.xml index 4e8678b376f..ca0a7e7eccf 100644 --- a/odoo/addons/test_translation_import/data/test_translation_import_data.xml +++ b/odoo/addons/test_translation_import/data/test_translation_import_data.xml @@ -4,4 +4,7 @@ Tableware
Knife
Spoon
 + + + diff --git a/odoo/addons/test_translation_import/models/models.py b/odoo/addons/test_translation_import/models/models.py index b0c7530a5dd..7218008f109 100644 --- a/odoo/addons/test_translation_import/models/models.py +++ b/odoo/addons/test_translation_import/models/models.py @@ -25,3 +25,11 @@ class TestTranslationImportModel1(models.Model): def get_code_named_placeholder_translation(self, *args, **kwargs): return _('Code, %(num)s, %(symbol)s, English', *args, **kwargs) + + +class TestTranslationImportModel2(models.Model): + _inherits = {'test.translation.import.model1': 'model1_id'} + _name = 'test.translation.import.model2' + _description = 'Translation Test 2' + + model1_id = fields.Many2one('test.translation.import.model1', required=True, ondelete='cascade') diff --git a/odoo/addons/test_translation_import/security/ir.model.access.csv b/odoo/addons/test_translation_import/security/ir.model.access.csv index 228691f2d88..1a6694da293 100644 --- a/odoo/addons/test_translation_import/security/ir.model.access.csv +++ b/odoo/addons/test_translation_import/security/ir.model.access.csv @@ -1,2 +1,3 @@ "id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" -"access_test_translation_import_model1","access.test.translation.import.model1","model_test_translation_import_model1",,0,0,0,0 \ No newline at end of file +"access_test_translation_import_model1","access.test.translation.import.model1","model_test_translation_import_model1",,0,0,0,0 +"access_test_translation_import_model2","access.test.translation.import.model2","model_test_translation_import_model2",,0,0,0,0 diff --git a/odoo/addons/test_translation_import/tests/test_term_count.py b/odoo/addons/test_translation_import/tests/test_term_count.py index cbfa00e68ea..41b66bb490b 100644 --- a/odoo/addons/test_translation_import/tests/test_term_count.py +++ b/odoo/addons/test_translation_import/tests/test_term_count.py @@ -5,7 +5,7 @@ import io from odoo.tests import common, tagged from odoo.tools.misc import file_open, mute_logger -from odoo.tools.translate import TranslationModuleReader, code_translations, CodeTranslations, PYTHON_TRANSLATION_COMMENT, JAVASCRIPT_TRANSLATION_COMMENT, WEB_TRANSLATION_COMMENT +from odoo.tools.translate import TranslationModuleReader, TranslationRecordReader, code_translations, CodeTranslations, PYTHON_TRANSLATION_COMMENT, JAVASCRIPT_TRANSLATION_COMMENT, WEB_TRANSLATION_COMMENT from odoo import Command from odoo.addons.base.models.ir_fields import BOOLEAN_TRANSLATIONS @@ -386,3 +386,35 @@ class TestTranslationFlow(common.TransactionCase): 'with spaces', 'hello \\"world\\"', }) + + def test_export_records(self): + self.env["base.language.install"].create({ + 'overwrite': True, + 'lang_ids': [(6, 0, [self.env.ref('base.lang_fr').id])], + }).lang_install() + + model1_ids = self.env.ref('test_translation_import.test_translation_import_model1_record1').ids + po_reader = TranslationRecordReader(self.env.cr, 'test.translation.import.model1', model1_ids, lang='fr_FR') + translations = {line[4]: line[5] for line in po_reader} + self.assertDictEqual( + translations, + { + 'Fork': 'Fourchette', + 'Knife': 'Couteau', + 'Spoon': 'Cuillère', + 'Tableware': 'Vaisselle', + } + ) + + model2_ids = self.env.ref('test_translation_import.test_translation_import_model2_record1').ids + po_reader = TranslationRecordReader(self.env.cr, 'test.translation.import.model2', model2_ids, lang='fr_FR') + translations = {line[4]: line[5] for line in po_reader} + self.assertDictEqual( + translations, + { + 'Fork': 'Fourchette', + 'Knife': 'Couteau', + 'Spoon': 'Cuillère', + 'Tableware': 'Vaisselle', + } + ) diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py index 745ec634bbc..5b3fc6e9a39 100644 --- a/odoo/tools/translate.py +++ b/odoo/tools/translate.py @@ -848,6 +848,13 @@ def trans_export(lang, modules, buffer, format, cr): writer = TranslationFileWriter(buffer, fileformat=format, lang=lang) writer.write_rows(reader) +# pylint: disable=redefined-builtin +def trans_export_records(lang, model_name, ids, buffer, format, cr): + reader = TranslationRecordReader(cr, model_name, ids, lang=lang) + writer = TranslationFileWriter(buffer, fileformat=format, lang=lang) + writer.write_rows(reader) + + def _push(callback, term, source_line): """ Sanity check before pushing translation terms """ term = (term or "").strip() @@ -974,33 +981,14 @@ def extract_spreadsheet_terms(fileobj, keywords, comment_tags, options): ImdInfo = namedtuple('ExternalId', ['name', 'model', 'res_id', 'module']) -class TranslationModuleReader: - """ Retrieve translated records per module - - :param cr: cursor to database to export - :param modules: list of modules to filter the exported terms, can be ['all'] - records with no external id are always ignored - :param lang: language code to retrieve the translations - retrieve source terms only if not set - """ - - def __init__(self, cr, modules=None, lang=None): +class TranslationReader: + def __init__(self, cr, lang=None): self._cr = cr - self._modules = modules or ['all'] self._lang = lang or 'en_US' self.env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {}) self._to_translate = [] - self._path_list = [(path, True) for path in odoo.addons.__path__] - self._installed_modules = [ - m['name'] - for m in self.env['ir.module.module'].search_read([('state', '=', 'installed')], fields=['name']) - ] - - self._export_translatable_records() - self._export_translatable_resources() def __iter__(self): - """ Export ir.translation values for all retrieved records """ for module, source, name, res_id, ttype, comments, _record_id, value in self._to_translate: yield (module, ttype, name, res_id, source, encode(odoo.tools.ustr(value)), comments) @@ -1021,6 +1009,35 @@ class TranslationModuleReader: return self._to_translate.append((module, source, name, res_id, ttype, tuple(comments or ()), record_id, value)) + def _export_imdinfo(self, model: str, imd_per_id: dict[int, ImdInfo]): + records = self._get_translatable_records(imd_per_id.values()) + if not records: + return + + for record in records: + module = imd_per_id[record.id].module + xml_name = "%s.%s" % (module, imd_per_id[record.id].name) + for field_name, field in record._fields.items(): + # ir_actions_actions.name is filtered because unlike other inherited fields, + # this field is inherited as postgresql inherited columns. + # From our business perspective, the parent column is no need to be translated, + # but it is need to be set to jsonb column, since the child columns need to be translated + # And export the parent field may make one value to be translated twice in transifex + if not field.translate or not field.store or str(field) == 'ir.actions.actions.name': + continue + name = model + "," + field_name + value_en = record[field_name] or '' + value_lang = record.with_context(lang=self._lang)[field_name] or '' + trans_type = 'model_terms' if callable(field.translate) else 'model' + try: + translation_dictionary = field.get_translation_dictionary(value_en, {self._lang: value_lang}) + except Exception: + _logger.exception("Failed to extract terms from %s %s", xml_name, name) + continue + for term_en, term_langs in translation_dictionary.items(): + term_lang = term_langs.get(self._lang) + self._push_translation(module, trans_type, name, xml_name, term_en, record_id=record.id, value=term_lang if term_lang != term_en else '') + def _get_translatable_records(self, imd_records): """ Filter the records that are translatable @@ -1072,6 +1089,84 @@ class TranslationModuleReader: return records +class TranslationRecordReader(TranslationReader): + """ Retrieve translations for specified records, the reader will + 1. create external ids for records without external ids + 2. export translations for stored translated and inherited translated fields + :param cr: cursor to database to export + :param model_name: model_name for the records to export + :param ids: ids of the records to export + :param field_names: field names to export, if not set, export all translatable fields + :param lang: language code to retrieve the translations retrieve source terms only if not set + """ + def __init__(self, cr, model_name, ids, field_names=None, lang=None): + super().__init__(cr, lang) + self._records = self.env[model_name].browse(ids) + self._field_names = field_names or list(self._records._fields.keys()) + + self._export_translatable_records(self._records, self._field_names) + + def _export_translatable_records(self, records, field_names): + """ Export translations of all stored/inherited translated fields. Create external id if needed. """ + if not records: + return + + fields = records._fields + + if records._inherits: + inherited_fields = defaultdict(list) + for field_name in field_names: + field = records._fields[field_name] + if field.translate and not field.store and field.inherited_field: + inherited_fields[field.inherited_field.model_name].append(field_name) + for parent_mname, parent_fname in records._inherits.items(): + if parent_mname in inherited_fields: + self._export_translatable_records(records[parent_fname], inherited_fields[parent_mname]) + + if not any(fields[field_name].translate and fields[field_name].store for field_name in field_names): + return + + records._BaseModel__ensure_xml_id() + + model_name = records._name + query = """SELECT min(concat(module, '.', name)), res_id + FROM ir_model_data + WHERE model = %s + AND res_id = ANY(%s) + GROUP BY model, res_id""" + + self._cr.execute(query, (model_name, records.ids)) + + imd_per_id = { + res_id: ImdInfo((tmp := module_xml_name.split('.', 1))[1], model_name, res_id, tmp[0]) + for module_xml_name, res_id in self._cr.fetchall() + } + + self._export_imdinfo(model_name, imd_per_id) + + +class TranslationModuleReader(TranslationReader): + """ Retrieve translated records per module + + :param cr: cursor to database to export + :param modules: list of modules to filter the exported terms, can be ['all'] + records with no external id are always ignored + :param lang: language code to retrieve the translations + retrieve source terms only if not set + """ + + def __init__(self, cr, modules=None, lang=None): + super().__init__(cr, lang) + self._modules = modules or ['all'] + self._path_list = [(path, True) for path in odoo.addons.__path__] + self._installed_modules = [ + m['name'] + for m in self.env['ir.module.module'].search_read([('state', '=', 'installed')], fields=['name']) + ] + + self._export_translatable_records() + self._export_translatable_resources() + def _export_translatable_records(self): """ Export translations of all translated records having an external id """ @@ -1093,31 +1188,7 @@ class TranslationModuleReader: records_per_model[model][res_id] = ImdInfo(xml_name, model, res_id, module) for model, imd_per_id in records_per_model.items(): - records = self._get_translatable_records(imd_per_id.values()) - if not records: - continue - - for record in records: - module = imd_per_id[record.id].module - xml_name = "%s.%s" % (module, imd_per_id[record.id].name) - for field_name, field in record._fields.items(): - # ir_actions_actions.name is filtered because unlike other inherited fields, - # this field is inherited as postgresql inherited columns. - # From our business perspective, the parent column is no need to be translated, - # but it is need to be set to jsonb column, since the child columns need to be translated - # And export the parent field may make one value to be translated twice in transifex - if field.translate and field.store and str(field) != 'ir.actions.actions.name': - name = model + "," + field_name - try: - value_en = record[field_name] or '' - value_lang = record.with_context(lang=self._lang)[field_name] or '' - except Exception: - _logger.exception("Failed to extract terms from %s", name) - continue - trans_type = 'model_terms' if callable(field.translate) else 'model' - for term_en, term_langs in field.get_translation_dictionary(value_en, {self._lang: value_lang}).items(): - term_lang = term_langs.get(self._lang) - self._push_translation(module, trans_type, name, xml_name, term_en, record_id=record.id, value=term_lang if term_lang != term_en else '') + self._export_imdinfo(model, imd_per_id) def _get_module_from_path(self, path): for (mp, rec) in self._path_list: @@ -1275,7 +1346,6 @@ class TranslationImporter: continue if row.get('type') == 'code': # ignore code translations continue - # TODO: CWG if the po file should not be trusted, we need to check each model term model_name = row.get('imd_model') module_name = row['module'] if model_name not in self.env: