From c142f1628d4e7b13a7de5a8cf0790d7425badc05 Mon Sep 17 00:00:00 2001 From: Martin Trigaux Date: Mon, 11 May 2020 12:46:42 +0000 Subject: [PATCH] [FIX] tools: reimport translations in csv Before this commit trying to reimport csv of a translation failed. The reason was that before 632fa044c3c20cd the parsing was common between po and csv while now it's split in two different parsers. The res_id column of the CSV is the external id of the record and is handled by IrTranslationImport. Use DictReader instead of csv_reader to easily add new keys and still be flexible on the given csv files. Match the POReader format with a imd_model and imd_name column. As for the PoFileReader, the code translations are unique and must be discarded in case of duplicate. Correct the error message if the imported file is not correct (was missing an argument) Fixes odoo/odoo#50975 closes odoo/odoo#51468 X-original-commit: 3cedadb5ba606e156f52e70ed08b25ac2df8d58b Signed-off-by: Martin Trigaux (mat) --- .../base/wizard/base_import_language.py | 2 +- .../tests/test_term_count.py | 29 +++++++++++++++++++ odoo/tools/translate.py | 25 +++++++++++++--- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/odoo/addons/base/wizard/base_import_language.py b/odoo/addons/base/wizard/base_import_language.py index be2da88318b..ae31b34447b 100644 --- a/odoo/addons/base/wizard/base_import_language.py +++ b/odoo/addons/base/wizard/base_import_language.py @@ -53,7 +53,7 @@ class BaseLanguageImport(models.TransientModel): raise UserError(_('File %r not imported due to a malformed file.\n\n' 'This issue can be caused by duplicates entries who are referring to the same field. ' 'Please check the content of the file you are trying to import.\n\n' - 'Technical Details:\n%s') % tools.ustr(e)) + 'Technical Details:\n%s') % (self.filename, tools.ustr(e))) except Exception as e: _logger.exception('File unsuccessfully imported, due to format mismatch.') raise UserError( diff --git a/odoo/addons/test_translation_import/tests/test_term_count.py b/odoo/addons/test_translation_import/tests/test_term_count.py index 9815e95a260..a417436c39f 100644 --- a/odoo/addons/test_translation_import/tests/test_term_count.py +++ b/odoo/addons/test_translation_import/tests/test_term_count.py @@ -320,6 +320,35 @@ class TestTranslationFlow(common.TransactionCase): self.assertEqual(init_translation_count, len(import_translation)) + def test_export_import_csv(self): + """ Ensure can reimport exported csv """ + self.env.ref("base.lang_fr").active = True + + module = self.env.ref('base.module_test_translation_import') + export = self.env["base.language.export"].create({ + 'lang': 'fr_FR', + 'format': 'csv', + 'modules': [(6, 0, [module.id])] + }) + export.act_getfile() + po_file = export.data + self.assertIsNotNone(po_file) + + self.env["ir.translation"].search([ + ('lang', '=', 'fr_FR'), + ('module', '=', 'test_translation_import') + ]).unlink() + + import_fr = self.env["base.language.import"].create({ + 'name': 'French', + 'code': 'fr_FR', + 'data': export.data, + 'filename': export.name, + 'overwrite': False, + }) + with mute_logger('odoo.addons.base.models.res_lang'): + import_fr.with_context(create_empty_translation=True).import_lang() + def test_export_static_templates(self): trans_static = [] po_reader = TranslationModuleReader(self.env.cr, ['test_translation_import']) diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py index ed3b3e66bd1..95524718f35 100644 --- a/odoo/tools/translate.py +++ b/odoo/tools/translate.py @@ -549,13 +549,30 @@ def TranslationFileReader(source, fileformat='po'): class CSVFileReader: def __init__(self, source): - self.source = pycompat.csv_reader(source, quotechar='"', delimiter=',') - # read the first line of the file (it contains columns titles) - self.fields = next(self.source) + _reader = codecs.getreader('utf-8') + self.source = csv.DictReader(_reader(source), quotechar='"', delimiter=',') + self.prev_code_src = "" def __iter__(self): for entry in self.source: - yield zip(self.fields, entry) + + # determine . from res_id + if entry["res_id"] and entry["res_id"].isnumeric(): + # res_id is an id or line number + entry["res_id"] = int(entry["res_id"]) + elif not entry.get("imd_name"): + # res_id is an external id and must follow . + entry["module"], entry["imd_name"] = entry["res_id"].split(".") + entry["res_id"] = None + entry["imd_model"] = entry["name"].split(":")[0] + + if entry["type"] == "code": + if entry["src"] == self.prev_code_src: + # skip entry due to unicity constrain on code translations + continue + self.prev_code_src = entry["src"] + + yield entry class PoFileReader: """ Iterate over po file to return Odoo translation entries """