From 723d8f8b509d3258b7a5e2b652796efa2ec98e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Lef=C3=A8vre?= Date: Thu, 22 Sep 2022 13:44:59 +0000 Subject: [PATCH] [FIX] tools: extract translatable terms from spreadsheet dashboard This commit extract translatable content from spreadsheet data. - chart titles and description - pivot and list names - argument of _t() functions - labels of links X-original-commit: d210a55f48b316fdf38fcbe5ae5a70cf0e2c3814 Part-of: odoo/odoo#101120 --- .../files/test_spreadsheet_dashboard.json | 175 ++++++++++++++++++ .../tests/test_term_count.py | 23 +++ odoo/tools/translate.py | 68 ++++++- 3 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 odoo/addons/test_translation_import/data/files/test_spreadsheet_dashboard.json diff --git a/odoo/addons/test_translation_import/data/files/test_spreadsheet_dashboard.json b/odoo/addons/test_translation_import/data/files/test_spreadsheet_dashboard.json new file mode 100644 index 00000000000..bd142dbec5c --- /dev/null +++ b/odoo/addons/test_translation_import/data/files/test_spreadsheet_dashboard.json @@ -0,0 +1,175 @@ +{ + "version": 12, + "sheets": [ + { + "id": "Sheet1", + "name": "Sheet1", + "colNumber": 26, + "rowNumber": 100, + "rows": {}, + "cols": {}, + "merges": [], + "cells": { + "A1": { + "content": "not exported" + }, + "A2": { + "content": "=_t(\"exported 1\")" + }, + "A3": { + "content": "=_t(\"exported 2\") + _t(\"exported 3\")" + }, + "A4": { + "content": "[link label](https://odoo.com)" + }, + "A5": { + "content": "_t(\"not a formula\")" + }, + "A6": { + "content": "=\"_t(\\\"inside an escaped string literal\\\")\"" + }, + "A7": { + "content": "=\"_t(\"inside a string literal\")\"" + }, + "A8": { + "content": "=_t(\"aa (\\\"inside\\\") bb\")" + }, + "A9": { + "content": "=CONCATENATE(\"a\", \"b\")" + }, + "A10": { + "content": "=_t( \"with spaces\" )" + }, + "A11": { + "content": "" + }, + "A12": { + "content": "=_t(\"123\")" + }, + "A13": { + "content": "=_t(456)" + }, + "A14": { + "content": "=_t(A1)" + }, + "A15": { + "content": "=_t(now())" + }, + "A16": { + "content": "=_t(\"hello \\\"world\\\"\")" + } + }, + "conditionalFormats": [], + "figures": [ + { + "id": "1", + "x": 0, + "y": 0, + "width": 100, + "height": 100, + "tag": "chart", + "data": { + "type": "bar", + "dataSetsHaveTitle": false, + "background": "#FFFFFF", + "dataSets": [ + "A4" + ], + "legendPosition": "none", + "verticalAxisPosition": "left", + "title": "Bar chart title", + "stackedBar": false + } + }, + { + "id": "2", + "x": 0, + "y": 0, + "width": 100, + "height": 100, + "tag": "chart", + "data": { + "baselineColorDown": "#DC6965", + "baselineColorUp": "#00A04A", + "baselineMode": "difference", + "title": "", + "type": "scorecard", + "background": "#FFFFFF", + "keyValue": "A5" + } + }, + { + "id": "2", + "x": 0, + "y": 0, + "width": 100, + "height": 100, + "tag": "chart", + "data": { + "baselineColorDown": "#DC6965", + "baselineColorUp": "#00A04A", + "baselineMode": "difference", + "title": "Scorecard chart", + "type": "scorecard", + "background": "#FFFFFF", + "baselineDescr": "Scorecard description", + "keyValue": "A6" + } + }, + { + "id": "3", + "x": 541, + "y": 61, + "width": 536, + "height": 335, + "tag": "chart", + "data": { + "title": "Opportunities", + "id": "3", + "background": "#FFFFFF", + "legendPosition": "top", + "metaData": { + "groupBy": [], + "measure": "__count", + "resModel": "crm.lead" + }, + "searchParams": { + "comparison": null, + "domain": [], + "groupBy": [], + "orderBy": [] + }, + "type": "odoo_bar", + "verticalAxisPosition": "left", + "stacked": true + } + } + ], + "areGridLinesVisible": true, + "isVisible": true + } + ], + "odooVersion": 4, + "lists": { + "1": { + "columns": [], + "domain": [], + "model": "crm.lead", + "orderBy": [], + "id": "1", + "name": "Pipeline" + } + }, + "pivots": { + "1": { + "colGroupBys": [], + "domain": [], + "id": "1", + "measures": [], + "model": "crm.lead", + "rowGroupBys": [], + "name": "Pipeline Analysis", + "sortedColumn": null + } + } +} diff --git a/odoo/addons/test_translation_import/tests/test_term_count.py b/odoo/addons/test_translation_import/tests/test_term_count.py index a0f0b9e7a21..d4b7df2987b 100644 --- a/odoo/addons/test_translation_import/tests/test_term_count.py +++ b/odoo/addons/test_translation_import/tests/test_term_count.py @@ -329,3 +329,26 @@ class TestTranslationFlow(common.TransactionCase): self.assertIn('text node', trans_static) self.assertIn('slot', trans_static) self.assertIn('slot 2', trans_static) + + def test_export_spreadsheet(self): + terms = [] + po_reader = TranslationModuleReader(self.env.cr, ['test_translation_import']) + for line in po_reader: + _module, _ttype, name, _res_id, source, _value, _comments = line + if name == "addons/test_translation_import/data/files/test_spreadsheet_dashboard.json": + terms.append(source) + self.assertEqual(set(terms), { + 'exported 1', + 'exported 2', + 'exported 3', + 'Bar chart title', + 'Scorecard description', + 'Scorecard chart', + 'Opportunities', + 'Pipeline', + 'Pipeline Analysis', + 'link label', + 'aa (\\"inside\\") bb', + 'with spaces', + 'hello \\"world\\"', + }) diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py index 9cb0f0b18d4..4a99d7bd2a7 100644 --- a/odoo/tools/translate.py +++ b/odoo/tools/translate.py @@ -5,9 +5,12 @@ import fnmatch import functools import inspect import io +import itertools +import json import locale import logging import os +from tokenize import generate_tokens, STRING, NEWLINE, INDENT, DEDENT import polib import re import tarfile @@ -889,6 +892,65 @@ def babel_extract_qweb(fileobj, keywords, comment_tags, options): _extract_translatable_qweb_terms(tree.getroot(), handle_text) return result + +def extract_formula_terms(formula): + """Extract strings in a spreadsheet formula which are arguments to '_t' functions + + >>> extract_formula_terms('=_t("Hello") + _t("Raoul")') + ["Hello", "Raoul"] + """ + tokens = generate_tokens(io.StringIO(formula).readline) + tokens = (token for token in tokens if token.type not in {NEWLINE, INDENT, DEDENT}) + for t1 in tokens: + if not t1.string == '_t': + continue + t2 = next(tokens, None) + if t2 and t2.string == '(': + t3 = next(tokens, None) + t4 = next(tokens, None) + if t4 and t4.string == ')' and t3 and t3.type == STRING: + yield t3.string[1:][:-1] # strip leading and trailing quotes + + +def extract_spreadsheet_terms(fileobj, keywords, comment_tags, options): + """Babel message extractor for spreadsheet data files. + + :param fileobj: the file-like object the messages should be extracted from + :param keywords: a list of keywords (i.e. function names) that should + be recognized as translation functions + :param comment_tags: a list of translator tags to search for and + include in the results + :param options: a dictionary of additional options (optional) + :return: an iterator over ``(lineno, funcname, message, comments)`` + tuples + """ + terms = [] + data = json.load(fileobj) + for sheet in data.get('sheets', []): + for cell in sheet['cells'].values(): + content = cell.get('content', '') + if content.startswith('='): + terms += extract_formula_terms(content) + else: + markdown_link = re.fullmatch(r'\[(.+)\]\(.+\)', content) + if markdown_link: + terms.append(markdown_link[1]) + for figure in sheet['figures']: + terms.append(figure['data']['title']) + if 'baselineDescr' in figure['data']: + terms.append(figure['data']['baselineDescr']) + pivots = data.get('pivots', {}).values() + lists = data.get('lists', {}).values() + for data_source in itertools.chain(lists, pivots): + terms.append(data_source['name']) + for global_filter in data.get('globalFilters', []): + terms.append(global_filter['label']) + return ( + (0, None, term, []) + for term in terms + if any(x.isalpha() for x in term) + ) + ImdInfo = namedtuple('ExternalId', ['name', 'model', 'res_id', 'module']) @@ -1090,6 +1152,7 @@ class TranslationModuleReader: - the python strings marked with _() or _lt() - the javascript strings marked with _t() or _lt() inside static/src/js/ - the strings inside Qweb files inside static/src/xml/ + - the spreadsheet data files """ # Also scan these non-addon paths @@ -1116,7 +1179,10 @@ class TranslationModuleReader: for fname in fnmatch.filter(files, '*.xml'): self._babel_extract_terms(fname, path, root, 'odoo.tools.translate:babel_extract_qweb', extra_comments=[WEB_TRANSLATION_COMMENT]) - + if fnmatch.fnmatch(root, '*/data/*'): + for fname in fnmatch.filter(files, '*_dashboard.json'): + self._babel_extract_terms(fname, path, root, 'odoo.tools.translate:extract_spreadsheet_terms', + extra_comments=[WEB_TRANSLATION_COMMENT]) if not recursive: # due to topdown, first iteration is in first level break