Martin Trigaux
ffbd1ff062
Nodes like <attribute name="t-on-dragenter.stop.prevent">() => state.dropzoneVisible = true</attribute> should not be exported in the translations files but <attribute name="string">Hello World</attribute> should be kept It was already the case in the xml processing method for server side templates. Since 16.0, client side QWeb views also use the same inheritance mechanism than the one on the server side so the exception needs to be replicated. X-original-commit: e05d1192481935b2cd7aae7e939f98e60e002899 Part-of: odoo/odoo#101053
1497 lines
61 KiB
Python
1497 lines
61 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
|
import codecs
|
|
import fnmatch
|
|
import functools
|
|
import inspect
|
|
import io
|
|
import locale
|
|
import logging
|
|
import os
|
|
import polib
|
|
import re
|
|
import tarfile
|
|
import tempfile
|
|
import threading
|
|
from collections import defaultdict, namedtuple
|
|
from datetime import datetime
|
|
from os.path import join
|
|
|
|
from pathlib import Path
|
|
from babel.messages import extract
|
|
from lxml import etree, html
|
|
from psycopg2.extras import Json
|
|
|
|
import odoo
|
|
from odoo.modules.module import get_resource_path
|
|
from . import config, pycompat
|
|
from .misc import file_open, get_iso_codes, SKIPPED_ELEMENT_TYPES
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
# used to notify web client that these translations should be loaded in the UI
|
|
WEB_TRANSLATION_COMMENT = "openerp-web"
|
|
|
|
SKIPPED_ELEMENTS = ('script', 'style', 'title')
|
|
|
|
_LOCALE2WIN32 = {
|
|
'af_ZA': 'Afrikaans_South Africa',
|
|
'sq_AL': 'Albanian_Albania',
|
|
'ar_SA': 'Arabic_Saudi Arabia',
|
|
'eu_ES': 'Basque_Spain',
|
|
'be_BY': 'Belarusian_Belarus',
|
|
'bs_BA': 'Bosnian_Bosnia and Herzegovina',
|
|
'bg_BG': 'Bulgarian_Bulgaria',
|
|
'ca_ES': 'Catalan_Spain',
|
|
'hr_HR': 'Croatian_Croatia',
|
|
'zh_CN': 'Chinese_China',
|
|
'zh_TW': 'Chinese_Taiwan',
|
|
'cs_CZ': 'Czech_Czech Republic',
|
|
'da_DK': 'Danish_Denmark',
|
|
'nl_NL': 'Dutch_Netherlands',
|
|
'et_EE': 'Estonian_Estonia',
|
|
'fa_IR': 'Farsi_Iran',
|
|
'ph_PH': 'Filipino_Philippines',
|
|
'fi_FI': 'Finnish_Finland',
|
|
'fr_FR': 'French_France',
|
|
'fr_BE': 'French_France',
|
|
'fr_CH': 'French_France',
|
|
'fr_CA': 'French_France',
|
|
'ga': 'Scottish Gaelic',
|
|
'gl_ES': 'Galician_Spain',
|
|
'ka_GE': 'Georgian_Georgia',
|
|
'de_DE': 'German_Germany',
|
|
'el_GR': 'Greek_Greece',
|
|
'gu': 'Gujarati_India',
|
|
'he_IL': 'Hebrew_Israel',
|
|
'hi_IN': 'Hindi',
|
|
'hu': 'Hungarian_Hungary',
|
|
'is_IS': 'Icelandic_Iceland',
|
|
'id_ID': 'Indonesian_Indonesia',
|
|
'it_IT': 'Italian_Italy',
|
|
'ja_JP': 'Japanese_Japan',
|
|
'kn_IN': 'Kannada',
|
|
'km_KH': 'Khmer',
|
|
'ko_KR': 'Korean_Korea',
|
|
'lo_LA': 'Lao_Laos',
|
|
'lt_LT': 'Lithuanian_Lithuania',
|
|
'lat': 'Latvian_Latvia',
|
|
'ml_IN': 'Malayalam_India',
|
|
'mi_NZ': 'Maori',
|
|
'mn': 'Cyrillic_Mongolian',
|
|
'no_NO': 'Norwegian_Norway',
|
|
'nn_NO': 'Norwegian-Nynorsk_Norway',
|
|
'pl': 'Polish_Poland',
|
|
'pt_PT': 'Portuguese_Portugal',
|
|
'pt_BR': 'Portuguese_Brazil',
|
|
'ro_RO': 'Romanian_Romania',
|
|
'ru_RU': 'Russian_Russia',
|
|
'sr_CS': 'Serbian (Cyrillic)_Serbia and Montenegro',
|
|
'sk_SK': 'Slovak_Slovakia',
|
|
'sl_SI': 'Slovenian_Slovenia',
|
|
#should find more specific locales for Spanish countries,
|
|
#but better than nothing
|
|
'es_AR': 'Spanish_Spain',
|
|
'es_BO': 'Spanish_Spain',
|
|
'es_CL': 'Spanish_Spain',
|
|
'es_CO': 'Spanish_Spain',
|
|
'es_CR': 'Spanish_Spain',
|
|
'es_DO': 'Spanish_Spain',
|
|
'es_EC': 'Spanish_Spain',
|
|
'es_ES': 'Spanish_Spain',
|
|
'es_GT': 'Spanish_Spain',
|
|
'es_HN': 'Spanish_Spain',
|
|
'es_MX': 'Spanish_Spain',
|
|
'es_NI': 'Spanish_Spain',
|
|
'es_PA': 'Spanish_Spain',
|
|
'es_PE': 'Spanish_Spain',
|
|
'es_PR': 'Spanish_Spain',
|
|
'es_PY': 'Spanish_Spain',
|
|
'es_SV': 'Spanish_Spain',
|
|
'es_UY': 'Spanish_Spain',
|
|
'es_VE': 'Spanish_Spain',
|
|
'sv_SE': 'Swedish_Sweden',
|
|
'ta_IN': 'English_Australia',
|
|
'th_TH': 'Thai_Thailand',
|
|
'tr_TR': 'Turkish_Turkey',
|
|
'uk_UA': 'Ukrainian_Ukraine',
|
|
'vi_VN': 'Vietnamese_Viet Nam',
|
|
'tlh_TLH': 'Klingon',
|
|
|
|
}
|
|
|
|
# these direct uses of CSV are ok.
|
|
import csv # pylint: disable=deprecated-module
|
|
class UNIX_LINE_TERMINATOR(csv.excel):
|
|
lineterminator = '\n'
|
|
|
|
csv.register_dialect("UNIX", UNIX_LINE_TERMINATOR)
|
|
|
|
|
|
# FIXME: holy shit this whole thing needs to be cleaned up hard it's a mess
|
|
def encode(s):
|
|
assert isinstance(s, str)
|
|
return s
|
|
|
|
|
|
# which elements are translated inline
|
|
TRANSLATED_ELEMENTS = {
|
|
'abbr', 'b', 'bdi', 'bdo', 'br', 'cite', 'code', 'data', 'del', 'dfn', 'em',
|
|
'font', 'i', 'ins', 'kbd', 'keygen', 'mark', 'math', 'meter', 'output',
|
|
'progress', 'q', 'ruby', 's', 'samp', 'small', 'span', 'strong', 'sub',
|
|
'sup', 'time', 'u', 'var', 'wbr', 'text',
|
|
}
|
|
|
|
# Which attributes must be translated. This is a dict, where the value indicates
|
|
# a condition for a node to have the attribute translatable.
|
|
TRANSLATED_ATTRS = dict.fromkeys({
|
|
'string', 'add-label', 'help', 'sum', 'avg', 'confirm', 'placeholder', 'alt', 'title', 'aria-label',
|
|
'aria-keyshortcuts', 'aria-placeholder', 'aria-roledescription', 'aria-valuetext',
|
|
'value_label', 'data-tooltip',
|
|
}, lambda e: True)
|
|
|
|
def translate_attrib_value(node):
|
|
# check if the value attribute of a node must be translated
|
|
classes = node.attrib.get('class', '').split(' ')
|
|
return (
|
|
(node.tag == 'input' and node.attrib.get('type', 'text') == 'text')
|
|
and 'datetimepicker-input' not in classes
|
|
or (node.tag == 'input' and node.attrib.get('type') == 'hidden')
|
|
and 'o_translatable_input_hidden' in classes
|
|
)
|
|
|
|
TRANSLATED_ATTRS.update(
|
|
value=translate_attrib_value,
|
|
text=lambda e: (e.tag == 'field' and e.attrib.get('widget', '') == 'url'),
|
|
**{f't-attf-{attr}': cond for attr, cond in TRANSLATED_ATTRS.items()},
|
|
)
|
|
|
|
avoid_pattern = re.compile(r"\s*<!DOCTYPE", re.IGNORECASE | re.MULTILINE | re.UNICODE)
|
|
|
|
|
|
def translate_xml_node(node, callback, parse, serialize):
|
|
""" Return the translation of the given XML/HTML node.
|
|
|
|
:param node:
|
|
:param callback: callback(text) returns translated text or None
|
|
:param parse: parse(text) returns a node (text is unicode)
|
|
:param serialize: serialize(node) returns unicode text
|
|
"""
|
|
|
|
def nonspace(text):
|
|
""" Return whether ``text`` is a string with non-space characters. """
|
|
return bool(text) and not text.isspace()
|
|
|
|
def translatable(node):
|
|
""" Return whether the given node can be translated as a whole. """
|
|
return (
|
|
node.tag in TRANSLATED_ELEMENTS
|
|
and not any(key.startswith("t-") for key in node.attrib)
|
|
and all(translatable(child) for child in node)
|
|
)
|
|
|
|
def hastext(node, pos=0):
|
|
""" Return whether the given node contains some text to translate at the
|
|
given child node position. The text may be before the child node,
|
|
inside it, or after it.
|
|
"""
|
|
return (
|
|
# there is some text before node[pos]
|
|
nonspace(node[pos-1].tail if pos else node.text)
|
|
or (
|
|
pos < len(node)
|
|
and translatable(node[pos])
|
|
and (
|
|
any( # attribute to translate
|
|
val and key in TRANSLATED_ATTRS and TRANSLATED_ATTRS[key](node[pos])
|
|
for key, val in node[pos].attrib.items()
|
|
)
|
|
# node[pos] contains some text to translate
|
|
or hastext(node[pos])
|
|
# node[pos] has no text, but there is some text after it
|
|
or hastext(node, pos + 1)
|
|
)
|
|
)
|
|
)
|
|
|
|
def process(node):
|
|
""" Translate the given node. """
|
|
if (
|
|
isinstance(node, SKIPPED_ELEMENT_TYPES)
|
|
or node.tag in SKIPPED_ELEMENTS
|
|
or node.get('t-translation', "").strip() == "off"
|
|
or node.tag == 'attribute' and node.get('name') not in TRANSLATED_ATTRS
|
|
or node.getparent() is None and avoid_pattern.match(node.text or "")
|
|
):
|
|
return
|
|
|
|
pos = 0
|
|
while True:
|
|
# check for some text to translate at the given position
|
|
if hastext(node, pos):
|
|
# move all translatable children nodes from the given position
|
|
# into a <div> element
|
|
div = etree.Element('div')
|
|
div.text = (node[pos-1].tail if pos else node.text) or ''
|
|
while pos < len(node) and translatable(node[pos]):
|
|
div.append(node[pos])
|
|
|
|
# translate the content of the <div> element as a whole
|
|
content = serialize(div)[5:-6]
|
|
original = content.strip()
|
|
translated = callback(original)
|
|
if translated:
|
|
result = content.replace(original, translated)
|
|
div = parse_html(f"<div>{result}</div>")
|
|
if pos:
|
|
node[pos-1].tail = div.text
|
|
else:
|
|
node.text = div.text
|
|
|
|
# move the content of the <div> element back inside node
|
|
while len(div) > 0:
|
|
node.insert(pos, div[0])
|
|
pos += 1
|
|
|
|
if pos >= len(node):
|
|
break
|
|
|
|
# node[pos] is not translatable as a whole, process it recursively
|
|
process(node[pos])
|
|
pos += 1
|
|
|
|
# translate the attributes of the node
|
|
for key, val in node.attrib.items():
|
|
if nonspace(val) and key in TRANSLATED_ATTRS and TRANSLATED_ATTRS[key](node):
|
|
node.set(key, callback(val.strip()) or val)
|
|
|
|
process(node)
|
|
|
|
return node
|
|
|
|
|
|
def parse_xml(text):
|
|
return etree.fromstring(text)
|
|
|
|
def serialize_xml(node):
|
|
return etree.tostring(node, method='xml', encoding='unicode')
|
|
|
|
_HTML_PARSER = etree.HTMLParser(encoding='utf8')
|
|
|
|
def parse_html(text):
|
|
return html.fragment_fromstring(text, parser=_HTML_PARSER)
|
|
|
|
def serialize_html(node):
|
|
return etree.tostring(node, method='html', encoding='unicode')
|
|
|
|
|
|
def xml_translate(callback, value):
|
|
""" Translate an XML value (string), using `callback` for translating text
|
|
appearing in `value`.
|
|
"""
|
|
if not value:
|
|
return value
|
|
|
|
try:
|
|
root = parse_xml(value)
|
|
result = translate_xml_node(root, callback, parse_xml, serialize_xml)
|
|
return serialize_xml(result)
|
|
except etree.ParseError:
|
|
# fallback for translated terms: use an HTML parser and wrap the term
|
|
root = parse_html(u"<div>%s</div>" % value)
|
|
result = translate_xml_node(root, callback, parse_xml, serialize_xml)
|
|
# remove tags <div> and </div> from result
|
|
return serialize_xml(result)[5:-6]
|
|
|
|
def xml_term_converter(value):
|
|
""" Convert the HTML fragment ``value`` to XML if necessary
|
|
"""
|
|
# wrap value inside a div and parse it as HTML
|
|
div = f"<div>{value}</div>"
|
|
root = etree.fromstring(div, etree.HTMLParser())
|
|
# root is html > body > div
|
|
# serialize div as XML and discard surrounding tags
|
|
return etree.tostring(root[0][0], encoding='unicode')[5:-6]
|
|
|
|
def html_translate(callback, value):
|
|
""" Translate an HTML value (string), using `callback` for translating text
|
|
appearing in `value`.
|
|
"""
|
|
if not value:
|
|
return value
|
|
|
|
try:
|
|
# value may be some HTML fragment, wrap it into a div
|
|
root = parse_html("<div>%s</div>" % value)
|
|
result = translate_xml_node(root, callback, parse_html, serialize_html)
|
|
# remove tags <div> and </div> from result
|
|
value = serialize_html(result)[5:-6]
|
|
except ValueError:
|
|
_logger.exception("Cannot translate malformed HTML, using source value instead")
|
|
|
|
return value
|
|
|
|
def html_term_converter(value):
|
|
""" Convert the HTML fragment ``value`` to XML if necessary
|
|
"""
|
|
# wrap value inside a div and parse it as HTML
|
|
div = f"<div>{value}</div>"
|
|
root = etree.fromstring(div, etree.HTMLParser())
|
|
# root is html > body > div
|
|
# serialize div as HTML and discard surrounding tags
|
|
return etree.tostring(root[0][0], encoding='unicode', method='html')[5:-6]
|
|
|
|
|
|
def get_text_content(term):
|
|
""" Return the textual content of the given term. """
|
|
return html.fromstring(term).text_content()
|
|
|
|
xml_translate.get_text_content = get_text_content
|
|
html_translate.get_text_content = get_text_content
|
|
|
|
xml_translate.term_converter = xml_term_converter
|
|
html_translate.term_converter = html_term_converter
|
|
|
|
def translate_sql_constraint(cr, key, lang):
|
|
cr.execute("""
|
|
SELECT COALESCE(c.message->>%s, c.message->>'en_US') as message
|
|
FROM ir_model_constraint c
|
|
WHERE name=%s and type='u'
|
|
""", (lang, key))
|
|
return cr.fetchone()[0]
|
|
|
|
class GettextAlias(object):
|
|
|
|
def _get_db(self):
|
|
# find current DB based on thread/worker db name (see netsvc)
|
|
db_name = getattr(threading.current_thread(), 'dbname', None)
|
|
if db_name:
|
|
return odoo.sql_db.db_connect(db_name)
|
|
|
|
def _get_cr(self, frame, allow_create=True):
|
|
# try, in order: cr, cursor, self.env.cr, self.cr,
|
|
# request.env.cr
|
|
if 'cr' in frame.f_locals:
|
|
return frame.f_locals['cr'], False
|
|
if 'cursor' in frame.f_locals:
|
|
return frame.f_locals['cursor'], False
|
|
s = frame.f_locals.get('self')
|
|
if hasattr(s, 'env'):
|
|
return s.env.cr, False
|
|
if hasattr(s, 'cr'):
|
|
return s.cr, False
|
|
try:
|
|
from odoo.http import request
|
|
return request.env.cr, False
|
|
except RuntimeError:
|
|
pass
|
|
if allow_create:
|
|
# create a new cursor
|
|
db = self._get_db()
|
|
if db is not None:
|
|
return db.cursor(), True
|
|
return None, False
|
|
|
|
def _get_uid(self, frame):
|
|
# try, in order: uid, user, self.env.uid
|
|
if 'uid' in frame.f_locals:
|
|
return frame.f_locals['uid']
|
|
if 'user' in frame.f_locals:
|
|
return int(frame.f_locals['user']) # user may be a record
|
|
s = frame.f_locals.get('self')
|
|
return s.env.uid
|
|
|
|
def _get_lang(self, frame):
|
|
# try, in order: context.get('lang'), kwargs['context'].get('lang'),
|
|
# self.env.lang, self.localcontext.get('lang'), request.env.lang
|
|
lang = None
|
|
if frame.f_locals.get('context'):
|
|
lang = frame.f_locals['context'].get('lang')
|
|
if not lang:
|
|
kwargs = frame.f_locals.get('kwargs', {})
|
|
if kwargs.get('context'):
|
|
lang = kwargs['context'].get('lang')
|
|
if not lang:
|
|
s = frame.f_locals.get('self')
|
|
if hasattr(s, 'env'):
|
|
lang = s.env.lang
|
|
if not lang:
|
|
if hasattr(s, 'localcontext'):
|
|
lang = s.localcontext.get('lang')
|
|
if not lang:
|
|
try:
|
|
from odoo.http import request
|
|
lang = request.env.lang
|
|
except RuntimeError:
|
|
pass
|
|
if not lang:
|
|
# Last resort: attempt to guess the language of the user
|
|
# Pitfall: some operations are performed in sudo mode, and we
|
|
# don't know the original uid, so the language may
|
|
# be wrong when the admin language differs.
|
|
(cr, dummy) = self._get_cr(frame, allow_create=False)
|
|
uid = self._get_uid(frame)
|
|
if cr and uid:
|
|
env = odoo.api.Environment(cr, uid, {})
|
|
lang = env['res.users'].context_get()['lang']
|
|
return lang
|
|
|
|
def __call__(self, source, *args, **kwargs):
|
|
translation = self._get_translation(source)
|
|
assert not (args and kwargs)
|
|
if args or kwargs:
|
|
try:
|
|
return translation % (args or kwargs)
|
|
except (TypeError, ValueError, KeyError):
|
|
bad = translation
|
|
# fallback: apply to source before logging exception (in case source fails)
|
|
translation = source % (args or kwargs)
|
|
_logger.exception('Bad translation %r for string %r', bad, source)
|
|
return translation
|
|
|
|
def _get_translation(self, source, module=None):
|
|
try:
|
|
frame = inspect.currentframe().f_back.f_back
|
|
lang = self._get_lang(frame)
|
|
if lang and lang != 'en_US':
|
|
if not module:
|
|
path = inspect.getfile(frame)
|
|
path_info = odoo.modules.get_resource_from_path(path)
|
|
module = path_info[0] if path_info else None
|
|
return code_translations.get_python_translations(module, lang).get(source, source)
|
|
else:
|
|
_logger.debug('no translation language detected, skipping translation for "%r" ', source)
|
|
except Exception:
|
|
_logger.debug('translation went wrong for "%r", skipped', source)
|
|
# if so, double-check the root/base translations filenames
|
|
return source
|
|
|
|
|
|
@functools.total_ordering
|
|
class _lt:
|
|
""" Lazy code translation
|
|
|
|
Similar to GettextAlias but the translation lookup will be done only at
|
|
__str__ execution.
|
|
|
|
A code using translated global variables such as:
|
|
|
|
LABEL = _lt("User")
|
|
|
|
def _compute_label(self):
|
|
context = {'lang': self.partner_id.lang}
|
|
self.user_label = LABEL
|
|
|
|
works as expected (unlike the classic GettextAlias implementation).
|
|
"""
|
|
|
|
__slots__ = ['_source', '_args', '_module']
|
|
|
|
def __init__(self, source, *args, **kwargs):
|
|
self._source = source
|
|
assert not (args and kwargs)
|
|
self._args = args or kwargs
|
|
|
|
frame = inspect.currentframe().f_back
|
|
path = inspect.getfile(frame)
|
|
path_info = odoo.modules.get_resource_from_path(path)
|
|
self._module = path_info[0] if path_info else None
|
|
|
|
def __str__(self):
|
|
# Call _._get_translation() like _() does, so that we have the same number
|
|
# of stack frames calling _get_translation()
|
|
translation = _._get_translation(self._source, self._module)
|
|
if self._args:
|
|
try:
|
|
return translation % self._args
|
|
except (TypeError, ValueError, KeyError):
|
|
bad = translation
|
|
# fallback: apply to source before logging exception (in case source fails)
|
|
translation = self._source % self._args
|
|
_logger.exception('Bad translation %r for string %r', bad, self._source)
|
|
return translation
|
|
|
|
def __eq__(self, other):
|
|
""" Prevent using equal operators
|
|
|
|
Prevent direct comparisons with ``self``.
|
|
One should compare the translation of ``self._source`` as ``str(self) == X``.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
def __lt__(self, other):
|
|
raise NotImplementedError()
|
|
|
|
def __add__(self, other):
|
|
# Call _._get_translation() like _() does, so that we have the same number
|
|
# of stack frames calling _get_translation()
|
|
if isinstance(other, str):
|
|
return _._get_translation(self._source) + other
|
|
elif isinstance(other, _lt):
|
|
return _._get_translation(self._source) + _._get_translation(other._source)
|
|
return NotImplemented
|
|
|
|
def __radd__(self, other):
|
|
# Call _._get_translation() like _() does, so that we have the same number
|
|
# of stack frames calling _get_translation()
|
|
if isinstance(other, str):
|
|
return other + _._get_translation(self._source)
|
|
return NotImplemented
|
|
|
|
_ = GettextAlias()
|
|
|
|
|
|
def quote(s):
|
|
"""Returns quoted PO term string, with special PO characters escaped"""
|
|
assert r"\n" not in s, "Translation terms may not include escaped newlines ('\\n'), please use only literal newlines! (in '%s')" % s
|
|
return '"%s"' % s.replace('\\','\\\\') \
|
|
.replace('"','\\"') \
|
|
.replace('\n', '\\n"\n"')
|
|
|
|
re_escaped_char = re.compile(r"(\\.)")
|
|
re_escaped_replacements = {'n': '\n', 't': '\t',}
|
|
|
|
def _sub_replacement(match_obj):
|
|
return re_escaped_replacements.get(match_obj.group(1)[1], match_obj.group(1)[1])
|
|
|
|
def unquote(str):
|
|
"""Returns unquoted PO term string, with special PO characters unescaped"""
|
|
return re_escaped_char.sub(_sub_replacement, str[1:-1])
|
|
|
|
def TranslationFileReader(source, fileformat='po'):
|
|
""" Iterate over translation file to return Odoo translation entries """
|
|
if fileformat == 'csv':
|
|
return CSVFileReader(source)
|
|
if fileformat == 'po':
|
|
return PoFileReader(source)
|
|
_logger.info('Bad file format: %s', fileformat)
|
|
raise Exception(_('Bad file format: %s', fileformat))
|
|
|
|
class CSVFileReader:
|
|
def __init__(self, source):
|
|
_reader = codecs.getreader('utf-8')
|
|
self.source = csv.DictReader(_reader(source), quotechar='"', delimiter=',')
|
|
self.prev_code_src = ""
|
|
|
|
def __iter__(self):
|
|
for entry in self.source:
|
|
|
|
# determine <module>.<imd_name> from res_id
|
|
if entry["res_id"] and entry["res_id"].isnumeric():
|
|
# res_id is an id or line number
|
|
entry["res_id"] = int(entry["res_id"])
|
|
elif not entry.get("imd_name"):
|
|
# res_id is an external id and must follow <module>.<name>
|
|
entry["module"], entry["imd_name"] = entry["res_id"].split(".")
|
|
entry["res_id"] = None
|
|
if entry["type"] == "model" or entry["type"] == "model_terms":
|
|
entry["imd_model"] = entry["name"].partition(',')[0]
|
|
|
|
if entry["type"] == "code":
|
|
if entry["src"] == self.prev_code_src:
|
|
# skip entry due to unicity constrain on code translations
|
|
continue
|
|
self.prev_code_src = entry["src"]
|
|
|
|
yield entry
|
|
|
|
class PoFileReader:
|
|
""" Iterate over po file to return Odoo translation entries """
|
|
def __init__(self, source):
|
|
|
|
def get_pot_path(source_name):
|
|
# when fileobj is a TemporaryFile, its name is an inter in P3, a string in P2
|
|
if isinstance(source_name, str) and source_name.endswith('.po'):
|
|
# Normally the path looks like /path/to/xxx/i18n/lang.po
|
|
# and we try to find the corresponding
|
|
# /path/to/xxx/i18n/xxx.pot file.
|
|
# (Sometimes we have 'i18n_extra' instead of just 'i18n')
|
|
path = Path(source_name)
|
|
filename = path.parent.parent.name + '.pot'
|
|
pot_path = path.with_name(filename)
|
|
return pot_path.exists() and str(pot_path) or False
|
|
return False
|
|
|
|
# polib accepts a path or the file content as a string, not a fileobj
|
|
if isinstance(source, str):
|
|
self.pofile = polib.pofile(source)
|
|
pot_path = get_pot_path(source)
|
|
else:
|
|
# either a BufferedIOBase or result from NamedTemporaryFile
|
|
self.pofile = polib.pofile(source.read().decode())
|
|
pot_path = get_pot_path(source.name)
|
|
|
|
if pot_path:
|
|
# Make a reader for the POT file
|
|
# (Because the POT comments are correct on GitHub but the
|
|
# PO comments tends to be outdated. See LP bug 933496.)
|
|
self.pofile.merge(polib.pofile(pot_path))
|
|
|
|
def __iter__(self):
|
|
for entry in self.pofile:
|
|
if entry.obsolete:
|
|
continue
|
|
|
|
# in case of moduleS keep only the first
|
|
match = re.match(r"(module[s]?): (\w+)", entry.comment)
|
|
_, module = match.groups()
|
|
comments = "\n".join([c for c in entry.comment.split('\n') if not c.startswith('module:')])
|
|
source = entry.msgid
|
|
translation = entry.msgstr
|
|
found_code_occurrence = False
|
|
for occurrence, line_number in entry.occurrences:
|
|
match = re.match(r'(model|model_terms):([\w.]+),([\w]+):(\w+)\.([^ ]+)', occurrence)
|
|
if match:
|
|
type, model_name, field_name, module, xmlid = match.groups()
|
|
yield {
|
|
'type': type,
|
|
'imd_model': model_name,
|
|
'name': model_name+','+field_name,
|
|
'imd_name': xmlid,
|
|
'res_id': None,
|
|
'src': source,
|
|
'value': translation,
|
|
'comments': comments,
|
|
'module': module,
|
|
}
|
|
continue
|
|
|
|
match = re.match(r'(code):([\w/.]+)', occurrence)
|
|
if match:
|
|
type, name = match.groups()
|
|
if found_code_occurrence:
|
|
# unicity constrain on code translation
|
|
continue
|
|
found_code_occurrence = True
|
|
yield {
|
|
'type': type,
|
|
'name': name,
|
|
'src': source,
|
|
'value': translation,
|
|
'comments': comments,
|
|
'res_id': int(line_number),
|
|
'module': module,
|
|
}
|
|
continue
|
|
|
|
match = re.match(r'(selection):([\w.]+),([\w]+)', occurrence)
|
|
if match:
|
|
_logger.info("Skipped deprecated occurrence %s", occurrence)
|
|
continue
|
|
|
|
match = re.match(r'(sql_constraint|constraint):([\w.]+)', occurrence)
|
|
if match:
|
|
_logger.info("Skipped deprecated occurrence %s", occurrence)
|
|
continue
|
|
_logger.error("malformed po file: unknown occurrence: %s", occurrence)
|
|
|
|
def TranslationFileWriter(target, fileformat='po', lang=None):
|
|
""" Iterate over translation file to return Odoo translation entries """
|
|
if fileformat == 'csv':
|
|
return CSVFileWriter(target)
|
|
|
|
if fileformat == 'po':
|
|
return PoFileWriter(target, lang=lang)
|
|
|
|
if fileformat == 'tgz':
|
|
return TarFileWriter(target, lang=lang)
|
|
|
|
raise Exception(_('Unrecognized extension: must be one of '
|
|
'.csv, .po, or .tgz (received .%s).') % fileformat)
|
|
|
|
|
|
class CSVFileWriter:
|
|
def __init__(self, target):
|
|
self.writer = pycompat.csv_writer(target, dialect='UNIX')
|
|
# write header first
|
|
self.writer.writerow(("module","type","name","res_id","src","value","comments"))
|
|
|
|
|
|
def write_rows(self, rows):
|
|
for module, type, name, res_id, src, trad, comments in rows:
|
|
comments = '\n'.join(comments)
|
|
self.writer.writerow((module, type, name, res_id, src, trad, comments))
|
|
|
|
|
|
class PoFileWriter:
|
|
""" Iterate over po file to return Odoo translation entries """
|
|
def __init__(self, target, lang):
|
|
|
|
self.buffer = target
|
|
self.lang = lang
|
|
self.po = polib.POFile()
|
|
|
|
def write_rows(self, rows):
|
|
# we now group the translations by source. That means one translation per source.
|
|
grouped_rows = {}
|
|
modules = set([])
|
|
for module, type, name, res_id, src, trad, comments in rows:
|
|
row = grouped_rows.setdefault(src, {})
|
|
row.setdefault('modules', set()).add(module)
|
|
if not row.get('translation') and trad != src:
|
|
row['translation'] = trad
|
|
row.setdefault('tnrs', []).append((type, name, res_id))
|
|
row.setdefault('comments', set()).update(comments)
|
|
modules.add(module)
|
|
|
|
for src, row in sorted(grouped_rows.items()):
|
|
if not self.lang:
|
|
# translation template, so no translation value
|
|
row['translation'] = ''
|
|
elif not row.get('translation'):
|
|
row['translation'] = ''
|
|
self.add_entry(row['modules'], sorted(row['tnrs']), src, row['translation'], row['comments'])
|
|
|
|
import odoo.release as release
|
|
self.po.header = "Translation of %s.\n" \
|
|
"This file contains the translation of the following modules:\n" \
|
|
"%s" % (release.description, ''.join("\t* %s\n" % m for m in modules))
|
|
now = datetime.utcnow().strftime('%Y-%m-%d %H:%M+0000')
|
|
self.po.metadata = {
|
|
'Project-Id-Version': "%s %s" % (release.description, release.version),
|
|
'Report-Msgid-Bugs-To': '',
|
|
'POT-Creation-Date': now,
|
|
'PO-Revision-Date': now,
|
|
'Last-Translator': '',
|
|
'Language-Team': '',
|
|
'MIME-Version': '1.0',
|
|
'Content-Type': 'text/plain; charset=UTF-8',
|
|
'Content-Transfer-Encoding': '',
|
|
'Plural-Forms': '',
|
|
}
|
|
|
|
# buffer expects bytes
|
|
self.buffer.write(str(self.po).encode())
|
|
|
|
def add_entry(self, modules, tnrs, source, trad, comments=None):
|
|
entry = polib.POEntry(
|
|
msgid=source,
|
|
msgstr=trad,
|
|
)
|
|
plural = len(modules) > 1 and 's' or ''
|
|
entry.comment = "module%s: %s" % (plural, ', '.join(modules))
|
|
if comments:
|
|
entry.comment += "\n" + "\n".join(comments)
|
|
|
|
code = False
|
|
for typy, name, res_id in tnrs:
|
|
if typy == 'code':
|
|
code = True
|
|
res_id = 0
|
|
if isinstance(res_id, int) or res_id.isdigit():
|
|
# second term of occurrence must be a digit
|
|
# occurrence line at 0 are discarded when rendered to string
|
|
entry.occurrences.append((u"%s:%s" % (typy, name), str(res_id)))
|
|
else:
|
|
entry.occurrences.append((u"%s:%s:%s" % (typy, name, res_id), ''))
|
|
if code:
|
|
entry.flags.append("python-format")
|
|
self.po.append(entry)
|
|
|
|
|
|
class TarFileWriter:
|
|
|
|
def __init__(self, target, lang):
|
|
self.tar = tarfile.open(fileobj=target, mode='w|gz')
|
|
self.lang = lang
|
|
|
|
def write_rows(self, rows):
|
|
rows_by_module = defaultdict(list)
|
|
for row in rows:
|
|
module = row[0]
|
|
rows_by_module[module].append(row)
|
|
|
|
for mod, modrows in rows_by_module.items():
|
|
with io.BytesIO() as buf:
|
|
po = PoFileWriter(buf, lang=self.lang)
|
|
po.write_rows(modrows)
|
|
buf.seek(0)
|
|
|
|
info = tarfile.TarInfo(
|
|
join(mod, 'i18n', '{basename}.{ext}'.format(
|
|
basename=self.lang or mod,
|
|
ext='po' if self.lang else 'pot',
|
|
)))
|
|
# addfile will read <size> bytes from the buffer so
|
|
# size *must* be set first
|
|
info.size = len(buf.getvalue())
|
|
|
|
self.tar.addfile(info, fileobj=buf)
|
|
|
|
self.tar.close()
|
|
|
|
# Methods to export the translation file
|
|
def trans_export(lang, modules, buffer, format, cr):
|
|
reader = TranslationModuleReader(cr, modules=modules, lang=lang)
|
|
writer = TranslationFileWriter(buffer, fileformat=format, lang=lang)
|
|
writer.write_rows(reader)
|
|
|
|
def _push(callback, term, source_line):
|
|
""" Sanity check before pushing translation terms """
|
|
term = (term or "").strip()
|
|
# Avoid non-char tokens like ':' '...' '.00' etc.
|
|
if len(term) > 8 or any(x.isalpha() for x in term):
|
|
callback(term, source_line)
|
|
|
|
def _extract_translatable_qweb_terms(element, callback):
|
|
""" Helper method to walk an etree document representing
|
|
a QWeb template, and call ``callback(term)`` for each
|
|
translatable term that is found in the document.
|
|
|
|
:param etree._Element element: root of etree document to extract terms from
|
|
:param Callable callback: a callable in the form ``f(term, source_line)``,
|
|
that will be called for each extracted term.
|
|
"""
|
|
# not using elementTree.iterparse because we need to skip sub-trees in case
|
|
# the ancestor element had a reason to be skipped
|
|
for el in element:
|
|
if isinstance(el, SKIPPED_ELEMENT_TYPES): continue
|
|
if (el.tag.lower() not in SKIPPED_ELEMENTS
|
|
and "t-js" not in el.attrib
|
|
and not ("t-jquery" in el.attrib and "t-operation" not in el.attrib)
|
|
and not (node.tag == 'attribute' and el.get('name') not in TRANSLATED_ATTRS)
|
|
and el.get("t-translation", '').strip() != "off"):
|
|
|
|
_push(callback, el.text, el.sourceline)
|
|
# Do not export terms contained on the Component directive of OWL
|
|
# attributes in this context are most of the time variables,
|
|
# not real HTML attributes.
|
|
# Node tags starting with a capital letter are considered OWL Components
|
|
# and a widespread convention and good practice for DOM tags is to write
|
|
# them all lower case.
|
|
# https://www.w3schools.com/html/html5_syntax.asp
|
|
# https://github.com/odoo/owl/blob/master/doc/reference/component.md#composition
|
|
if not el.tag[0].isupper() and 't-component' not in el.attrib:
|
|
for att in ('title', 'alt', 'label', 'placeholder', 'aria-label'):
|
|
if att in el.attrib:
|
|
_push(callback, el.attrib[att], el.sourceline)
|
|
_extract_translatable_qweb_terms(el, callback)
|
|
_push(callback, el.tail, el.sourceline)
|
|
|
|
|
|
def babel_extract_qweb(fileobj, keywords, comment_tags, options):
|
|
"""Babel message extractor for qweb template files.
|
|
|
|
:param fileobj: the file-like object the messages should be extracted from
|
|
:param keywords: a list of keywords (i.e. function names) that should
|
|
be recognized as translation functions
|
|
:param comment_tags: a list of translator tags to search for and
|
|
include in the results
|
|
:param options: a dictionary of additional options (optional)
|
|
:return: an iterator over ``(lineno, funcname, message, comments)``
|
|
tuples
|
|
:rtype: Iterable
|
|
"""
|
|
result = []
|
|
def handle_text(text, lineno):
|
|
result.append((lineno, None, text, []))
|
|
tree = etree.parse(fileobj)
|
|
_extract_translatable_qweb_terms(tree.getroot(), handle_text)
|
|
return result
|
|
|
|
ImdInfo = namedtuple('ExternalId', ['name', 'model', 'res_id', 'module'])
|
|
|
|
|
|
class TranslationModuleReader:
|
|
""" Retrieve translated records per module
|
|
|
|
:param cr: cursor to database to export
|
|
:param modules: list of modules to filter the exported terms, can be ['all']
|
|
records with no external id are always ignored
|
|
:param lang: language code to retrieve the translations
|
|
retrieve source terms only if not set
|
|
"""
|
|
|
|
def __init__(self, cr, modules=None, lang=None):
|
|
self._cr = cr
|
|
self._modules = modules or ['all']
|
|
self._lang = lang or 'en_US'
|
|
self.env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
|
|
self._to_translate = []
|
|
self._path_list = [(path, True) for path in odoo.addons.__path__]
|
|
self._installed_modules = [
|
|
m['name']
|
|
for m in self.env['ir.module.module'].search_read([('state', '=', 'installed')], fields=['name'])
|
|
]
|
|
|
|
self._export_translatable_records()
|
|
self._export_translatable_resources()
|
|
|
|
def __iter__(self):
|
|
""" Export ir.translation values for all retrieved records """
|
|
for module, source, name, res_id, ttype, comments, _record_id, value in self._to_translate:
|
|
yield (module, ttype, name, res_id, source, encode(odoo.tools.ustr(value)), comments)
|
|
|
|
def _push_translation(self, module, ttype, name, res_id, source, comments=None, record_id=None, value=None):
|
|
""" Insert a translation that will be used in the file generation
|
|
In po file will create an entry
|
|
#: <ttype>:<name>:<res_id>
|
|
#, <comment>
|
|
msgid "<source>"
|
|
record_id is the database id of the record being translated
|
|
"""
|
|
# empty and one-letter terms are ignored, they probably are not meant to be
|
|
# translated, and would be very hard to translate anyway.
|
|
sanitized_term = (source or '').strip()
|
|
# remove non-alphanumeric chars
|
|
sanitized_term = re.sub(r'\W+', '', sanitized_term)
|
|
if not sanitized_term or len(sanitized_term) <= 1:
|
|
return
|
|
self._to_translate.append((module, source, name, res_id, ttype, tuple(comments or ()), record_id, value))
|
|
|
|
def _get_translatable_records(self, imd_records):
|
|
""" Filter the records that are translatable
|
|
|
|
A record is considered as untranslatable if:
|
|
- it does not exist
|
|
- the model is flagged with _translate=False
|
|
- it is a field of a model flagged with _translate=False
|
|
- it is a selection of a field of a model flagged with _translate=False
|
|
|
|
:param records: a list of namedtuple ImdInfo belonging to the same model
|
|
"""
|
|
model = next(iter(imd_records)).model
|
|
if model not in self.env:
|
|
_logger.error("Unable to find object %r", model)
|
|
return self.env["_unknown"].browse()
|
|
|
|
if not self.env[model]._translate:
|
|
return self.env[model].browse()
|
|
|
|
res_ids = [r.res_id for r in imd_records]
|
|
records = self.env[model].browse(res_ids).exists()
|
|
if len(records) < len(res_ids):
|
|
missing_ids = set(res_ids) - set(records.ids)
|
|
missing_records = [f"{r.module}.{r.name}" for r in imd_records if r.res_id in missing_ids]
|
|
_logger.warning("Unable to find records of type %r with external ids %s", model, ', '.join(missing_records))
|
|
if not records:
|
|
return records
|
|
|
|
if model == 'ir.model.fields.selection':
|
|
fields = defaultdict(list)
|
|
for selection in records:
|
|
fields[selection.field_id] = selection
|
|
for field, selection in fields.items():
|
|
field_name = field.name
|
|
field_model = self.env.get(field.model)
|
|
if (field_model is None or not field_model._translate or
|
|
field_name not in field_model._fields):
|
|
# the selection is linked to a model with _translate=False, remove it
|
|
records -= selection
|
|
elif model == 'ir.model.fields':
|
|
for field in records:
|
|
field_name = field.name
|
|
field_model = self.env.get(field.model)
|
|
if (field_model is None or not field_model._translate or
|
|
field_name not in field_model._fields):
|
|
# the field is linked to a model with _translate=False, remove it
|
|
records -= field
|
|
|
|
return records
|
|
|
|
|
|
def _export_translatable_records(self):
|
|
""" Export translations of all translated records having an external id """
|
|
|
|
query = """SELECT min(name), model, res_id, module
|
|
FROM ir_model_data
|
|
WHERE module = ANY(%s)
|
|
GROUP BY model, res_id, module
|
|
ORDER BY module, model, min(name)"""
|
|
|
|
if 'all' not in self._modules:
|
|
query_param = list(self._modules)
|
|
else:
|
|
query_param = self._installed_modules
|
|
|
|
self._cr.execute(query, (query_param,))
|
|
|
|
records_per_model = defaultdict(dict)
|
|
for (xml_name, model, res_id, module) in self._cr.fetchall():
|
|
records_per_model[model][res_id] = ImdInfo(xml_name, model, res_id, module)
|
|
|
|
for model, imd_per_id in records_per_model.items():
|
|
records = self._get_translatable_records(imd_per_id.values())
|
|
if not records:
|
|
continue
|
|
|
|
for record in records:
|
|
module = imd_per_id[record.id].module
|
|
xml_name = "%s.%s" % (module, imd_per_id[record.id].name)
|
|
for field_name, field in record._fields.items():
|
|
# ir_actions_actions.name is filtered because unlike other inherited fields,
|
|
# this field is inherited as postgresql inherited columns.
|
|
# From our business perspective, the parent column is no need to be translated,
|
|
# but it is need to be set to jsonb column, since the child columns need to be translated
|
|
# And export the parent field may make one value to be translated twice in transifex
|
|
if field.translate and not (field.model_name == 'ir.actions.actions' and field.name == 'name'):
|
|
name = model + "," + field_name
|
|
try:
|
|
value_en = record[field_name] or ''
|
|
value_lang = record.with_context(lang=self._lang)[field_name] or ''
|
|
except Exception:
|
|
continue
|
|
trans_type = 'model_terms' if callable(field.translate) else 'model'
|
|
for term_en, term_langs in field.get_translation_dictionary(value_en, {self._lang: value_lang}).items():
|
|
term_lang = term_langs.get(self._lang)
|
|
self._push_translation(module, trans_type, name, xml_name, term_en, record_id=record.id, value=term_lang if term_lang != term_en else '')
|
|
|
|
def _get_module_from_path(self, path):
|
|
for (mp, rec) in self._path_list:
|
|
mp = os.path.join(mp, '')
|
|
dirname = os.path.join(os.path.dirname(path), '')
|
|
if rec and path.startswith(mp) and dirname != mp:
|
|
path = path[len(mp):]
|
|
return path.split(os.path.sep)[0]
|
|
return 'base' # files that are not in a module are considered as being in 'base' module
|
|
|
|
def _verified_module_filepaths(self, fname, path, root):
|
|
fabsolutepath = join(root, fname)
|
|
frelativepath = fabsolutepath[len(path):]
|
|
display_path = "addons%s" % frelativepath
|
|
module = self._get_module_from_path(fabsolutepath)
|
|
if ('all' in self._modules or module in self._modules) and module in self._installed_modules:
|
|
if os.path.sep != '/':
|
|
display_path = display_path.replace(os.path.sep, '/')
|
|
return module, fabsolutepath, frelativepath, display_path
|
|
return None, None, None, None
|
|
|
|
def _babel_extract_terms(self, fname, path, root, extract_method="python", trans_type='code',
|
|
extra_comments=None, extract_keywords={'_': None}):
|
|
|
|
module, fabsolutepath, _, display_path = self._verified_module_filepaths(fname, path, root)
|
|
if not module:
|
|
return
|
|
extra_comments = extra_comments or []
|
|
src_file = open(fabsolutepath, 'rb')
|
|
options = {}
|
|
if extract_method == 'python':
|
|
options['encoding'] = 'UTF-8'
|
|
translations = code_translations.get_python_translations(module, self._lang)
|
|
else:
|
|
translations = code_translations.get_web_translations(module, self._lang)
|
|
try:
|
|
for extracted in extract.extract(extract_method, src_file, keywords=extract_keywords, options=options):
|
|
# Babel 0.9.6 yields lineno, message, comments
|
|
# Babel 1.3 yields lineno, message, comments, context
|
|
lineno, message, comments = extracted[:3]
|
|
value = translations.get(message, '')
|
|
self._push_translation(module, trans_type, display_path, lineno,
|
|
encode(message), comments + extra_comments, value=value)
|
|
except Exception:
|
|
_logger.exception("Failed to extract terms from %s", fabsolutepath)
|
|
finally:
|
|
src_file.close()
|
|
|
|
def _export_translatable_resources(self):
|
|
""" Export translations for static terms
|
|
|
|
This will include:
|
|
- the python strings marked with _() or _lt()
|
|
- the javascript strings marked with _t() or _lt() inside static/src/js/
|
|
- the strings inside Qweb files inside static/src/xml/
|
|
"""
|
|
|
|
# Also scan these non-addon paths
|
|
for bin_path in ['osv', 'report', 'modules', 'service', 'tools']:
|
|
self._path_list.append((os.path.join(config['root_path'], bin_path), True))
|
|
# non-recursive scan for individual files in root directory but without
|
|
# scanning subdirectories that may contain addons
|
|
self._path_list.append((config['root_path'], False))
|
|
_logger.debug("Scanning modules at paths: %s", self._path_list)
|
|
|
|
for (path, recursive) in self._path_list:
|
|
_logger.debug("Scanning files of modules at %s", path)
|
|
for root, dummy, files in os.walk(path, followlinks=True):
|
|
for fname in fnmatch.filter(files, '*.py'):
|
|
self._babel_extract_terms(fname, path, root,
|
|
extract_keywords={'_': None, '_lt': None})
|
|
if fnmatch.fnmatch(root, '*/static/src*'):
|
|
# Javascript source files
|
|
for fname in fnmatch.filter(files, '*.js'):
|
|
self._babel_extract_terms(fname, path, root, 'javascript',
|
|
extra_comments=[WEB_TRANSLATION_COMMENT],
|
|
extract_keywords={'_t': None, '_lt': None})
|
|
# QWeb template files
|
|
for fname in fnmatch.filter(files, '*.xml'):
|
|
self._babel_extract_terms(fname, path, root, 'odoo.tools.translate:babel_extract_qweb',
|
|
extra_comments=[WEB_TRANSLATION_COMMENT])
|
|
|
|
if not recursive:
|
|
# due to topdown, first iteration is in first level
|
|
break
|
|
|
|
|
|
def trans_load(cr, filename, lang, verbose=True, overwrite=False):
|
|
try:
|
|
with file_open(filename, mode='rb') as fileobj:
|
|
_logger.info("loading %s", filename)
|
|
fileformat = os.path.splitext(filename)[-1][1:].lower()
|
|
return trans_load_data(cr, fileobj, fileformat, lang,
|
|
verbose=verbose,
|
|
overwrite=overwrite)
|
|
except IOError:
|
|
if verbose:
|
|
_logger.error("couldn't read translation file %s", filename)
|
|
return None
|
|
|
|
def trans_load_data(cr, fileobj, fileformat, lang,
|
|
verbose=True, overwrite=False):
|
|
"""Load translations.
|
|
|
|
:param cr:
|
|
:param fileobj: buffer open to a translation file
|
|
:param fileformat: format of the `fielobj` file, one of 'po' or 'csv'
|
|
:param lang: language code of the translations contained in `fileobj`
|
|
language must be present and activated in the database
|
|
:param verbose: increase log output
|
|
:param overwrite: if a translation already exists for a term, replace it with
|
|
the one in `fileobj`
|
|
"""
|
|
if verbose:
|
|
_logger.info('loading translation file for language %s', lang)
|
|
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {'lang': lang})
|
|
if not env['res.lang']._lang_get(lang):
|
|
_logger.error("Couldn't read translation for lang '%s', language not found", lang)
|
|
return None
|
|
try:
|
|
fileobj.seek(0)
|
|
reader = TranslationFileReader(fileobj, fileformat=fileformat)
|
|
_trans_load_data(cr, reader, lang, overwrite)
|
|
|
|
if verbose:
|
|
_logger.info("translation file loaded successfully")
|
|
|
|
except IOError:
|
|
iso_lang = get_iso_codes(lang)
|
|
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat)
|
|
_logger.exception("couldn't read translation file %s", filename)
|
|
|
|
|
|
def _trans_load_data(cr, reader, lang, overwrite=False, force_overwrite=False, xml_ids=None):
|
|
# logical overwrite: if a translation should be updated when it has already been in the DB before
|
|
# logical overwrite = force_overwrite or (not noudpate and overwrite)
|
|
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {'lang': lang})
|
|
if force_overwrite:
|
|
overwrite = True
|
|
|
|
# load all translation in to dict
|
|
deep_defaultdict = lambda: defaultdict(deep_defaultdict)
|
|
# {model_name: {xmlid: {field_name, {src: value}}}}
|
|
translations = deep_defaultdict()
|
|
# [(module_name, imd_name), (module_name, imd_name), ...] only for model_terms translation
|
|
module_imd_names = []
|
|
# {model_name: {field_name: [xmlid, xmlid, ...]}}
|
|
model_name_to_field_name_to_xmlids = defaultdict(lambda: defaultdict(list))
|
|
|
|
for row in reader:
|
|
if not row.get('value') or not row.get('src'): # ignore empty translations
|
|
continue
|
|
if row.get('type') == 'code': # ignore code translations
|
|
continue
|
|
# TODO: CWG if the po file should not be trusted, we need to check each model term
|
|
model_name = row.get('imd_model')
|
|
module_name = row['module']
|
|
imd_name = row['imd_name']
|
|
if model_name in env:
|
|
field_name = row['name'].split(',')[1]
|
|
field = env[model_name]._fields.get(field_name)
|
|
if not field or not field.translate or not field.store:
|
|
continue
|
|
xmlid = module_name + '.' + row['imd_name']
|
|
if xml_ids and xmlid not in xml_ids:
|
|
continue
|
|
if row['type'] == 'model_terms':
|
|
module_imd_names.append((module_name, imd_name))
|
|
model_name_to_field_name_to_xmlids[model_name][field_name].append(xmlid)
|
|
translations[model_name][xmlid][field_name][row['src']] = row['value']
|
|
|
|
if not translations:
|
|
return
|
|
env.flush_all()
|
|
|
|
# get mapping xmlid_to_id, id_to_xmlid
|
|
xmlid_to_id = {}
|
|
for sub_module_imd_names in cr.split_for_in_conditions(module_imd_names):
|
|
query = "SELECT module || '.' || name, res_id, noupdate, model FROM ir_model_data WHERE "
|
|
query += " OR ".join(["module = %s AND name = %s"] * len(sub_module_imd_names))
|
|
cr.execute(query, [param for params in sub_module_imd_names for param in params])
|
|
# {xmlid: (id, noupdate, model_name)}
|
|
xmlid_to_id.update((
|
|
(xmlid, (id_, not force_overwrite and noupdate, model_name))
|
|
for xmlid, id_, noupdate, model_name in cr.fetchall()
|
|
))
|
|
id_to_xmlid = {
|
|
(model_name, id_): (xmlid, noupdate)
|
|
for xmlid, (id_, noupdate, model_name) in xmlid_to_id.items()
|
|
}
|
|
|
|
# translate model terms
|
|
for model_name, model_dictionary in translations.items():
|
|
if model_name not in env:
|
|
continue
|
|
Model = env[model_name]
|
|
model_table = Model._table
|
|
field_name_to_xmlids = model_name_to_field_name_to_xmlids[model_name]
|
|
fields = Model._fields
|
|
field_type = defaultdict(list)
|
|
for field_name in field_name_to_xmlids.keys():
|
|
translate = fields[field_name].translate
|
|
field_type['no_translate' if not translate else 'model_terms' if callable(translate) else 'model'].append(field_name)
|
|
if field_type['no_translate']:
|
|
pass # ignore illegal translations
|
|
for field_name in field_type['model_terms']:
|
|
record_ids = {
|
|
it[0]
|
|
for it in map(xmlid_to_id.get, field_name_to_xmlids[field_name])
|
|
if it
|
|
}
|
|
field = fields[field_name]
|
|
for sub_ids in cr.split_for_in_conditions(record_ids):
|
|
cr.execute(f'SELECT id, "{field_name}" FROM "{model_table}" WHERE id IN %s', (sub_ids,))
|
|
for id_, values in cr.fetchall():
|
|
if not values:
|
|
continue
|
|
value_en = values.get('en_US')
|
|
if not value_en:
|
|
continue
|
|
value_lang = values.get(lang, value_en)
|
|
xmlid, noupdate = id_to_xmlid[(model_name, id_)]
|
|
field_dictionary = {k: v for k, v in model_dictionary[xmlid][field_name].items()} # len(field.get_trans_terms(v)) == 1
|
|
translation_dictionary = field.get_translation_dictionary(value_en, {lang: value_lang})
|
|
|
|
# update translation_dictionary using new translations
|
|
for term_en, term in translation_dictionary.items():
|
|
if not overwrite and noupdate and term[lang] != term_en:
|
|
continue
|
|
term[lang] = field_dictionary.get(term_en, term[lang])
|
|
|
|
new_value_lang = field.translate(lambda term: translation_dictionary.get(term, {}).get(lang), value_en)
|
|
values[lang] = new_value_lang
|
|
env.cache.update_raw(Model.browse(id_), field, [values], dirty=True)
|
|
for field_name in field_type['model']:
|
|
field_translations = []
|
|
for xmlid in field_name_to_xmlids[field_name]:
|
|
src, value = list(model_dictionary[xmlid][field_name].items())[0]
|
|
field_translations.extend([xmlid.split('.')[-1], src, Json({lang: value})])
|
|
if field_translations:
|
|
if not force_overwrite:
|
|
value_query = f"""CASE WHEN {overwrite} is True AND imd.noupdate is False
|
|
THEN m."{field_name}" || t.value
|
|
ELSE t.value || m."{field_name}"END"""
|
|
else:
|
|
value_query = f'm."{field_name}" || t.value'
|
|
env.cr.execute(f"""
|
|
UPDATE "{model_table}" AS m
|
|
SET "{field_name}" = {value_query}
|
|
FROM (
|
|
VALUES {', '.join(['(%s, %s, %s::jsonb)'] * int(len(field_translations) / 3))}
|
|
) AS t(imd_name, src, value)
|
|
JOIN "ir_model_data" AS imd
|
|
ON imd."model" = '{model_name}' AND imd.name = t.imd_name
|
|
WHERE imd."res_id" = m."id" AND m."{field_name}"->>'en_US' = t.src
|
|
""", field_translations)
|
|
env.invalidate_all()
|
|
|
|
|
|
def get_locales(lang=None):
|
|
if lang is None:
|
|
lang = locale.getdefaultlocale()[0]
|
|
|
|
if os.name == 'nt':
|
|
lang = _LOCALE2WIN32.get(lang, lang)
|
|
|
|
def process(enc):
|
|
ln = locale._build_localename((lang, enc))
|
|
yield ln
|
|
nln = locale.normalize(ln)
|
|
if nln != ln:
|
|
yield nln
|
|
|
|
for x in process('utf8'): yield x
|
|
|
|
prefenc = locale.getpreferredencoding()
|
|
if prefenc:
|
|
for x in process(prefenc): yield x
|
|
|
|
prefenc = {
|
|
'latin1': 'latin9',
|
|
'iso-8859-1': 'iso8859-15',
|
|
'cp1252': '1252',
|
|
}.get(prefenc.lower())
|
|
if prefenc:
|
|
for x in process(prefenc): yield x
|
|
|
|
yield lang
|
|
|
|
|
|
def resetlocale():
|
|
# locale.resetlocale is bugged with some locales.
|
|
for ln in get_locales():
|
|
try:
|
|
return locale.setlocale(locale.LC_ALL, ln)
|
|
except locale.Error:
|
|
continue
|
|
|
|
|
|
def load_language(cr, lang):
|
|
""" Loads a translation terms for a language.
|
|
|
|
Used mainly to automate language loading at db initialization.
|
|
|
|
:param cr:
|
|
:param str lang: language ISO code with optional underscore (``_``) and
|
|
l10n flavor (ex: 'fr', 'fr_BE', but not 'fr-BE')
|
|
"""
|
|
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
|
|
lang_ids = env['res.lang'].with_context(active_test=False).search([('code', '=', lang)]).ids
|
|
installer = env['base.language.install'].create({'lang_ids': [(6, 0, lang_ids)]})
|
|
installer.lang_install()
|
|
|
|
|
|
class CodeTranslations:
|
|
def __init__(self):
|
|
# {(module_name, lang): {src: value}}
|
|
self.python_translations = {}
|
|
# {(module_name, lang): {'message': [{'id': src, 'string': value}]}
|
|
self.web_translations = {}
|
|
|
|
def _get_po_paths(self, mod, lang):
|
|
lang_base = lang.split('_')[0]
|
|
po_paths = [get_resource_path(mod, 'i18n', lang_base + '.po'),
|
|
get_resource_path(mod, 'i18n', lang + '.po'),
|
|
get_resource_path(mod, 'i18n_extra', lang_base + '.po'),
|
|
get_resource_path(mod, 'i18n_extra', lang + '.po')]
|
|
return [path for path in po_paths if path]
|
|
|
|
def _trans_load_code_python(self, fileobj, fileformat, lang):
|
|
python_translations = {}
|
|
fileobj.seek(0)
|
|
reader = TranslationFileReader(fileobj, fileformat=fileformat)
|
|
for row in reader:
|
|
if row.get('value') and row.get('src') and row.get('type') == 'code':
|
|
python_translations[row['src']] = row['value']
|
|
return python_translations
|
|
|
|
def _trans_load_code_webclient(self, fileobj, fileformat, lang):
|
|
# current, we assume the fileobj is from the source code, which only contains the translation for the current module
|
|
# don't use it in the import logic
|
|
webclient_translations = {}
|
|
fileobj.seek(0)
|
|
reader = TranslationFileReader(fileobj, fileformat=fileformat)
|
|
for row in reader:
|
|
if row.get('value') and row.get('src') and row.get('type') == 'code' and WEB_TRANSLATION_COMMENT in row[
|
|
'comments']:
|
|
webclient_translations[row['src']] = row['value']
|
|
return webclient_translations
|
|
|
|
def _load_python_translations(self, module_name, lang):
|
|
po_paths = self._get_po_paths(module_name, lang)
|
|
translations = {}
|
|
for po_path in po_paths:
|
|
try:
|
|
with file_open(po_path, mode='rb') as fileobj:
|
|
p = self._trans_load_code_python(fileobj, 'po', lang)
|
|
translations.update(p)
|
|
except IOError:
|
|
iso_lang = get_iso_codes(lang)
|
|
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', 'po')
|
|
_logger.exception("couldn't read translation file %s", filename)
|
|
self.python_translations[(module_name, lang)] = translations
|
|
|
|
def _load_web_translations(self, module_name, lang):
|
|
po_paths = self._get_po_paths(module_name, lang)
|
|
translations = {}
|
|
for po_path in po_paths:
|
|
try:
|
|
with file_open(po_path, mode='rb') as fileobj:
|
|
p = self._trans_load_code_webclient(fileobj, 'po', lang)
|
|
translations.update(p)
|
|
except IOError:
|
|
iso_lang = get_iso_codes(lang)
|
|
filename = '[lang: %s][format: %s]' % (iso_lang or 'new', 'po')
|
|
_logger.exception("couldn't read translation file %s", filename)
|
|
self.web_translations[(module_name, lang)] = {
|
|
"messages": [{"id": src, "string": value} for src, value in translations.items()]
|
|
}
|
|
|
|
def get_python_translations(self, module_name, lang):
|
|
if (module_name, lang) not in self.python_translations:
|
|
self._load_python_translations(module_name, lang)
|
|
return self.python_translations[(module_name, lang)]
|
|
|
|
def get_web_translations(self, module_name, lang):
|
|
if (module_name, lang) not in self.web_translations:
|
|
self._load_web_translations(module_name, lang)
|
|
return self.web_translations[(module_name, lang)]
|
|
|
|
|
|
code_translations = CodeTranslations()
|
|
|
|
|
|
def _get_translation_upgrade_queries(cr, field):
|
|
""" Return a pair of lists ``migrate_queries, cleanup_queries`` of SQL queries. The queries in
|
|
``migrate_queries`` do migrate the data from table ``_ir_translation`` to the corresponding
|
|
field's column, while the queries in ``cleanup_queries`` remove the corresponding data from
|
|
table ``_ir_translation``.
|
|
"""
|
|
Model = odoo.registry(cr.dbname)[field.model_name]
|
|
translation_name = f"{field.model_name},{field.name}"
|
|
migrate_queries = []
|
|
cleanup_queries = []
|
|
|
|
if field.translate is True:
|
|
query = f"""
|
|
WITH t AS (
|
|
SELECT res_id, jsonb_object_agg(lang, value) AS value
|
|
FROM _ir_translation
|
|
WHERE type = 'model' AND name = %s AND state = 'translated'
|
|
GROUP BY res_id
|
|
)
|
|
UPDATE {Model._table} m
|
|
SET "{field.name}" = m."{field.name}" || t.value
|
|
FROM t
|
|
WHERE t.res_id = m.id
|
|
"""
|
|
migrate_queries.append(cr.mogrify(query, [translation_name]).decode())
|
|
|
|
query = "DELETE FROM _ir_translation WHERE type = 'model' AND name = %s"
|
|
cleanup_queries.append(cr.mogrify(query, [translation_name]).decode())
|
|
|
|
# upgrade model_terms translation: one update per field per record
|
|
if callable(field.translate):
|
|
cr.execute(f"""
|
|
WITH t0 AS (
|
|
-- aggregate translations by source term --
|
|
SELECT res_id, lang, jsonb_object_agg(src, value) AS value
|
|
FROM _ir_translation
|
|
WHERE type = 'model_terms' AND name = %s AND state = 'translated'
|
|
GROUP BY res_id, lang
|
|
),
|
|
t AS (
|
|
-- aggregate translations by lang --
|
|
SELECT res_id, jsonb_object_agg(lang, value) AS value
|
|
FROM t0
|
|
GROUP BY res_id
|
|
)
|
|
SELECT t.res_id, m."{field.name}"->>'en_US', t.value
|
|
FROM t
|
|
JOIN "{Model._table}" m ON t.res_id = m.id
|
|
""", [translation_name])
|
|
for id_, source_value, translations in cr.fetchall():
|
|
if not source_value:
|
|
continue
|
|
new_value = {
|
|
lang: field.translate(terms_mapping.get, source_value)
|
|
for lang, terms_mapping in translations.items()
|
|
}
|
|
if "en_US" not in new_value:
|
|
new_value["en_US"] = field.translate(lambda v: None, source_value)
|
|
query = f'UPDATE "{Model._table}" SET "{field.name}" = %s WHERE id = %s'
|
|
migrate_queries.append(cr.mogrify(query, [Json(new_value), id_]).decode())
|
|
|
|
query = "DELETE FROM _ir_translation WHERE type = 'model_terms' AND name = %s"
|
|
cleanup_queries.append(cr.mogrify(query, [translation_name]).decode())
|
|
|
|
return migrate_queries, cleanup_queries
|