pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] tools: remove custom pofile reader

Browse Source 
Use polib library that handles this correctly
The complexity of the parser is moved to the library
This commit is contained in:
Martin Trigaux
2019-06-05 09:44:11 +00:00
parent 3c568aec0b
commit 632fa044c3
10 changed files with 222 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
debian/control
Vendored
+1
View File
@@ -30,6 +30,7 @@ Depends:
python3-mako,
python3-mock,
python3-passlib,
python3-polib,
python3-psutil,
python3-psycopg2,
python3-pydot,
odoo/addons/test_translation_import/i18n/fr.po
+14 -3
View File
@@ -23,6 +23,12 @@ msgstr ""
msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
msgstr "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB in french"
#. module: test_translation_import
#: selection:test.translation.import,import_type:0
msgid "Bar"
msgstr "Bar in french"
# Note: the line in pot is 21 and is 14 here
#. module: test_translation_import
#: code:addons/test_translation_import/models.py:14
#, python-format
@@ -30,15 +36,20 @@ msgid "Ijkl"
msgstr "Ijkl in french"
#. module: test_translation_import
#: model:ir.model,name:test_translation_import.model_test_translation_import
msgid "test.translation.import"
msgstr "test.translation.import in french"
#: selection:test.translation.import,import_type:0
msgid "Foo"
msgstr "Foo in french"
#. module: test_translation_import
#: model:ir.model.fields,help:test_translation_import.field_test_translation_import__name
msgid "Efgh"
msgstr "Efgh in french"
#. module: test_translation_import
#: model:ir.model,name:test_translation_import.model_test_translation_import
msgid "Test: Translation Import"
msgstr "Test: Import de traductions"
#. module: test_translation_import
#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
odoo/addons/test_translation_import/i18n/fr_BE.po
+2 -2
View File
@@ -21,8 +21,8 @@ msgstr "Ijkl in belgian french"
#. module: test_translation_import
#: model:ir.model,name:test_translation_import.model_test_translation_import
msgid "test.translation.import"
msgstr "test.translation.import in belgian french"
msgid "Test: Translation Import"
msgstr "Test: Import de traductions in belgian french"
#. module: test_translation_import
#: model:ir.model.fields,help:test_translation_import.field_test_translation_import__name
odoo/addons/test_translation_import/i18n/test_translation_import.pot
+36 -12
View File
@@ -23,21 +23,15 @@ msgstr ""
"Plural-Forms: \n"
#. module: test_translation_import
#: code:addons/test_translation_import/models.py:15
#: code:addons/test_translation_import/models.py:21
#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__name
#, python-format
msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
msgstr ""
#. module: test_translation_import
#: code:addons/test_translation_import/models.py:14
#, python-format
msgid "Ijkl"
msgstr ""
#. module: test_translation_import
#: model:ir.model,name:test_translation_import.model_test_translation_import
msgid "test.translation.import"
#: selection:test.translation.import,import_type:0
msgid "Bar"
msgstr ""
#. module: test_translation_import
@@ -46,9 +40,27 @@ msgid "Efgh"
msgstr ""
#. module: test_translation_import
#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
msgid "Test translation import"
#: selection:test.translation.import,import_type:0
msgid "Foo"
msgstr ""
#. module: test_translation_import
#: code:addons/test_translation_import/models.py:18
#: code:addons/test_translation_import/models.py:21
#, python-format
msgid "Ijkl"
msgstr ""
#. module: test_translation_import
#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__import_type
msgid "Import Type"
msgstr ""
#. module: test_translation_import
#: code:addons/test_translation_import/models.py:23
#: code:addons/test_translation_import/tests/test_term_count.py:158
#, python-format
msgid "Klingon"
msgstr ""
#. module: test_translation_import
@@ -56,6 +68,12 @@ msgstr ""
msgid "Test translation"
msgstr ""
#. module: test_translation_import
#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
msgid "Test translation import"
msgstr ""
#. module: test_translation_import
#: code:addons/base/models/arthur.py:42
#: code:addons/base/models/arthur.py:43
@@ -73,3 +91,9 @@ msgstr ""
#: code:addons/base/models/trillian.py:42
msgid "Test translation with two code type and model"
msgstr ""
#. module: test_translation_import
#: model:ir.model,name:test_translation_import.model_test_translation_import
msgid "Test: Translation Import"
msgstr ""
odoo/addons/test_translation_import/models.py
+4
View File
@@ -10,6 +10,10 @@ class m(models.TransientModel):
name = fields.Char('1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB',
size=32, help='Efgh')
other_name = fields.Char('Test translation with two code type and model')
import_type = fields.Selection([
('foo', 'Foo'),
('bar', 'Bar'),
])
_('Ijkl')
odoo/addons/test_translation_import/tests/test_term_count.py
+25 -3
View File
@@ -17,11 +17,32 @@ class TestTermCount(common.TransactionCase):
Just make sure we have as many translation entries as we wanted.
"""
odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', module_name='test_translation_import', verbose=False)
ids = self.env['ir.translation'].search([
translations = self.env['ir.translation'].search([
('lang', '=', 'fr_FR'),
('src', '=', '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB'),
])
self.assertEqual(len(ids), 2)
], order='type')
self.assertEqual(len(translations), 2)
self.assertEqual(translations[0].type, 'code')
self.assertEqual(translations[0].module, 'test_translation_import')
self.assertEqual(translations[0].name, 'addons/test_translation_import/models.py')
self.assertEqual(translations[0].comments, '')
self.assertEqual(translations[0].res_id, 21)
self.assertEqual(translations[1].type, 'model')
self.assertEqual(translations[1].module, 'test_translation_import')
self.assertEqual(translations[1].name, 'ir.model.fields,field_description')
self.assertEqual(translations[1].comments, '')
field = self.env['ir.model.fields'].search([('model', '=', 'test.translation.import'), ('name', '=', 'name')])
self.assertEqual(translations[1].res_id, field.id)
translations = self.env['ir.translation'].search([
('lang', '=', 'fr_FR'),
('type', '=', 'selection'),
('module', '=', 'test_translation_import'),
], order='src')
self.assertEqual(len(translations), 2)
self.assertEqual(translations[0].name, 'test.translation.import,import_type')
self.assertEqual(translations[0].res_id, 0)
def test_count_term_module(self):
"""
@@ -34,6 +55,7 @@ class TestTermCount(common.TransactionCase):
('module', '=', 'test_translation_import'),
])
self.assertEqual(len(translations), 1)
self.assertEqual(translations.res_id, 21)
def test_noupdate(self):
"""
odoo/tools/translate.py
+137 -217
View File
@@ -7,6 +7,7 @@ import io
import locale
import logging
import os
import polib
import re
import tarfile
import tempfile
@@ -15,6 +16,7 @@ from collections import defaultdict
from datetime import datetime
from os.path import join
from pathlib import Path
from babel.messages import extract
from lxml import etree, html
@@ -476,138 +478,147 @@ def unquote(str):
"""Returns unquoted PO term string, with special PO characters unescaped"""
return re_escaped_char.sub(_sub_replacement, str[1:-1])
def TranslationFileReader(source, fileformat='po'):
""" Iterate over translation file to return Odoo translation entries """
if fileformat == 'csv':
return CSVFileReader(source)
if fileformat == 'po':
return PoFileReader(source)
_logger.info('Bad file format: %s', fileformat)
raise Exception(_('Bad file format: %s') % fileformat)
class CSVFileReader:
def __init__(self, source):
self.source = pycompat.csv_reader(source, quotechar='"', delimiter=',')
# read the first line of the file (it contains columns titles)
self.fields = next(self.source)
def __iter__(self):
for entry in self.source:
yield zip(self.fields, entry)
class PoFileReader:
""" Iterate over po file to return Odoo translation entries """
def __init__(self, source):
def get_pot_path(source_name):
# when fileobj is a TemporaryFile, its name is an inter in P3, a string in P2
if isinstance(source_name, str) and source_name.endswith('.po'):
# Normally the path looks like /path/to/xxx/i18n/lang.po
# and we try to find the corresponding
# /path/to/xxx/i18n/xxx.pot file.
# (Sometimes we have 'i18n_extra' instead of just 'i18n')
path = Path(source_name)
filename = path.parent.parent.name + '.pot'
pot_path = path.with_name(filename)
return pot_path.exists() and str(pot_path) or False
return False
# polib accepts a path or the file content as a string, not a fileobj
if isinstance(source, str):
self.pofile = polib.pofile(source)
pot_path = get_pot_path(source)
else:
# either a BufferedIOBase or result from NamedTemporaryFile
self.pofile = polib.pofile(source.read().decode())
pot_path = get_pot_path(source.name)
if pot_path:
# Make a reader for the POT file
# (Because the POT comments are correct on GitHub but the
# PO comments tends to be outdated. See LP bug 933496.)
self.pofile.merge(polib.pofile(pot_path))
def __iter__(self):
for entry in self.pofile:
if entry.obsolete:
continue
# in case of moduleS keep only the first
match = re.match(r"(module[s]?): (\w+)", entry.comment)
_, module = match.groups()
comments = "\n".join([c for c in entry.comment.split('\n') if not c.startswith('module:')])
source = entry.msgid
translation = entry.msgstr
found_code_occurrence = False
for occurrence, line_number in entry.occurrences:
match = re.match(r'(model|model_terms):([\w.]+),([\w]+):(\w+)\.(\w+)', occurrence)
if match:
type, model_name, field_name, module, xmlid = match.groups()
yield {
'type': type,
'imd_model': model_name,
'name': model_name+','+field_name,
'imd_name': xmlid,
'res_id': None,
'src': source,
'value': translation,
'comments': comments,
'module': module,
}
continue
match = re.match(r'(code):([\w/.]+)', occurrence)
if match:
type, name = match.groups()
if found_code_occurrence:
# unicity constrain on code translation
continue
found_code_occurrence = True
yield {
'type': type,
'name': name,
'src': source,
'value': translation,
'comments': comments,
'res_id': line_number,
'module': module,
}
continue
match = re.match(r'(selection):([\w.]+),([\w]+)', occurrence)
if match:
type, model_name, field_name = match.groups()
yield {
'type': type,
'model': model_name,
'name': model_name+','+field_name,
'src': source,
'value': translation,
'comments': comments,
'res_id': line_number,
'module': module,
}
continue
match = re.match(r'(sql_constraint|constraint):([\w.]+)', occurrence)
if match:
type, model = match.groups()
yield {
'type': type,
'name': model,
'src': source,
'value': translation,
'comments': comments,
'res_id': line_number,
'module': module,
}
continue
_logger.error("malformed po file: unknown occurrence: %s", occurrence)
# class to handle po files
class PoFile(object):
def __init__(self, buffer):
# TextIOWrapper closes its underlying buffer on close *and* can't
# handle actual file objects (on python 2)
self.buffer = codecs.StreamReaderWriter(
stream=buffer,
Reader=codecs.getreader('utf-8'),
Writer=codecs.getwriter('utf-8')
)
def __iter__(self):
self.buffer.seek(0)
self.lines = self._get_lines()
self.lines_count = len(self.lines)
self.first = True
self.extra_lines= []
return self
def _get_lines(self):
lines = self.buffer.readlines()
# remove the BOM (Byte Order Mark):
if len(lines):
lines[0] = lines[0].lstrip(u"\ufeff")
lines.append('') # ensure that the file ends with at least an empty line
return lines
def cur_line(self):
return self.lines_count - len(self.lines)
def next(self):
trans_type = name = res_id = source = trad = module = None
if self.extra_lines:
trans_type, name, res_id, source, trad, comments = self.extra_lines.pop(0)
if not res_id:
res_id = '0'
else:
comments = []
targets = []
line = None
fuzzy = False
while not line:
if 0 == len(self.lines):
raise StopIteration()
line = self.lines.pop(0).strip()
while line.startswith('#'):
if line.startswith('#~ '):
break
if line.startswith('#.'):
line = line[2:].strip()
if not line.startswith('module:'):
comments.append(line)
else:
module = line[7:].strip()
elif line.startswith('#:'):
# Process the `reference` comments. Each line can specify
# multiple targets (e.g. model, view, code, selection,
# ...). For each target, we will return an additional
# entry.
for lpart in line[2:].strip().split(' '):
trans_info = lpart.strip().split(':',2)
if trans_info and len(trans_info) == 2:
# looks like the translation trans_type is missing, which is not
# unexpected because it is not a GetText standard. Default: 'code'
trans_info[:0] = ['code']
if trans_info and len(trans_info) == 3:
# this is a ref line holding the destination info (model, field, record)
targets.append(trans_info)
elif line.startswith('#,') and (line[2:].strip() == 'fuzzy'):
fuzzy = True
line = self.lines.pop(0).strip()
if not self.lines:
raise StopIteration()
while not line:
# allow empty lines between comments and msgid
line = self.lines.pop(0).strip()
if line.startswith('#~ '):
while line.startswith('#~ ') or not line.strip():
if 0 == len(self.lines):
raise StopIteration()
line = self.lines.pop(0)
# This has been a deprecated entry, don't return anything
return next(self)
if not line.startswith('msgid'):
raise Exception("malformed file: bad line: %s" % line)
source = unquote(line[6:])
line = self.lines.pop(0).strip()
if not source and self.first:
self.first = False
# if the source is "" and it's the first msgid, it's the special
# msgstr with the information about the translate and the
# translator; we skip it
self.extra_lines = []
while line:
line = self.lines.pop(0).strip()
return next(self)
while not line.startswith('msgstr'):
if not line:
raise Exception('malformed file at %d'% self.cur_line())
source += unquote(line)
line = self.lines.pop(0).strip()
trad = unquote(line[7:])
line = self.lines.pop(0).strip()
while line:
trad += unquote(line)
line = self.lines.pop(0).strip()
if targets and not fuzzy:
# Use the first target for the current entry (returned at the
# end of this next() call), and keep the others to generate
# additional entries (returned the next next() calls).
trans_type, name, res_id = targets.pop(0)
code = trans_type == 'code'
for t, n, r in targets:
if t == 'code' and code: continue
if t == 'code':
code = True
self.extra_lines.append((t, n, r, source, trad, comments))
if name is None:
if not fuzzy:
_logger.warning('Missing "#:" formatted comment at line %d for the following source:\n\t%s',
self.cur_line(), source[:30])
return next(self)
return trans_type, name, res_id, source, trad, '\n'.join(comments), module
__next__ = next
def write_infos(self, modules):
import odoo.release as release
self.buffer.write(u"# Translation of %(project)s.\n" \
@@ -1032,60 +1043,9 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
# lets create the language with locale information
Lang.load_lang(lang=lang, lang_name=lang_name)
# Parse also the POT: it will possibly provide additional targets.
# (Because the POT comments are correct on Launchpad but not the
# PO comments due to a Launchpad limitation. See LP bug 933496.)
pot_reader = []
use_pot_reference = False
# now, the serious things: we read the language file
fileobj.seek(0)
if fileformat == 'csv':
reader = pycompat.csv_reader(fileobj, quotechar='"', delimiter=',')
# read the first line of the file (it contains columns titles)
fields = next(reader)
elif fileformat == 'po':
reader = PoFile(fileobj)
fields = ['type', 'name', 'res_id', 'src', 'value', 'comments', 'module']
# Make a reader for the POT file and be somewhat defensive for the
# stable branch.
# when fileobj is a TemporaryFile, its name is an interget in P3, a string in P2
if isinstance(fileobj.name, str) and fileobj.name.endswith('.po'):
try:
# Normally the path looks like /path/to/xxx/i18n/lang.po
# and we try to find the corresponding
# /path/to/xxx/i18n/xxx.pot file.
# (Sometimes we have 'i18n_extra' instead of just 'i18n')
addons_module_i18n, _ignored = os.path.split(fileobj.name)
addons_module, i18n_dir = os.path.split(addons_module_i18n)
addons, module = os.path.split(addons_module)
pot_handle = file_open(os.path.join(
addons, module, i18n_dir, module + '.pot'), mode='rb')
pot_reader = PoFile(pot_handle)
use_pot_reference = True
except:
pass
else:
_logger.info('Bad file format: %s', fileformat)
raise Exception(_('Bad file format: %s') % fileformat)
# Read the POT references, and keep them indexed by source string.
class Target(object):
def __init__(self):
self.value = None
self.targets = set() # set of (type, name, res_id)
self.comments = None
pot_targets = defaultdict(Target)
for type, name, res_id, src, _ignored, comments, module in pot_reader:
if type is not None:
target = pot_targets[src]
target.targets.add((type, name, type != 'code' and res_id or 0))
target.comments = comments
reader = TranslationFileReader(fileobj, fileformat=fileformat)
# read the rest of the file
irt_cursor = Translation._get_import_cursor()
@@ -1098,43 +1058,14 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
'comments', 'imd_model', 'imd_name', 'module'))
dic['lang'] = lang
dic.update(zip(fields, row))
dic.update(row)
# do not import empty values
if not env.context.get('create_empty_translation', False) and not dic['value']:
return
if use_pot_reference:
# discard the target from the POT targets.
src = dic['src']
target_key = (dic['type'], dic['name'], dic['type'] != 'code' and dic['res_id'] or 0)
target = pot_targets.get(src)
if not target or target_key not in target.targets:
_logger.info("Translation '%s' (%s, %s, %s) not found in reference pot, skipping",
src[:60], dic['type'], dic['name'], dic['res_id'])
return
target.value = dic['value']
target.targets.discard(target_key)
# This would skip terms that fail to specify a res_id
res_id = dic['res_id']
if not res_id and dic['type'] != 'code':
return
if isinstance(res_id, int) or \
(isinstance(res_id, str) and res_id.isdigit()):
dic['res_id'] = int(res_id)
if module_name:
dic['module'] = module_name
else:
# res_id is an xml id
dic['res_id'] = None
dic['imd_model'] = dic['name'].split(',')[0]
if '.' in res_id:
dic['module'], dic['imd_name'] = res_id.split('.', 1)
else:
dic['module'], dic['imd_name'] = module_name, res_id
if dic['type'] == 'code' and module_name:
dic['module'] = module_name
irt_cursor.push(dic)
@@ -1143,17 +1074,6 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
for row in reader:
process_row(row)
if use_pot_reference:
# Then process the entries implied by the POT file (which is more
# correct w.r.t. the targets) if some of them remain.
pot_rows = []
for src, target in pot_targets.items():
if target.value:
for type, name, res_id in target.targets:
pot_rows.append((type, name, res_id, src, target.value, target.comments))
for row in pot_rows:
process_row(row)
irt_cursor.finish()
Translation.clear_caches()
if verbose:
requirements.txt
+1
View File
@@ -21,6 +21,7 @@ num2words==0.5.6
ofxparse==0.16
passlib==1.6.5
Pillow==4.0.0
polib==1.1.0
psutil==4.3.1; sys_platform != 'win32'
psycopg2==2.7.3.1; sys_platform != 'win32'
pydot==1.2.3
setup.cfg
+1
View File
@@ -28,6 +28,7 @@ requires =
python3-pillow
python3-psutil
python3-psycopg2
python3-polib
python3-pydot
python3-pyldap
python3-pyparsing
setup/win32/winpy_requirements.txt
+1
View File
@@ -15,6 +15,7 @@ num2words==0.5.4
ofxparse==0.16
passlib==1.6.5
Pillow>=3.4.1
polib>=1.1.0
psutil>=4.3.1
psycopg2==2.7.1
pydot==1.2.3