[IMP] tools: remove custom pofile reader

Use polib library that handles this correctly
The complexity of the parser is moved to the library

debian/control

@@ -30,6 +30,7 @@ Depends:
  python3-mako,
  python3-mock,
  python3-passlib,
+ python3-polib,
  python3-psutil,
  python3-psycopg2,
  python3-pydot,

odoo/addons/test_translation_import/i18n/fr.po

@@ -23,6 +23,12 @@ msgstr ""
 msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
 msgstr "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB in french"
 
+#. module: test_translation_import
+#: selection:test.translation.import,import_type:0
+msgid "Bar"
+msgstr "Bar in french"
+
+# Note: the line in pot is 21 and is 14 here
 #. module: test_translation_import
 #: code:addons/test_translation_import/models.py:14
 #, python-format
@@ -30,15 +36,20 @@ msgid "Ijkl"
 msgstr "Ijkl in french"
 
 #. module: test_translation_import
-#: model:ir.model,name:test_translation_import.model_test_translation_import
+#: selection:test.translation.import,import_type:0
-msgid "test.translation.import"
+msgid "Foo"
-msgstr "test.translation.import in french"
+msgstr "Foo in french"
 
 #. module: test_translation_import
 #: model:ir.model.fields,help:test_translation_import.field_test_translation_import__name
 msgid "Efgh"
 msgstr "Efgh in french"
 
+#. module: test_translation_import
+#: model:ir.model,name:test_translation_import.model_test_translation_import
+msgid "Test: Translation Import"
+msgstr "Test: Import de traductions"
+
 #. module: test_translation_import
 #: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
 #: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import

odoo/addons/test_translation_import/i18n/fr_BE.po

@@ -21,8 +21,8 @@ msgstr "Ijkl in belgian french"
 
 #. module: test_translation_import
 #: model:ir.model,name:test_translation_import.model_test_translation_import
-msgid "test.translation.import"
+msgid "Test: Translation Import"
-msgstr "test.translation.import in belgian french"
+msgstr "Test: Import de traductions in belgian french"
 
 #. module: test_translation_import
 #: model:ir.model.fields,help:test_translation_import.field_test_translation_import__name

odoo/addons/test_translation_import/i18n/test_translation_import.pot

@@ -23,21 +23,15 @@ msgstr ""
 "Plural-Forms: \n"
 
 #. module: test_translation_import
-#: code:addons/test_translation_import/models.py:15
+#: code:addons/test_translation_import/models.py:21
 #: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__name
 #, python-format
 msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
 msgstr ""
 
 #. module: test_translation_import
-#: code:addons/test_translation_import/models.py:14
+#: selection:test.translation.import,import_type:0
-#, python-format
+msgid "Bar"
-msgid "Ijkl"
-msgstr ""
-
-#. module: test_translation_import
-#: model:ir.model,name:test_translation_import.model_test_translation_import
-msgid "test.translation.import"
 msgstr ""
 
 #. module: test_translation_import
@@ -46,9 +40,27 @@ msgid "Efgh"
 msgstr ""
 
 #. module: test_translation_import
-#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
+#: selection:test.translation.import,import_type:0
-#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
+msgid "Foo"
-msgid "Test translation import"
+msgstr ""
+
+#. module: test_translation_import
+#: code:addons/test_translation_import/models.py:18
+#: code:addons/test_translation_import/models.py:21
+#, python-format
+msgid "Ijkl"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__import_type
+msgid "Import Type"
+msgstr ""
+
+#. module: test_translation_import
+#: code:addons/test_translation_import/models.py:23
+#: code:addons/test_translation_import/tests/test_term_count.py:158
+#, python-format
+msgid "Klingon"
 msgstr ""
 
 #. module: test_translation_import
@@ -56,6 +68,12 @@ msgstr ""
 msgid "Test translation"
 msgstr ""
 
+#. module: test_translation_import
+#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
+#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
+msgid "Test translation import"
+msgstr ""
+
 #. module: test_translation_import
 #: code:addons/base/models/arthur.py:42
 #: code:addons/base/models/arthur.py:43
@@ -73,3 +91,9 @@ msgstr ""
 #: code:addons/base/models/trillian.py:42
 msgid "Test translation with two code type and model"
 msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model,name:test_translation_import.model_test_translation_import
+msgid "Test: Translation Import"
+msgstr ""
+

odoo/addons/test_translation_import/models.py

@@ -10,6 +10,10 @@ class m(models.TransientModel):
     name = fields.Char('1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB',
                        size=32, help='Efgh')
     other_name = fields.Char('Test translation with two code type and model')
+    import_type = fields.Selection([
+        ('foo', 'Foo'),
+        ('bar', 'Bar'),
+    ])
 
     _('Ijkl')
 

odoo/addons/test_translation_import/tests/test_term_count.py

@@ -17,11 +17,32 @@ class TestTermCount(common.TransactionCase):
         Just make sure we have as many translation entries as we wanted.
         """
         odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', module_name='test_translation_import', verbose=False)
-        ids = self.env['ir.translation'].search([
+        translations = self.env['ir.translation'].search([
             ('lang', '=', 'fr_FR'),
             ('src', '=', '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB'),
-        ])
+        ], order='type')
-        self.assertEqual(len(ids), 2)
+        self.assertEqual(len(translations), 2)
+        self.assertEqual(translations[0].type, 'code')
+        self.assertEqual(translations[0].module, 'test_translation_import')
+        self.assertEqual(translations[0].name, 'addons/test_translation_import/models.py')
+        self.assertEqual(translations[0].comments, '')
+        self.assertEqual(translations[0].res_id, 21)
+        self.assertEqual(translations[1].type, 'model')
+        self.assertEqual(translations[1].module, 'test_translation_import')
+        self.assertEqual(translations[1].name, 'ir.model.fields,field_description')
+        self.assertEqual(translations[1].comments, '')
+        field = self.env['ir.model.fields'].search([('model', '=', 'test.translation.import'), ('name', '=', 'name')])
+        self.assertEqual(translations[1].res_id, field.id)
+
+        translations = self.env['ir.translation'].search([
+            ('lang', '=', 'fr_FR'),
+            ('type', '=', 'selection'),
+            ('module', '=', 'test_translation_import'),
+        ], order='src')
+        self.assertEqual(len(translations), 2)
+        self.assertEqual(translations[0].name, 'test.translation.import,import_type')
+        self.assertEqual(translations[0].res_id, 0)
+
 
     def test_count_term_module(self):
         """
@@ -34,6 +55,7 @@ class TestTermCount(common.TransactionCase):
             ('module', '=', 'test_translation_import'),
         ])
         self.assertEqual(len(translations), 1)
+        self.assertEqual(translations.res_id, 21)
 
     def test_noupdate(self):
         """

odoo/tools/translate.py

@@ -7,6 +7,7 @@ import io
 import locale
 import logging
 import os
+import polib
 import re
 import tarfile
 import tempfile
@@ -15,6 +16,7 @@ from collections import defaultdict
 from datetime import datetime
 from os.path import join
 
+from pathlib import Path
 from babel.messages import extract
 from lxml import etree, html
 
@@ -476,138 +478,147 @@ def unquote(str):
     """Returns unquoted PO term string, with special PO characters unescaped"""
     return re_escaped_char.sub(_sub_replacement, str[1:-1])
 
+def TranslationFileReader(source, fileformat='po'):
+    """ Iterate over translation file to return Odoo translation entries """
+    if fileformat == 'csv':
+        return CSVFileReader(source)
+    if fileformat == 'po':
+        return PoFileReader(source)
+    _logger.info('Bad file format: %s', fileformat)
+    raise Exception(_('Bad file format: %s') % fileformat)
+
+class CSVFileReader:
+    def __init__(self, source):
+        self.source = pycompat.csv_reader(source, quotechar='"', delimiter=',')
+        # read the first line of the file (it contains columns titles)
+        self.fields = next(self.source)
+
+    def __iter__(self):
+        for entry in self.source:
+            yield zip(self.fields, entry)
+
+class PoFileReader:
+    """ Iterate over po file to return Odoo translation entries """
+    def __init__(self, source):
+
+        def get_pot_path(source_name):
+            # when fileobj is a TemporaryFile, its name is an inter in P3, a string in P2
+            if isinstance(source_name, str) and source_name.endswith('.po'):
+                # Normally the path looks like /path/to/xxx/i18n/lang.po
+                # and we try to find the corresponding
+                # /path/to/xxx/i18n/xxx.pot file.
+                # (Sometimes we have 'i18n_extra' instead of just 'i18n')
+                path = Path(source_name)
+                filename = path.parent.parent.name + '.pot'
+                pot_path = path.with_name(filename)
+                return pot_path.exists() and str(pot_path) or False
+            return False
+
+        # polib accepts a path or the file content as a string, not a fileobj
+        if isinstance(source, str):
+            self.pofile = polib.pofile(source)
+            pot_path = get_pot_path(source)
+        else:
+            # either a BufferedIOBase or result from NamedTemporaryFile
+            self.pofile = polib.pofile(source.read().decode())
+            pot_path = get_pot_path(source.name)
+
+        if pot_path:
+            # Make a reader for the POT file
+            # (Because the POT comments are correct on GitHub but the
+            # PO comments tends to be outdated. See LP bug 933496.)
+            self.pofile.merge(polib.pofile(pot_path))
+
+    def __iter__(self):
+        for entry in self.pofile:
+            if entry.obsolete:
+                continue
+
+            # in case of moduleS keep only the first
+            match = re.match(r"(module[s]?): (\w+)", entry.comment)
+            _, module = match.groups()
+            comments = "\n".join([c for c in entry.comment.split('\n') if not c.startswith('module:')])
+            source = entry.msgid
+            translation = entry.msgstr
+            found_code_occurrence = False
+            for occurrence, line_number in entry.occurrences:
+                match = re.match(r'(model|model_terms):([\w.]+),([\w]+):(\w+)\.(\w+)', occurrence)
+                if match:
+                    type, model_name, field_name, module, xmlid = match.groups()
+                    yield {
+                        'type': type,
+                        'imd_model': model_name,
+                        'name': model_name+','+field_name,
+                        'imd_name': xmlid,
+                        'res_id': None,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'module': module,
+                    }
+                    continue
+
+                match = re.match(r'(code):([\w/.]+)', occurrence)
+                if match:
+                    type, name = match.groups()
+                    if found_code_occurrence:
+                        # unicity constrain on code translation
+                        continue
+                    found_code_occurrence = True
+                    yield {
+                        'type': type,
+                        'name': name,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'res_id': line_number,
+                        'module': module,
+                    }
+                    continue
+
+                match = re.match(r'(selection):([\w.]+),([\w]+)', occurrence)
+                if match:
+                    type, model_name, field_name = match.groups()
+                    yield {
+                        'type': type,
+                        'model': model_name,
+                        'name': model_name+','+field_name,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'res_id': line_number,
+                        'module': module,
+                    }
+                    continue
+
+                match = re.match(r'(sql_constraint|constraint):([\w.]+)', occurrence)
+                if match:
+                    type, model = match.groups()
+                    yield {
+                        'type': type,
+                        'name': model,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'res_id': line_number,
+                        'module': module,
+                    }
+                    continue
+
+                _logger.error("malformed po file: unknown occurrence: %s", occurrence)
+
+
 # class to handle po files
 class PoFile(object):
+
     def __init__(self, buffer):
         # TextIOWrapper closes its underlying buffer on close *and* can't
-        # handle actual file objects (on python 2)
         self.buffer = codecs.StreamReaderWriter(
             stream=buffer,
             Reader=codecs.getreader('utf-8'),
             Writer=codecs.getwriter('utf-8')
         )
 
-    def __iter__(self):
-        self.buffer.seek(0)
-        self.lines = self._get_lines()
-        self.lines_count = len(self.lines)
-
-        self.first = True
-        self.extra_lines= []
-        return self
-
-    def _get_lines(self):
-        lines = self.buffer.readlines()
-        # remove the BOM (Byte Order Mark):
-        if len(lines):
-            lines[0] = lines[0].lstrip(u"\ufeff")
-
-        lines.append('') # ensure that the file ends with at least an empty line
-        return lines
-
-    def cur_line(self):
-        return self.lines_count - len(self.lines)
-
-    def next(self):
-        trans_type = name = res_id = source = trad = module = None
-        if self.extra_lines:
-            trans_type, name, res_id, source, trad, comments = self.extra_lines.pop(0)
-            if not res_id:
-                res_id = '0'
-        else:
-            comments = []
-            targets = []
-            line = None
-            fuzzy = False
-            while not line:
-                if 0 == len(self.lines):
-                    raise StopIteration()
-                line = self.lines.pop(0).strip()
-            while line.startswith('#'):
-                if line.startswith('#~ '):
-                    break
-                if line.startswith('#.'):
-                    line = line[2:].strip()
-                    if not line.startswith('module:'):
-                        comments.append(line)
-                    else:
-                        module = line[7:].strip()
-                elif line.startswith('#:'):
-                    # Process the `reference` comments. Each line can specify
-                    # multiple targets (e.g. model, view, code, selection,
-                    # ...). For each target, we will return an additional
-                    # entry.
-                    for lpart in line[2:].strip().split(' '):
-                        trans_info = lpart.strip().split(':',2)
-                        if trans_info and len(trans_info) == 2:
-                            # looks like the translation trans_type is missing, which is not
-                            # unexpected because it is not a GetText standard. Default: 'code'
-                            trans_info[:0] = ['code']
-                        if trans_info and len(trans_info) == 3:
-                            # this is a ref line holding the destination info (model, field, record)
-                            targets.append(trans_info)
-                elif line.startswith('#,') and (line[2:].strip() == 'fuzzy'):
-                    fuzzy = True
-                line = self.lines.pop(0).strip()
-            if not self.lines:
-                raise StopIteration()
-            while not line:
-                # allow empty lines between comments and msgid
-                line = self.lines.pop(0).strip()
-            if line.startswith('#~ '):
-                while line.startswith('#~ ') or not line.strip():
-                    if 0 == len(self.lines):
-                        raise StopIteration()
-                    line = self.lines.pop(0)
-                # This has been a deprecated entry, don't return anything
-                return next(self)
-
-            if not line.startswith('msgid'):
-                raise Exception("malformed file: bad line: %s" % line)
-            source = unquote(line[6:])
-            line = self.lines.pop(0).strip()
-            if not source and self.first:
-                self.first = False
-                # if the source is "" and it's the first msgid, it's the special
-                # msgstr with the information about the translate and the
-                # translator; we skip it
-                self.extra_lines = []
-                while line:
-                    line = self.lines.pop(0).strip()
-                return next(self)
-
-            while not line.startswith('msgstr'):
-                if not line:
-                    raise Exception('malformed file at %d'% self.cur_line())
-                source += unquote(line)
-                line = self.lines.pop(0).strip()
-
-            trad = unquote(line[7:])
-            line = self.lines.pop(0).strip()
-            while line:
-                trad += unquote(line)
-                line = self.lines.pop(0).strip()
-
-            if targets and not fuzzy:
-                # Use the first target for the current entry (returned at the
-                # end of this next() call), and keep the others to generate
-                # additional entries (returned the next next() calls).
-                trans_type, name, res_id = targets.pop(0)
-                code = trans_type == 'code'
-                for t, n, r in targets:
-                    if t == 'code' and code: continue
-                    if t == 'code':
-                        code = True
-                    self.extra_lines.append((t, n, r, source, trad, comments))
-
-        if name is None:
-            if not fuzzy:
-                _logger.warning('Missing "#:" formatted comment at line %d for the following source:\n\t%s',
-                                self.cur_line(), source[:30])
-            return next(self)
-        return trans_type, name, res_id, source, trad, '\n'.join(comments), module
-    __next__ = next
-
     def write_infos(self, modules):
         import odoo.release as release
         self.buffer.write(u"# Translation of %(project)s.\n" \
@@ -1032,60 +1043,9 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
             # lets create the language with locale information
             Lang.load_lang(lang=lang, lang_name=lang_name)
 
-        # Parse also the POT: it will possibly provide additional targets.
-        # (Because the POT comments are correct on Launchpad but not the
-        # PO comments due to a Launchpad limitation. See LP bug 933496.)
-        pot_reader = []
-        use_pot_reference = False
-
         # now, the serious things: we read the language file
         fileobj.seek(0)
-        if fileformat == 'csv':
+        reader = TranslationFileReader(fileobj, fileformat=fileformat)
-            reader = pycompat.csv_reader(fileobj, quotechar='"', delimiter=',')
-            # read the first line of the file (it contains columns titles)
-            fields = next(reader)
-
-        elif fileformat == 'po':
-            reader = PoFile(fileobj)
-            fields = ['type', 'name', 'res_id', 'src', 'value', 'comments', 'module']
-
-            # Make a reader for the POT file and be somewhat defensive for the
-            # stable branch.
-
-            # when fileobj is a TemporaryFile, its name is an interget in P3, a string in P2
-            if isinstance(fileobj.name, str) and fileobj.name.endswith('.po'):
-                try:
-                    # Normally the path looks like /path/to/xxx/i18n/lang.po
-                    # and we try to find the corresponding
-                    # /path/to/xxx/i18n/xxx.pot file.
-                    # (Sometimes we have 'i18n_extra' instead of just 'i18n')
-                    addons_module_i18n, _ignored = os.path.split(fileobj.name)
-                    addons_module, i18n_dir = os.path.split(addons_module_i18n)
-                    addons, module = os.path.split(addons_module)
-                    pot_handle = file_open(os.path.join(
-                        addons, module, i18n_dir, module + '.pot'), mode='rb')
-                    pot_reader = PoFile(pot_handle)
-                    use_pot_reference = True
-                except:
-                    pass
-
-        else:
-            _logger.info('Bad file format: %s', fileformat)
-            raise Exception(_('Bad file format: %s') % fileformat)
-
-        # Read the POT references, and keep them indexed by source string.
-        class Target(object):
-            def __init__(self):
-                self.value = None
-                self.targets = set()            # set of (type, name, res_id)
-                self.comments = None
-
-        pot_targets = defaultdict(Target)
-        for type, name, res_id, src, _ignored, comments, module in pot_reader:
-            if type is not None:
-                target = pot_targets[src]
-                target.targets.add((type, name, type != 'code' and res_id or 0))
-                target.comments = comments
 
         # read the rest of the file
         irt_cursor = Translation._get_import_cursor()
@@ -1098,43 +1058,14 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
             dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
                                  'comments', 'imd_model', 'imd_name', 'module'))
             dic['lang'] = lang
-            dic.update(zip(fields, row))
+            dic.update(row)
 
             # do not import empty values
             if not env.context.get('create_empty_translation', False) and not dic['value']:
                 return
 
-            if use_pot_reference:
+            if dic['type'] == 'code' and module_name:
-                # discard the target from the POT targets.
+                dic['module'] = module_name
-                src = dic['src']
-                target_key = (dic['type'], dic['name'], dic['type'] != 'code' and dic['res_id'] or 0)
-                target = pot_targets.get(src)
-                if not target or target_key not in target.targets:
-                    _logger.info("Translation '%s' (%s, %s, %s) not found in reference pot, skipping",
-                        src[:60], dic['type'], dic['name'], dic['res_id'])
-                    return
-
-                target.value = dic['value']
-                target.targets.discard(target_key)
-
-            # This would skip terms that fail to specify a res_id
-            res_id = dic['res_id']
-            if not res_id and dic['type'] != 'code':
-                return
-
-            if isinstance(res_id, int) or \
-                    (isinstance(res_id, str) and res_id.isdigit()):
-                dic['res_id'] = int(res_id)
-                if module_name:
-                    dic['module'] = module_name
-            else:
-                # res_id is an xml id
-                dic['res_id'] = None
-                dic['imd_model'] = dic['name'].split(',')[0]
-                if '.' in res_id:
-                    dic['module'], dic['imd_name'] = res_id.split('.', 1)
-                else:
-                    dic['module'], dic['imd_name'] = module_name, res_id
 
             irt_cursor.push(dic)
 
@@ -1143,17 +1074,6 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
         for row in reader:
             process_row(row)
 
-        if use_pot_reference:
-            # Then process the entries implied by the POT file (which is more
-            # correct w.r.t. the targets) if some of them remain.
-            pot_rows = []
-            for src, target in pot_targets.items():
-                if target.value:
-                    for type, name, res_id in target.targets:
-                        pot_rows.append((type, name, res_id, src, target.value, target.comments))
-            for row in pot_rows:
-                process_row(row)
-
         irt_cursor.finish()
         Translation.clear_caches()
         if verbose:

requirements.txt

@@ -21,6 +21,7 @@ num2words==0.5.6
 ofxparse==0.16
 passlib==1.6.5
 Pillow==4.0.0
+polib==1.1.0
 psutil==4.3.1; sys_platform != 'win32'
 psycopg2==2.7.3.1; sys_platform != 'win32'
 pydot==1.2.3

setup.cfg

@@ -28,6 +28,7 @@ requires =
   python3-pillow
   python3-psutil
   python3-psycopg2
+  python3-polib
   python3-pydot
   python3-pyldap
   python3-pyparsing

setup/win32/winpy_requirements.txt

@@ -15,6 +15,7 @@ num2words==0.5.4
 ofxparse==0.16
 passlib==1.6.5
 Pillow>=3.4.1
+polib>=1.1.0
 psutil>=4.3.1
 psycopg2==2.7.1
 pydot==1.2.3