[MERGE] Use polib to parse .po files

Remove ugly custom po files parser and use polib for reader and writer. Refactor reader/writer to use a class and have an abstraction for csv or po files. Task-id: 1986334 closes odoo/odoo#32165 Signed-off-by: Martin Trigaux (mat) <mat@odoo.com>
2019-06-06 09:45:37 +02:00
parent 7bb0c8b6c5 55dd375c84
commit a7621137cb
16 changed files with 367 additions and 368 deletions
@@ -56,7 +56,7 @@
      	<field name="name">Citroen</field>
      	<field name="image" type="base64" file="fleet/static/img/brand_citroen-image.png"/>
      </record>
-      <record id="brand_corre la licorne" model="fleet.vehicle.model.brand">
+      <record id="brand_corre_la_licorne" model="fleet.vehicle.model.brand">
      	<field name="name">Corre La Licorne</field>
      	<field name="image" type="base64" file="fleet/static/img/brand_corre-la-licorne-image.png"/>
      </record>
@@ -128,7 +128,7 @@
      	<field name="name">Lancia</field>
      	<field name="image" type="base64" file="fleet/static/img/brand_lancia-image.png"/>
      </record>
-      <record id="brand_land rover" model="fleet.vehicle.model.brand">
+      <record id="brand_land_rover" model="fleet.vehicle.model.brand">
      	<field name="name">Land Rover</field>
      	<field name="image" type="base64" file="fleet/static/img/brand_land-rover-image.png"/>
      </record>
@@ -236,7 +236,7 @@
      	<field name="name">Subaru</field>
      	<field name="image" type="base64" file="fleet/static/img/brand_subaru-image.png"/>
      </record>
-      <record id="brand_tesla motors" model="fleet.vehicle.model.brand">
+      <record id="brand_tesla_motors" model="fleet.vehicle.model.brand">
      	<field name="name">Tesla Motors</field>
      	<field name="image" type="base64" file="fleet/static/img/brand_tesla-motors-image.png"/>
      </record>
@@ -2655,7 +2655,7 @@ msgid "eCommerce Shipping Costs"
 msgstr ""

 #. module: website_sale
-#: model_terms:ir.ui.view,arch_db:website_sale.search count
+#: model_terms:ir.ui.view,arch_db:website_sale.search_count_box
 msgid "found)"
 msgstr ""

@@ -109,7 +109,7 @@
        </t>
    </template>

-    <template id="search count" inherit_id="website.website_search_box" active="False" customize_show="True" name="Show # found">
+    <template id="search_count_box" inherit_id="website.website_search_box" active="False" customize_show="True" name="Show # found">
        <xpath expr="//button[hasclass('oe_search_button')]" position="inside">
            <span t-if='search' class='oe_search_found'> <small>(<t t-esc="search_count"/> found)</small></span>
        </xpath>
@@ -11,5 +11,5 @@ class TestUi(HttpCase):

    def test_01_admin_shop_sale_coupon_tour(self):
        # pre enable "Show # found" option to avoid race condition...
-        self.env.ref("website_sale.search count").write({"active": True})
+        self.env.ref("website_sale.search_count_box").write({"active": True})
        self.start_tour("/", 'shop_sale_coupon', login="admin")
@@ -30,6 +30,7 @@ Depends:
 python3-mako,
 python3-mock,
 python3-passlib,
+ python3-polib,
 python3-psutil,
 python3-psycopg2,
 python3-pydot,
@@ -504,7 +504,7 @@ state_mx_nay,mx,"Nayarit","NAY"
 state_mx_nl,mx,"Nuevo León","NLE"
 state_mx_oax,mx,"Oaxaca","OAX"
 state_mx_pue,mx,"Puebla","PUE"
-state_mx_q roo,mx,"Quintana Roo","ROO"
+state_mx_q_roo,mx,"Quintana Roo","ROO"
 state_mx_qro,mx,"Querétaro","QUE"
 state_mx_sin,mx,"Sinaloa","SIN"
 state_mx_slp,mx,"San Luis Potosí","SLP"
@@ -55,37 +55,6 @@ class IrTranslationImport(object):
        """ Feed a translation, as a dictionary, into the cursor """
        params = dict(trans_dict, state="translated")

-        if params['type'] == 'view':
-            # ugly hack for QWeb views - pending refactoring of translations in master
-            if params['imd_model'] == 'website':
-                params['imd_model'] = "ir.ui.view"
-            # non-QWeb views do not need a matching res_id in case they do not
-            # have an xml id -> force to 0 to avoid dropping them
-            elif params['res_id'] is None and not params['imd_name']:
-                # maybe we should insert this translation for all views of the
-                # given model?
-                params['res_id'] = 0
-
-        # backward compatibility: convert 'field', 'help', 'view' into 'model'
-        if params['type'] == 'field':
-            model, field = params['name'].split(',')
-            params['type'] = 'model'
-            params['name'] = 'ir.model.fields,field_description'
-            params['imd_model'] = 'ir.model.fields'
-            params['imd_name'] = 'field_%s__%s' % (model.replace('.', '_'), field)
-
-        elif params['type'] == 'help':
-            model, field = params['name'].split(',')
-            params['type'] = 'model'
-            params['name'] = 'ir.model.fields,help'
-            params['imd_model'] = 'ir.model.fields'
-            params['imd_name'] = 'field_%s__%s' % (model.replace('.', '_'), field)
-
-        elif params['type'] == 'view':
-            params['type'] = 'model'
-            params['name'] = 'ir.ui.view,arch_db'
-            params['imd_model'] = "ir.ui.view"
-
        self._rows.append((params['name'], params['lang'], params['res_id'],
                           params['src'], params['type'], params['imd_model'],
                           params['module'], params['imd_name'], params['value'],
@@ -23,6 +23,12 @@ msgstr ""
 msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
 msgstr "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB in french"

+#. module: test_translation_import
+#: selection:test.translation.import,import_type:0
+msgid "Bar"
+msgstr "Bar in french"
+
+# Note: the line in pot is 21 and is 14 here
 #. module: test_translation_import
 #: code:addons/test_translation_import/models.py:14
 #, python-format
@@ -30,15 +36,20 @@ msgid "Ijkl"
 msgstr "Ijkl in french"

 #. module: test_translation_import
-#: model:ir.model,name:test_translation_import.model_test_translation_import
-msgid "test.translation.import"
-msgstr "test.translation.import in french"
+#: selection:test.translation.import,import_type:0
+msgid "Foo"
+msgstr "Foo in french"

 #. module: test_translation_import
 #: model:ir.model.fields,help:test_translation_import.field_test_translation_import__name
 msgid "Efgh"
 msgstr "Efgh in french"

+#. module: test_translation_import
+#: model:ir.model,name:test_translation_import.model_test_translation_import
+msgid "Test: Translation Import"
+msgstr "Test: Import de traductions"
+
 #. module: test_translation_import
 #: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
 #: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
@@ -21,8 +21,8 @@ msgstr "Ijkl in belgian french"

 #. module: test_translation_import
 #: model:ir.model,name:test_translation_import.model_test_translation_import
-msgid "test.translation.import"
-msgstr "test.translation.import in belgian french"
+msgid "Test: Translation Import"
+msgstr "Test: Import de traductions in belgian french"

 #. module: test_translation_import
 #: model:ir.model.fields,help:test_translation_import.field_test_translation_import__name
@@ -30,14 +30,29 @@ msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
 msgstr ""

 #. module: test_translation_import
-#: code:addons/test_translation_import/models.py:14
+#: code:addons/test_translation_import/models.py:21
 #, python-format
-msgid "Ijkl"
+msgid "Accounting"
 msgstr ""

 #. module: test_translation_import
-#: model:ir.model,name:test_translation_import.model_test_translation_import
-msgid "test.translation.import"
+#: selection:test.translation.import,import_type:0
+msgid "Bar"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__create_uid
+msgid "Created by"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__create_date
+msgid "Created on"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__display_name
+msgid "Display Name"
 msgstr ""

 #. module: test_translation_import
@@ -45,25 +60,64 @@ msgstr ""
 msgid "Efgh"
 msgstr ""

+#. module: test_translation_import
+#: selection:test.translation.import,import_type:0
+msgid "Foo"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__id
+msgid "ID"
+msgstr ""
+
+#. module: test_translation_import
+#: code:addons/test_translation_import/models.py:21
+#, python-format
+msgid "Ijkl"
+msgstr ""
+
+#. module: test_translation_import
+#: code:addons/test_translation_import/models.py:19
+#, python-format
+msgid "Klingon"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import____last_update
+msgid "Last Modified on"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__write_uid
+msgid "Last Updated by"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__write_date
+msgid "Last Updated on"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.ui.menu,name:test_translation_import.menu_test_translation
+msgid "Test translation"
+msgstr ""
+
 #. module: test_translation_import
 #: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
 #: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
 msgid "Test translation import"
 msgstr ""

-#. module: test_translation_import
-#: model:ir.ui.menu,name:test_translation_import.menu_test_translation
-msgid "Test translation"
-msgstr ""
-
 #. module: test_translation_import
 #: code:addons/base/models/arthur.py:42
 #: code:addons/base/models/arthur.py:43
+#, python-format
 msgid "Test translation with two code lines"
 msgstr ""

 #. module: test_translation_import
 #: code:addons/base/models/ford.py:42
+#, python-format
 msgid "Test translation with a code type but different line number in pot"
 msgstr ""

@@ -71,5 +125,11 @@ msgstr ""
 #: model:ir.model.fields,field_description:test_translation_import.field_test_translation_import__other_name
 #: code:addons/base/models/marvin.py:42
 #: code:addons/base/models/trillian.py:42
+#, python-format
 msgid "Test translation with two code type and model"
 msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model,name:test_translation_import.model_test_translation_import
+msgid "Test: Translation Import"
+msgstr ""
@@ -10,6 +10,10 @@ class m(models.TransientModel):
    name = fields.Char('1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB',
                       size=32, help='Efgh')
    other_name = fields.Char('Test translation with two code type and model')
+    import_type = fields.Selection([
+        ('foo', 'Foo'),
+        ('bar', 'Bar'),
+    ])

    _('Ijkl')

@@ -17,11 +17,32 @@ class TestTermCount(common.TransactionCase):
        Just make sure we have as many translation entries as we wanted.
        """
        odoo.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', module_name='test_translation_import', verbose=False)
-        ids = self.env['ir.translation'].search([
+        translations = self.env['ir.translation'].search([
            ('lang', '=', 'fr_FR'),
            ('src', '=', '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB'),
-        ])
-        self.assertEqual(len(ids), 2)
+        ], order='type')
+        self.assertEqual(len(translations), 2)
+        self.assertEqual(translations[0].type, 'code')
+        self.assertEqual(translations[0].module, 'test_translation_import')
+        self.assertEqual(translations[0].name, 'addons/test_translation_import/models.py')
+        self.assertEqual(translations[0].comments, '')
+        self.assertEqual(translations[0].res_id, 15)
+        self.assertEqual(translations[1].type, 'model')
+        self.assertEqual(translations[1].module, 'test_translation_import')
+        self.assertEqual(translations[1].name, 'ir.model.fields,field_description')
+        self.assertEqual(translations[1].comments, '')
+        field = self.env['ir.model.fields'].search([('model', '=', 'test.translation.import'), ('name', '=', 'name')])
+        self.assertEqual(translations[1].res_id, field.id)
+
+        translations = self.env['ir.translation'].search([
+            ('lang', '=', 'fr_FR'),
+            ('type', '=', 'selection'),
+            ('module', '=', 'test_translation_import'),
+        ], order='src')
+        self.assertEqual(len(translations), 2)
+        self.assertEqual(translations[0].name, 'test.translation.import,import_type')
+        self.assertEqual(translations[0].res_id, 0)
+

    def test_count_term_module(self):
        """
@@ -34,6 +55,7 @@ class TestTermCount(common.TransactionCase):
            ('module', '=', 'test_translation_import'),
        ])
        self.assertEqual(len(translations), 1)
+        self.assertEqual(translations.res_id, 21)

    def test_noupdate(self):
        """
@@ -7,6 +7,7 @@ import io
 import locale
 import logging
 import os
+import polib
 import re
 import tarfile
 import tempfile
@@ -15,6 +16,7 @@ from collections import defaultdict
 from datetime import datetime
 from os.path import join

+from pathlib import Path
 from babel.messages import extract
 from lxml import etree, html

@@ -476,257 +478,275 @@ def unquote(str):
    """Returns unquoted PO term string, with special PO characters unescaped"""
    return re_escaped_char.sub(_sub_replacement, str[1:-1])

-# class to handle po files
-class PoFile(object):
-    def __init__(self, buffer):
-        # TextIOWrapper closes its underlying buffer on close *and* can't
-        # handle actual file objects (on python 2)
-        self.buffer = codecs.StreamReaderWriter(
-            stream=buffer,
-            Reader=codecs.getreader('utf-8'),
-            Writer=codecs.getwriter('utf-8')
-        )
+def TranslationFileReader(source, fileformat='po'):
+    """ Iterate over translation file to return Odoo translation entries """
+    if fileformat == 'csv':
+        return CSVFileReader(source)
+    if fileformat == 'po':
+        return PoFileReader(source)
+    _logger.info('Bad file format: %s', fileformat)
+    raise Exception(_('Bad file format: %s') % fileformat)
+
+class CSVFileReader:
+    def __init__(self, source):
+        self.source = pycompat.csv_reader(source, quotechar='"', delimiter=',')
+        # read the first line of the file (it contains columns titles)
+        self.fields = next(self.source)

    def __iter__(self):
-        self.buffer.seek(0)
-        self.lines = self._get_lines()
-        self.lines_count = len(self.lines)
+        for entry in self.source:
+            yield zip(self.fields, entry)

-        self.first = True
-        self.extra_lines= []
-        return self
+class PoFileReader:
+    """ Iterate over po file to return Odoo translation entries """
+    def __init__(self, source):

-    def _get_lines(self):
-        lines = self.buffer.readlines()
-        # remove the BOM (Byte Order Mark):
-        if len(lines):
-            lines[0] = lines[0].lstrip(u"\ufeff")
+        def get_pot_path(source_name):
+            # when fileobj is a TemporaryFile, its name is an inter in P3, a string in P2
+            if isinstance(source_name, str) and source_name.endswith('.po'):
+                # Normally the path looks like /path/to/xxx/i18n/lang.po
+                # and we try to find the corresponding
+                # /path/to/xxx/i18n/xxx.pot file.
+                # (Sometimes we have 'i18n_extra' instead of just 'i18n')
+                path = Path(source_name)
+                filename = path.parent.parent.name + '.pot'
+                pot_path = path.with_name(filename)
+                return pot_path.exists() and str(pot_path) or False
+            return False

-        lines.append('') # ensure that the file ends with at least an empty line
-        return lines
-
-    def cur_line(self):
-        return self.lines_count - len(self.lines)
-
-    def next(self):
-        trans_type = name = res_id = source = trad = module = None
-        if self.extra_lines:
-            trans_type, name, res_id, source, trad, comments = self.extra_lines.pop(0)
-            if not res_id:
-                res_id = '0'
+        # polib accepts a path or the file content as a string, not a fileobj
+        if isinstance(source, str):
+            self.pofile = polib.pofile(source)
+            pot_path = get_pot_path(source)
        else:
-            comments = []
-            targets = []
-            line = None
-            fuzzy = False
-            while not line:
-                if 0 == len(self.lines):
-                    raise StopIteration()
-                line = self.lines.pop(0).strip()
-            while line.startswith('#'):
-                if line.startswith('#~ '):
-                    break
-                if line.startswith('#.'):
-                    line = line[2:].strip()
-                    if not line.startswith('module:'):
-                        comments.append(line)
-                    else:
-                        module = line[7:].strip()
-                elif line.startswith('#:'):
-                    # Process the `reference` comments. Each line can specify
-                    # multiple targets (e.g. model, view, code, selection,
-                    # ...). For each target, we will return an additional
-                    # entry.
-                    for lpart in line[2:].strip().split(' '):
-                        trans_info = lpart.strip().split(':',2)
-                        if trans_info and len(trans_info) == 2:
-                            # looks like the translation trans_type is missing, which is not
-                            # unexpected because it is not a GetText standard. Default: 'code'
-                            trans_info[:0] = ['code']
-                        if trans_info and len(trans_info) == 3:
-                            # this is a ref line holding the destination info (model, field, record)
-                            targets.append(trans_info)
-                elif line.startswith('#,') and (line[2:].strip() == 'fuzzy'):
-                    fuzzy = True
-                line = self.lines.pop(0).strip()
-            if not self.lines:
-                raise StopIteration()
-            while not line:
-                # allow empty lines between comments and msgid
-                line = self.lines.pop(0).strip()
-            if line.startswith('#~ '):
-                while line.startswith('#~ ') or not line.strip():
-                    if 0 == len(self.lines):
-                        raise StopIteration()
-                    line = self.lines.pop(0)
-                # This has been a deprecated entry, don't return anything
-                return next(self)
+            # either a BufferedIOBase or result from NamedTemporaryFile
+            self.pofile = polib.pofile(source.read().decode())
+            pot_path = get_pot_path(source.name)

-            if not line.startswith('msgid'):
-                raise Exception("malformed file: bad line: %s" % line)
-            source = unquote(line[6:])
-            line = self.lines.pop(0).strip()
-            if not source and self.first:
-                self.first = False
-                # if the source is "" and it's the first msgid, it's the special
-                # msgstr with the information about the translate and the
-                # translator; we skip it
-                self.extra_lines = []
-                while line:
-                    line = self.lines.pop(0).strip()
-                return next(self)
+        if pot_path:
+            # Make a reader for the POT file
+            # (Because the POT comments are correct on GitHub but the
+            # PO comments tends to be outdated. See LP bug 933496.)
+            self.pofile.merge(polib.pofile(pot_path))

-            while not line.startswith('msgstr'):
-                if not line:
-                    raise Exception('malformed file at %d'% self.cur_line())
-                source += unquote(line)
-                line = self.lines.pop(0).strip()
+    def __iter__(self):
+        for entry in self.pofile:
+            if entry.obsolete:
+                continue

-            trad = unquote(line[7:])
-            line = self.lines.pop(0).strip()
-            while line:
-                trad += unquote(line)
-                line = self.lines.pop(0).strip()
+            # in case of moduleS keep only the first
+            match = re.match(r"(module[s]?): (\w+)", entry.comment)
+            _, module = match.groups()
+            comments = "\n".join([c for c in entry.comment.split('\n') if not c.startswith('module:')])
+            source = entry.msgid
+            translation = entry.msgstr
+            found_code_occurrence = False
+            for occurrence, line_number in entry.occurrences:
+                match = re.match(r'(model|model_terms):([\w.]+),([\w]+):(\w+)\.(\w+)', occurrence)
+                if match:
+                    type, model_name, field_name, module, xmlid = match.groups()
+                    yield {
+                        'type': type,
+                        'imd_model': model_name,
+                        'name': model_name+','+field_name,
+                        'imd_name': xmlid,
+                        'res_id': None,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'module': module,
+                    }
+                    continue

-            if targets and not fuzzy:
-                # Use the first target for the current entry (returned at the
-                # end of this next() call), and keep the others to generate
-                # additional entries (returned the next next() calls).
-                trans_type, name, res_id = targets.pop(0)
-                code = trans_type == 'code'
-                for t, n, r in targets:
-                    if t == 'code' and code: continue
-                    if t == 'code':
-                        code = True
-                    self.extra_lines.append((t, n, r, source, trad, comments))
+                match = re.match(r'(code):([\w/.]+)', occurrence)
+                if match:
+                    type, name = match.groups()
+                    if found_code_occurrence:
+                        # unicity constrain on code translation
+                        continue
+                    found_code_occurrence = True
+                    yield {
+                        'type': type,
+                        'name': name,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'res_id': int(line_number),
+                        'module': module,
+                    }
+                    continue

-        if name is None:
-            if not fuzzy:
-                _logger.warning('Missing "#:" formatted comment at line %d for the following source:\n\t%s',
-                                self.cur_line(), source[:30])
-            return next(self)
-        return trans_type, name, res_id, source, trad, '\n'.join(comments), module
-    __next__ = next
+                match = re.match(r'(selection):([\w.]+),([\w]+)', occurrence)
+                if match:
+                    type, model_name, field_name = match.groups()
+                    yield {
+                        'type': type,
+                        'model': model_name,
+                        'name': model_name+','+field_name,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'res_id': int(line_number),
+                        'module': module,
+                    }
+                    continue

-    def write_infos(self, modules):
+                match = re.match(r'(sql_constraint|constraint):([\w.]+)', occurrence)
+                if match:
+                    type, model = match.groups()
+                    yield {
+                        'type': type,
+                        'name': model,
+                        'src': source,
+                        'value': translation,
+                        'comments': comments,
+                        'res_id': int(line_number),
+                        'module': module,
+                    }
+                    continue
+
+                _logger.error("malformed po file: unknown occurrence: %s", occurrence)
+
+def TranslationFileWriter(target, fileformat='po', lang=None, modules=None):
+    """ Iterate over translation file to return Odoo translation entries """
+    if fileformat == 'csv':
+        return CSVFileWriter(target)
+
+    if fileformat == 'po':
+        return PoFileWriter(target, modules=modules, lang=lang)
+
+    if fileformat == 'tgz':
+        return TarFileWriter(target, lang=lang)
+
+    raise Exception(_('Unrecognized extension: must be one of '
+                      '.csv, .po, or .tgz (received .%s).') % fileformat)
+
+
+class CSVFileWriter:
+    def __init__(self, target):
+        self.writer = pycompat.csv_writer(target, dialect='UNIX')
+        # write header first
+        self.writer.writerow(("module","type","name","res_id","src","value","comments"))
+
+
+    def write_rows(self, rows):
+        for module, type, name, res_id, src, trad, comments in rows:
+            comments = '\n'.join(comments)
+            self.writer.writerow((module, type, name, res_id, src, trad, comments))
+
+
+class PoFileWriter:
+    """ Iterate over po file to return Odoo translation entries """
+    def __init__(self, target, modules, lang):
        import odoo.release as release
-        self.buffer.write(u"# Translation of %(project)s.\n" \
-                          "# This file contains the translation of the following modules:\n" \
-                          "%(modules)s" \
-                          "#\n" \
-                          "msgid \"\"\n" \
-                          "msgstr \"\"\n" \
-                          '''"Project-Id-Version: %(project)s %(version)s\\n"\n''' \
-                          '''"Report-Msgid-Bugs-To: \\n"\n''' \
-                          '''"POT-Creation-Date: %(now)s\\n"\n'''        \
-                          '''"PO-Revision-Date: %(now)s\\n"\n'''         \
-                          '''"Last-Translator: <>\\n"\n''' \
-                          '''"Language-Team: \\n"\n'''   \
-                          '''"MIME-Version: 1.0\\n"\n''' \
-                          '''"Content-Type: text/plain; charset=UTF-8\\n"\n'''   \
-                          '''"Content-Transfer-Encoding: \\n"\n'''       \
-                          '''"Plural-Forms: \\n"\n'''    \
-                          "\n"

-                          % { 'project': release.description,
-                              'version': release.version,
-                              'modules': ''.join("#\t* %s\n" % m for m in modules),
-                              'now': datetime.utcnow().strftime('%Y-%m-%d %H:%M')+"+0000",
-                            }
-                          )
+        self.buffer = target
+        self.lang = lang
+        self.po = polib.POFile()

-    def write(self, modules, tnrs, source, trad, comments=None):
+        self.po.header = "Translation of %s.\n" \
+                    "This file contains the translation of the following modules:\n" \
+                    "%s" % (release.description, ''.join("\t* %s\n" % m for m in modules))
+        now = datetime.utcnow().strftime('%Y-%m-%d %H:%M+0000')
+        self.po.metadata = {
+            'Project-Id-Version': "%s %s" % (release.description, release.version),
+            'Report-Msgid-Bugs-To': '',
+            'POT-Creation-Date': now,
+            'PO-Revision-Date': now,
+            'Last-Translator': '',
+            'Language-Team': '',
+            'MIME-Version': '1.0',
+            'Content-Type': 'text/plain; charset=UTF-8',
+            'Content-Transfer-Encoding': '',
+            'Plural-Forms': '',
+        }

-        plurial = len(modules) > 1 and 's' or ''
-        self.buffer.write(u"#. module%s: %s\n" % (plurial, ', '.join(modules)))
+    def write_rows(self, rows):
+        # we now group the translations by source. That means one translation per source.
+        grouped_rows = {}
+        for module, type, name, res_id, src, trad, comments in rows:
+            row = grouped_rows.setdefault(src, {})
+            row.setdefault('modules', set()).add(module)
+            if not row.get('translation') and trad != src:
+                row['translation'] = trad
+            row.setdefault('tnrs', []).append((type, name, res_id))
+            row.setdefault('comments', set()).update(comments)

+        for src, row in sorted(grouped_rows.items()):
+            if not self.lang:
+                # translation template, so no translation value
+                row['translation'] = ''
+            elif not row.get('translation'):
+                row['translation'] = ''
+            self.add_entry(row['modules'], row['tnrs'], src, row['translation'], row['comments'])
+
+        # buffer expects bytes
+        self.buffer.write(str(self.po).encode())
+
+    def add_entry(self, modules, tnrs, source, trad, comments=None):
+        entry = polib.POEntry(
+            msgid=source,
+            msgstr=trad,
+        )
+        plural = len(modules) > 1 and 's' or ''
+        entry.comment = "module%s: %s" % (plural, ', '.join(modules))
        if comments:
-            self.buffer.write(u''.join(('#. %s\n' % c for c in comments)))
+            entry.comment += "\n" + "\n".join(comments)

        code = False
        for typy, name, res_id in tnrs:
-            self.buffer.write(u"#: %s:%s:%s\n" % (typy, name, res_id))
            if typy == 'code':
                code = True
-
+            if isinstance(res_id, int) or res_id.isdigit():
+                # second term of occurrence must be a digit
+                # occurrence line at 0 are discarded when rendered to string
+                entry.occurrences.append((u"%s:%s" % (typy, name), str(res_id)))
+            else:
+                entry.occurrences.append((u"%s:%s:%s" % (typy, name, res_id), ''))
        if code:
-            # only strings in python code are python formatted
-            self.buffer.write(u"#, python-format\n")
+            entry.flags.append("python-format")
+        self.po.append(entry)

-        msg = (
-            u"msgid %s\n"
-            u"msgstr %s\n\n"
-        ) % (
-            quote(str(source)), 
-            quote(str(trad))
-        )
-        self.buffer.write(msg)

+class TarFileWriter:
+
+    def __init__(self, target, lang):
+        self.tar = tarfile.open(fileobj=target, mode='w|gz')
+        self.lang = lang
+
+    def write_rows(self, rows):
+        rows_by_module = defaultdict(list)
+        for row in rows:
+            module = row[0]
+            rows_by_module[module].append(row)
+
+        for mod, modrows in rows_by_module.items():
+            with io.BytesIO() as buf:
+                po = PoFileWriter(buf, modules=[mod], lang=self.lang)
+                po.write_rows(modrows)
+                buf.seek(0)
+
+                info = tarfile.TarInfo(
+                    join(mod, 'i18n', '{basename}.{ext}'.format(
+                        basename=self.lang or mod,
+                        ext='po' if self.lang else 'pot',
+                    )))
+                # addfile will read <size> bytes from the buffer so
+                # size *must* be set first
+                info.size = len(buf.getvalue())
+
+                self.tar.addfile(info, fileobj=buf)
+
+        self.tar.close()

 # Methods to export the translation file

 def trans_export(lang, modules, buffer, format, cr):

-    def _process(format, modules, rows, buffer, lang):
-        if format == 'csv':
-            writer = pycompat.csv_writer(buffer, dialect='UNIX')
-            # write header first
-            writer.writerow(("module","type","name","res_id","src","value","comments"))
-            for module, type, name, res_id, src, trad, comments in rows:
-                comments = '\n'.join(comments)
-                writer.writerow((module, type, name, res_id, src, trad, comments))
-
-        elif format == 'po':
-            writer = PoFile(buffer)
-            writer.write_infos(modules)
-
-            # we now group the translations by source. That means one translation per source.
-            grouped_rows = {}
-            for module, type, name, res_id, src, trad, comments in rows:
-                row = grouped_rows.setdefault(src, {})
-                row.setdefault('modules', set()).add(module)
-                if not row.get('translation') and trad != src:
-                    row['translation'] = trad
-                row.setdefault('tnrs', []).append((type, name, res_id))
-                row.setdefault('comments', set()).update(comments)
-
-            for src, row in sorted(grouped_rows.items()):
-                if not lang:
-                    # translation template, so no translation value
-                    row['translation'] = ''
-                elif not row.get('translation'):
-                    row['translation'] = ''
-                writer.write(row['modules'], row['tnrs'], src, row['translation'], row['comments'])
-
-        elif format == 'tgz':
-            rows_by_module = defaultdict(list)
-            for row in rows:
-                module = row[0]
-                rows_by_module[module].append(row)
-
-            with tarfile.open(fileobj=buffer, mode='w|gz') as tar:
-                for mod, modrows in rows_by_module.items():
-                    with io.BytesIO() as buf:
-                        _process('po', [mod], modrows, buf, lang)
-                        buf.seek(0)
-
-                        info = tarfile.TarInfo(
-                            join(mod, 'i18n', '{basename}.{ext}'.format(
-                                basename=lang or mod,
-                                ext='po' if lang else 'pot',
-                            )))
-                        # addfile will read <size> bytes from the buffer so
-                        # size *must* be set first
-                        info.size = len(buf.getvalue())
-
-                        tar.addfile(info, fileobj=buf)
-        else:
-            raise Exception(_('Unrecognized extension: must be one of '
-                '.csv, .po, or .tgz (received .%s).') % format)
-
    translations = trans_generate(lang, modules, cr)
    modules = set(t[0] for t in translations)
-    _process(format, modules, translations, buffer, lang)
+    writer = TranslationFileWriter(buffer, fileformat=format, lang=lang, modules=modules)
+    writer.write_rows(translations)
    del translations


@@ -1032,60 +1052,9 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
            # lets create the language with locale information
            Lang.load_lang(lang=lang, lang_name=lang_name)

-        # Parse also the POT: it will possibly provide additional targets.
-        # (Because the POT comments are correct on Launchpad but not the
-        # PO comments due to a Launchpad limitation. See LP bug 933496.)
-        pot_reader = []
-        use_pot_reference = False
-
        # now, the serious things: we read the language file
        fileobj.seek(0)
-        if fileformat == 'csv':
-            reader = pycompat.csv_reader(fileobj, quotechar='"', delimiter=',')
-            # read the first line of the file (it contains columns titles)
-            fields = next(reader)
-
-        elif fileformat == 'po':
-            reader = PoFile(fileobj)
-            fields = ['type', 'name', 'res_id', 'src', 'value', 'comments', 'module']
-
-            # Make a reader for the POT file and be somewhat defensive for the
-            # stable branch.
-
-            # when fileobj is a TemporaryFile, its name is an interget in P3, a string in P2
-            if isinstance(fileobj.name, str) and fileobj.name.endswith('.po'):
-                try:
-                    # Normally the path looks like /path/to/xxx/i18n/lang.po
-                    # and we try to find the corresponding
-                    # /path/to/xxx/i18n/xxx.pot file.
-                    # (Sometimes we have 'i18n_extra' instead of just 'i18n')
-                    addons_module_i18n, _ignored = os.path.split(fileobj.name)
-                    addons_module, i18n_dir = os.path.split(addons_module_i18n)
-                    addons, module = os.path.split(addons_module)
-                    pot_handle = file_open(os.path.join(
-                        addons, module, i18n_dir, module + '.pot'), mode='rb')
-                    pot_reader = PoFile(pot_handle)
-                    use_pot_reference = True
-                except:
-                    pass
-
-        else:
-            _logger.info('Bad file format: %s', fileformat)
-            raise Exception(_('Bad file format: %s') % fileformat)
-
-        # Read the POT references, and keep them indexed by source string.
-        class Target(object):
-            def __init__(self):
-                self.value = None
-                self.targets = set()            # set of (type, name, res_id)
-                self.comments = None
-
-        pot_targets = defaultdict(Target)
-        for type, name, res_id, src, _ignored, comments, module in pot_reader:
-            if type is not None:
-                target = pot_targets[src]
-                target.targets.add((type, name, type != 'code' and res_id or 0))
-                target.comments = comments
+        reader = TranslationFileReader(fileobj, fileformat=fileformat)

        # read the rest of the file
        irt_cursor = Translation._get_import_cursor()
@@ -1098,43 +1067,14 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
            dic = dict.fromkeys(('type', 'name', 'res_id', 'src', 'value',
                                 'comments', 'imd_model', 'imd_name', 'module'))
            dic['lang'] = lang
-            dic.update(zip(fields, row))
+            dic.update(row)

            # do not import empty values
            if not env.context.get('create_empty_translation', False) and not dic['value']:
                return

-            if use_pot_reference:
-                # discard the target from the POT targets.
-                src = dic['src']
-                target_key = (dic['type'], dic['name'], dic['type'] != 'code' and dic['res_id'] or 0)
-                target = pot_targets.get(src)
-                if not target or target_key not in target.targets:
-                    _logger.info("Translation '%s' (%s, %s, %s) not found in reference pot, skipping",
-                        src[:60], dic['type'], dic['name'], dic['res_id'])
-                    return
-
-                target.value = dic['value']
-                target.targets.discard(target_key)
-
-            # This would skip terms that fail to specify a res_id
-            res_id = dic['res_id']
-            if not res_id and dic['type'] != 'code':
-                return
-
-            if isinstance(res_id, int) or \
-                    (isinstance(res_id, str) and res_id.isdigit()):
-                dic['res_id'] = int(res_id)
-                if module_name:
-                    dic['module'] = module_name
-            else:
-                # res_id is an xml id
-                dic['res_id'] = None
-                dic['imd_model'] = dic['name'].split(',')[0]
-                if '.' in res_id:
-                    dic['module'], dic['imd_name'] = res_id.split('.', 1)
-                else:
-                    dic['module'], dic['imd_name'] = module_name, res_id
+            if dic['type'] == 'code' and module_name:
+                dic['module'] = module_name

            irt_cursor.push(dic)

@@ -1143,17 +1083,6 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
        for row in reader:
            process_row(row)

-        if use_pot_reference:
-            # Then process the entries implied by the POT file (which is more
-            # correct w.r.t. the targets) if some of them remain.
-            pot_rows = []
-            for src, target in pot_targets.items():
-                if target.value:
-                    for type, name, res_id in target.targets:
-                        pot_rows.append((type, name, res_id, src, target.value, target.comments))
-            for row in pot_rows:
-                process_row(row)
-
        irt_cursor.finish()
        Translation.clear_caches()
        if verbose:
@@ -21,6 +21,7 @@ num2words==0.5.6
 ofxparse==0.16
 passlib==1.6.5
 Pillow==4.0.0
+polib==1.1.0
 psutil==4.3.1; sys_platform != 'win32'
 psycopg2==2.7.3.1; sys_platform != 'win32'
 pydot==1.2.3
@@ -28,6 +28,7 @@ requires =
  python3-pillow
  python3-psutil
  python3-psycopg2
+  python3-polib
  python3-pydot
  python3-pyldap
  python3-pyparsing
@@ -15,6 +15,7 @@ num2words==0.5.4
 ofxparse==0.16
 passlib==1.6.5
 Pillow>=3.4.1
+polib>=1.1.0
 psutil>=4.3.1
 psycopg2==2.7.1
 pydot==1.2.3