[FIX] mail: UTF-8 text/xml attachment and omitted charset

When parsing an email containing an xml attachment, the `email` python
module will decode the base64 attachment using the charset or ascii if
the charset is missing.

In some cases, the payload is in UTF-8 but the charset is omitted. This
results in replacement characters for the non ASCII characters.

The solution is to force the charset to UTF-8, since it is a superset of
ASCII that should not be a problem.

NB1: Omitting the charset for text/xml is not recommended. See the RFC
(section 6.4): https://www.ietf.org/rfc/rfc2376.txt

opw-3144519

closes odoo/odoo#126474

X-original-commit: e3a5b46c7b4f5030e165a6d943ec2275aea5d583
Signed-off-by: Julien Castiaux (juc) <juc@odoo.com>
Signed-off-by: Julien Van Roy (juvr) <juvr@odoo.com>

addons/mail/models/mail_thread.py

@@ -1420,6 +1420,10 @@ class MailThread(models.AbstractModel):
                     continue  # skip container
 
                 filename = part.get_filename()  # I may not properly handle all charsets
+                if part.get_content_type() == 'text/xml' and not part.get_param('charset'):
+                    # for text/xml with omitted charset, the charset is assumed to be ASCII by the `email` module
+                    # although the payload might be in UTF8
+                    part.set_charset('utf-8')
                 encoding = part.get_content_charset()  # None if attachment
 
                 content = part.get_content()

addons/test_mail/data/test_mail_data.py

@@ -326,6 +326,50 @@ Q2hhdXNz77+977+9ZSBkZSBCcnV4ZWxsZXM8L2NiYzpTdHJlZXROYW1lPg0KPC9JbnZvaWNlPg0K
 --00000000000005d9da05fa394cc0--
 """
 
+MAIL_MULTIPART_OMITTED_CHARSET = """Return-Path: <whatever-2a840@postmaster.twitter.com>
+To: {to}
+cc: {cc}
+Received: by mail1.openerp.com (Postfix, from userid 10002)
+    id 5DF9ABFB2A; Fri, 10 Aug 2012 16:16:39 +0200 (CEST)
+From: {email_from}
+Subject: {subject}
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+    boundary="00000000000005d9da05fa394cc0"
+Date: Fri, 10 Aug 2012 14:16:26 +0000
+Message-ID: {msg_id}
+{extra}
+
+--00000000000005d9da05fa394cc0
+Content-Type: multipart/alternative; boundary="00000000000005d9d905fa394cbe"
+
+--00000000000005d9d905fa394cbe
+Content-Type: text/plain; charset="UTF-8"
+
+Dear customer,
+
+Please find attached the UBL attachment of your invoice
+
+Cheers,
+
+--00000000000005d9d905fa394cbe
+Content-Type: text/html; charset="UTF-8"
+
+<div dir="ltr">Dear customer,<div><br></div><div>Please find attached the UBL attachment of your invoice</div><div><br></div><div>Cheers,</div></div>
+
+--00000000000005d9d905fa394cbe--
+
+--00000000000005d9da05fa394cc0
+Content-Disposition: attachment; filename="bis3.xml"
+Content-Transfer-Encoding: base64
+Content-Type: text/xml; name="bis3.xml"
+Content-ID: <f_lgxgdqx40>
+X-Attachment-Id: f_lgxgdqx40
+
+PEludm9pY2U+Q2hhdXNzw6llIGRlIEJydXhlbGxlczwvSW52b2ljZT4=
+--00000000000005d9da05fa394cc0--
+"""
+
 
 MAIL_SINGLE_BINARY = """X-Original-To: raoul@grosbedon.fr
 Delivered-To: raoul@grosbedon.fr

addons/test_mail/tests/test_mail_gateway.py

@@ -1763,6 +1763,14 @@ class TestMailgateway(MailCommon):
         # This explains the multiple "�" in the attachment.
         self.assertIn("Chauss������e de Bruxelles", record.message_ids.attachment_ids.raw.decode())
 
+    def test_message_process_file_omitted_charset(self):
+        """ For incoming email containing an xml attachment with omitted charset and containing an UTF8 payload we
+        should parse the attachment using UTF-8.
+        """
+        record = self.format_and_process(test_mail_data.MAIL_MULTIPART_OMITTED_CHARSET, self.email_from, 'groups@test.com')
+        self.assertEqual(record.message_ids.attachment_ids.name, 'bis3.xml')
+        self.assertEqual("<Invoice>Chaussée de Bruxelles</Invoice>", record.message_ids.attachment_ids.raw.decode())
+
     def test_message_route_reply_model_none(self):
         """
         Test the message routing and reply functionality when the model is None.