From e553f0b37242ebef9891c7a30ac58f4e1cd8b9d0 Mon Sep 17 00:00:00 2001 From: Julien Van Roy Date: Wed, 10 May 2023 13:20:19 +0000 Subject: [PATCH] [FIX] mail,account_edi: fix creation of invoice upon email reception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When receiving an email on a mailbox with an alias that triggers the creation of invoices, 4 bugs could occur. 1. If the xml received contains replacement characters (U+FFFD �), and the charset of the part of the email is "US-ASCII" the encoding of the string will fail, preventing the rest of the flow to be completed. Be more resilient, encode the string and ignores these characters if this case occurs. NB: sometimes, the charset is omitted for a Content-type: text/xml. This is valid but not recommended (see: https://www.ietf.org/rfc/rfc2376.txt). In this case, the default used is "US-ASCII". This means that any non-ascii char will be lost (they are replaced by the replacement character: �, see: https://github.com/python/cpython/blob/3.10/Lib/email/contentmanager.py#L67) when decoding the attachment. 2. When the xml attachment is created in Odoo, the mimetype is 'text/plain' (rather than 'application/xml'). Thus, the `_decode_attachment` needs to be more flexible when guessing the type of the attachment (to know which function to use to read the content of the attachment and create the invoice). 3. When creating an invoice from an email with an xml attachment, the xml is attached as the `message_main_attachment_id`. It's only later on that the content of the xml is read and we possibly find the PDF in base64 inside. When creating the PDF attachment, it was not set as the `message_main_attachment_id`, so the PDF was not rendered on the right part of the invoice form view. Add a clause to replace the `message_main_attachment_id` in such a case. 4. When the xml attachment represents a credit note, the move_type of the invoice created by the email alias needs to be changed. Indeed, the invoice is created before decoding the attachment, so we can only change the `move_type` later. opw-3144519 opw-3149649 closes odoo/odoo#121076 X-original-commit: 1e193a92b9c84e75b958985f8067873b90f686e0 Signed-off-by: Laurent Smet Signed-off-by: Julien Van Roy --- addons/account/models/ir_attachment.py | 7 +-- .../models/account_edi_common.py | 18 ++++++- .../tests/test_xml_ubl_be.py | 15 ++++++ addons/mail/models/mail_thread.py | 5 +- addons/test_mail/data/test_mail_data.py | 50 +++++++++++++++++++ addons/test_mail/tests/test_mail_gateway.py | 21 ++++++++ 6 files changed, 110 insertions(+), 6 deletions(-) diff --git a/addons/account/models/ir_attachment.py b/addons/account/models/ir_attachment.py index ddaf4b40ac7..516f955675d 100644 --- a/addons/account/models/ir_attachment.py +++ b/addons/account/models/ir_attachment.py @@ -97,9 +97,10 @@ class IrAttachment(models.Model): """ def is_xml(attachment): - # XML attachments received by mail have a 'text/plain' mimetype. - # Therefore, if content start with ' +To: {to} +cc: {cc} +Received: by mail1.openerp.com (Postfix, from userid 10002) + id 5DF9ABFB2A; Fri, 10 Aug 2012 16:16:39 +0200 (CEST) +From: {email_from} +Subject: {subject} +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="00000000000005d9da05fa394cc0" +Date: Fri, 10 Aug 2012 14:16:26 +0000 +Message-ID: {msg_id} +{extra} + +--00000000000005d9da05fa394cc0 +Content-Type: multipart/alternative; boundary="00000000000005d9d905fa394cbe" + +--00000000000005d9d905fa394cbe +Content-Type: text/plain; charset="UTF-8" + +Dear customer, + +Please find attached the Peppol Bis 3 attachment of your invoice (with an +encoding error in the address) + +Cheers, + +--00000000000005d9d905fa394cbe +Content-Type: text/html; charset="UTF-8" + +
Dear customer,

Please find attached the Peppol Bis 3 attachment of your invoice (with an encoding error in the address)

Cheers,
+ +--00000000000005d9d905fa394cbe-- + +--00000000000005d9da05fa394cc0 +Content-Type: text/xml; charset="US-ASCII"; + name="bis3_with_error_encoding_address.xml" +Content-Disposition: attachment; + filename="bis3_with_error_encoding_address.xml" +Content-Transfer-Encoding: base64 +Content-ID: +X-Attachment-Id: f_lgxgdqx40 + +PEludm9pY2UgeG1sbnM6Y2JjPSJ1cm46b2FzaXM6bmFtZXM6c3BlY2lmaWNhdGlvbjp1Ymw6c2No +ZW1hOnhzZDpDb21tb25CYXNpY0NvbXBvbmVudHMtMiIgeG1sbnM9InVybjpvYXNpczpuYW1lczpz +cGVjaWZpY2F0aW9uOnVibDpzY2hlbWE6eHNkOkludm9pY2UtMiI+DQo8Y2JjOlN0cmVldE5hbWU+ +Q2hhdXNz77+977+9ZSBkZSBCcnV4ZWxsZXM8L2NiYzpTdHJlZXROYW1lPg0KPC9JbnZvaWNlPg0K +--00000000000005d9da05fa394cc0-- +""" + MAIL_SINGLE_BINARY = """X-Original-To: raoul@grosbedon.fr Delivered-To: raoul@grosbedon.fr diff --git a/addons/test_mail/tests/test_mail_gateway.py b/addons/test_mail/tests/test_mail_gateway.py index bb2d4db482a..534dccef77c 100644 --- a/addons/test_mail/tests/test_mail_gateway.py +++ b/addons/test_mail/tests/test_mail_gateway.py @@ -1742,6 +1742,27 @@ class TestMailgateway(MailCommon): records = self.env['mail.test.gateway'].search([('name', 'ilike', 'Whitelist test alias loop %')]) self.assertEqual(len(records), 10, msg='Email whitelisted should not have the restriction') + # -------------------------------------------------- + # Corner cases / Bugs during message process + # -------------------------------------------------- + + def test_message_process_file_encoding_ascii(self): + """ Incoming email containing an xml attachment with unknown characters (�) but an ASCII charset should not + raise an Exception. UTF-8 is used as a safe fallback. + """ + record = self.format_and_process(test_mail_data.MAIL_MULTIPART_INVALID_ENCODING, self.email_from, 'groups@test.com') + + self.assertEqual(record.message_ids.attachment_ids.name, 'bis3_with_error_encoding_address.xml') + # NB: the xml received by email contains b"Chauss\xef\xbf\xbd\xef\xbf\xbde" with "\xef\xbf\xbd" being the + # replacement character � in UTF-8. + # When calling `_message_parse_extract_payload`, `part.get_content()` will be called on the attachment part of + # the email, triggering the decoding of the base64 attachment, so b"Chauss\xef\xbf\xbd\xef\xbf\xbde" is + # first retrieved. Then, `get_text_content` in `email` tries to decode this using the charset of the email + # part, i.e: `content.decode('us-ascii', errors='replace')`. So the errors are replaced using the Unicode + # replacement marker and the string "Chauss������e" is used to create the attachment. + # This explains the multiple "�" in the attachment. + self.assertIn("Chauss������e de Bruxelles", record.message_ids.attachment_ids.raw.decode()) + @tagged('mail_gateway') class TestMailThreadCC(MailCommon):