From 77aa313b141b9b4c79fa8e24ed0984a9d5c955fe Mon Sep 17 00:00:00 2001 From: Nicolas Martinelli Date: Wed, 16 May 2018 15:20:33 +0200 Subject: [PATCH] [FIX] mail: XHTML In case an email received contains XHTML, `fromstring` fails because of the following error: > Unicode strings with encoding declaration are not supported. Please > use bytes input or XML fragments without declaration. opw-1832683 --- addons/mail/models/mail_thread.py | 8 +- addons/mail/tests/test_mail_gateway.py | 248 +++++++++++++++++++++++++ 2 files changed, 255 insertions(+), 1 deletion(-) diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 6b41b8769d9..7050f2bfc89 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -1373,7 +1373,13 @@ class MailThread(models.AbstractModel): located in tools. """ if not body: return body, attachments - root = lxml.html.fromstring(body) + try: + root = lxml.html.fromstring(body) + except ValueError: + # In case the email client sent XHTML, fromstring will fail because 'Unicode strings + # with encoding declaration are not supported'. + root = lxml.html.fromstring(body.encode('utf-8')) + postprocessed = False to_remove = [] for node in root.iter(): diff --git a/addons/mail/tests/test_mail_gateway.py b/addons/mail/tests/test_mail_gateway.py index 12eac4437ac..93d873ce0cc 100644 --- a/addons/mail/tests/test_mail_gateway.py +++ b/addons/mail/tests/test_mail_gateway.py @@ -439,6 +439,248 @@ Web: http://www.odoo --------------A6B5FD5F68F4D73ECD739009--""" + +MAIL_XHTML = """Return-Path: +Received: from xxxx.internal (xxxx.xxxx.internal [1.1.1.1]) + by xxxx (xxxx 1.1.1-111-g972eecc-slipenbois) with LMTPA; + Fri, 13 Apr 2018 22:11:52 -0400 +X-Cyrus-Session-Id: sloti35d1t38-1111111-11111111111-5-11111111111111111111 +X-Sieve: CMU Sieve 1.0 +X-Spam-known-sender: no ("Email failed DMARC policy for domain"); in-addressbook +X-Spam-score: 0.0 +X-Spam-hits: ALL_TRUSTED -1, BAYES_00 -1.9, FREEMAIL_FROM 0.001, + HTML_FONT_LOW_CONTRAST 0.001, HTML_MESSAGE 0.001, SPF_SOFTFAIL 0.665, + LANGUAGES en, BAYES_USED global, SA_VERSION 1.1.0 +X-Spam-source: IP='1.1.1.1', Host='unk', Country='unk', FromHeader='com', + MailFrom='com' +X-Spam-charsets: plain='utf-8', html='utf-8' +X-IgnoreVacation: yes ("Email failed DMARC policy for domain") +X-Resolved-to: catchall@xxxx.xxxx +X-Delivered-to: catchall@xxxx.xxxx +X-Mail-from: xxxx@xxxx.com +Received: from mx4 ([1.1.1.1]) + by xxxx.internal (LMTPProxy); Fri, 13 Apr 2018 22:11:52 -0400 +Received: from xxxx.xxxx.com (localhost [127.0.0.1]) + by xxxx.xxxx.internal (Postfix) with ESMTP id E1111C1111; + Fri, 13 Apr 2018 22:11:51 -0400 (EDT) +Received: from xxxx.xxxx.internal (localhost [127.0.0.1]) + by xxxx.xxxx.com (Authentication Milter) with ESMTP + id BBDD1111D1A; + Fri, 13 Apr 2018 22:11:51 -0400 +ARC-Authentication-Results: i=1; xxxx.xxxx.com; arc=none (no signatures found); + dkim=pass (2048-bit rsa key sha256) header.d=xxxx.com header.i=@xxxx.com header.b=P1aaAAaa x-bits=2048 x-keytype=rsa x-algorithm=sha256 x-selector=fm2; + dmarc=fail (p=none,d=none) header.from=xxxx.com; + iprev=pass policy.iprev=1.1.1.1 (out1-smtp.xxxx.com); + spf=softfail smtp.mailfrom=xxxx@xxxx.com smtp.helo=out1-smtp.xxxx.com; + x-aligned-from=pass (Address match); + x-cm=none score=0; + x-ptr=pass x-ptr-helo=out1-smtp.xxxx.com x-ptr-lookup=out1-smtp.xxxx.com; + x-return-mx=pass smtp.domain=xxxx.com smtp.result=pass smtp_is_org_domain=yes header.domain=xxxx.com header.result=pass header_is_org_domain=yes; + x-tls=pass version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128; + x-vs=clean score=0 state=0 +Authentication-Results: xxxx.xxxx.com; + arc=none (no signatures found); + dkim=pass (2048-bit rsa key sha256) header.d=xxxx.com header.i=@xxxx.com header.b=P1awJPiy x-bits=2048 x-keytype=rsa x-algorithm=sha256 x-selector=fm2; + dmarc=fail (p=none,d=none) header.from=xxxx.com; + iprev=pass policy.iprev=66.111.4.25 (out1-smtp.xxxx.com); + spf=softfail smtp.mailfrom=xxxx@xxxx.com smtp.helo=out1-smtp.xxxx.com; + x-aligned-from=pass (Address match); + x-cm=none score=0; + x-ptr=pass x-ptr-helo=out1-smtp.xxxx.com x-ptr-lookup=out1-smtp.xxxx.com; + x-return-mx=pass smtp.domain=xxxx.com smtp.result=pass smtp_is_org_domain=yes header.domain=xxxx.com header.result=pass header_is_org_domain=yes; + x-tls=pass version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128; + x-vs=clean score=0 state=0 +X-ME-VSCategory: clean +X-ME-CMScore: 0 +X-ME-CMCategory: none +Received-SPF: softfail + (gmail.com ... _spf.xxxx.com: Sender is not authorized by default to use 'xxxx@xxxx.com' in 'mfrom' identity, however domain is not currently prepared for false failures (mechanism '~all' matched)) + receiver=xxxx.xxxx.com; + identity=mailfrom; + envelope-from="xxxx@xxxx.com"; + helo=out1-smtp.xxxx.com; + client-ip=1.1.1.1 +Received: from xxxx.xxxx.internal (gateway1.xxxx.internal [1.1.1.1]) + (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) + (No client certificate requested) + by xxxx.xxxx.internal (Postfix) with ESMTPS; + Fri, 13 Apr 2018 22:11:51 -0400 (EDT) +Received: from compute3.internal (xxxx.xxxx.internal [10.202.2.43]) + by xxxx.xxxx.internal (Postfix) with ESMTP id 8BD5B21BBD; + Fri, 13 Apr 2018 22:11:51 -0400 (EDT) +Received: from xxxx ([10.202.2.163]) + by xxxx.internal (MEProxy); Fri, 13 Apr 2018 22:11:51 -0400 +X-ME-Sender: +Received: from [1.1.1.1] (unknown [1.1.1.1]) + by mail.xxxx.com (Postfix) with ESMTPA id BF5E1111D + for ; Fri, 13 Apr 2018 22:11:50 -0400 (EDT) +From: "xxxx xxxx" +To: "xxxx" +Subject: Re: xxxx (Ref PO1) +Date: Sat, 14 Apr 2018 02:11:42 +0000 +Message-Id: +In-Reply-To: <829228111124527.1111111602.256611118262939-openerp-129-xxxx.xxxx@ip-1-1-1-1> +References: <867911111953277.1523671337.187951111160400-openerp-129-xxxx.xxxx@ip-1-1-1-1> + <867911111953277.1523671337.256611118262939-openerp-129-xxxx.xxxx@ip-1-1-1-1> +Reply-To: "xxxx xxxx" +User-Agent: eM_Client/7.0.26687.0 +Mime-Version: 1.0 +Content-Type: multipart/alternative; + boundary="------=_MB48E455BD-1111-42EC-1111-886CDF48905E" + +--------=_MB48E455BD-1111-42EC-1111-886CDF48905E +Content-Type: text/plain; format=flowed; charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +xxxx + + +------ Original Message ------ +From: "xxxx" +To: "xxxx" +Sent: 4/13/2018 7:06:43 PM +Subject: xxxx + +>xxxx + +--------=_MB48E455BD-1111-42EC-1111-886CDF48905E +Content-Type: text/html; charset=utf-8 +Content-Transfer-Encoding: quoted-printable + +
this is a reply to PO200109 from emClient
--
xxxx
xxxx
xxxx



+
------ Original Message ------
+
From: "xxxx" <xxxx= +@xxxx.com>
+
To: "xxxx" <a= +xxxx@xxxx.com>
+
Sent: 4/13/2018 7:06:43 PM
+
Subject: xxxx

+
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + mangez des saucisses + + + 3D= +
+
+ + + + +
+

xxxx.=20 +,

+

+xxxx. +

+ +

You can reply = +to this email if you have any questions.

+

Thank you,

+
+
+ + + + + +
+ xxxx
+ +1-801-980-4240 + 		+ info@aust-mfg.com= +
+ + http://www.xxxx.com + +
+
+ Powered by Odo= +o. +
+ =20 + 
xxxx.
+
+
+ +--------=_MB48E455BD-2850-42EC-B1CA-886CDF48905E--""" + + class TestMailgateway(TestMail): def setUp(self): @@ -512,6 +754,12 @@ class TestMailgateway(TestMail): """ self.env['mail.thread'].message_process('mail.channel', MAIL_EML_ATTACHMENT) + @mute_logger('odoo.addons.mail.models.mail_thread') + def test_message_parse_xhtml(self): + """ Test that the parsing of mail with embedded emails as eml(msg) which generates empty attachments, can be processed. + """ + self.env['mail.thread'].message_process('mail.channel', MAIL_XHTML) + @mute_logger('odoo.addons.mail.models.mail_thread') def test_message_process_cid(self): new_groups = self.format_and_process(MAIL_MULTIPART_IMAGE, subject='My Frogs', to='groups@example.com')