From f0ae04ab3ddeb05ad02c0358ae174993b19a4c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Mon, 28 Oct 2019 14:47:51 +0000 Subject: [PATCH 1/3] [FIX] tools: correctly format emails when having an encoding issue It seems the emails > < are missing. By the way one manual formataddr is replaced by an email_formatted field. Result is the same but let us use fields doing it for us. --- addons/mail/models/mail_thread.py | 2 +- odoo/tools/mail.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 260fbfc9c24..499b0461b32 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -1483,7 +1483,7 @@ class MailThread(models.AbstractModel): if partner and partner.id in [val[0] for val in result[self.ids[0]]]: # already existing partner ID -> skip return result if partner and partner.email: # complete profile: id, name - result[self.ids[0]].append((partner.id, tools.formataddr((partner.name, partner.email)), reason)) + result[self.ids[0]].append((partner.id, partner.email_formatted, reason)) elif partner: # incomplete profile: id, name result[self.ids[0]].append((partner.id, '%s' % (partner.name), reason)) else: # unknown partner, we are probably managing an email address diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py index e1c999a041f..3e881d0d841 100644 --- a/odoo/tools/mail.py +++ b/odoo/tools/mail.py @@ -539,7 +539,7 @@ def formataddr(pair): except UnicodeEncodeError: # non-ascii name, transcode the name in a safe format # rfc2047 - MIME Message Header Extensions for Non-ASCII Text - return "=?utf-8?b?{name}?= {addr}".format( + return "=?utf-8?b?{name}?= <{addr}>".format( name=base64.b64encode(name.encode('utf-8')).decode('ascii'), addr=address) else: From 7b65a63c97d185d9b6e6f1b0bfbc7968d1e4de49 Mon Sep 17 00:00:00 2001 From: Julien Castiaux Date: Tue, 29 Oct 2019 14:06:18 +0000 Subject: [PATCH 2/3] [IMP] tools: restore P2 formataddr default behavior Python 2 `email.tools.formataddr` doesn't encode not RFC-2822 compliant realname to base64 or quoted-printable. This is the wanted behavior to output formatted email addresses on screen. Python 3 `email.tools.formataddr` does encode the realname to be RFC-2822 compliant. This is the wanted behavior when connecting to a smtp server to send emails. That method has been deprecated by pep-594 as the new python email API is capable of automatically formatting email address in a RFC compliant way. As the method was still in use in a lot of modules as the preferred way to format email addresses, a refined P3-like implementation as been included in the tool suite. This commit changes the default behavior so it mimics P2 implementation with an easy way to use the P3 behavior. Task: 2003936 --- odoo/addons/base/tests/test_mail.py | 27 ++++++++++++++++++++++++++- odoo/tools/mail.py | 11 ++++++----- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/odoo/addons/base/tests/test_mail.py b/odoo/addons/base/tests/test_mail.py index d47c9e6ae87..8dbb2cddb65 100644 --- a/odoo/addons/base/tests/test_mail.py +++ b/odoo/addons/base/tests/test_mail.py @@ -6,7 +6,10 @@ from unittest.mock import patch from odoo.tests.common import BaseCase from odoo.tests.common import SavepointCase -from odoo.tools import html_sanitize, append_content_to_html, plaintext2html, email_split, misc +from odoo.tools import ( + html_sanitize, append_content_to_html, plaintext2html, email_split, + misc, formataddr, +) from . import test_mail_examples @@ -328,6 +331,28 @@ class TestEmailTools(BaseCase): for text, expected in cases: self.assertEqual(email_split(text), expected, 'email_split is broken') + def test_email_formataddr(self): + email = 'joe@example.com' + cases = [ + # (name, address), charsets expected + (('', email), ['ascii', 'utf-8'], 'joe@example.com'), + (('joe', email), ['ascii', 'utf-8'], '"joe" '), + (('joe doe', email), ['ascii', 'utf-8'], '"joe doe" '), + (('joe"doe', email), ['ascii', 'utf-8'], '"joe\\"doe" '), + (('joé', email), ['ascii'], '=?utf-8?b?am/DqQ==?= '), + (('joé', email), ['utf-8'], '"joé" '), + (('', 'joé@example.com'), ['ascii', 'utf-8'], UnicodeEncodeError), # need SMTPUTF8 support + (('', 'joe@examplé.com'), ['ascii', 'utf-8'], UnicodeEncodeError), # need IDNA support + ] + + for pair, charsets, expected in cases: + for charset in charsets: + with self.subTest(pair=pair, charset=charset): + if isinstance(expected, str): + self.assertEqual(formataddr(pair, charset), expected) + else: + self.assertRaises(expected, formataddr, pair, charset) + class EmailConfigCase(SavepointCase): @patch.dict("odoo.tools.config.options", {"email_from": "settings@example.com"}) diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py index 3e881d0d841..03ac8760ece 100644 --- a/odoo/tools/mail.py +++ b/odoo/tools/mail.py @@ -516,9 +516,10 @@ def email_escape_char(email_address): def decode_message_header(message, header, separator=' '): return separator.join(h for h in message.get_all(header, []) if h) -def formataddr(pair): - """Takes a 2-tuple of the form (realname, email_address) and returns - the string value suitable for an RFC 2822 From, To or Cc header. +def formataddr(pair, charset='utf-8'): + """Pretty format a 2-tuple of the form (realname, email_address). + + Set the charset to ascii to get a RFC-2822 compliant email. The email address is considered valid and is left unmodified. @@ -535,9 +536,9 @@ def formataddr(pair): address.encode('ascii') if name: try: - name.encode('ascii') + name.encode(charset) except UnicodeEncodeError: - # non-ascii name, transcode the name in a safe format + # charset mismatch, encode as utf-8/base64 # rfc2047 - MIME Message Header Extensions for Non-ASCII Text return "=?utf-8?b?{name}?= <{addr}>".format( name=base64.b64encode(name.encode('utf-8')).decode('ascii'), From 7bf2dd43393be3758b3d8d6eeb94c877517c1bcf Mon Sep 17 00:00:00 2001 From: Julien Castiaux Date: Tue, 29 Oct 2019 13:41:04 +0000 Subject: [PATCH 3/3] [FIX] ir.mail.server: squash redundant CRs (bpo-34424) Problem description: Using the chatter, send an email to someone having non-ascii characters in their name: the body of the received email looks like it contains a mix of the original body and headers. Python encodes the name into either base64 or quoted-printable but leaves some redundant carriage returns at the end of the header. Those redundant carriage returns are not RFC conformant but are interpreted as newlines by some email clients, thus are read like a headers/body separator and the next headers are considered part of the body, corrupting the email structure. This problem is internal to Python and has been fixed in 3.8, cfr bpo-34424 [1], and later backported to 3.7.4. As we also support 3.6 and earlier 3.7 and it is not possible to easily monkey-patch the function, we fixed it by squashing duplicate carriage returns. (There is no case where a series of bare CRs can occur in a normal RFC5322 email message) Task: 2003936 [1]: https://bugs.python.org/issue34424 --- odoo/addons/base/models/ir_mail_server.py | 10 ++++++- odoo/addons/base/tests/test_mail.py | 34 +++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/odoo/addons/base/models/ir_mail_server.py b/odoo/addons/base/models/ir_mail_server.py index 65b6ee47c1e..8c5399236c7 100644 --- a/odoo/addons/base/models/ir_mail_server.py +++ b/odoo/addons/base/models/ir_mail_server.py @@ -9,6 +9,7 @@ import email.policy import logging import re import smtplib +import sys import threading import html2text @@ -414,7 +415,14 @@ class IrMailServer(models.Model): smtp = smtp or self.connect( smtp_server, smtp_port, smtp_user, smtp_password, smtp_encryption, smtp_debug, mail_server_id=mail_server_id) - smtp.sendmail(smtp_from, smtp_to_list, message.as_string()) + + message_str = message.as_string() + # header folding code is buggy and adds redundant carriage + # returns, it got fixed in 3.7.4 thanks to bpo-34424 + if sys.version_info < (3, 7, 4): + message_str = re.sub('\r+', '\r', message_str) + + smtp.sendmail(smtp_from, smtp_to_list, message_str) # do not quit() a pre-established smtp_session if not smtp_session: smtp.quit() diff --git a/odoo/addons/base/tests/test_mail.py b/odoo/addons/base/tests/test_mail.py index 8dbb2cddb65..e1b80c52c28 100644 --- a/odoo/addons/base/tests/test_mail.py +++ b/odoo/addons/base/tests/test_mail.py @@ -3,9 +3,11 @@ from unittest.mock import patch +import email.policy +import email.message +import threading -from odoo.tests.common import BaseCase -from odoo.tests.common import SavepointCase +from odoo.tests.common import BaseCase, SavepointCase, TransactionCase from odoo.tools import ( html_sanitize, append_content_to_html, plaintext2html, email_split, misc, formataddr, @@ -374,3 +376,31 @@ class EmailConfigCase(SavepointCase): "The body of an email", ) self.assertEqual(message["From"], "settings@example.com") + + +class TestEmailMessage(TransactionCase): + def test_as_string(self): + """Ensure all email sent are bpo-34424 free""" + + class FakeSMTP: + """SMTP stub""" + def __init__(this): + this.email_sent = False + + def sendmail(this, smtp_from, smtp_to_list, message_str): + this.email_sent = True + message_truth = ( + r'From: .+? \r\n' + r'To: .+? \r\n' + r'\r\n' + ) + self.assertRegex(message_str, message_truth) + + msg = email.message.EmailMessage(policy=email.policy.SMTP) + msg['From'] = '"Joé Doe" ' + msg['To'] = '"Joé Doe" ' + + smtp = FakeSMTP() + self.patch(threading.currentThread(), 'testing', False) + self.env['ir.mail_server'].send_email(msg, smtp_session=smtp) + self.assertTrue(smtp.email_sent)