pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] mail: gateway: CIDs

Browse Source 
Finally add support of CIDs in incoming emails. Inline images are recognized
and added as attachments. Image links are updated from src=cid: to src=link
using the /web/image controller.

It has been decided to keep the image in atttachments instead of putting them
in base64, like proposed in various PRs. Indeed we prefer to store this data
into the attachments instead of directly putting it in the mail_message table.
Moreover this enable the display of attachments in the chatter and record views.
This commit is contained in:
Thibault Delavallée
2016-03-03 15:56:09 +01:00
parent a127a7aed2
commit f4481ff74e
2 changed files with 178 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/mail/models/mail_thread.py
+62 -19
View File
@@ -12,6 +12,8 @@ import re
import socket
import time
import xmlrpclib
from collections import namedtuple
from email.message import Message
from email.utils import formataddr
from werkzeug import url_encode
@@ -81,6 +83,7 @@ class MailThread(models.AbstractModel):
_mail_flat_thread = True # flatten the discussino history
_mail_post_access = 'write' # access required on the document to post on it
_mail_mass_mailing = False # enable mass mailing on this model
_Attachment = namedtuple('Attachment', ('fname', 'content', 'info'))
message_is_follower = fields.Boolean(
'Is Follower', compute='_compute_is_follower', search='_search_is_follower')
@@ -1303,6 +1306,13 @@ class MailThread(models.AbstractModel):
postprocessed = True
if node.getparent() is not None:
to_remove.append(node)
if node.tag == 'img' and node.get('src', '').startswith('cid:'):
cid = node.get('src').split(':', 1)[1]
related_attachment = [attach for attach in attachments if len(attach) == 2 and attach[2] == cid]
if related_attachment:
node.set('data-filename', related_attachment[0])
postprocessed = True
for node in to_remove:
node.getparent().remove(node)
if postprocessed:
@@ -1314,7 +1324,7 @@ class MailThread(models.AbstractModel):
attachments = []
body = u''
if save_original:
attachments.append(('original_email.eml', message.as_string()))
attachments.append(self._Attachment('original_email.eml', message.as_string(), {}))
# Be careful, content-type may contain tricky content like in the
# following example so test the MIME type with startswith()
@@ -1344,19 +1354,25 @@ class MailThread(models.AbstractModel):
# original get_filename is not able to decode iso-8859-1 (for instance).
# therefore, iso encoded attachements are not able to be decoded properly with get_filename
# code here partially copy the original get_filename method, but handle more encoding
filename=part.get_param('filename', None, 'content-disposition')
filename = part.get_param('filename', None, 'content-disposition')
if not filename:
filename=part.get_param('name', None)
filename = part.get_param('name', None)
if filename:
if isinstance(filename, tuple):
# RFC2231
filename=email.utils.collapse_rfc2231_value(filename).strip()
filename = email.utils.collapse_rfc2231_value(filename).strip()
else:
filename=decode(filename)
filename = decode(filename)
encoding = part.get_content_charset() # None if attachment
# 0) Inline Attachments -> attachments, with a third part in the tuple to match cid / attachment
if filename and part.get('content-id'):
inner_cid = part.get('content-id').strip('><')
attachments.append(self._Attachment(filename, part.get_payload(decode=True), {'cid': inner_cid}))
continue
# 1) Explicit Attachments -> attachments
if filename or part.get('content-disposition', '').strip().startswith('attachment'):
attachments.append((filename or 'attachment', part.get_payload(decode=True)))
attachments.append(self._Attachment(filename or 'attachment', part.get_payload(decode=True), {}))
continue
# 2) text/plain -> <pre/>
if part.get_content_type() == 'text/plain' and (not alternative or not body):
@@ -1374,7 +1390,7 @@ class MailThread(models.AbstractModel):
body = tools.append_content_to_html(body, html, plaintext=False)
# 4) Anything else -> attachment
else:
attachments.append((filename or 'attachment', part.get_payload(decode=True)))
attachments.append(self._Attachment(filename or 'attachment', part.get_payload(decode=True), {}))
body, attachments = self._message_extract_payload_postprocess(message, body, attachments)
return body, attachments
@@ -1601,7 +1617,6 @@ class MailThread(models.AbstractModel):
]).write({'author_id': partner_info['partner_id']})
return result
@api.model
def _message_preprocess_attachments(self, attachments, attachment_ids, attach_model, attach_res_id):
""" Preprocess attachments for mail_thread.message_post() or mail_mail.create().
@@ -1611,17 +1626,30 @@ class MailThread(models.AbstractModel):
:param str attach_model: the model of the attachments parent record
:param integer attach_res_id: the id of the attachments parent record
"""
return self._message_post_process_attachments(attachments, attachment_ids, {'model': attach_model, 'res_id': attach_res_id})
def _message_post_process_attachments(self, attachments, attachment_ids, message_data):
IrAttachment, parameter_attachments = self.env['ir.attachment'], self.env['ir.attachment']
m2m_attachment_ids = []
cid_mapping = {}
if attachment_ids:
filtered_attachment_ids = self.env['ir.attachment'].sudo().search([
('res_model', '=', 'mail.compose.message'),
('create_uid', '=', self._uid),
('id', 'in', attachment_ids)])
if filtered_attachment_ids:
filtered_attachment_ids.write({'res_model': attach_model, 'res_id': attach_res_id})
filtered_attachment_ids.write({'res_model': message_data['model'], 'res_id': message_data['res_id']})
m2m_attachment_ids += [(4, id) for id in attachment_ids]
# Handle attachments parameter, that is a dictionary of attachments
for name, content in attachments:
for attachment in attachments:
if len(attachment) == 2:
name, content = attachment
elif len(attachment) == 3:
name, content, info = attachment
if info and info.get('cid'):
cid_mapping[info['cid']] = name
else:
continue
if isinstance(content, unicode):
content = content.encode('utf-8')
data_attach = {
@@ -1629,10 +1657,26 @@ class MailThread(models.AbstractModel):
'datas': base64.b64encode(str(content)),
'datas_fname': name,
'description': name,
'res_model': attach_model,
'res_id': attach_res_id,
'res_model': message_data['model'],
'res_id': message_data['res_id'],
}
m2m_attachment_ids.append((0, 0, data_attach))
parameter_attachments |= IrAttachment.create(data_attach)
m2m_attachment_ids += [(4, attach.id) for attach in parameter_attachments]
if cid_mapping and message_data.get('body'):
root = lxml.html.fromstring(tools.ustr(message_data['body']))
postprocessed = False
for node in root.iter('img'):
if node.get('src', '').startswith('cid:'):
fname = cid_mapping.get(node.get('src').split('cid:')[1], node.get('data-filename', ''))
attachment = parameter_attachments.filtered(lambda attachment: attachment.name == fname)
if attachment:
node.set('src', '/web/image/%s' % attachment.id)
postprocessed = True
if postprocessed:
body = lxml.html.tostring(root, pretty_print=False, encoding='UTF-8')
message_data['body'] = body
return m2m_attachment_ids
@api.multi
@@ -1706,11 +1750,6 @@ class MailThread(models.AbstractModel):
private_followers -= set([author_id])
partner_ids |= private_followers
# 3. Attachments
# - HACK TDE FIXME: Chatter: attachments linked to the document (not done JS-side), load the message
attachment_ids = self._message_preprocess_attachments(
attachments, kwargs.pop('attachment_ids', []), model, self.ids and self.ids[0] or None)
# 4: mail.message.subtype
subtype_id = kwargs.get('subtype_id', False)
if not subtype_id:
@@ -1754,11 +1793,15 @@ class MailThread(models.AbstractModel):
'subject': subject or False,
'message_type': message_type,
'parent_id': parent_id,
'attachment_ids': attachment_ids,
'subtype_id': subtype_id,
'partner_ids': [(4, pid) for pid in partner_ids],
})
# 3. Attachments
# - HACK TDE FIXME: Chatter: attachments linked to the document (not done JS-side), load the message
attachment_ids = self._message_post_process_attachments(attachments, kwargs.pop('attachment_ids', []), values)
values['attachment_ids'] = attachment_ids
# Avoid warnings about non-existing fields
for x in ('from', 'to', 'cc'):
values.pop(x, None)
addons/mail/tests/test_mail_gateway.py
+116
View File
@@ -176,6 +176,115 @@ Content-Type: text/html;
"""
MAIL_MULTIPART_IMAGE = """X-Original-To: raoul@example.com
Delivered-To: micheline@example.com
Received: by mail1.example.com (Postfix, from userid 99999)
id 9DFB7BF509; Thu, 17 Dec 2015 15:22:56 +0100 (CET)
X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on mail1.example.com
X-Spam-Level: *
X-Spam-Status: No, score=1.1 required=5.0 tests=FREEMAIL_FROM,
HTML_IMAGE_ONLY_08,HTML_MESSAGE,RCVD_IN_DNSWL_LOW,RCVD_IN_MSPIKE_H3,
RCVD_IN_MSPIKE_WL,T_DKIM_INVALID autolearn=no autolearn_force=no version=3.4.0
Received: from mail-lf0-f44.example.com (mail-lf0-f44.example.com [209.85.215.44])
by mail1.example.com (Postfix) with ESMTPS id 1D80DBF509
for <micheline@example.com>; Thu, 17 Dec 2015 15:22:56 +0100 (CET)
Authentication-Results: mail1.example.com; dkim=pass
reason="2048-bit key; unprotected key"
header.d=example.com header.i=@example.com header.b=kUkTIIlt;
dkim-adsp=pass; dkim-atps=neutral
Received: by mail-lf0-f44.example.com with SMTP id z124so47959461lfa.3
for <micheline@example.com>; Thu, 17 Dec 2015 06:22:56 -0800 (PST)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
d=example.com; s=20120113;
h=mime-version:date:message-id:subject:from:to:content-type;
bh=GdrEuMrz6vxo/Z/F+mJVho/1wSe6hbxLx2SsP8tihzw=;
b=kUkTIIlt6fe4dftKHPNBkdHU2rO052o684R0e2bqH7roGUQFb78scYE+kqX0wo1zlk
zhKPVBR1TqTsYlqcHu+D3aUzai7L/Q5m40sSGn7uYGkZJ6m1TwrWNqVIgTZibarqvy94
NWhrjjK9gqd8segQdSjCgTipNSZME4bJCzPyBg/D5mqe07FPBJBGoF9SmIzEBhYeqLj1
GrXjb/D8J11aOyzmVvyt+bT+oeLUJI8E7qO5g2eQkMncyu+TyIXaRofOOBA14NhQ+0nS
w5O9rzzqkKuJEG4U2TJ2Vi2nl2tHJW2QPfTtFgcCzGxQ0+5n88OVlbGTLnhEIJ/SYpem
O5EA==
MIME-Version: 1.0
X-Received: by 10.25.167.197 with SMTP id q188mr22222517lfe.129.1450362175493;
Thu, 17 Dec 2015 06:22:55 -0800 (PST)
Received: by 10.25.209.145 with HTTP; Thu, 17 Dec 2015 06:22:55 -0800 (PST)
Date: Thu, 17 Dec 2015 15:22:55 +0100
Message-ID: <CAP76m_UB=aLqWEFccnq86AhkpwRB3aZoGL9vMffX7co3YEro_A@mail.gmail.com>
Subject: {subject}
From: =?UTF-8?Q?Thibault_Delavall=C3=A9e?= <raoul@example.com>
To: {to}
Content-Type: multipart/related; boundary=001a11416b9e9b229a05272b7052
--001a11416b9e9b229a05272b7052
Content-Type: multipart/alternative; boundary=001a11416b9e9b229805272b7051
--001a11416b9e9b229805272b7051
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
Premi=C3=A8re image, orang=C3=A9e.
[image: Inline image 1]
Seconde image, rosa=C3=A7=C3=A9e.
[image: Inline image 2]
Troisi=C3=A8me image, verte!=C2=B5
[image: Inline image 3]
J'esp=C3=A8re que tout se passera bien.
--=20
Thibault Delavall=C3=A9e
--001a11416b9e9b229805272b7051
Content-Type: text/html; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
<div dir=3D"ltr"><div>Premi=C3=A8re image, orang=C3=A9e.</div><div><br></di=
v><div><img src=3D"cid:ii_151b519fc025fdd3" alt=3D"Inline image 1" width=3D=
"2" height=3D"2"><br></div><div><br></div><div>Seconde image, rosa=C3=A7=C3=
=A9e.</div><div><br></div><div><img src=3D"cid:ii_151b51a290ed6a91" alt=3D"=
Inline image 2" width=3D"2" height=3D"2"></div><div><br></div><div>Troisi=
=C3=A8me image, verte!=C2=B5</div><div><br></div><div><img src=3D"cid:ii_15=
1b51a37e5eb7a6" alt=3D"Inline image 3" width=3D"10" height=3D"10"><br></div=
><div><br></div><div>J&#39;esp=C3=A8re que tout se passera bien.</div>-- <b=
r><div class=3D"gmail_signature">Thibault Delavall=C3=A9e</div>
</div>
--001a11416b9e9b229805272b7051--
--001a11416b9e9b229a05272b7052
Content-Type: image/gif; name="=?UTF-8?B?b3JhbmfDqWUuZ2lm?="
Content-Disposition: inline; filename="=?UTF-8?B?b3JhbmfDqWUuZ2lm?="
Content-Transfer-Encoding: base64
Content-ID: <ii_151b519fc025fdd3>
X-Attachment-Id: ii_151b519fc025fdd3
R0lGODdhAgACALMAAAAAAP///wAAAP//AP8AAP+AAAD/AAAAAAAA//8A/wAAAAAAAAAAAAAAAAAA
AAAAACwAAAAAAgACAAAEA7DIEgA7
--001a11416b9e9b229a05272b7052
Content-Type: image/gif; name="=?UTF-8?B?dmVydGUhwrUuZ2lm?="
Content-Disposition: inline; filename="=?UTF-8?B?dmVydGUhwrUuZ2lm?="
Content-Transfer-Encoding: base64
Content-ID: <ii_151b51a37e5eb7a6>
X-Attachment-Id: ii_151b51a37e5eb7a6
R0lGODlhCgAKALMAAAAAAIAAAACAAICAAAAAgIAAgACAgMDAwICAgP8AAAD/AP//AAAA//8A/wD/
/////ywAAAAACgAKAAAEClDJSau9OOvNe44AOw==
--001a11416b9e9b229a05272b7052
Content-Type: image/gif; name="=?UTF-8?B?cm9zYcOnw6llLmdpZg==?="
Content-Disposition: inline; filename="=?UTF-8?B?cm9zYcOnw6llLmdpZg==?="
Content-Transfer-Encoding: base64
Content-ID: <ii_151b51a290ed6a91>
X-Attachment-Id: ii_151b51a290ed6a91
R0lGODdhAgACALMAAAAAAP///wAAAP//AP8AAP+AAAD/AAAAAAAA//8A/wAAAP+AgAAAAAAAAAAA
AAAAACwAAAAAAgACAAAEA3DJFQA7
--001a11416b9e9b229a05272b7052--
"""
class TestMailgateway(TestMail):
def setUp(self):
@@ -230,6 +339,13 @@ class TestMailgateway(TestMail):
res.get('body', ''),
'message_parse: second part of the html version should be in body after parsing multipart/mixed')
@mute_logger('openerp.addons.mail.models.mail_thread')
def test_message_process_cid(self):
new_groups = self.format_and_process(MAIL_MULTIPART_IMAGE, subject='My Frogs', to='groups@example.com')
message = new_groups.message_ids[0]
for attachment in message.attachment_ids:
self.assertIn('/web/image/%s' % attachment.id, message.body)
@mute_logger('openerp.addons.mail.models.mail_thread', 'openerp.models')
def test_message_process_alias_basic(self):
""" Incoming email on an alias creating a new record + message_new + message details """