From b259eb69d2085a961c1ad00d8bba3cb8a92123c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Thu, 3 Dec 2015 12:59:19 +0100 Subject: [PATCH] [CLEAN] mail, tools: factorize decode email tools mail.mail and mail.thread have methods to decode headers. This commit move them in mail tools. It is indeed simplier to have all email tools in the same file. Moreover they have been renamed to avoid confusion with standard library methods: decode and decode_header already exist. * decode in mail_message is now decode_smtp_header * decode_header in mail_thread is now decode_message_header A method to data from references is also moved into tools. A regex was already defined in tools. Now the method using it is already put in tools. Mailgateway will now simply call this method. --- addons/hr/models/mail_thread.py | 8 +-- addons/mail/models/mail_message.py | 11 ----- addons/mail/models/mail_thread.py | 60 +++++++++-------------- addons/mass_mailing/models/mail_thread.py | 11 ++--- openerp/tools/mail.py | 38 ++++++++++++-- 5 files changed, 67 insertions(+), 61 deletions(-) diff --git a/addons/hr/models/mail_thread.py b/addons/hr/models/mail_thread.py index d27cbbd2afc..2e756883856 100644 --- a/addons/hr/models/mail_thread.py +++ b/addons/hr/models/mail_thread.py @@ -4,11 +4,11 @@ import logging from odoo import api, models -from odoo.addons.mail.models.mail_thread import decode_header -from odoo.tools import email_split +from odoo.tools import decode_message_header, email_split _logger = logging.getLogger(__name__) + class MailThread(models.AbstractModel): _inherit = "mail.thread" @@ -18,13 +18,13 @@ class MailThread(models.AbstractModel): if res: alias = route[4] - email_from = decode_header(message, 'From') + email_from = decode_message_header(message, 'From') message_id = message.get('Message-Id') # Identically equal to the definition in mail module because sub methods are local # variables and cannot be called with super def _create_bounce_email(body_html): - bounce_to = decode_header(message, 'Return-Path') or email_from + bounce_to = decode_message_header(message, 'Return-Path') or email_from bounce_mail_values = { 'body_html': body_html, 'subject': 'Re: %s' % message.get('subject'), diff --git a/addons/mail/models/mail_message.py b/addons/mail/models/mail_message.py index 5aa52d755c8..8d6a933221b 100644 --- a/addons/mail/models/mail_message.py +++ b/addons/mail/models/mail_message.py @@ -3,7 +3,6 @@ import logging -from email.header import decode_header from email.utils import formataddr from odoo import _, api, fields, models, SUPERUSER_ID, tools @@ -14,16 +13,6 @@ from odoo.osv import expression _logger = logging.getLogger(__name__) -def decode(text): - """Returns unicode() string conversion of the the given encoded smtp header text""" - # TDE proposal: move to tools ? - if text: - text = decode_header(text.replace('\r', '')) - # The joining space will not be needed as of Python 3.3 - # See https://hg.python.org/cpython/rev/8c03fe231877 - return ' '.join([tools.ustr(x[0], x[1]) for x in text]) - - class Message(models.Model): """ Messages model: system notification (replacing res.log notifications), comments (OpenChatter discussion) and incoming emails. """ diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 3980ebd4b3f..194d7dc25e4 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -9,7 +9,6 @@ import json import lxml import logging import pytz -import re import socket import time import xmlrpclib @@ -21,20 +20,12 @@ from lxml import etree from werkzeug import url_encode from odoo import _, api, exceptions, fields, models, tools -from odoo.addons.mail.models.mail_message import decode from odoo.tools.safe_eval import safe_eval _logger = logging.getLogger(__name__) -mail_header_msgid_re = re.compile('<[^<>]+>') - - -def decode_header(message, header, separator=' '): - return separator.join(map(decode, filter(None, message.get_all(header, [])))) - - class MailThread(models.AbstractModel): ''' mail_thread model is meant to be inherited by any model that needs to act as a discussion topic on which messages can be attached. Public @@ -795,7 +786,7 @@ class MailThread(models.AbstractModel): """ Find partners related to some header fields of the message. :param string message: an email.message instance """ - s = ', '.join([decode(message.get(h)) for h in header_fields if message.get(h)]) + s = ', '.join([tools.decode_smtp_header(message.get(h)) for h in header_fields if message.get(h)]) return filter(lambda x: x, self._find_partner_from_emails(tools.email_split(s))) @api.model @@ -819,13 +810,13 @@ class MailThread(models.AbstractModel): assert len(route) == 5, 'A route should contain 5 elements: model, thread_id, custom_values, uid, alias record' message_id = message.get('Message-Id') - email_from = decode_header(message, 'From') + email_from = tools.decode_message_header(message, 'From') author_id = message_dict.get('author_id') model, thread_id, alias = route[0], route[1], route[4] record_set = None def _create_bounce_email(): - bounce_to = decode_header(message, 'Return-Path') or email_from + bounce_to = tools.decode_smtp_header(message.get('Return-Path')) or email_from bounce_mail_values = { 'body_html': '

Hello,

' '

The following email sent to %s cannot be accepted because this is ' @@ -979,10 +970,10 @@ class MailThread(models.AbstractModel): # Get email.message.Message variables for future processing message_id = message.get('Message-Id') - email_from = decode_header(message, 'From') - email_to = decode_header(message, 'To') - references = decode_header(message, 'References') - in_reply_to = decode_header(message, 'In-Reply-To').strip() + email_from = tools.decode_message_header(message, 'From') + email_to = tools.decode_message_header(message, 'To') + references = tools.decode_message_header(message, 'References') + in_reply_to = tools.decode_message_header(message, 'In-Reply-To').strip() thread_references = references or in_reply_to # 0. First check if this is a bounce message or not. @@ -996,8 +987,8 @@ class MailThread(models.AbstractModel): return [] # 1. message is a reply to an existing message (exact match of message_id) - ref_match = thread_references and tools.reference_re.search(thread_references) - msg_references = mail_header_msgid_re.findall(thread_references) + ref_match, reply_model, reply_thread_id, reply_hostname = tools.email_references(thread_references) + msg_references = tools.mail_header_msgid_re.findall(thread_references) mail_messages = MailMessage.sudo().search([('message_id', 'in', msg_references)], limit=1) if ref_match and mail_messages: model, thread_id = mail_messages.model, mail_messages.res_id @@ -1017,9 +1008,6 @@ class MailThread(models.AbstractModel): # 2. message is a reply to an existign thread (6.1 compatibility) if ref_match: - reply_thread_id = int(ref_match.group(1)) - reply_model = ref_match.group(2) or fallback_model - reply_hostname = ref_match.group(3) local_hostname = socket.gethostname() # do not match forwarded emails from another OpenERP system (thread_id collision!) if local_hostname == reply_hostname: @@ -1071,11 +1059,11 @@ class MailThread(models.AbstractModel): # Delivered-To is a safe bet in most modern MTAs, but we have to fallback on To + Cc values # for all the odd MTAs out there, as there is no standard header for the envelope's `rcpt_to` value. rcpt_tos = \ - ','.join([decode_header(message, 'Delivered-To'), - decode_header(message, 'To'), - decode_header(message, 'Cc'), - decode_header(message, 'Resent-To'), - decode_header(message, 'Resent-Cc')]) + ','.join([tools.decode_message_header(message, 'Delivered-To'), + tools.decode_message_header(message, 'To'), + tools.decode_message_header(message, 'Cc'), + tools.decode_message_header(message, 'Resent-To'), + tools.decode_message_header(message, 'Resent-Cc')]) local_parts = [e.split('@')[0] for e in tools.email_split(rcpt_tos)] if local_parts: aliases = Alias.search([('alias_name', 'in', local_parts)]) @@ -1105,7 +1093,7 @@ class MailThread(models.AbstractModel): # 5. Fallback to the provided parameters, if they work if not thread_id: # Legacy: fallback to matching [ID] in the Subject - match = tools.res_re.search(decode_header(message, 'Subject')) + match = tools.res_re.search(tools.decode_message_header(message, 'Subject')) thread_id = match and match.group(1) # Convert into int (bug spotted in 7.0 because of str) try: @@ -1355,7 +1343,7 @@ class MailThread(models.AbstractModel): # RFC2231 filename = email.utils.collapse_rfc2231_value(filename).strip() else: - filename = decode(filename) + filename = tools.decode_smtp_header(filename) encoding = part.get_content_charset() # None if attachment # 0) Inline Attachments -> attachments, with a third part in the tuple to match cid / attachment @@ -1432,19 +1420,19 @@ class MailThread(models.AbstractModel): msg_dict['message_id'] = message_id if message.get('Subject'): - msg_dict['subject'] = decode(message.get('Subject')) + msg_dict['subject'] = tools.decode_smtp_header(message.get('Subject')) # Envelope fields not stored in mail.message but made available for message_new() - msg_dict['from'] = decode(message.get('from')) - msg_dict['to'] = decode(message.get('to')) - msg_dict['cc'] = decode(message.get('cc')) - msg_dict['email_from'] = decode(message.get('from')) + msg_dict['from'] = tools.decode_smtp_header(message.get('from')) + msg_dict['to'] = tools.decode_smtp_header(message.get('to')) + msg_dict['cc'] = tools.decode_smtp_header(message.get('cc')) + msg_dict['email_from'] = tools.decode_smtp_header(message.get('from')) partner_ids = self._message_find_partners(message, ['To', 'Cc']) msg_dict['partner_ids'] = [(4, partner_id) for partner_id in partner_ids] if message.get('Date'): try: - date_hdr = decode(message.get('Date')) + date_hdr = tools.decode_smtp_header(message.get('Date')) parsed_date = dateutil.parser.parse(date_hdr, fuzzy=True) if parsed_date.utcoffset() is None: # naive datetime, so we arbitrarily decide to make it @@ -1461,13 +1449,13 @@ class MailThread(models.AbstractModel): msg_dict['date'] = stored_date.strftime(tools.DEFAULT_SERVER_DATETIME_FORMAT) if message.get('In-Reply-To'): - parent_ids = self.env['mail.message'].search([('message_id', '=', decode(message['In-Reply-To'].strip()))], limit=1) + parent_ids = self.env['mail.message'].search([('message_id', '=', tools.decode_smtp_header(message['In-Reply-To'].strip()))], limit=1) if parent_ids: msg_dict['parent_id'] = parent_ids.id msg_dict['internal'] = parent_ids.subtype_id and parent_ids.subtype_id.internal or False if message.get('References') and 'parent_id' not in msg_dict: - msg_list = mail_header_msgid_re.findall(decode(message['References'])) + msg_list = tools.mail_header_msgid_re.findall(tools.decode_smtp_header(message['References'])) parent_ids = self.env['mail.message'].search([('message_id', 'in', [x.strip() for x in msg_list])], limit=1) if parent_ids: msg_dict['parent_id'] = parent_ids.id diff --git a/addons/mass_mailing/models/mail_thread.py b/addons/mass_mailing/models/mail_thread.py index 1c0c8a38478..0931271509d 100644 --- a/addons/mass_mailing/models/mail_thread.py +++ b/addons/mass_mailing/models/mail_thread.py @@ -5,10 +5,7 @@ import logging import re from odoo import api, models - -from odoo.addons.mail.models.mail_message import decode -from odoo.addons.mail.models.mail_thread import decode_header - +from odoo.tools import decode_smtp_header, decode_message_header _logger = logging.getLogger(__name__) @@ -24,8 +21,8 @@ class MailThread(models.AbstractModel): return False to end the routing process. """ bounce_alias = self.env['ir.config_parameter'].get_param("mail.bounce.alias") message_id = message.get('Message-Id') - email_from = decode_header(message, 'From') - email_to = decode_header(message, 'To') + email_from = decode_message_header(message, 'From') + email_to = decode_message_header(message, 'To') # 0. Verify whether this is a bounced email (wrong destination,...) -> use it to collect data, such as dead leads if bounce_alias and bounce_alias in email_to: @@ -70,6 +67,6 @@ class MailThread(models.AbstractModel): by using the References header of the incoming message and looking for matching message_id in mail.mail.statistics. """ if message.get('References'): - message_ids = [x.strip() for x in decode(message['References']).split()] + message_ids = [x.strip() for x in decode_smtp_header(message['References']).split()] self.env['mail.mail.statistics'].set_replied(mail_message_ids=message_ids) return super(MailThread, self).message_route_process(message, message_dict, routes) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index 1f8e8d6f67f..3c9953677fd 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -1,21 +1,21 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -from lxml import etree import cgi import logging -import lxml.html import lxml.html.clean as clean import random import re import socket import threading import time + +from email.header import decode_header from email.utils import getaddresses, formataddr +from lxml import etree import openerp from openerp.loglevels import ustr -from openerp.tools.translate import _ _logger = logging.getLogger(__name__) @@ -403,7 +403,10 @@ email_re = re.compile(r"""([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63})""", # matches a string containing only one email single_email_re = re.compile(r"""^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63}$""", re.VERBOSE) +# legacy behavior: match [ID] in the Subject, used with the fallback model of the mail gateway res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE) + +# update command in emails body command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) # Updated in 7.0 to match the model name as well @@ -411,6 +414,8 @@ command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) # group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?[^>]*@([^>]*)>", re.UNICODE) +mail_header_msgid_re = re.compile('<[^<>]+>') + def generate_tracking_message_id(res_id): """Returns a string that can be used in the Message-ID RFC822 header field @@ -487,3 +492,30 @@ def email_split_and_format(text): # is strictly required in RFC2822's `addr-spec`. if addr[1] if '@' in addr[1]] + +def email_references(references): + ref_match, model, thread_id, hostname = False, False, False, False + if references: + ref_match = reference_re.search(references) + if ref_match: + model = ref_match.group(2) + thread_id = int(ref_match.group(1)) + hostname = ref_match.group(3) + return (ref_match, model, thread_id, hostname) + +# was mail_message.decode() +def decode_smtp_header(smtp_header): + """Returns unicode() string conversion of the given encoded smtp header + text. email.header decode_header method return a decoded string and its + charset for each decoded par of the header. This method unicodes the + decoded header and join them in a complete string. """ + if smtp_header: + text = decode_header(smtp_header.replace('\r', '')) + # The joining space will not be needed as of Python 3.3 + # See https://hg.python.org/cpython/rev/8c03fe231877 + return ' '.join([ustr(x[0], x[1]) for x in text]) + return u'' + +# was mail_thread.decode_header() +def decode_message_header(message, header, separator=' '): + return separator.join(map(decode_smtp_header, filter(None, message.get_all(header, []))))