diff --git a/addons/hr/models/mail_thread.py b/addons/hr/models/mail_thread.py index d27cbbd2afc..2e756883856 100644 --- a/addons/hr/models/mail_thread.py +++ b/addons/hr/models/mail_thread.py @@ -4,11 +4,11 @@ import logging from odoo import api, models -from odoo.addons.mail.models.mail_thread import decode_header -from odoo.tools import email_split +from odoo.tools import decode_message_header, email_split _logger = logging.getLogger(__name__) + class MailThread(models.AbstractModel): _inherit = "mail.thread" @@ -18,13 +18,13 @@ class MailThread(models.AbstractModel): if res: alias = route[4] - email_from = decode_header(message, 'From') + email_from = decode_message_header(message, 'From') message_id = message.get('Message-Id') # Identically equal to the definition in mail module because sub methods are local # variables and cannot be called with super def _create_bounce_email(body_html): - bounce_to = decode_header(message, 'Return-Path') or email_from + bounce_to = decode_message_header(message, 'Return-Path') or email_from bounce_mail_values = { 'body_html': body_html, 'subject': 'Re: %s' % message.get('subject'), diff --git a/addons/mail/models/mail_message.py b/addons/mail/models/mail_message.py index 5aa52d755c8..8d6a933221b 100644 --- a/addons/mail/models/mail_message.py +++ b/addons/mail/models/mail_message.py @@ -3,7 +3,6 @@ import logging -from email.header import decode_header from email.utils import formataddr from odoo import _, api, fields, models, SUPERUSER_ID, tools @@ -14,16 +13,6 @@ from odoo.osv import expression _logger = logging.getLogger(__name__) -def decode(text): - """Returns unicode() string conversion of the the given encoded smtp header text""" - # TDE proposal: move to tools ? - if text: - text = decode_header(text.replace('\r', '')) - # The joining space will not be needed as of Python 3.3 - # See https://hg.python.org/cpython/rev/8c03fe231877 - return ' '.join([tools.ustr(x[0], x[1]) for x in text]) - - class Message(models.Model): """ Messages model: system notification (replacing res.log notifications), comments (OpenChatter discussion) and incoming emails. """ diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 3980ebd4b3f..194d7dc25e4 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -9,7 +9,6 @@ import json import lxml import logging import pytz -import re import socket import time import xmlrpclib @@ -21,20 +20,12 @@ from lxml import etree from werkzeug import url_encode from odoo import _, api, exceptions, fields, models, tools -from odoo.addons.mail.models.mail_message import decode from odoo.tools.safe_eval import safe_eval _logger = logging.getLogger(__name__) -mail_header_msgid_re = re.compile('<[^<>]+>') - - -def decode_header(message, header, separator=' '): - return separator.join(map(decode, filter(None, message.get_all(header, [])))) - - class MailThread(models.AbstractModel): ''' mail_thread model is meant to be inherited by any model that needs to act as a discussion topic on which messages can be attached. Public @@ -795,7 +786,7 @@ class MailThread(models.AbstractModel): """ Find partners related to some header fields of the message. :param string message: an email.message instance """ - s = ', '.join([decode(message.get(h)) for h in header_fields if message.get(h)]) + s = ', '.join([tools.decode_smtp_header(message.get(h)) for h in header_fields if message.get(h)]) return filter(lambda x: x, self._find_partner_from_emails(tools.email_split(s))) @api.model @@ -819,13 +810,13 @@ class MailThread(models.AbstractModel): assert len(route) == 5, 'A route should contain 5 elements: model, thread_id, custom_values, uid, alias record' message_id = message.get('Message-Id') - email_from = decode_header(message, 'From') + email_from = tools.decode_message_header(message, 'From') author_id = message_dict.get('author_id') model, thread_id, alias = route[0], route[1], route[4] record_set = None def _create_bounce_email(): - bounce_to = decode_header(message, 'Return-Path') or email_from + bounce_to = tools.decode_smtp_header(message.get('Return-Path')) or email_from bounce_mail_values = { 'body_html': '

Hello,

' '

The following email sent to %s cannot be accepted because this is ' @@ -979,10 +970,10 @@ class MailThread(models.AbstractModel): # Get email.message.Message variables for future processing message_id = message.get('Message-Id') - email_from = decode_header(message, 'From') - email_to = decode_header(message, 'To') - references = decode_header(message, 'References') - in_reply_to = decode_header(message, 'In-Reply-To').strip() + email_from = tools.decode_message_header(message, 'From') + email_to = tools.decode_message_header(message, 'To') + references = tools.decode_message_header(message, 'References') + in_reply_to = tools.decode_message_header(message, 'In-Reply-To').strip() thread_references = references or in_reply_to # 0. First check if this is a bounce message or not. @@ -996,8 +987,8 @@ class MailThread(models.AbstractModel): return [] # 1. message is a reply to an existing message (exact match of message_id) - ref_match = thread_references and tools.reference_re.search(thread_references) - msg_references = mail_header_msgid_re.findall(thread_references) + ref_match, reply_model, reply_thread_id, reply_hostname = tools.email_references(thread_references) + msg_references = tools.mail_header_msgid_re.findall(thread_references) mail_messages = MailMessage.sudo().search([('message_id', 'in', msg_references)], limit=1) if ref_match and mail_messages: model, thread_id = mail_messages.model, mail_messages.res_id @@ -1017,9 +1008,6 @@ class MailThread(models.AbstractModel): # 2. message is a reply to an existign thread (6.1 compatibility) if ref_match: - reply_thread_id = int(ref_match.group(1)) - reply_model = ref_match.group(2) or fallback_model - reply_hostname = ref_match.group(3) local_hostname = socket.gethostname() # do not match forwarded emails from another OpenERP system (thread_id collision!) if local_hostname == reply_hostname: @@ -1071,11 +1059,11 @@ class MailThread(models.AbstractModel): # Delivered-To is a safe bet in most modern MTAs, but we have to fallback on To + Cc values # for all the odd MTAs out there, as there is no standard header for the envelope's `rcpt_to` value. rcpt_tos = \ - ','.join([decode_header(message, 'Delivered-To'), - decode_header(message, 'To'), - decode_header(message, 'Cc'), - decode_header(message, 'Resent-To'), - decode_header(message, 'Resent-Cc')]) + ','.join([tools.decode_message_header(message, 'Delivered-To'), + tools.decode_message_header(message, 'To'), + tools.decode_message_header(message, 'Cc'), + tools.decode_message_header(message, 'Resent-To'), + tools.decode_message_header(message, 'Resent-Cc')]) local_parts = [e.split('@')[0] for e in tools.email_split(rcpt_tos)] if local_parts: aliases = Alias.search([('alias_name', 'in', local_parts)]) @@ -1105,7 +1093,7 @@ class MailThread(models.AbstractModel): # 5. Fallback to the provided parameters, if they work if not thread_id: # Legacy: fallback to matching [ID] in the Subject - match = tools.res_re.search(decode_header(message, 'Subject')) + match = tools.res_re.search(tools.decode_message_header(message, 'Subject')) thread_id = match and match.group(1) # Convert into int (bug spotted in 7.0 because of str) try: @@ -1355,7 +1343,7 @@ class MailThread(models.AbstractModel): # RFC2231 filename = email.utils.collapse_rfc2231_value(filename).strip() else: - filename = decode(filename) + filename = tools.decode_smtp_header(filename) encoding = part.get_content_charset() # None if attachment # 0) Inline Attachments -> attachments, with a third part in the tuple to match cid / attachment @@ -1432,19 +1420,19 @@ class MailThread(models.AbstractModel): msg_dict['message_id'] = message_id if message.get('Subject'): - msg_dict['subject'] = decode(message.get('Subject')) + msg_dict['subject'] = tools.decode_smtp_header(message.get('Subject')) # Envelope fields not stored in mail.message but made available for message_new() - msg_dict['from'] = decode(message.get('from')) - msg_dict['to'] = decode(message.get('to')) - msg_dict['cc'] = decode(message.get('cc')) - msg_dict['email_from'] = decode(message.get('from')) + msg_dict['from'] = tools.decode_smtp_header(message.get('from')) + msg_dict['to'] = tools.decode_smtp_header(message.get('to')) + msg_dict['cc'] = tools.decode_smtp_header(message.get('cc')) + msg_dict['email_from'] = tools.decode_smtp_header(message.get('from')) partner_ids = self._message_find_partners(message, ['To', 'Cc']) msg_dict['partner_ids'] = [(4, partner_id) for partner_id in partner_ids] if message.get('Date'): try: - date_hdr = decode(message.get('Date')) + date_hdr = tools.decode_smtp_header(message.get('Date')) parsed_date = dateutil.parser.parse(date_hdr, fuzzy=True) if parsed_date.utcoffset() is None: # naive datetime, so we arbitrarily decide to make it @@ -1461,13 +1449,13 @@ class MailThread(models.AbstractModel): msg_dict['date'] = stored_date.strftime(tools.DEFAULT_SERVER_DATETIME_FORMAT) if message.get('In-Reply-To'): - parent_ids = self.env['mail.message'].search([('message_id', '=', decode(message['In-Reply-To'].strip()))], limit=1) + parent_ids = self.env['mail.message'].search([('message_id', '=', tools.decode_smtp_header(message['In-Reply-To'].strip()))], limit=1) if parent_ids: msg_dict['parent_id'] = parent_ids.id msg_dict['internal'] = parent_ids.subtype_id and parent_ids.subtype_id.internal or False if message.get('References') and 'parent_id' not in msg_dict: - msg_list = mail_header_msgid_re.findall(decode(message['References'])) + msg_list = tools.mail_header_msgid_re.findall(tools.decode_smtp_header(message['References'])) parent_ids = self.env['mail.message'].search([('message_id', 'in', [x.strip() for x in msg_list])], limit=1) if parent_ids: msg_dict['parent_id'] = parent_ids.id diff --git a/addons/mass_mailing/models/mail_thread.py b/addons/mass_mailing/models/mail_thread.py index 1c0c8a38478..0931271509d 100644 --- a/addons/mass_mailing/models/mail_thread.py +++ b/addons/mass_mailing/models/mail_thread.py @@ -5,10 +5,7 @@ import logging import re from odoo import api, models - -from odoo.addons.mail.models.mail_message import decode -from odoo.addons.mail.models.mail_thread import decode_header - +from odoo.tools import decode_smtp_header, decode_message_header _logger = logging.getLogger(__name__) @@ -24,8 +21,8 @@ class MailThread(models.AbstractModel): return False to end the routing process. """ bounce_alias = self.env['ir.config_parameter'].get_param("mail.bounce.alias") message_id = message.get('Message-Id') - email_from = decode_header(message, 'From') - email_to = decode_header(message, 'To') + email_from = decode_message_header(message, 'From') + email_to = decode_message_header(message, 'To') # 0. Verify whether this is a bounced email (wrong destination,...) -> use it to collect data, such as dead leads if bounce_alias and bounce_alias in email_to: @@ -70,6 +67,6 @@ class MailThread(models.AbstractModel): by using the References header of the incoming message and looking for matching message_id in mail.mail.statistics. """ if message.get('References'): - message_ids = [x.strip() for x in decode(message['References']).split()] + message_ids = [x.strip() for x in decode_smtp_header(message['References']).split()] self.env['mail.mail.statistics'].set_replied(mail_message_ids=message_ids) return super(MailThread, self).message_route_process(message, message_dict, routes) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index 1f8e8d6f67f..3c9953677fd 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -1,21 +1,21 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -from lxml import etree import cgi import logging -import lxml.html import lxml.html.clean as clean import random import re import socket import threading import time + +from email.header import decode_header from email.utils import getaddresses, formataddr +from lxml import etree import openerp from openerp.loglevels import ustr -from openerp.tools.translate import _ _logger = logging.getLogger(__name__) @@ -403,7 +403,10 @@ email_re = re.compile(r"""([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63})""", # matches a string containing only one email single_email_re = re.compile(r"""^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,63}$""", re.VERBOSE) +# legacy behavior: match [ID] in the Subject, used with the fallback model of the mail gateway res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE) + +# update command in emails body command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) # Updated in 7.0 to match the model name as well @@ -411,6 +414,8 @@ command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) # group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?[^>]*@([^>]*)>", re.UNICODE) +mail_header_msgid_re = re.compile('<[^<>]+>') + def generate_tracking_message_id(res_id): """Returns a string that can be used in the Message-ID RFC822 header field @@ -487,3 +492,30 @@ def email_split_and_format(text): # is strictly required in RFC2822's `addr-spec`. if addr[1] if '@' in addr[1]] + +def email_references(references): + ref_match, model, thread_id, hostname = False, False, False, False + if references: + ref_match = reference_re.search(references) + if ref_match: + model = ref_match.group(2) + thread_id = int(ref_match.group(1)) + hostname = ref_match.group(3) + return (ref_match, model, thread_id, hostname) + +# was mail_message.decode() +def decode_smtp_header(smtp_header): + """Returns unicode() string conversion of the given encoded smtp header + text. email.header decode_header method return a decoded string and its + charset for each decoded par of the header. This method unicodes the + decoded header and join them in a complete string. """ + if smtp_header: + text = decode_header(smtp_header.replace('\r', '')) + # The joining space will not be needed as of Python 3.3 + # See https://hg.python.org/cpython/rev/8c03fe231877 + return ' '.join([ustr(x[0], x[1]) for x in text]) + return u'' + +# was mail_thread.decode_header() +def decode_message_header(message, header, separator=' '): + return separator.join(map(decode_smtp_header, filter(None, message.get_all(header, []))))