[IMP] mail: bounce management

Handle bounces using bounce alias directly in mailgateway. Previously bounces
were used only in mass mailing to update campaign statistics. Now the
mailgateway tries to find data from standard delivery status emails. This
data is used to update state of emails sent to customers, to display it in
the Chatter. It is also used to increment the message_bounce counter on the
bounced partner and bounced record, if any and if the field exists.

Previous more generic code that detect bounces is kept. This code is more specific
to standard delivery failure notifications by parsing its content.

addons/mail/data/mail_data.xml

@@ -39,6 +39,12 @@
             <field name="value">catchall</field>
         </record>
 
+        <!-- Bounce Email Alias -->
+        <record id="icp_mail_bounce_alias" model="ir.config_parameter">
+            <field name="key">mail.bounce.alias</field>
+            <field name="value">bounce</field>
+        </record>
+
         <!-- Discussion subtype for messaging / Chatter -->
         <record id="mt_comment" model="mail.message.subtype">
             <field name="name">Discussions</field>

addons/mail/models/mail_thread.py

@@ -9,6 +9,7 @@ import json
 import lxml
 import logging
 import pytz
+import re
 import socket
 import time
 import xmlrpclib
@@ -977,6 +978,7 @@ class MailThread(models.AbstractModel):
             raise TypeError('message must be an email.message.Message at this point')
         MailMessage = self.env['mail.message']
         Alias, dest_aliases = self.env['mail.alias'], self.env['mail.alias']
+        bounce_alias = self.env['ir.config_parameter'].get_param("mail.bounce.alias")
         fallback_model = model
 
         # get email.message.Message variables for future processing
@@ -1005,6 +1007,49 @@ class MailThread(models.AbstractModel):
             tools.decode_message_header(message, 'Resent-Cc')])
         rcpt_tos_localparts = [e.split('@')[0] for e in tools.email_split(rcpt_tos)]
 
+        # 0. Verify whether this is a bounced email and use it to collect bounce data and update notifications for customers
+        if bounce_alias and bounce_alias in email_to_localpart:
+            # Bounce regex: typical form of bounce is bounce_alias+128-crm.lead-34@domain
+            # group(1) = the mail ID; group(2) = the model (if any); group(3) = the record ID
+            bounce_re = re.compile("%s\+(\d+)-?([\w.]+)?-?(\d+)?" % re.escape(bounce_alias), re.UNICODE)
+            bounce_match = bounce_re.search(email_to)
+
+            if bounce_match:
+                bounced_mail_id, bounced_model, bounced_thread_id = bounce_match.group(1), bounce_match.group(2), int(bounce_match.group(3))
+
+                email_part = next((part for part in message.walk() if part.get_content_type() == 'message/rfc822'), None)
+                dsn_part = next((part for part in message.walk() if part.get_content_type() == 'message/delivery-status'), None)
+
+                partner, partner_address = self.env['res.partner'], False
+                if dsn_part and len(dsn_part.get_payload()) > 1:
+                    dsn = dsn_part.get_payload()[1]
+                    final_recipient_data = tools.decode_message_header(dsn, 'Final-Recipient')
+                    partner_address = final_recipient_data.split(';', 1)[1].strip()
+                    if partner_address:
+                        partner = partner.sudo().search([('email', 'like', partner_address)])
+                        partner.message_receive_bounce(partner_address, partner, mail_id=bounced_mail_id)
+
+                mail_message = self.env['mail.message']
+                if email_part:
+                    email = email_part.get_payload()[0]
+                    bounced_message_id = tools.mail_header_msgid_re.findall(tools.decode_message_header(email, 'Message-Id'))
+                    mail_message = MailMessage.sudo().search([('message_id', 'in', bounced_message_id)])
+
+                if partner and mail_message:
+                    notifications = self.env['mail.notification'].sudo().search([
+                        ('mail_message_id', '=', mail_message.id),
+                        ('res_partner_id', '=', partner.id)])
+                    notifications.write({
+                        'email_status': 'bounce'
+                    })
+
+                if bounced_model in self.env and hasattr(self.env[bounced_model], 'message_receive_bounce') and bounced_thread_id:
+                    self.env[bounced_model].browse(bounced_thread_id).message_receive_bounce(partner_address, partner, mail_id=bounced_mail_id)
+
+                _logger.info('Routing mail from %s to %s with Message-Id %s: bounced mail from mail %s, model: %s, thread_id: %s: dest %s (partner %s)',
+                             email_from, email_to, message_id, bounced_mail_id, bounced_model, bounced_thread_id, partner_address, partner.id)
+                return []
+
         # 0. First check if this is a bounce message or not.
         #    See http://datatracker.ietf.org/doc/rfc3462/?include_text=1
         #    As all MTA does not respect this RFC (googlemail is one of them),
@@ -1255,6 +1300,22 @@ class MailThread(models.AbstractModel):
             self.write(update_vals)
         return True
 
+    @api.multi
+    def message_receive_bounce(self, email, partner, mail_id=None):
+        """Called by ``message_process`` when a bounce email (such as Undelivered
+        Mail Returned to Sender) is received for an existing thread. The default
+        behavior is to check is an integer  ``message_bounce`` column exists.
+        If it is the case, its content is incremented.
+
+        :param mail_id: ID of the sent email that bounced. It may not exist anymore
+                        but it could be usefull if the information was kept. This is
+                        used notably in mass mailing.
+        :param RecordSet partner: partner matching the bounced email address, if any
+        :param string email: email that caused the bounce """
+        if 'message_bounce' in self._fields:
+            for record in self:
+                record.message_bounce = record.message_bounce + 1
+
     def _message_extract_payload_postprocess(self, message, body, attachments):
         """ Perform some cleaning / postprocess in the body and attachments
         extracted from the email. Note that this processing is specific to the

addons/mass_mailing/__openerp__.py

@@ -22,7 +22,6 @@ professional emails and reuse templates in a few clicks.
     ],
     'data': [
         'security/mass_mailing_security.xml',
-        'data/mail_data.xml',
         'data/mass_mailing_data.xml',
         'wizard/mail_compose_message_views.xml',
         'wizard/test_mailing_views.xml',

addons/mass_mailing/data/mail_data.xml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<odoo>
-    <data noupdate="1">
-
-        <!-- Bounce Email Alias -->
-        <record id="icp_mail_bounce_alias" model="ir.config_parameter">
-            <field name="key">mail.bounce.alias</field>
-            <field name="value">bounce</field>
-        </record>
-
-    </data>
-</odoo>

addons/mass_mailing/models/mail_thread.py

@@ -4,62 +4,31 @@
 import logging
 import re
 
-from odoo import api, models
+from odoo import api, models, tools
 from odoo.tools import decode_smtp_header, decode_message_header
 
 _logger = logging.getLogger(__name__)
 
 
 class MailThread(models.AbstractModel):
-    """ Update MailThread to add the feature of bounced emails and replied emails
-    in message_process. """
+    """ Update MailThread to add the support of bounce management in mass mailing statistics. """
     _inherit = 'mail.thread'
 
-    def message_route_check_bounce(self, message):
-        """ Override to verify that the email_to is the bounce alias. If it is the
-        case, log the bounce, set the parent and related document as bounced and
-        return False to end the routing process. """
-        bounce_alias = self.env['ir.config_parameter'].get_param("mail.bounce.alias")
-        message_id = message.get('Message-Id')
-        email_from = decode_message_header(message, 'From')
-        email_to = decode_message_header(message, 'To')
-
-        # 0. Verify whether this is a bounced email (wrong destination,...) -> use it to collect data, such as dead leads
-        if bounce_alias and bounce_alias in email_to:
-            # Bounce regex
-            # Typical form of bounce is bounce_alias-128-crm.lead-34@domain
-            # group(1) = the mail ID; group(2) = the model (if any); group(3) = the record ID
-            bounce_re = re.compile("%s\+(\d+)-?([\w.]+)?-?(\d+)?" % re.escape(bounce_alias), re.UNICODE)
-            bounce_match = bounce_re.search(email_to)
-            if bounce_match:
-                bounced_model, bounced_thread_id = None, False
-                bounced_mail_id = bounce_match.group(1)
-                statistics = self.env['mail.mail.statistics'].set_bounced(mail_mail_ids=[bounced_mail_id])
-                for stat in statistics:
-                    bounced_model = stat.model
-                    bounced_thread_id = stat.res_id
-                _logger.info('Routing mail from %s to %s with Message-Id %s: bounced mail from mail %s, model: %s, thread_id: %s',
-                             email_from, email_to, message_id, bounced_mail_id, bounced_model, bounced_thread_id)
-                if bounced_model and bounced_model in self.env and hasattr(self.env[bounced_model], 'message_receive_bounce') and bounced_thread_id:
-                    self.env[bounced_model].browse(bounced_thread_id).message_receive_bounce(mail_id=bounced_mail_id)
-                return False
-
-        return True
-
     @api.model
     def message_route(self, message, message_dict, model=None, thread_id=None, custom_values=None):
-        if not self.message_route_check_bounce(message):
-            return []
-        return super(MailThread, self).message_route(message, message_dict, model, thread_id, custom_values)
+        """ Override to udpate mass mailing statistics based on bounce emails """
+        bounce_alias = self.env['ir.config_parameter'].get_param("mail.bounce.alias")
+        email_to = decode_message_header(message, 'To')
+        email_to_localpart = (tools.email_split(email_to) or [''])[0].split('@', 1)[0].lower()
 
-    def message_receive_bounce(self, mail_id=None):
-        """Called by ``message_process`` when a bounce email (such as Undelivered
-        Mail Returned to Sender) is received for an existing thread. The default
-        behavior is to check is an integer  ``message_bounce`` column exists.
-        If it is the case, its content is incremented. """
-        if 'message_bounce' in self._fields:
-            for mail_thread in self:
-                mail_thread.message_bounce = mail_thread.message_bounce + 1
+        if bounce_alias and bounce_alias in email_to_localpart:
+            bounce_re = re.compile("%s\+(\d+)-?([\w.]+)?-?(\d+)?" % re.escape(bounce_alias), re.UNICODE)
+            bounce_match = bounce_re.search(email_to)
+            if bounce_match:
+                bounced_mail_id = bounce_match.group(1)
+                self.env['mail.mail.statistics'].set_bounced(mail_mail_ids=[bounced_mail_id])
+
+        return super(MailThread, self).message_route(message, message_dict, model, thread_id, custom_values)
 
     @api.model
     def message_route_process(self, message, message_dict, routes):

addons/mass_mailing/models/mass_mailing.py

@@ -98,14 +98,6 @@ class MassMailingContact(models.Model):
     def message_get_default_recipients(self):
         return dict((record.id, {'partner_ids': [], 'email_to': record.email, 'email_cc': False}) for record in self)
 
-    def message_receive_bounce(self, mail_id=None):
-        """Called by ``message_process`` when a bounce email (such as Undelivered
-        Mail Returned to Sender) is received for an existing thread. As contacts
-        do not inherit form mail.thread, we have to define this method to be able
-        to track bounces (see mail.thread for more details). """
-        for contact in self:
-            contact.message_bounce = contact.message_bounce + 1
-
 
 class MassMailingStage(models.Model):