[IMP] mail: improve notification email previews

Prepare an informative and nice-looking preview from the main content of the
email body, avoiding buttons/images alt etc.
Links, images, tables are removed.

Whitespace is added after the preview to avoid including the full message in
the preview (with markup).

Tags and attributes are also added to increase compliance with HTML5 standard,
including accessibility.

One additional SQL request is required to fetch the preview sub-template.

Task-2413355

closes odoo/odoo#86266

Signed-off-by: Thibault Delavallee (tde) <tde@openerp.com>

addons/mail/data/mail_templates_email_layouts.xml

@@ -2,6 +2,13 @@
 <odoo>
     <data>
         <template id="message_notification_email">
+<html t-att-lang="lang">
+<header>
+    <meta http-equiv="Content-Type" content="text/html charset=UTF-8" />
+</header>
+<body>
+<t t-set="subtype_internal" t-value="subtype.internal"/>
+<t t-call="mail.notification_preview"/>
 <div>
 <div t-if="has_button_access" itemscope="itemscope" itemtype="http://schema.org/EmailMessage">
     <div itemprop="potentialAction" itemscope="itemscope" itemtype="http://schema.org/ViewAction">
@@ -12,7 +19,7 @@
 </div>
 <div t-if="has_button_access or len(actions) &gt; 0 or not is_discussion"
         summary="o_mail_notification" style="padding: 0px; width:600px;">
-    <table cellspacing="0" cellpadding="0" border="0" style="width: 600px; margin-top: 5px;">
+    <table role="presentation" cellspacing="0" cellpadding="0" border="0" style="width: 600px; margin-top: 5px;">
     <tbody><tr>
     <td valign="center">
         <a t-if="has_button_access"
@@ -36,7 +43,7 @@
     <td colspan="2" style="text-align:center;">
         <hr width="100%"
             style="background-color:rgb(204,204,204);border:medium none;clear:both;display:block;font-size:0px;min-height:1px;line-height:0; margin:4px 0 12px 0;"/>
-        <p t-if="subtype.internal" style="background-color: #f2dede; padding: 5px; margin-bottom: 16px;">
+        <p t-if="subtype_internal" style="background-color: #f2dede; padding: 5px; margin-bottom: 16px;">
             <strong>Internal communication</strong>: Replying will post an internal note. Followers won't receive any email notification.
         </p>
     </td>
@@ -46,7 +53,7 @@
 <div t-out="message.body"/>
 <ul t-if="tracking_values">
     <t t-foreach="tracking_values" t-as="tracking">
-        <li><t t-out="tracking[0]"/>: <t t-out="tracking[1]"/> -&gt; <t t-out="tracking[2]"/></li>
+        <li><t t-out="tracking[0]"/>: <t t-out="tracking[1]"/> &#8594; <t t-out="tracking[2]"/></li>
     </t>
 </ul>
 <div t-if="email_add_signature and not is_html_empty(signature)" t-out="signature" style="font-size: 13px;"/>
@@ -63,16 +70,24 @@
     <a target="_blank" href="https://www.odoo.com?utm_source=db&amp;utm_medium=email" style="text-decoration:none; color: #875A7B;">Odoo</a>.
 </p>
 </div>
+</body></html>
         </template>
 
         <template id="mail_notification_light">
-<table border="0" cellpadding="0" cellspacing="0" style="padding-top: 16px; background-color: #F1F1F1; font-family:Verdana, Arial,sans-serif; color: #454748; width: 100%; border-collapse:separate;"><tr><td align="center">
-<table border="0" cellpadding="0" cellspacing="0" width="590" style="padding: 24px; background-color: white; color: #454748; border-collapse:separate;">
+<html t-att-lang="lang">
+<header>
+    <meta http-equiv="Content-Type" content="text/html charset=UTF-8" />
+</header>
+<body>
+<t t-set="subtype_internal" t-value="False"/>
+<t t-call="mail.notification_preview"/>
+<table role="presentation" border="0" cellpadding="0" cellspacing="0" style="padding-top: 16px; background-color: #F1F1F1; font-family:Verdana, Arial,sans-serif; color: #454748; width: 100%; border-collapse:separate;"><tr><td align="center">
+<table role="presentation" border="0" cellpadding="0" cellspacing="0" width="590" style="padding: 24px; background-color: white; color: #454748; border-collapse:separate;">
 <tbody>
     <!-- HEADER -->
     <tr>
         <td align="center" style="min-width: 590px;">
-            <table border="0" cellpadding="0" cellspacing="0" width="100%" style="background-color: white; padding: 0; border-collapse:separate;">
+            <table role="presentation" border="0" cellpadding="0" cellspacing="0" width="100%" style="background-color: white; padding: 0; border-collapse:separate;">
                 <tr><td valign="middle">
                     <span style="font-size: 10px;">Your <t t-out="model_description or 'document'"/></span>
                     <br/>
@@ -125,17 +140,26 @@
         Powered by <a target="_blank" href="https://www.odoo.com?utm_source=db&amp;utm_medium=email" style="color: #875A7B;">Odoo</a>
 </td></tr>
 </table>
+</body>
+</html>
         </template>
 
         <!-- Used mainly in conjunction of portal models (SO, PO, Invoices) although not mandatory -->
         <template id="mail_notification_paynow" name="Mail: Pay Now mail notification template">
-<table border="0" cellpadding="0" cellspacing="0" style="background-color: #FFFFFF; font-family:Verdana, Arial,sans-serif; color: #454748; width: 100%; border-collapse:separate;"><tr><td>
-<table border="0" cellpadding="0" cellspacing="0" style="max-width:900px; width:100%; background-color: #FFFFFF; color: #454748; border-collapse:separate;">
+<html t-att-lang="lang">
+<header>
+    <meta http-equiv="Content-Type" content="text/html charset=UTF-8" />
+</header>
+<body>
+<t t-set="subtype_internal" t-value="False"/>
+<t t-call="mail.notification_preview"/>
+<table role="presentation" border="0" cellpadding="0" cellspacing="0" style="background-color: #FFFFFF; font-family:Verdana, Arial,sans-serif; color: #454748; width: 100%; border-collapse:separate;"><tr><td>
+<table role="presentation" border="0" cellpadding="0" cellspacing="0" style="max-width:900px; width:100%; background-color: #FFFFFF; color: #454748; border-collapse:separate;">
 <tbody>
     <!-- HEADER -->
     <tr>
         <td align="center">
-            <table border="0" cellpadding="0" cellspacing="5" width="100%" style="background-color: #FFFFFF; padding: 0; border-collapse:separate;">
+            <table role="presentation" border="0" cellpadding="0" cellspacing="5" width="100%" style="background-color: #FFFFFF; padding: 0; border-collapse:separate;">
                 <tr><td>
                     <img t-att-src="'/logo.png?company=%s' % (company.id or 0)" height="48" style="padding: 0; margin: 0"  t-att-alt="'%s' % company.name"/>
                 </td></tr>
@@ -222,6 +246,21 @@
 </table>
 </td></tr>
 </table>
+</body>
+</html>
+        </template>
+
+        <template id="notification_preview">
+<div style="display: none; max-height: 0px; overflow: hidden;">
+    <t t-if="tracking_values">
+        <t t-out="tracking_values[0][0]"/>: <t t-out="tracking_values[0][1]"/> &#8594; <t t-out="tracking_values[0][2]"/>
+        <t t-if="len(tracking_values) > 1"> |...</t>
+        <t t-if="message.preview"> | </t>
+    </t>
+    <t t-if="subtype_internal">Internal communication: </t><t t-out="message.preview"/>
+    <!--Trailing whitespace to push back email content so that it doesn't appear in preview. Specific characters to use may change over time -->
+    <t t-out="'&#847; &#8203; ' * 140"/>
+</div>
         </template>
     </data>
 </odoo>

addons/mail/models/mail_message.py

@@ -3,9 +3,9 @@
 
 import logging
 import re
-from collections import defaultdict
-
+import textwrap
 from binascii import Error as binascii_error
+from collections import defaultdict
 
 from odoo import _, api, Command, fields, models, modules, tools
 from odoo.exceptions import AccessError
@@ -89,6 +89,9 @@ class Message(models.Model):
     description = fields.Char(
         'Short description', compute="_compute_description",
         help='Message description: either the subject, or the beginning of the body')
+    preview = fields.Char(
+        'Preview', compute='_compute_preview',
+        help='The text-only beginning of the body used as email preview.')
     attachment_ids = fields.Many2many(
         'ir.attachment', 'message_attachment_rel',
         'message_id', 'attachment_id',
@@ -178,13 +181,18 @@ class Message(models.Model):
     canned_response_ids = fields.One2many('mail.shortcode', 'message_ids', string="Canned Responses", store=False)
     reaction_ids = fields.One2many('mail.message.reaction', 'message_id', string="Reactions", groups="base.group_system")
 
+    @api.depends('body', 'subject')
     def _compute_description(self):
         for message in self:
             if message.subject:
                 message.description = message.subject
             else:
-                plaintext_ct = '' if not message.body else tools.html2plaintext(message.body)
-                message.description = plaintext_ct[:30] + '%s' % (' [...]' if len(plaintext_ct) >= 30 else '')
+                message.description = message._get_message_preview(max_char=30)
+
+    @api.depends('body')
+    def _compute_preview(self):
+        for message in self:
+            message.preview = message._get_message_preview()
 
     @api.depends('author_id', 'author_guest_id')
     @api.depends_context('guest', 'uid')
@@ -1027,6 +1035,15 @@ class Message(models.Model):
     # TOOLS
     # ------------------------------------------------------
 
+    def _get_message_preview(self, max_char=190):
+        """Returns an unformatted version of the message body. Unless `max_char=0` is passed,
+        output will be capped at max_char characters with a ' [...]' suffix if applicable.
+        Default `max_char` is the longest known mail client preview length (Outlook 2013)."""
+        self.ensure_one()
+
+        plaintext_ct = tools.html_to_inner_content(self.body)
+        return textwrap.shorten(plaintext_ct, max_char) if max_char else plaintext_ct
+
     @api.model
     def _get_record_name(self, values):
         """ Return the related document name, using name_get. It is done using

addons/test_mail/tests/test_message_post.py

@@ -295,9 +295,10 @@ class TestMessageNotify(TestMessagePostCommon):
         admin_mails = [mail for mail in self._mails if self.partner_admin.name in mail.get('email_to')[0]]
         self.assertEqual(len(admin_mails), 1, 'There should be exactly one email sent to admin')
         admin_mail_body = admin_mails[0].get('body')
-        admin_access_link = admin_mail_body[admin_mail_body.index('model='):admin_mail_body.index('/>') - 1] if 'model=' in admin_mail_body else None
 
-        self.assertIsNotNone(admin_access_link, 'The email sent to admin should contain an access link')
+        self.assertTrue('model=' in admin_mail_body, 'The email sent to admin should contain an access link')
+        admin_access_link = admin_mail_body[
+            admin_mail_body.index('model='):admin_mail_body.index('/>', admin_mail_body.index('model=')) - 1]
         self.assertIn('model=%s' % self.test_record._name, admin_access_link, 'The access link should contain a valid model argument')
         self.assertIn('res_id=%d' % self.test_record.id, admin_access_link, 'The access link should contain a valid res_id argument')
 

addons/test_mail/tests/test_performance.py

@@ -575,7 +575,7 @@ class TestMailAPIPerformance(BaseMailPerformance):
     def test_message_post_one_email_notification(self):
         record = self.env['mail.test.simple'].create({'name': 'Test'})
 
-        with self.assertQueryCount(__system__=30, employee=33):
+        with self.assertQueryCount(__system__=31, employee=34):
             record.message_post(
                 body='<p>Test Post Performances with an email ping</p>',
                 partner_ids=self.customer.ids,
@@ -726,7 +726,7 @@ class TestMailComplexPerformance(BaseMailPerformance):
         self.container.message_subscribe(self.user_portal.partner_id.ids)
         record = self.container.with_user(self.env.user)
 
-        with self.assertQueryCount(__system__=53, employee=54):  # about 20 (19?) queries per additional customer group
+        with self.assertQueryCount(__system__=54, employee=55):  # about 20 (19?) queries per additional customer group
             record.message_post(
                 body='<p>Test Post Performances</p>',
                 message_type='comment',
@@ -743,7 +743,7 @@ class TestMailComplexPerformance(BaseMailPerformance):
         record = self.container.with_user(self.env.user)
         template_id = self.env.ref('test_mail.mail_test_container_tpl').id
 
-        with self.assertQueryCount(__system__=61, employee=62):  # about 20 (19 ?) queries per additional customer group
+        with self.assertQueryCount(__system__=62, employee=63):  # about 20 (19 ?) queries per additional customer group
             record.message_post_with_template(template_id, message_type='comment', composition_mode='comment')
 
         self.assertEqual(record.message_ids[0].body, '<p>Adding stuff on %s</p>' % record.name)

odoo/addons/base/tests/test_mail.py

@@ -11,7 +11,7 @@ import threading
 from odoo.addons.base.models.ir_mail_server import extract_rfc2822_addresses
 from odoo.tests.common import BaseCase, TransactionCase
 from odoo.tools import (
-    is_html_empty, html_sanitize, append_content_to_html, plaintext2html,
+    is_html_empty, html_to_inner_content, html_sanitize, append_content_to_html, plaintext2html,
     email_split, email_domain_normalize,
     misc, formataddr,
     prepend_html_content,
@@ -306,6 +306,19 @@ class TestHtmlTools(BaseCase):
             html = plaintext2html(content, container_tag)
             self.assertEqual(html, expected, 'plaintext2html is broken')
 
+    def test_html_html_to_inner_content(self):
+        cases = [
+            ('<div><p>First <br/>Second <br/>Third Paragraph</p><p>--<br/>Signature paragraph with a <a href="./link">link</a></p></div>',
+             'First Second Third Paragraph -- Signature paragraph with a link'),
+            ('<p>Now =&gt; processing&nbsp;entities&#8203;and extra whitespace too.  </p>',
+             'Now =&gt; processing\xa0entities\u200band extra whitespace too.'),
+            ('<div>Look what happens with <p>unmatched tags</div>', 'Look what happens with unmatched tags'),
+            ('<div>Look what happens with <p unclosed tags</div> Are we good?', 'Look what happens with Are we good?')
+        ]
+        for content, expected in cases:
+            text = html_to_inner_content(content)
+            self.assertEqual(text, expected, 'html_html_to_inner_content is broken')
+
     def test_append_to_html(self):
         test_samples = [
             ('<!DOCTYPE...><HTML encoding="blah">some <b>content</b></HtMl>', '--\nYours truly', True, True, False,

odoo/tools/mail.py

@@ -260,6 +260,8 @@ URL_REGEX = r'(\bhref=[\'"](?!mailto:|tel:|sms:)([^\'"]+)[\'"])'
 TEXT_URL_REGEX = r'https?://[\w@:%.+&~#=/-]+(?:\?\S+)?'
 # retrieve inner content of the link
 HTML_TAG_URL_REGEX = URL_REGEX + r'([^<>]*>([^<>]+)<\/)?'
+HTML_TAGS_REGEX = re.compile('<.*?>')
+HTML_NEWLINES_REGEX = re.compile('<(div|p|br|tr)[^>]*>|\n')
 
 
 def validate_url(url):
@@ -295,6 +297,21 @@ def html_keep_url(text):
     return final
 
 
+def html_to_inner_content(html):
+    """Returns unformatted text after removing html tags and excessive whitespace from a
+    string/Markup. Passed strings will first be sanitized.
+    """
+    if is_html_empty(html):
+        return ''
+    if not isinstance(html, markupsafe.Markup):
+        html = html_sanitize(html)
+    processed = re.sub(HTML_NEWLINES_REGEX, ' ', html)
+    processed = re.sub(HTML_TAGS_REGEX, '', processed)
+    processed = re.sub(r' {2,}|\t', ' ', processed)
+    processed = processed.strip()
+    return processed
+
+
 def html2plaintext(html, body_id=None, encoding='utf-8'):
     """ From an HTML text, convert the HTML to plain text.
     If @param body_id is provided then this is the tag where the