From 207675a1743cdd60ba459d4e119af953096dbf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Tue, 6 Nov 2012 12:54:20 +0100 Subject: [PATCH 01/22] [REN] html_sanitize -> mail, to have a central file for methods about mail (html sanitizing, html management, emails related functions, ...). bzr revid: tde@openerp.com-20121106115420-1h8mvzohol9zjysa --- openerp/tests/__init__.py | 4 +-- .../{test_html_sanitize.py => test_mail.py} | 36 ++++++++++++++++--- openerp/tools/__init__.py | 2 +- openerp/tools/{html_sanitize.py => mail.py} | 0 4 files changed, 35 insertions(+), 7 deletions(-) rename openerp/tests/{test_html_sanitize.py => test_mail.py} (67%) rename openerp/tools/{html_sanitize.py => mail.py} (100%) diff --git a/openerp/tests/__init__.py b/openerp/tests/__init__.py index c4b77a606a9..8f16fc76650 100644 --- a/openerp/tests/__init__.py +++ b/openerp/tests/__init__.py @@ -8,7 +8,7 @@ Tests can be explicitely added to the `fast_suite` or `checks` lists or not. See the :ref:`test-framework` section in the :ref:`features` list. """ -from . import test_expression, test_html_sanitize, test_ir_sequence, test_orm,\ +from . import test_expression, test_mail, test_ir_sequence, test_orm,\ test_fields, test_basecase, \ test_view_validation, test_uninstall, test_misc, test_db_cursor @@ -18,7 +18,7 @@ fast_suite = [ checks = [ test_expression, - test_html_sanitize, + test_mail, test_db_cursor, test_orm, test_fields, diff --git a/openerp/tests/test_html_sanitize.py b/openerp/tests/test_mail.py similarity index 67% rename from openerp/tests/test_html_sanitize.py rename to openerp/tests/test_mail.py index cb46b325144..14d49a3eefc 100755 --- a/openerp/tests/test_html_sanitize.py +++ b/openerp/tests/test_mail.py @@ -1,5 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Business Applications +# Copyright (c) 2012-TODAY OpenERP S.A. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + import unittest from openerp.tools.html_sanitize import html_sanitize @@ -21,21 +41,29 @@ test12

test link """ + class TestSanitizer(unittest.TestCase): + # TDE note: could be improved by actually checking the output def test_simple(self): x = "yop" self.assertEqual(x, html_sanitize(x)) - + def test_trailing_text(self): x = 'lala

yop

xxx' self.assertEqual(x, html_sanitize(x)) - + def test_no_exception(self): html_sanitize(test_case) - + def test_unicode(self): html_sanitize("Merci à l'intérêt pour notre produit.nous vous contacterons bientôt. Merci") + +class TestCleaner(unittest.TestCase): + + def test(self): + print 'poinpoinpoinponi' + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/openerp/tools/__init__.py b/openerp/tools/__init__.py index a0ca411a9df..af14189bbae 100644 --- a/openerp/tools/__init__.py +++ b/openerp/tools/__init__.py @@ -33,7 +33,7 @@ from pdf_utils import * from yaml_import import * from sql import * from float_utils import * -from html_sanitize import * +from mail import * #.apidoc title: Tools diff --git a/openerp/tools/html_sanitize.py b/openerp/tools/mail.py similarity index 100% rename from openerp/tools/html_sanitize.py rename to openerp/tools/mail.py From da6ad1e27c827d08e35099e051559dd09c0f8fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Tue, 6 Nov 2012 12:57:44 +0100 Subject: [PATCH 02/22] [REF] tools: moved email and mail related functions from misc to mail. bzr revid: tde@openerp.com-20121106115744-pcxgyqv3lf8zfedw --- openerp/tools/mail.py | 163 +++++++++++++++++++++++++++++++++++++++++ openerp/tools/misc.py | 167 ------------------------------------------ 2 files changed, 163 insertions(+), 167 deletions(-) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index 6ea7b90e2ba..6509c508900 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -90,3 +90,166 @@ def append_to(elements, dest_node): children[-1].tail = element else: dest_node.append(element) + + +#---------------------------------------------------------- +# Emails +#---------------------------------------------------------- + +email_re = re.compile(r""" + ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part + @ # mandatory @ sign + [a-zA-Z0-9][\w\.-]* # domain must start with a letter ... Ged> why do we include a 0-9 then? + \. + [a-z]{2,3} # TLD + ) + """, re.VERBOSE) +res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE) +command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) + +# Updated in 7.0 to match the model name as well +# Typical form of references is +# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain +reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE) + +def html2plaintext(html, body_id=None, encoding='utf-8'): + """ From an HTML text, convert the HTML to plain text. + If @param body_id is provided then this is the tag where the + body (not necessarily ) starts. + """ + ## (c) Fry-IT, www.fry-it.com, 2007 + ## + ## download here: http://www.peterbe.com/plog/html2plaintext + + html = ustr(html) + + from lxml.etree import tostring, fromstring, HTMLParser + tree = fromstring(html, parser=HTMLParser()) + + if body_id is not None: + source = tree.xpath('//*[@id=%s]'%(body_id,)) + else: + source = tree.xpath('//body') + if len(source): + tree = source[0] + + url_index = [] + i = 0 + for link in tree.findall('.//a'): + url = link.get('href') + if url: + i += 1 + link.tag = 'span' + link.text = '%s [%s]' % (link.text, i) + url_index.append(url) + + html = ustr(tostring(tree, encoding=encoding)) + + html = html.replace('','*').replace('','*') + html = html.replace('','*').replace('','*') + html = html.replace('

','*').replace('

','*') + html = html.replace('

','**').replace('

','**') + html = html.replace('

','**').replace('

','**') + html = html.replace('','/').replace('','/') + html = html.replace('', '\n') + html = html.replace('

', '\n') + html = re.sub('', '\n', html) + html = re.sub('<.*?>', ' ', html) + html = html.replace(' ' * 2, ' ') + + # strip all lines + html = '\n'.join([x.strip() for x in html.splitlines()]) + html = html.replace('\n' * 2, '\n') + + for i, url in enumerate(url_index): + if i == 0: + html += '\n\n' + html += ustr('[%s] %s\n') % (i+1, url) + + return html + +def generate_tracking_message_id(res_id): + """Returns a string that can be used in the Message-ID RFC822 header field + + Used to track the replies related to a given object thanks to the "In-Reply-To" + or "References" fields that Mail User Agents will set. + """ + try: + rnd = random.SystemRandom().random() + except NotImplementedError: + rnd = random.random() + rndstr = ("%.15f" % rnd)[2:] + return "<%.15f.%s-openerp-%s@%s>" % (time.time(), rndstr, res_id, socket.gethostname()) + +def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False, + attachments=None, message_id=None, references=None, openobject_id=False, debug=False, subtype='plain', headers=None, + smtp_server=None, smtp_port=None, ssl=False, smtp_user=None, smtp_password=None, cr=None, uid=None): + """Low-level function for sending an email (deprecated). + + :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead. + :param email_from: A string used to fill the `From` header, if falsy, + config['email_from'] is used instead. Also used for + the `Reply-To` header if `reply_to` is not provided + :param email_to: a sequence of addresses to send the mail to. + """ + + # If not cr, get cr from current thread database + if not cr: + db_name = getattr(threading.currentThread(), 'dbname', None) + if db_name: + cr = pooler.get_db_only(db_name).cursor() + else: + raise Exception("No database cursor found, please pass one explicitly") + + # Send Email + try: + mail_server_pool = pooler.get_pool(cr.dbname).get('ir.mail_server') + res = False + # Pack Message into MIME Object + email_msg = mail_server_pool.build_email(email_from, email_to, subject, body, email_cc, email_bcc, reply_to, + attachments, message_id, references, openobject_id, subtype, headers=headers) + + res = mail_server_pool.send_email(cr, uid or 1, email_msg, mail_server_id=None, + smtp_server=smtp_server, smtp_port=smtp_port, smtp_user=smtp_user, smtp_password=smtp_password, + smtp_encryption=('ssl' if ssl else None), smtp_debug=debug) + except Exception: + _logger.exception("tools.email_send failed to deliver email") + return False + finally: + cr.close() + return res + +def email_split(text): + """ Return a list of the email addresses found in ``text`` """ + if not text: return [] + return re.findall(r'([^ ,<@]+@[^> ,]+)', text) + +def append_content_to_html(html, content, plaintext=True): + """Append extra content at the end of an HTML snippet, trying + to locate the end of the HTML document (, , or + EOF), and wrapping the provided content in a 
 block
+       unless ``plaintext`` is False. A side-effect of this
+       method is to coerce all HTML tags to lowercase in ``html``,
+       and strip enclosing  or  tags in content if
+       ``plaintext`` is False.
+       
+       :param str html: html tagsoup (doesn't have to be XHTML)
+       :param str content: extra content to append
+       :param bool plaintext: whether content is plaintext and should
+           be wrapped in a 
 tag.
+    """
+    html = ustr(html)
+    if plaintext:
+        content = u'\n
%s
\n' % ustr(content)
+    else:
+        content = re.sub(r'(?i)(||)', '', content)
+        content = u'\n%s\n'% ustr(content)
+    # Force all tags to lowercase
+    html = re.sub(r'(])',
+        lambda m: '%s%s%s' % (m.group(1),m.group(2).lower(),m.group(3)), html)
+    insert_location = html.find('')
+    if insert_location == -1:
+        insert_location = html.find('')
+    if insert_location == -1:
+        return '%s%s' % (html, content)
+    return '%s%s%s' % (html[:insert_location], content, html[insert_location:])  
diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py
index caf28e54481..73c5315d4ed 100644
--- a/openerp/tools/misc.py
+++ b/openerp/tools/misc.py
@@ -43,12 +43,7 @@ from itertools import islice, izip
 from lxml import etree
 from which import which
 from threading import local
-try:
-    from html2text import html2text
-except ImportError:
-    html2text = None
 
-import openerp.loglevels as loglevels
 import openerp.pooler as pooler
 from config import config
 from cache import *
@@ -274,168 +269,6 @@ def reverse_enumerate(l):
     """
     return izip(xrange(len(l)-1, -1, -1), reversed(l))
 
-#----------------------------------------------------------
-# Emails
-#----------------------------------------------------------
-email_re = re.compile(r"""
-    ([a-zA-Z][\w\.-]*[a-zA-Z0-9]     # username part
-    @                                # mandatory @ sign
-    [a-zA-Z0-9][\w\.-]*              # domain must start with a letter ... Ged> why do we include a 0-9 then?
-     \.
-     [a-z]{2,3}                      # TLD
-    )
-    """, re.VERBOSE)
-res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE)
-command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE)
-
-# Updated in 7.0 to match the model name as well
-# Typical form of references is 
-# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain
-reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE)
-
-def html2plaintext(html, body_id=None, encoding='utf-8'):
-    """ From an HTML text, convert the HTML to plain text.
-    If @param body_id is provided then this is the tag where the
-    body (not necessarily ) starts.
-    """
-    ## (c) Fry-IT, www.fry-it.com, 2007
-    ## 
-    ## download here: http://www.peterbe.com/plog/html2plaintext
-
-    html = ustr(html)
-
-    from lxml.etree import tostring, fromstring, HTMLParser
-    tree = fromstring(html, parser=HTMLParser())
-
-    if body_id is not None:
-        source = tree.xpath('//*[@id=%s]'%(body_id,))
-    else:
-        source = tree.xpath('//body')
-    if len(source):
-        tree = source[0]
-
-    url_index = []
-    i = 0
-    for link in tree.findall('.//a'):
-        url = link.get('href')
-        if url:
-            i += 1
-            link.tag = 'span'
-            link.text = '%s [%s]' % (link.text, i)
-            url_index.append(url)
-
-    html = ustr(tostring(tree, encoding=encoding))
-
-    html = html.replace('','*').replace('','*')
-    html = html.replace('','*').replace('','*')
-    html = html.replace('
','*').replace('
','*')
-    html = html.replace('
','**').replace('
','**')
-    html = html.replace('
','**').replace('
','**')
-    html = html.replace('','/').replace('','/')
-    html = html.replace('', '\n')
-    html = html.replace('
', '\n')
-    html = re.sub('', '\n', html)
-    html = re.sub('<.*?>', ' ', html)
-    html = html.replace(' ' * 2, ' ')
-
-    # strip all lines
-    html = '\n'.join([x.strip() for x in html.splitlines()])
-    html = html.replace('\n' * 2, '\n')
-
-    for i, url in enumerate(url_index):
-        if i == 0:
-            html += '\n\n'
-        html += ustr('[%s] %s\n') % (i+1, url)
-
-    return html
-
-def generate_tracking_message_id(res_id):
-    """Returns a string that can be used in the Message-ID RFC822 header field
-    
-       Used to track the replies related to a given object thanks to the "In-Reply-To"
-       or "References" fields that Mail User Agents will set.
-    """
-    try:
-        rnd = random.SystemRandom().random()
-    except NotImplementedError:
-        rnd = random.random()
-    rndstr = ("%.15f" % rnd)[2:] 
-    return "<%.15f.%s-openerp-%s@%s>" % (time.time(), rndstr, res_id, socket.gethostname())
-
-def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False,
-               attachments=None, message_id=None, references=None, openobject_id=False, debug=False, subtype='plain', headers=None,
-               smtp_server=None, smtp_port=None, ssl=False, smtp_user=None, smtp_password=None, cr=None, uid=None):
-    """Low-level function for sending an email (deprecated).
-
-    :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead. 
-    :param email_from: A string used to fill the `From` header, if falsy,
-                       config['email_from'] is used instead.  Also used for
-                       the `Reply-To` header if `reply_to` is not provided
-    :param email_to: a sequence of addresses to send the mail to.
-    """
-
-    # If not cr, get cr from current thread database
-    if not cr:
-        db_name = getattr(threading.currentThread(), 'dbname', None)
-        if db_name:
-            cr = pooler.get_db_only(db_name).cursor()
-        else:
-            raise Exception("No database cursor found, please pass one explicitly")
-
-    # Send Email
-    try:
-        mail_server_pool = pooler.get_pool(cr.dbname).get('ir.mail_server')
-        res = False
-        # Pack Message into MIME Object
-        email_msg = mail_server_pool.build_email(email_from, email_to, subject, body, email_cc, email_bcc, reply_to,
-                   attachments, message_id, references, openobject_id, subtype, headers=headers)
-
-        res = mail_server_pool.send_email(cr, uid or 1, email_msg, mail_server_id=None,
-                       smtp_server=smtp_server, smtp_port=smtp_port, smtp_user=smtp_user, smtp_password=smtp_password,
-                       smtp_encryption=('ssl' if ssl else None), smtp_debug=debug)
-    except Exception:
-        _logger.exception("tools.email_send failed to deliver email")
-        return False
-    finally:
-        cr.close()
-    return res
-
-def email_split(text):
-    """ Return a list of the email addresses found in ``text`` """
-    if not text: return []
-    return re.findall(r'([^ ,<@]+@[^> ,]+)', text)
-
-def append_content_to_html(html, content, plaintext=True):
-    """Append extra content at the end of an HTML snippet, trying
-       to locate the end of the HTML document (, , or
-       EOF), and wrapping the provided content in a 
 block
-       unless ``plaintext`` is False. A side-effect of this
-       method is to coerce all HTML tags to lowercase in ``html``,
-       and strip enclosing  or  tags in content if
-       ``plaintext`` is False.
-       
-       :param str html: html tagsoup (doesn't have to be XHTML)
-       :param str content: extra content to append
-       :param bool plaintext: whether content is plaintext and should
-           be wrapped in a 
 tag.
-    """
-    html = ustr(html)
-    if plaintext:
-        content = u'\n
%s
\n' % ustr(content)
-    else:
-        content = re.sub(r'(?i)(||)', '', content)
-        content = u'\n%s\n'% ustr(content)
-    # Force all tags to lowercase
-    html = re.sub(r'(])',
-        lambda m: '%s%s%s' % (m.group(1),m.group(2).lower(),m.group(3)), html)
-    insert_location = html.find('')
-    if insert_location == -1:
-        insert_location = html.find('')
-    if insert_location == -1:
-        return '%s%s' % (html, content)
-    return '%s%s%s' % (html[:insert_location], content, html[insert_location:])  
-
-
 #----------------------------------------------------------
 # SMS
 #----------------------------------------------------------

From 5f53f5253bba48a4ee52724317d3117b1c0dbca2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= 
Date: Tue, 6 Nov 2012 12:58:51 +0100
Subject: [PATCH 03/22] [FIX] tools: some import cleaning and fixing.

bzr revid: tde@openerp.com-20121106115851-5f5sfhtp2g3qrm1o
---
 openerp/tools/mail.py | 2 ++
 openerp/tools/misc.py | 3 ---
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py
index 6509c508900..87e5e069fd8 100644
--- a/openerp/tools/mail.py
+++ b/openerp/tools/mail.py
@@ -20,7 +20,9 @@
 ##############################################################################
 
 import lxml.html
+import openerp.pooler as pooler
 import operator
+import random
 import re
 
 from openerp.loglevels import ustr
diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py
index 73c5315d4ed..15af8967409 100644
--- a/openerp/tools/misc.py
+++ b/openerp/tools/misc.py
@@ -30,8 +30,6 @@ from functools import wraps
 import subprocess
 import logging
 import os
-import random
-import re
 import socket
 import sys
 import threading
@@ -44,7 +42,6 @@ from lxml import etree
 from which import which
 from threading import local
 
-import openerp.pooler as pooler
 from config import config
 from cache import *
 

From 1d52a68dd95ff6add09966ac0ed807fda498e3b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= 
Date: Tue, 6 Nov 2012 13:17:37 +0100
Subject: [PATCH 04/22] [IMP] tools: mail: added html_cleaner, to clean html
 code to display in chatter (remove quotes, signatures, ...).

bzr revid: tde@openerp.com-20121106121737-ej57bijc7ypksu8v
---
 openerp/tools/mail.py | 58 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py
index 87e5e069fd8..abd5d5e736a 100644
--- a/openerp/tools/mail.py
+++ b/openerp/tools/mail.py
@@ -19,6 +19,8 @@
 #
 ##############################################################################
 
+from lxml.html.soupparser import fromstring
+from lxml.etree import tostring
 import lxml.html
 import openerp.pooler as pooler
 import operator
@@ -94,6 +96,62 @@ def append_to(elements, dest_node):
             dest_node.append(element)
 
 
+#----------------------------------------------------------
+# HTML Cleaner
+#----------------------------------------------------------
+
+def html_email_clean(html):
+    """ html_email_clean: clean the html to display in the web client.
+        - strip email quotes (remove blockquote nodes)
+        - strip signatures (remove --\n{\n)Blahblah), by replacing 
 by
+            \n to avoid ignoring signatures converted into html
+    """
+    modified_html = ''
+
+    # 1.  -> \n, because otherwise the tree is obfuscated
+    br_tags = re.compile(r'([<]\s*br\s*\/?[>])')
+    idx = 0
+    for item in re.finditer(br_tags, html):
+        modified_html += html[idx:item.start()] + '\n'
+        idx = item.end()
+    modified_html += html[idx:]
+
+    # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre
+    root = fromstring(modified_html)
+    if not len(root) and root.text is None and root.tail is None:
+        modified_html = '
%s
' % modified_html
+        root = fromstring(modified_html)
+
+    # 3. remove blockquotes
+    quotes = [el for el in root.iterchildren(tag='blockquote')]
+    for node in quotes:
+        node.getparent().remove(node)
+
+    # 4. strip signatures
+    signature = re.compile(r'([-]{2}[\s]?[\r\n]{1,2}[^\z]+)')
+    for elem in root.getiterator():
+        if elem.text:
+            match = re.search(signature, elem.text)
+            if match:
+                elem.text = elem.text[:match.start()] + elem.text[match.end():]
+        if elem.tail:
+            match = re.search(signature, elem.tail)
+            if match:
+                elem.tail = elem.tail[:match.start()] + elem.tail[match.end():]
+
+    # 5. \n back to 

+    for el in root.iterchildren():
+        if el.tag == 'pre':
+            continue
+        if el.text:
+            el.text = el.text.replace('\n', '
')
+        if el.tail:
+            el.tail = el.tail.replace('\n', '
')
+
+    new_html = tostring(root, pretty_print=True)
+    return new_html
+
+
 #----------------------------------------------------------
 # Emails
 #----------------------------------------------------------

From 56b0b518a22267f182155166beb8288286d94368 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= 
Date: Tue, 6 Nov 2012 13:18:24 +0100
Subject: [PATCH 05/22] [IMP] tests: moved append_to_html into right test file.
 Added basic struct and templates for testing html_email_clean.

bzr revid: tde@openerp.com-20121106121824-wb8qf9snma7qffqe
---
 openerp/tests/test_mail.py | 190 +++++++++++++++++++++++++++++++++++--
 openerp/tests/test_misc.py |  12 ---
 2 files changed, 183 insertions(+), 19 deletions(-)

diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py
index 14d49a3eefc..26ccdd39b28 100755
--- a/openerp/tests/test_mail.py
+++ b/openerp/tests/test_mail.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+# This test can be run stand-alone with something like:
+# > PYTHONPATH=. python2 openerp/tests/test_misc.py
 ##############################################################################
 #
 #    OpenERP, Open Source Business Applications
@@ -20,8 +22,8 @@
 #
 ##############################################################################
 
-import unittest
-from openerp.tools.html_sanitize import html_sanitize
+import unittest2
+from openerp.tools.mail import html_sanitize, html_email_clean, append_content_to_html
 
 test_case = """
 test1
@@ -41,8 +43,180 @@ test12

test link """ +GMAIL_REPLY = """ + + + + +
Réponse via thunderbird, classique.
+
+ On 11/05/2012 10:51 AM, Thibault Delavallée wrote:
+
+
+
Plop !
+
    +
  • Vive les lapins rapides !
    +
    • +
  • Nouille
    • +
  • Frites
    • +
+

+
+
Clairement, hein ?
+ --
+ Thibault Delavallée
+
+
+
+ 
-- 
+Thibault Delavallée
+
+ +""" -class TestSanitizer(unittest.TestCase): +GAMIL_REPLY_SAN = """
Réponse via thunderbird, classique.

+ On 11/05/2012 10:51 AM, Thibault Delavallée wrote:
+
+
Plop !
+
  • Vive les lapins rapides !
    • +
  • Nouille
    • +
  • Frites
    • +

+
Clairement, hein ?
+ --
+ Thibault Delavallée
+ 

-- 
+Thibault Delavallée
+
""" + + + + +GMAIL_REPLY2_SAN = """
Je réponds, hop, via thunderbird. Mais + je vais répodnre aussi au milieu du thread.

+ On 11/05/2012 10:53 AM, Thibault Delavallée wrote:
+
Reply rapide de gmail.
+
+ Jamais.

+


2012/11/5 Thibault Delavallée <tde@openerp.com>
+
+
Réponse via thunderbird, classique. +
+


+ On 11/05/2012 10:51 AM, Thibault Delavallée wrote:
+
+
+
+
+
+
Plop !
+
  • Vive les lapins rapides !
    • +
  • Nouille
    • +
+
+
+
+
+
+
+
+ je rajotuerais bien pommes de terre dans la liste.
+
+
+
+
+
+
+
+
  • Frites
    • +

+
Clairement, hein ?
+ --
+ Thibault Delavallée
+

+
+ + 
-- 
+Thibault Delavallée
+
+
+
+
+


+ --
+ Thibault Delavallée
+
+ 

-- 
+Thibault Delavallée
+
""" + + + +MAIL_TEMPLATE = """Hey XYZ, + +I've fixed that error and tested it a couple of times, it seems to be working +fine now. + +On Fri, Feb 18, 2011 at 7:44 AM, Joe David wrote: +Initial thread starts here... + +-- +Thanks +joe@david.com +""" + +GMAIL_TPL = """Salut Bob! +Le 28 oct. 2012 à 00:02, Thibault Delavallée a écrit : + +> MatrixPlus le 22/02 ? Si c'est ce que tu dis, je suis intéressé, oui ! + +Non, pas MatrixPlus, juste Matrix. + +Et oui, le 22/02 est la date qui doit vous convenir le deux. + +Bon, on tente de s’organiser un truc se samedi-là et je réserve des places. + +En passant, t’as contacté les gens du KdM pour le truc pour Bénisexe ? + +> 2012/10/27 Édouard Gilbert : +>> Diantre, me disè-je en envoyant un message similaire à Koukouiles, que ne voilà un concert qui pourrait intéresser Thibault ? +>> +>> L’ONL donne, en février et à Lille, Matrix en ciné-concert. Il passe le film sans musique mais c’est pas grave parce qu’il y a un orchestre qui la joue en même temps. Intéressé ? +>> +>> On peut trouver plus d’info sur le site de l’ONL, onlille.com , mais il est tellement mal foutu que je ne trouve pas de lien direct. Je te laisse chercher. +>> +>> Édouard GILBERT +>> edouard.gilbert@gmail.com +>> +> +> +> +> -- +> Thibault Delavallée + +Édouard GILBERT +edouard.gilbert@gmail.com +""" + + +class TestAppendContentToHtml(unittest2.TestCase): + """ Test some of our generic utility functions """ + + def test_append_to_html(self): + test_samples = [ + ('some content', '--\nYours truly', True, + 'some content\n
--\nYours truly
\n'), + ('some content', '\n\n

--

\n

Yours truly

\n\n', False, + 'some content\n\n\n

--

\n

Yours truly

\n\n\n'), + ] + for html, content, flag, expected in test_samples: + self.assertEqual(append_content_to_html(html, content, flag), expected, 'append_content_to_html is broken') + + +class TestSanitizer(unittest2.TestCase): # TDE note: could be improved by actually checking the output def test_simple(self): @@ -60,10 +234,12 @@ class TestSanitizer(unittest.TestCase): html_sanitize("Merci à l'intérêt pour notre produit.nous vous contacterons bientôt. Merci") -class TestCleaner(unittest.TestCase): +class TestCleaner(unittest2.TestCase): + + def test_gmail(self): + test_case = """youplaboum""" + html_email_clean(test_case) - def test(self): - print 'poinpoinpoinponi' if __name__ == '__main__': - unittest.main() + unittest2.main() diff --git a/openerp/tests/test_misc.py b/openerp/tests/test_misc.py index 54ae69899a6..9540c0222f3 100644 --- a/openerp/tests/test_misc.py +++ b/openerp/tests/test_misc.py @@ -4,18 +4,6 @@ import unittest2 from ..tools import misc -class append_content_to_html(unittest2.TestCase): - """ Test some of our generic utility functions """ - - def test_append_to_html(self): - test_samples = [ - ('some content', '--\nYours truly', True, - 'some content\n
--\nYours truly
\n'), - ('some content', '\n\n

--

\n

Yours truly

\n\n', False, - 'some content\n\n\n

--

\n

Yours truly

\n\n\n'), - ] - for html, content, flag, expected in test_samples: - self.assertEqual(misc.append_content_to_html(html,content,flag), expected, 'append_content_to_html is broken') class test_countingstream(unittest2.TestCase): def test_empty_stream(self): From 8a1a612fbae9f86f4cb3af2920e59e3802f1840e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Tue, 6 Nov 2012 13:27:58 +0100 Subject: [PATCH 06/22] [FIX] fields.py: html_sanitize now in mail file. bzr revid: tde@openerp.com-20121106122758-iwyxmedgnznpjyna --- openerp/osv/fields.py | 2 +- openerp/osv/osv.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index 2d51ad7d6db..ee95cece722 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -45,7 +45,7 @@ import openerp.tools as tools from openerp.tools.translate import _ from openerp.tools import float_round, float_repr import simplejson -from openerp.tools.html_sanitize import html_sanitize +from openerp.tools.mail import html_sanitize from openerp import SUPERUSER_ID _logger = logging.getLogger(__name__) diff --git a/openerp/osv/osv.py b/openerp/osv/osv.py index 94f179c7897..d59998193a2 100644 --- a/openerp/osv/osv.py +++ b/openerp/osv/osv.py @@ -120,6 +120,7 @@ class object_proxy(object): raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.') return f(self, dbname, *args, **kwargs) except orm.except_orm, inst: + raise raise except_osv(inst.name, inst.value) except except_osv: raise From c596792bc26f4068e926140c3f737053ec6a7e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Tue, 6 Nov 2012 13:34:07 +0100 Subject: [PATCH 07/22] [FIX] tools: re-added html2text import, because of its use in ir_mail_server. Not sure it belongs here. bzr revid: tde@openerp.com-20121106123407-2s4ya7ln7detui6y --- openerp/tools/misc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py index 15af8967409..4e3db23bc2d 100644 --- a/openerp/tools/misc.py +++ b/openerp/tools/misc.py @@ -41,6 +41,10 @@ from itertools import islice, izip from lxml import etree from which import which from threading import local +try: + from html2text import html2text +except ImportError: + html2text = None from config import config from cache import * From f2d53fd215d54abbfa8ee578a96d7271e6806784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Tue, 6 Nov 2012 13:41:09 +0100 Subject: [PATCH 08/22] [FIX] tools: mail: fixed some import, cleaned some code. bzr revid: tde@openerp.com-20121106124109-63svzi72lx2a13yq --- openerp/tools/mail.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index abd5d5e736a..b4e5754bfe4 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -21,14 +21,21 @@ from lxml.html.soupparser import fromstring from lxml.etree import tostring +import logging import lxml.html import openerp.pooler as pooler import operator import random import re +import socket +import threading +import time from openerp.loglevels import ustr +_logger = logging.getLogger(__name__) + + def html_sanitize(src): if not src: return src @@ -230,7 +237,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'): def generate_tracking_message_id(res_id): """Returns a string that can be used in the Message-ID RFC822 header field - + Used to track the replies related to a given object thanks to the "In-Reply-To" or "References" fields that Mail User Agents will set. """ @@ -238,7 +245,7 @@ def generate_tracking_message_id(res_id): rnd = random.SystemRandom().random() except NotImplementedError: rnd = random.random() - rndstr = ("%.15f" % rnd)[2:] + rndstr = ("%.15f" % rnd)[2:] return "<%.15f.%s-openerp-%s@%s>" % (time.time(), rndstr, res_id, socket.gethostname()) def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=None, reply_to=False, @@ -246,7 +253,7 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non smtp_server=None, smtp_port=None, ssl=False, smtp_user=None, smtp_password=None, cr=None, uid=None): """Low-level function for sending an email (deprecated). - :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead. + :deprecate: since OpenERP 6.1, please use ir.mail_server.send_email() instead. :param email_from: A string used to fill the `From` header, if falsy, config['email_from'] is used instead. Also used for the `Reply-To` header if `reply_to` is not provided @@ -281,7 +288,8 @@ def email_send(email_from, email_to, subject, body, email_cc=None, email_bcc=Non def email_split(text): """ Return a list of the email addresses found in ``text`` """ - if not text: return [] + if not text: + return [] return re.findall(r'([^ ,<@]+@[^> ,]+)', text) def append_content_to_html(html, content, plaintext=True): @@ -292,7 +300,7 @@ def append_content_to_html(html, content, plaintext=True): method is to coerce all HTML tags to lowercase in ``html``, and strip enclosing or tags in content if ``plaintext`` is False. - + :param str html: html tagsoup (doesn't have to be XHTML) :param str content: extra content to append :param bool plaintext: whether content is plaintext and should @@ -303,13 +311,13 @@ def append_content_to_html(html, content, plaintext=True): content = u'\n
%s
\n' % ustr(content) else: content = re.sub(r'(?i)(||)', '', content) - content = u'\n%s\n'% ustr(content) + content = u'\n%s\n' % ustr(content) # Force all tags to lowercase html = re.sub(r'(])', - lambda m: '%s%s%s' % (m.group(1),m.group(2).lower(),m.group(3)), html) + lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html) insert_location = html.find('') if insert_location == -1: insert_location = html.find('') if insert_location == -1: return '%s%s' % (html, content) - return '%s%s%s' % (html[:insert_location], content, html[insert_location:]) + return '%s%s%s' % (html[:insert_location], content, html[insert_location:]) From a80fab63036d43f71e971bbbe5c2aa1263da4ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Tue, 6 Nov 2012 14:37:51 +0100 Subject: [PATCH 09/22] [FIX] tools: mail: use beautifulsoup parser, if not present, html parser. bzr revid: tde@openerp.com-20121106133751-ds0g2c7k8pu4d6rl --- openerp/tools/mail.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index b4e5754bfe4..2cbb094bf8e 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -19,8 +19,11 @@ # ############################################################################## -from lxml.html.soupparser import fromstring -from lxml.etree import tostring +from lxml.etree import tostring, Element +try: + from lxml.html.soupparser import fromstring as parser_fromstring +except ImportError: + from lxml.html import fromstring as parser_fromstring import logging import lxml.html import openerp.pooler as pooler @@ -124,10 +127,10 @@ def html_email_clean(html): modified_html += html[idx:] # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre - root = fromstring(modified_html) + root = parser_fromstring(modified_html) if not len(root) and root.text is None and root.tail is None: modified_html = '
%s
' % modified_html - root = fromstring(modified_html) + root = parser_fromstring(modified_html) # 3. remove blockquotes quotes = [el for el in root.iterchildren(tag='blockquote')] From 3ba739cbf510bf1ed164d1439d85d412fe77a0ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Wed, 7 Nov 2012 09:58:07 +0100 Subject: [PATCH 10/22] [IMP] tools: mail: improved basic html_email_clean. bzr revid: tde@openerp.com-20121107085807-td2okmbr61pm4g1e --- openerp/tools/mail.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index 2cbb094bf8e..0ae4b373fee 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -19,11 +19,11 @@ # ############################################################################## -from lxml.etree import tostring, Element -try: - from lxml.html.soupparser import fromstring as parser_fromstring -except ImportError: - from lxml.html import fromstring as parser_fromstring +from lxml import etree +# try: +# from lxml.html.soupparser import fromstring as parser_fromstring +# except ImportError: +# from lxml.html import fromstring as parser_fromstring import logging import lxml.html import openerp.pooler as pooler @@ -122,15 +122,16 @@ def html_email_clean(html): br_tags = re.compile(r'([<]\s*br\s*\/?[>])') idx = 0 for item in re.finditer(br_tags, html): - modified_html += html[idx:item.start()] + '\n' + modified_html += html[idx:item.start()] + '__BR_TAG__' idx = item.end() modified_html += html[idx:] + html = modified_html # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre - root = parser_fromstring(modified_html) + root = lxml.html.fromstring(html) if not len(root) and root.text is None and root.tail is None: - modified_html = '
%s
' % modified_html - root = parser_fromstring(modified_html) + html = '
%s
' % html + root = lxml.html.fromstring(html) # 3. remove blockquotes quotes = [el for el in root.iterchildren(tag='blockquote')] @@ -150,16 +151,21 @@ def html_email_clean(html): elem.tail = elem.tail[:match.start()] + elem.tail[match.end():] # 5. \n back to
- for el in root.iterchildren(): - if el.tag == 'pre': - continue - if el.text: - el.text = el.text.replace('\n', '
') - if el.tail: - el.tail = el.tail.replace('\n', '
') + html = etree.tostring(root, pretty_print=True) + html = html.replace('__BR_TAG__', '
') - new_html = tostring(root, pretty_print=True) - return new_html + # 6. Misc cleaning : + # - ClEditor seems to love using

-> replace with
+ modified_html = '' + br_div_tags = re.compile(r'(
\s*\s*<\/div>)') + idx = 0 + for item in re.finditer(br_div_tags, html): + modified_html += html[idx:item.start()] + '
' + idx = item.end() + modified_html += html[idx:] + html = modified_html + + return html #---------------------------------------------------------- From 4d89845de6efc17a6f8dc0e5f5c446043d884793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Wed, 7 Nov 2012 10:32:01 +0100 Subject: [PATCH 11/22] [IMP] tools: tests: cleaner: cleaned a bit test data. Will have to be improved. bzr revid: tde@openerp.com-20121107093201-s0pco15msilt4erz --- openerp/tests/test_mail.py | 110 ++++++++++--------------------------- openerp/tools/mail.py | 4 ++ 2 files changed, 33 insertions(+), 81 deletions(-) diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py index 26ccdd39b28..ab82fc39caf 100755 --- a/openerp/tests/test_mail.py +++ b/openerp/tests/test_mail.py @@ -43,42 +43,8 @@ test12

test link """ -GMAIL_REPLY = """ - - - - -
Réponse via thunderbird, classique.
-
- On 11/05/2012 10:51 AM, Thibault Delavallée wrote:
-
-
-
Plop !
-
    -
  • Vive les lapins rapides !
    -
    • -
  • Nouille
    • -
  • Frites
    • -
-

-
-
Clairement, hein ?
- --
- Thibault Delavallée
-
-
-
- 
-- 
-Thibault Delavallée
-
- -""" - -GAMIL_REPLY_SAN = """
Réponse via thunderbird, classique.

- On 11/05/2012 10:51 AM, Thibault Delavallée wrote:
+GMAIL_REPLY_SAN = """
Réponse via thunderbird, classique.

+ On 11/05/2012 10:51 AM, Raoul Tartopoils wrote:
Plop !
  • Vive les lapins rapides !
    • @@ -87,17 +53,14 @@ GAMIL_REPLY_SAN = """
    Réponse via thunderbird, classique.


Clairement, hein ?
--
- Thibault Delavallée
+ Raoul Tartopoils


-- 
-Thibault Delavallée
+Raoul Tartopoils
""" - - - GMAIL_REPLY2_SAN = """
Je réponds, hop, via thunderbird. Mais je vais répodnre aussi au milieu du thread.

- On 11/05/2012 10:53 AM, Thibault Delavallée wrote:
+ On 11/05/2012 10:53 AM, Raoul Tartopoils wrote:
Reply rapide de gmail.

Jamais.

@@ -106,7 +69,7 @@ GMAIL_REPLY2_SAN = """
Je réponds, hop, via thunderbird. Mais
Réponse via thunderbird, classique.


- On 11/05/2012 10:51 AM, Thibault Delavallée wrote:
+ On 11/05/2012 10:51 AM, Raoul Tartopoils wrote:
@@ -135,70 +98,52 @@ GMAIL_REPLY2_SAN = """
Je réponds, hop, via thunderbird. Mais

Clairement, hein ?
--
- Thibault Delavallée
+ Raoul Tartopoils


-- 
-Thibault Delavallée
+Raoul Tartopoils



--
- Thibault Delavallée
+ Raoul Tartopoils


-- 
-Thibault Delavallée
+Raoul Tartopoils
""" +TEXT_TPL = """Salut Raoul! +Le 28 oct. 2012 à 00:02, Raoul Grosbedon a écrit : -MAIL_TEMPLATE = """Hey XYZ, +> C'est sûr que je suis intéressé (quote)! -I've fixed that error and tested it a couple of times, it seems to be working -fine now. +Trouloulou pouet pouet. -On Fri, Feb 18, 2011 at 7:44 AM, Joe David wrote: -Initial thread starts here... +Je ne vais quand même pas écrire de vrais mails, non mais ho. --- -Thanks -joe@david.com -""" - -GMAIL_TPL = """Salut Bob! -Le 28 oct. 2012 à 00:02, Thibault Delavallée a écrit : - -> MatrixPlus le 22/02 ? Si c'est ce que tu dis, je suis intéressé, oui ! - -Non, pas MatrixPlus, juste Matrix. - -Et oui, le 22/02 est la date qui doit vous convenir le deux. - -Bon, on tente de s’organiser un truc se samedi-là et je réserve des places. - -En passant, t’as contacté les gens du KdM pour le truc pour Bénisexe ? - -> 2012/10/27 Édouard Gilbert : ->> Diantre, me disè-je en envoyant un message similaire à Koukouiles, que ne voilà un concert qui pourrait intéresser Thibault ? +> 2012/10/27 Bert Tartopoils : +>> Diantre, me disè-je en envoyant un message similaire à Martine, mais comment vas-tu (quote)? >> ->> L’ONL donne, en février et à Lille, Matrix en ciné-concert. Il passe le film sans musique mais c’est pas grave parce qu’il y a un orchestre qui la joue en même temps. Intéressé ? +>> A la base le contenu était un vrai mail, mais je l'ai quand même réécrit pour ce test, histoire de dire que, quand même, on ne met pas n'importe quoi ici. (quote) >> ->> On peut trouver plus d’info sur le site de l’ONL, onlille.com , mais il est tellement mal foutu que je ne trouve pas de lien direct. Je te laisse chercher. +>> Et sinon bon courage pour trouver tes clefs (quote). >> ->> Édouard GILBERT ->> edouard.gilbert@gmail.com +>> Bert TARTOPOILS +>> bert.tartopoils@miam.miam >> > > > > -- -> Thibault Delavallée +> Raoul Grosbedon -Édouard GILBERT -edouard.gilbert@gmail.com +Bert TARTOPOILS +bert.tartopoils@miam.miam """ @@ -237,8 +182,11 @@ class TestSanitizer(unittest2.TestCase): class TestCleaner(unittest2.TestCase): def test_gmail(self): - test_case = """youplaboum""" - html_email_clean(test_case) + # Test1: blahblah + new_html = html_email_clean(GMAIL_REPLY_SAN) + self.assertNotIn(new_html, 'blockquote') + self.assertNotIn(new_html, 'Vive les lapins rapides !') + self.assertNotIn(new_html, 'Bert Tartopoils') if __name__ == '__main__': diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index 0ae4b373fee..d95d47c3b91 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -115,6 +115,10 @@ def html_email_clean(html): - strip email quotes (remove blockquote nodes) - strip signatures (remove --\n{\n)Blahblah), by replacing
by \n to avoid ignoring signatures converted into html + + :param string html: sanitized html; tags like html or head should not + be present in the html string. This method therefore takes as input + html code coming from a sanitized source, like fields.html. """ modified_html = '' From 688b6babed38acd00ad68a190fe980bba9f66bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Wed, 7 Nov 2012 11:50:31 +0100 Subject: [PATCH 12/22] [FIX] tools: html_email_clean: remove blockquote in any node, not just firs childs of root. bzr revid: tde@openerp.com-20121107105031-xf0u71q3chx0chnv --- openerp/tools/mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index d95d47c3b91..1290ebbdec6 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -138,7 +138,7 @@ def html_email_clean(html): root = lxml.html.fromstring(html) # 3. remove blockquotes - quotes = [el for el in root.iterchildren(tag='blockquote')] + quotes = [el for el in root.getiterator(tag='blockquote')] for node in quotes: node.getparent().remove(node) From b36547e3efca8d799c7250cd2549325a4a18f7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Wed, 7 Nov 2012 16:41:29 +0100 Subject: [PATCH 13/22] [CLEAN] tools: mail: removed commented code; moved some code. bzr revid: tde@openerp.com-20121107154129-nhz7qjv1ksq521ur --- openerp/tools/mail.py | 48 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index 1290ebbdec6..cebc6313b08 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -20,10 +20,7 @@ ############################################################################## from lxml import etree -# try: -# from lxml.html.soupparser import fromstring as parser_fromstring -# except ImportError: -# from lxml.html import fromstring as parser_fromstring +import cgi import logging import lxml.html import openerp.pooler as pooler @@ -39,6 +36,10 @@ from openerp.loglevels import ustr _logger = logging.getLogger(__name__) +#---------------------------------------------------------- +# HTML Sanitizer +#---------------------------------------------------------- + def html_sanitize(src): if not src: return src @@ -173,25 +174,9 @@ def html_email_clean(html): #---------------------------------------------------------- -# Emails +# HTML/Text management #---------------------------------------------------------- -email_re = re.compile(r""" - ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part - @ # mandatory @ sign - [a-zA-Z0-9][\w\.-]* # domain must start with a letter ... Ged> why do we include a 0-9 then? - \. - [a-z]{2,3} # TLD - ) - """, re.VERBOSE) -res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE) -command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) - -# Updated in 7.0 to match the model name as well -# Typical form of references is -# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain -reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE) - def html2plaintext(html, body_id=None, encoding='utf-8'): """ From an HTML text, convert the HTML to plain text. If @param body_id is provided then this is the tag where the @@ -248,6 +233,27 @@ def html2plaintext(html, body_id=None, encoding='utf-8'): return html + +#---------------------------------------------------------- +# Emails +#---------------------------------------------------------- + +email_re = re.compile(r""" + ([a-zA-Z][\w\.-]*[a-zA-Z0-9] # username part + @ # mandatory @ sign + [a-zA-Z0-9][\w\.-]* # domain must start with a letter ... Ged> why do we include a 0-9 then? + \. + [a-z]{2,3} # TLD + ) + """, re.VERBOSE) +res_re = re.compile(r"\[([0-9]+)\]", re.UNICODE) +command_re = re.compile("^Set-([a-z]+) *: *(.+)$", re.I + re.UNICODE) + +# Updated in 7.0 to match the model name as well +# Typical form of references is +# group(1) = the record ID ; group(2) = the model (if any) ; group(3) = the domain +reference_re = re.compile("<.*-open(?:object|erp)-(\\d+)(?:-([\w.]+))?.*@(.*)>", re.UNICODE) + def generate_tracking_message_id(res_id): """Returns a string that can be used in the Message-ID RFC822 header field From e6a90846779dade003230c1ceccdc42485dc70d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Wed, 7 Nov 2012 17:41:17 +0100 Subject: [PATCH 14/22] [IMP] tools: added text2html function, basically turning plaintext into html. Added a test for this function. Updated signature for demo data. bzr revid: tde@openerp.com-20121107164117-2caoqna2aclzowpk --- openerp/addons/base/base_data.xml | 2 +- openerp/addons/base/base_demo.xml | 2 +- openerp/tests/test_mail.py | 14 +++++++++- openerp/tools/mail.py | 45 +++++++++++++++++++++++++------ 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/openerp/addons/base/base_data.xml b/openerp/addons/base/base_data.xml index 4c429fe56ff..943ae73435d 100644 --- a/openerp/addons/base/base_data.xml +++ b/openerp/addons/base/base_data.xml @@ -76,7 +76,7 @@ - Administrator + --\nAdministrator diff --git a/openerp/addons/base/base_demo.xml b/openerp/addons/base/base_demo.xml index ce675ebb99e..e1ef2009f71 100644 --- a/openerp/addons/base/base_demo.xml +++ b/openerp/addons/base/base_demo.xml @@ -10,7 +10,7 @@ demo demo - Mr Demo + --\nMr Demo diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py index ab82fc39caf..5a0c062b3a0 100755 --- a/openerp/tests/test_mail.py +++ b/openerp/tests/test_mail.py @@ -23,7 +23,7 @@ ############################################################################## import unittest2 -from openerp.tools.mail import html_sanitize, html_email_clean, append_content_to_html +from openerp.tools.mail import html_sanitize, html_email_clean, append_content_to_html, text2html test_case = """ test1 @@ -189,5 +189,17 @@ class TestCleaner(unittest2.TestCase): self.assertNotIn(new_html, 'Bert Tartopoils') +class TestText2Html(unittest2.TestCase): + + def test_text2html(self): + cases = [ + ("First \nSecond \nThird\n \nParagraph\n\r--\nSignature paragraph", 'div', + "

First
Second
Third

Paragraph

--
Signature paragraph

"), + ] + for content, container_tag, expected in cases: + html = text2html(content, container_tag) + self.assertEqual(html, expected, 'text2html is broken') + + if __name__ == '__main__': unittest2.main() diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index cebc6313b08..861f6d82243 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -192,7 +192,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'): tree = fromstring(html, parser=HTMLParser()) if body_id is not None: - source = tree.xpath('//*[@id=%s]'%(body_id,)) + source = tree.xpath('//*[@id=%s]' % (body_id,)) else: source = tree.xpath('//body') if len(source): @@ -210,12 +210,12 @@ def html2plaintext(html, body_id=None, encoding='utf-8'): html = ustr(tostring(tree, encoding=encoding)) - html = html.replace('','*').replace('','*') - html = html.replace('','*').replace('','*') - html = html.replace('

','*').replace('

','*') - html = html.replace('

','**').replace('

','**') - html = html.replace('

','**').replace('

','**') - html = html.replace('','/').replace('','/') + html = html.replace('', '*').replace('', '*') + html = html.replace('', '*').replace('', '*') + html = html.replace('

', '*').replace('

', '*') + html = html.replace('

', '**').replace('

', '**') + html = html.replace('

', '**').replace('

', '**') + html = html.replace('', '/').replace('', '/') html = html.replace('', '\n') html = html.replace('

', '\n') html = re.sub('', '\n', html) @@ -229,10 +229,39 @@ def html2plaintext(html, body_id=None, encoding='utf-8'): for i, url in enumerate(url_index): if i == 0: html += '\n\n' - html += ustr('[%s] %s\n') % (i+1, url) + html += ustr('[%s] %s\n') % (i + 1, url) return html +def text2html(text, container_tag='div'): + """ Convert plaintext into html. Content of the text is escaped to manage + html entities, using cgi.escape(). + - all \n,\r are replaced by
+ - enclose content into

+ - 2 or more consecutive
are considered as paragraph breaks + + :param string container_tag: container of the html; by default the + content is embedded into a

+ """ + text = cgi.escape(text) + + # 1. replace \n and \r + text = text.replace('\n', '
') + text = text.replace('\r', '
') + + # 2-3: form paragraphs + idx = 0 + final = '

' + br_tags = re.compile(r'(([<]\s*[bB][rR]\s*\/?[>]\s*){2,})') + for item in re.finditer(br_tags, text): + final += text[idx:item.start()] + '

' + idx = item.end() + final += text[idx:] + '

' + + # 4. container + if container_tag: + final = '<%s>%s' % (container_tag, final, container_tag) + return final #---------------------------------------------------------- # Emails From 89083480fa2dd3fa6eb9ebd549867eaa838e2e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Thu, 8 Nov 2012 11:17:15 +0100 Subject: [PATCH 15/22] [IMP] [DEMO] admin, demo: updated demo data (email, signature). bzr revid: tde@openerp.com-20121108101715-3ipopg5otdq9u0vx --- openerp/addons/base/base_data.xml | 4 +++- openerp/addons/base/base_demo.xml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openerp/addons/base/base_data.xml b/openerp/addons/base/base_data.xml index 943ae73435d..9bd9b9ef19c 100644 --- a/openerp/addons/base/base_data.xml +++ b/openerp/addons/base/base_data.xml @@ -67,6 +67,7 @@ Administrator + admin@example.com @@ -76,7 +77,8 @@ - --\nAdministrator + -- +Administrator diff --git a/openerp/addons/base/base_demo.xml b/openerp/addons/base/base_demo.xml index e1ef2009f71..1dc27a3d05e 100644 --- a/openerp/addons/base/base_demo.xml +++ b/openerp/addons/base/base_demo.xml @@ -3,6 +3,7 @@ Demo User + demo@example.com @@ -10,7 +11,8 @@ demo demo - --\nMr Demo + -- +Mr Demo From 95f07f7773e3ce672a2cc7ae66e4554af1bd96d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Thu, 8 Nov 2012 17:10:36 +0100 Subject: [PATCH 16/22] [FIX] demo data: moved email address of admin from data to demo. bzr revid: tde@openerp.com-20121108161036-97lynkhqdhpjbfa8 --- openerp/addons/base/base_data.xml | 1 - openerp/addons/base/base_demo.xml | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/openerp/addons/base/base_data.xml b/openerp/addons/base/base_data.xml index 9bd9b9ef19c..1b253b82041 100644 --- a/openerp/addons/base/base_data.xml +++ b/openerp/addons/base/base_data.xml @@ -67,7 +67,6 @@ Administrator - admin@example.com diff --git a/openerp/addons/base/base_demo.xml b/openerp/addons/base/base_demo.xml index 1dc27a3d05e..65ade7ec3b9 100644 --- a/openerp/addons/base/base_demo.xml +++ b/openerp/addons/base/base_demo.xml @@ -17,6 +17,10 @@ Mr Demo + + admin@example.com + + 1.5289 From 62c419e66fe49484dc51060d41cff5280ea83597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Fri, 9 Nov 2012 13:35:21 +0100 Subject: [PATCH 17/22] [IMP] tools: mail: improved append_content_to_html, including plaintext2html. Updated tests. bzr revid: tde@openerp.com-20121109123521-e04g4zrhtv947l1r --- openerp/addons/base/base_demo.xml | 1 - openerp/osv/fields.py | 2 +- openerp/tests/test_mail.py | 195 +++++++++++++++--------------- openerp/tools/mail.py | 101 ++++++++++------ 4 files changed, 161 insertions(+), 138 deletions(-) diff --git a/openerp/addons/base/base_demo.xml b/openerp/addons/base/base_demo.xml index e0013289928..89176d1811b 100644 --- a/openerp/addons/base/base_demo.xml +++ b/openerp/addons/base/base_demo.xml @@ -3,7 +3,6 @@ Demo User - demo@example.com demo@example.com diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index ee95cece722..0d12a9a8bc3 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -44,8 +44,8 @@ import openerp import openerp.tools as tools from openerp.tools.translate import _ from openerp.tools import float_round, float_repr +from openerp.tools import html_sanitize import simplejson -from openerp.tools.mail import html_sanitize from openerp import SUPERUSER_ID _logger = logging.getLogger(__name__) diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py index 5a0c062b3a0..0d6c72ff915 100755 --- a/openerp/tests/test_mail.py +++ b/openerp/tests/test_mail.py @@ -23,9 +23,9 @@ ############################################################################## import unittest2 -from openerp.tools.mail import html_sanitize, html_email_clean, append_content_to_html, text2html +from openerp.tools import html_sanitize, html_email_clean, append_content_to_html, plaintext2html -test_case = """ +HTML_SOURCE = """ test1
test2
@@ -43,88 +43,65 @@ test12

test link """ -GMAIL_REPLY_SAN = """
Réponse via thunderbird, classique.

- On 11/05/2012 10:51 AM, Raoul Tartopoils wrote:
-
-
Plop !
-
  • Vive les lapins rapides !
    • -
  • Nouille
    • -
  • Frites
    • -

-
Clairement, hein ?
- --
- Raoul Tartopoils
- 

-- 
-Raoul Tartopoils
-
""" +TEXT_MAIL1 = """I contact you about our meeting for tomorrow. Here is the schedule I propose: +9 AM: brainstorming about our new amazing business app +9.45 AM: summary +10 AM: meeting with Fabien to present our app +Is everything ok for you ? +-- +Administrator""" -GMAIL_REPLY2_SAN = """
Je réponds, hop, via thunderbird. Mais - je vais répodnre aussi au milieu du thread.

- On 11/05/2012 10:53 AM, Raoul Tartopoils wrote:
-
Reply rapide de gmail.
-
- Jamais.

-


2012/11/5 Thibault Delavallée <tde@openerp.com>
-
-
Réponse via thunderbird, classique. -
-


- On 11/05/2012 10:51 AM, Raoul Tartopoils wrote:
-
-
-
-
-
-
Plop !
-
  • Vive les lapins rapides !
    • -
  • Nouille
    • -
-
-
-
-
-
+HTML_MAIL1 = """
+I contact you about our meeting for tomorrow. Here is the schedule I propose: +
+
    +
  • 9 AM: brainstorming about our new amazing business app
    • +
  • 9.45 AM: summary
    • +
  • 10 AM: meeting with Fabien to present our app
    • +
+
Is everything ok for you ?
""" + +GMAIL_REPLY1_SAN = """Hello,

Ok for me. I am replying directly in gmail, without signature.

Kind regards,

Demo.

On Thu, Nov 8, 2012 at 5:29 PM, <dummy@example.com> wrote:
I contact you about our meeting for tomorrow. Here is the schedule I propose:
  • 9 AM: brainstorming about our new amazing business app</span></li>
    • +
  • 9.45 AM: summary
  • 10 AM: meeting with Fabien to present our app
Is everything ok for you ?
+

--
Administrator

+ + +

""" + +THUNDERBIRD_16_REPLY1_SAN = """
On 11/08/2012 05:29 PM, + dummy@example.com wrote:
+
+
I contact you about our meeting for tomorrow. Here is the + schedule I propose:
+
+
  • 9 AM: brainstorming about our new amazing business + app</span></li>
    • +
  • 9.45 AM: summary
    • +
  • 10 AM: meeting with Fabien to present our app
    • +
+
Is everything ok for you ?
+
+

--
+ Administrator

+
+
- je rajotuerais bien pommes de terre dans la liste.
-
-
-
-
-
-
-
-
  • Frites
    • -

-
Clairement, hein ?
- --
- Raoul Tartopoils
-

-
- - 
-- 
-Raoul Tartopoils
-
-
-
-
-


- --
- Raoul Tartopoils
-
- 

-- 
-Raoul Tartopoils
+    Ok for me. I am replying directly below your mail, using
+    Thunderbird, with a signature.


+    Did you receive my email about my new laptop, by the way ?


+    Raoul.
-- 
+Raoul Grosbedonnée
 
"""
 
-
 TEXT_TPL = """Salut Raoul!
 Le 28 oct. 2012 à 00:02, Raoul Grosbedon a écrit :
 
 > C'est sûr que je suis intéressé (quote)!
 
-Trouloulou pouet pouet.
-
-Je ne vais quand même pas écrire de vrais mails, non mais ho.
+Trouloulou pouet pouet. Je ne vais quand même pas écrire de vrais mails, non mais ho.
 
 > 2012/10/27 Bert Tartopoils :
 >> Diantre, me disè-je en envoyant un message similaire à Martine, mais comment vas-tu (quote)?
@@ -138,7 +115,6 @@ Je ne vais quand même pas écrire de vrais mails, non mais ho.
 >> 
 > 
 > 
-> 
 > -- 
 > Raoul Grosbedon
 
@@ -147,21 +123,8 @@ bert.tartopoils@miam.miam
 """
 
 
-class TestAppendContentToHtml(unittest2.TestCase):
-    """ Test some of our generic utility functions """
-
-    def test_append_to_html(self):
-        test_samples = [
-            ('some content', '--\nYours truly', True,
-             'some content\n
--\nYours truly
\n'),
-            ('some content', '\n\n
--
\n
Yours truly
\n\n', False,
-             'some content\n\n\n
--
\n
Yours truly
\n\n\n'),
-        ]
-        for html, content, flag, expected in test_samples:
-            self.assertEqual(append_content_to_html(html, content, flag), expected, 'append_content_to_html is broken')
-
-
 class TestSanitizer(unittest2.TestCase):
+    """ Test the html sanitizer """
     # TDE note: could be improved by actually checking the output
 
     def test_simple(self):
@@ -173,33 +136,67 @@ class TestSanitizer(unittest2.TestCase):
         self.assertEqual(x, html_sanitize(x))
 
     def test_no_exception(self):
-        html_sanitize(test_case)
+        html_sanitize(HTML_SOURCE)
 
     def test_unicode(self):
         html_sanitize("Merci à l'intérêt pour notre produit.nous vous contacterons bientôt. Merci")
 
 
 class TestCleaner(unittest2.TestCase):
+    """ Test the email cleaner function that filter the content of incoming emails """
 
-    def test_gmail(self):
-        # Test1: blahblah
-        new_html = html_email_clean(GMAIL_REPLY_SAN)
-        self.assertNotIn(new_html, 'blockquote')
-        self.assertNotIn(new_html, 'Vive les lapins rapides !')
-        self.assertNotIn(new_html, 'Bert Tartopoils')
+    def test_html_email_clean(self):
+        # Test1: reply through gmail: quote in blockquote, signature --\nAdministrator
+        new_html = html_email_clean(GMAIL_REPLY1_SAN)
+        self.assertNotIn('blockquote', new_html, 'html_email_cleaner did not remove a blockquote')
+        self.assertNotIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+        self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+        self.assertIn('Ok for me', new_html, 'html_email_cleaner erased too much content')
+
+        # Test2: reply through Tunderbird 16.0.2
+        new_html = html_email_clean(THUNDERBIRD_16_REPLY1_SAN)
+        self.assertNotIn('blockquote', new_html, 'html_email_cleaner did not remove a blockquote')
+        self.assertNotIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+        self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+        self.assertNotIn('Grosbedonn', new_html, 'html_email_cleaner did not erase the signature')
+        self.assertIn('Ok for me', new_html, 'html_email_cleaner erased too much content')
+
+        # Test3: text email
+        new_html = html_email_clean(TEXT_MAIL1)
+        self.assertIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+        self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+
+        # Test4: more complex text email
+        new_html = html_email_clean(TEXT_TPL)
+        self.assertNotIn('quote', new_html, 'html_email_cleaner did not remove correctly plaintext quotes')
 
 
-class TestText2Html(unittest2.TestCase):
+class TestAppendContentToHtml(unittest2.TestCase):
+    """ Test some of our generic utility functions about html """
 
-    def test_text2html(self):
+    def test_plaintext2html(self):
         cases = [
             ("First \nSecond \nThird\n \nParagraph\n\r--\nSignature paragraph", 'div',
              "
First 
Second 
Third
Paragraph
--
Signature paragraph
"),
+            ("First
It should be escaped
\nSignature", False,
+             "
First<p>It should be escaped</p>
Signature
")
         ]
         for content, container_tag, expected in cases:
-            html = text2html(content, container_tag)
+            html = plaintext2html(content, container_tag)
             self.assertEqual(html, expected, 'text2html is broken')
 
+    def test_append_to_html(self):
+        test_samples = [
+            ('some content', '--\nYours truly', True, True, False,
+             'some content\n
--\nYours truly
\n'),
+            ('some content', '--\nYours truly', True, False, False,
+             'some content\n
--
Yours truly
\n'),
+            ('some content', '\n\n
--
\n
Yours truly
\n\n', False, False, False,
+             'some content\n\n\n
--
\n
Yours truly
\n\n\n'),
+        ]
+        for html, content, plaintext_flag, preserve_flag, container_tag, expected in test_samples:
+            self.assertEqual(append_content_to_html(html, content, plaintext_flag, preserve_flag, container_tag), expected, 'append_content_to_html is broken')
+
 
 if __name__ == '__main__':
     unittest2.main()
diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py
index 861f6d82243..020ab2313b4 100644
--- a/openerp/tools/mail.py
+++ b/openerp/tools/mail.py
@@ -121,6 +121,7 @@ def html_email_clean(html):
             be present in the html string. This method therefore takes as input
             html code coming from a sanitized source, like fields.html.
     """
+    html = ustr(html)
     modified_html = ''
 
     # 1.  -> \n, because otherwise the tree is obfuscated
@@ -131,6 +132,7 @@ def html_email_clean(html):
         idx = item.end()
     modified_html += html[idx:]
     html = modified_html
+    # TDE note: seems to have lots of 

 in emails... needs to be checks, could be cleaned
 
     # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre
     root = lxml.html.fromstring(html)
@@ -138,9 +140,28 @@ def html_email_clean(html):
         html = '
%s
' % html
         root = lxml.html.fromstring(html)
 
+    # 2.5 remove quoted text in nodes
+    quote_tags = re.compile(r'(\n(>)+[^\n\r]*)')
+    for node in root.getiterator():
+        if not node.text:
+            continue
+        idx = 0
+        text = ''
+        for item in re.finditer(quote_tags, node.text):
+            print item
+            text += node.text[idx:item.start()]
+            idx = item.end()
+        text += node.text[idx:]
+        node.text = text
+
     # 3. remove blockquotes
     quotes = [el for el in root.getiterator(tag='blockquote')]
     for node in quotes:
+        # copy the node tail into parent text
+        if node.tail:
+            parent = node.getparent()
+            parent.text = parent.text or '' + node.tail
+        # remove the node
         node.getparent().remove(node)
 
     # 4. strip signatures
@@ -187,9 +208,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'):
     ## download here: http://www.peterbe.com/plog/html2plaintext
 
     html = ustr(html)
-
-    from lxml.etree import tostring, fromstring, HTMLParser
-    tree = fromstring(html, parser=HTMLParser())
+    tree = etree.fromstring(html, parser=etree.HTMLParser())
 
     if body_id is not None:
         source = tree.xpath('//*[@id=%s]' % (body_id,))
@@ -208,7 +227,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'):
             link.text = '%s [%s]' % (link.text, i)
             url_index.append(url)
 
-    html = ustr(tostring(tree, encoding=encoding))
+    html = ustr(etree.tostring(tree, encoding=encoding))
 
     html = html.replace('', '*').replace('', '*')
     html = html.replace('', '*').replace('', '*')
@@ -233,7 +252,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'):
 
     return html
 
-def text2html(text, container_tag='div'):
+def plaintext2html(text, container_tag=False):
     """ Convert plaintext into html. Content of the text is escaped to manage
         html entities, using cgi.escape().
         - all \n,\r are replaced by 

@@ -243,7 +262,7 @@ def text2html(text, container_tag='div'):
         :param string container_tag: container of the html; by default the
             content is embedded into a 

     """
-    text = cgi.escape(text)
+    text = cgi.escape(ustr(text))
 
     # 1. replace \n and \r
     text = text.replace('\n', '
')
@@ -261,7 +280,45 @@ def text2html(text, container_tag='div'):
     # 4. container
     if container_tag:
         final = '<%s>%s' % (container_tag, final, container_tag)
-    return final
+    return ustr(final)
+
+def append_content_to_html(html, content, plaintext=True, preserve=False, container_tag=False):
+    """ Append extra content at the end of an HTML snippet, trying
+        to locate the end of the HTML document (, , or
+        EOF), and converting the provided content in html unless ``plaintext``
+        is False.
+        Content conversion can be done in two ways:
+        - wrapping it into a pre (preserve=True)
+        - use plaintext2html (preserve=False, using container_tag to wrap the
+            whole content)
+        A side-effect of this method is to coerce all HTML tags to
+        lowercase in ``html``, and strip enclosing  or  tags in
+        content if ``plaintext`` is False.
+
+        :param str html: html tagsoup (doesn't have to be XHTML)
+        :param str content: extra content to append
+        :param bool plaintext: whether content is plaintext and should
+            be wrapped in a 
 tag.
+        :param bool preserve: if content is plaintext, wrap it into a 
+            instead of converting it into html
+    """
+    html = ustr(html)
+    if plaintext and preserve:
+        content = u'\n
%s
\n' % ustr(content)
+    elif plaintext:
+        content = '\n%s\n' % plaintext2html(content, container_tag)
+    else:
+        content = re.sub(r'(?i)(||)', '', content)
+        content = u'\n%s\n' % ustr(content)
+    # Force all tags to lowercase
+    html = re.sub(r'(])',
+        lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html)
+    insert_location = html.find('')
+    if insert_location == -1:
+        insert_location = html.find('')
+    if insert_location == -1:
+        return '%s%s' % (html, content)
+    return '%s%s%s' % (html[:insert_location], content, html[insert_location:])
 
 #----------------------------------------------------------
 # Emails
@@ -339,33 +396,3 @@ def email_split(text):
     if not text:
         return []
     return re.findall(r'([^ ,<@]+@[^> ,]+)', text)
-
-def append_content_to_html(html, content, plaintext=True):
-    """Append extra content at the end of an HTML snippet, trying
-       to locate the end of the HTML document (, , or
-       EOF), and wrapping the provided content in a 
 block
-       unless ``plaintext`` is False. A side-effect of this
-       method is to coerce all HTML tags to lowercase in ``html``,
-       and strip enclosing  or  tags in content if
-       ``plaintext`` is False.
-
-       :param str html: html tagsoup (doesn't have to be XHTML)
-       :param str content: extra content to append
-       :param bool plaintext: whether content is plaintext and should
-           be wrapped in a 
 tag.
-    """
-    html = ustr(html)
-    if plaintext:
-        content = u'\n
%s
\n' % ustr(content)
-    else:
-        content = re.sub(r'(?i)(||)', '', content)
-        content = u'\n%s\n' % ustr(content)
-    # Force all tags to lowercase
-    html = re.sub(r'(])',
-        lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html)
-    insert_location = html.find('')
-    if insert_location == -1:
-        insert_location = html.find('')
-    if insert_location == -1:
-        return '%s%s' % (html, content)
-    return '%s%s%s' % (html[:insert_location], content, html[insert_location:])

From 7508a5c7243f7cdc17dd94866721555972a0d830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= 
Date: Wed, 14 Nov 2012 11:38:17 +0100
Subject: [PATCH 18/22] [CLEAN] html_email_clean: cleaned a bit the code.

bzr revid: tde@openerp.com-20121114103817-t19i2sv817jsur41
---
 openerp/tools/mail.py | 34 ++++++++++++----------------------
 1 file changed, 12 insertions(+), 22 deletions(-)

diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py
index 020ab2313b4..5f8c65c1d9b 100644
--- a/openerp/tools/mail.py
+++ b/openerp/tools/mail.py
@@ -121,17 +121,20 @@ def html_email_clean(html):
             be present in the html string. This method therefore takes as input
             html code coming from a sanitized source, like fields.html.
     """
+    def _replace_matching_regex(regex, source, replace=''):
+        dest = ''
+        idx = 0
+        for item in re.finditer(regex, source):
+            dest += source[idx:item.start()] + replace
+            idx = item.end()
+        dest += source[idx:]
+        return dest
+
     html = ustr(html)
-    modified_html = ''
 
     # 1.  -> \n, because otherwise the tree is obfuscated
     br_tags = re.compile(r'([<]\s*br\s*\/?[>])')
-    idx = 0
-    for item in re.finditer(br_tags, html):
-        modified_html += html[idx:item.start()] + '__BR_TAG__'
-        idx = item.end()
-    modified_html += html[idx:]
-    html = modified_html
+    html = _replace_matching_regex(br_tags, html, '__BR_TAG__')
     # TDE note: seems to have lots of 

 in emails... needs to be checks, could be cleaned
 
     # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre
@@ -145,14 +148,7 @@ def html_email_clean(html):
     for node in root.getiterator():
         if not node.text:
             continue
-        idx = 0
-        text = ''
-        for item in re.finditer(quote_tags, node.text):
-            print item
-            text += node.text[idx:item.start()]
-            idx = item.end()
-        text += node.text[idx:]
-        node.text = text
+        node.text = _replace_matching_regex(quote_tags, node.text)
 
     # 3. remove blockquotes
     quotes = [el for el in root.getiterator(tag='blockquote')]
@@ -182,14 +178,8 @@ def html_email_clean(html):
 
     # 6. Misc cleaning :
     # - ClEditor seems to love using 

 -> replace with 

-    modified_html = ''
     br_div_tags = re.compile(r'(
\s*\s*<\/div>)')
-    idx = 0
-    for item in re.finditer(br_div_tags, html):
-        modified_html += html[idx:item.start()] + '
'
-        idx = item.end()
-    modified_html += html[idx:]
-    html = modified_html
+    html = _replace_matching_regex(br_div_tags, html, '
')
 
     return html
 

From cbf3c82e88c8dfb5b9829a60262caa35d085c241 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= 
Date: Wed, 14 Nov 2012 11:38:49 +0100
Subject: [PATCH 19/22] [IMP] tools: added a true test for html_sanitize, some
 function renaming.

bzr revid: tde@openerp.com-20121114103849-hbxni04dpog1fxju
---
 openerp/tests/test_mail.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py
index 0d6c72ff915..975a8b846ee 100755
--- a/openerp/tests/test_mail.py
+++ b/openerp/tests/test_mail.py
@@ -124,8 +124,7 @@ bert.tartopoils@miam.miam
 
 
 class TestSanitizer(unittest2.TestCase):
-    """ Test the html sanitizer """
-    # TDE note: could be improved by actually checking the output
+    """ Test the html sanitizer that filters html to remove unwanted attributes """
 
     def test_simple(self):
         x = "yop"
@@ -135,15 +134,19 @@ class TestSanitizer(unittest2.TestCase):
         x = 'lala
yop
xxx'
         self.assertEqual(x, html_sanitize(x))
 
-    def test_no_exception(self):
-        html_sanitize(HTML_SOURCE)
+    def test_html(self):
+        sanitized_html = html_sanitize(HTML_SOURCE)
+        for tag in ['', '
', '', '', '', '', '
  • ', '
    ', '
Date: Wed, 14 Nov 2012 12:20:21 +0100
Subject: [PATCH 20/22] [REM] osv: removed temporary override of raise, added
 for debug purpose.

bzr revid: tde@openerp.com-20121114112021-trre94clvs8nuo43
---
 openerp/osv/osv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/openerp/osv/osv.py b/openerp/osv/osv.py
index 4e875d29a21..7703406ec96 100644
--- a/openerp/osv/osv.py
+++ b/openerp/osv/osv.py
@@ -120,7 +120,6 @@ class object_proxy(object):
                     raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
                 return f(self, dbname, *args, **kwargs)
             except orm.except_orm, inst:
-                raise
                 raise except_osv(inst.name, inst.value)
             except except_osv:
                 raise

From 8e15ab0dff77edbb754732385872f8d30cad520e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= 
Date: Wed, 14 Nov 2012 14:51:59 +0100
Subject: [PATCH 21/22] [FIX] Typo in tests.

bzr revid: tde@openerp.com-20121114135159-nouaj72drtot5hiv
---
 openerp/tests/test_mail.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py
index 975a8b846ee..0ad6504a7ae 100755
--- a/openerp/tests/test_mail.py
+++ b/openerp/tests/test_mail.py
@@ -186,7 +186,7 @@ class TestHtmlTools(unittest2.TestCase):
         ]
         for content, container_tag, expected in cases:
             html = plaintext2html(content, container_tag)
-            self.assertEqual(html, expected, 'text2html is broken')
+            self.assertEqual(html, expected, 'plaintext2html is broken')
 
     def test_append_to_html(self):
         test_samples = [

From 23743683ea0e0e0be74007f3e411b7847bcec179 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= 
Date: Wed, 14 Nov 2012 15:14:52 +0100
Subject: [PATCH 22/22] [IMP] tools: html_email_clean: a bit more robust.

bzr revid: tde@openerp.com-20121114141452-n5jy3yzbmlbd3811
---
 openerp/tools/mail.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py
index 5f8c65c1d9b..39d918da97b 100644
--- a/openerp/tools/mail.py
+++ b/openerp/tools/mail.py
@@ -133,9 +133,8 @@ def html_email_clean(html):
     html = ustr(html)
 
     # 1.  -> \n, because otherwise the tree is obfuscated
-    br_tags = re.compile(r'([<]\s*br\s*\/?[>])')
+    br_tags = re.compile(r'([<]\s*[bB][rR]\s*\/?[>])')
     html = _replace_matching_regex(br_tags, html, '__BR_TAG__')
-    # TDE note: seems to have lots of 

     in emails... needs to be checks, could be cleaned
 
     # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre
     root = lxml.html.fromstring(html)