diff --git a/openerp/addons/base/base_demo.xml b/openerp/addons/base/base_demo.xml index e0013289928..89176d1811b 100644 --- a/openerp/addons/base/base_demo.xml +++ b/openerp/addons/base/base_demo.xml @@ -3,7 +3,6 @@ Demo User - demo@example.com demo@example.com diff --git a/openerp/osv/fields.py b/openerp/osv/fields.py index ee95cece722..0d12a9a8bc3 100644 --- a/openerp/osv/fields.py +++ b/openerp/osv/fields.py @@ -44,8 +44,8 @@ import openerp import openerp.tools as tools from openerp.tools.translate import _ from openerp.tools import float_round, float_repr +from openerp.tools import html_sanitize import simplejson -from openerp.tools.mail import html_sanitize from openerp import SUPERUSER_ID _logger = logging.getLogger(__name__) diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py index 5a0c062b3a0..0d6c72ff915 100755 --- a/openerp/tests/test_mail.py +++ b/openerp/tests/test_mail.py @@ -23,9 +23,9 @@ ############################################################################## import unittest2 -from openerp.tools.mail import html_sanitize, html_email_clean, append_content_to_html, text2html +from openerp.tools import html_sanitize, html_email_clean, append_content_to_html, plaintext2html -test_case = """ +HTML_SOURCE = """ test1
test2
@@ -43,88 +43,65 @@ test12

test link """ -GMAIL_REPLY_SAN = """
Réponse via thunderbird, classique.

- On 11/05/2012 10:51 AM, Raoul Tartopoils wrote:
-
-
Plop !
-
  • Vive les lapins rapides !
    • -
  • Nouille
    • -
  • Frites
    • -

-
Clairement, hein ?
- --
- Raoul Tartopoils
- 

-- 
-Raoul Tartopoils
-
""" +TEXT_MAIL1 = """I contact you about our meeting for tomorrow. Here is the schedule I propose: +9 AM: brainstorming about our new amazing business app +9.45 AM: summary +10 AM: meeting with Fabien to present our app +Is everything ok for you ? +-- +Administrator""" -GMAIL_REPLY2_SAN = """
Je réponds, hop, via thunderbird. Mais - je vais répodnre aussi au milieu du thread.

- On 11/05/2012 10:53 AM, Raoul Tartopoils wrote:
-
Reply rapide de gmail.
-
- Jamais.

-


2012/11/5 Thibault Delavallée <tde@openerp.com>
-
-
Réponse via thunderbird, classique. -
-


- On 11/05/2012 10:51 AM, Raoul Tartopoils wrote:
-
-
-
-
-
-
Plop !
-
  • Vive les lapins rapides !
    • -
  • Nouille
    • -
-
-
-
-
-
+HTML_MAIL1 = """
+I contact you about our meeting for tomorrow. Here is the schedule I propose: +
+
    +
  • 9 AM: brainstorming about our new amazing business app
    • +
  • 9.45 AM: summary
    • +
  • 10 AM: meeting with Fabien to present our app
    • +
+
Is everything ok for you ?
""" + +GMAIL_REPLY1_SAN = """Hello,

Ok for me. I am replying directly in gmail, without signature.

Kind regards,

Demo.

On Thu, Nov 8, 2012 at 5:29 PM, <dummy@example.com> wrote:
I contact you about our meeting for tomorrow. Here is the schedule I propose:
  • 9 AM: brainstorming about our new amazing business app</span></li>
    • +
  • 9.45 AM: summary
  • 10 AM: meeting with Fabien to present our app
Is everything ok for you ?
+

--
Administrator

+ + +

""" + +THUNDERBIRD_16_REPLY1_SAN = """
On 11/08/2012 05:29 PM, + dummy@example.com wrote:
+
+
I contact you about our meeting for tomorrow. Here is the + schedule I propose:
+
+
  • 9 AM: brainstorming about our new amazing business + app</span></li>
    • +
  • 9.45 AM: summary
    • +
  • 10 AM: meeting with Fabien to present our app
    • +
+
Is everything ok for you ?
+
+

--
+ Administrator

+
+
- je rajotuerais bien pommes de terre dans la liste.
-
-
-
-
-
-
-
-
  • Frites
    • -

-
Clairement, hein ?
- --
- Raoul Tartopoils
-

-
- - 
-- 
-Raoul Tartopoils
-
-
-
-
-


- --
- Raoul Tartopoils
-
- 

-- 
-Raoul Tartopoils
+    Ok for me. I am replying directly below your mail, using
+    Thunderbird, with a signature.


+    Did you receive my email about my new laptop, by the way ?


+    Raoul.
-- 
+Raoul Grosbedonnée
 
"""
 
-
 TEXT_TPL = """Salut Raoul!
 Le 28 oct. 2012 à 00:02, Raoul Grosbedon a écrit :
 
 > C'est sûr que je suis intéressé (quote)!
 
-Trouloulou pouet pouet.
-
-Je ne vais quand même pas écrire de vrais mails, non mais ho.
+Trouloulou pouet pouet. Je ne vais quand même pas écrire de vrais mails, non mais ho.
 
 > 2012/10/27 Bert Tartopoils :
 >> Diantre, me disè-je en envoyant un message similaire à Martine, mais comment vas-tu (quote)?
@@ -138,7 +115,6 @@ Je ne vais quand même pas écrire de vrais mails, non mais ho.
 >> 
 > 
 > 
-> 
 > -- 
 > Raoul Grosbedon
 
@@ -147,21 +123,8 @@ bert.tartopoils@miam.miam
 """
 
 
-class TestAppendContentToHtml(unittest2.TestCase):
-    """ Test some of our generic utility functions """
-
-    def test_append_to_html(self):
-        test_samples = [
-            ('some content', '--\nYours truly', True,
-             'some content\n
--\nYours truly
\n'),
-            ('some content', '\n\n
--
\n
Yours truly
\n\n', False,
-             'some content\n\n\n
--
\n
Yours truly
\n\n\n'),
-        ]
-        for html, content, flag, expected in test_samples:
-            self.assertEqual(append_content_to_html(html, content, flag), expected, 'append_content_to_html is broken')
-
-
 class TestSanitizer(unittest2.TestCase):
+    """ Test the html sanitizer """
     # TDE note: could be improved by actually checking the output
 
     def test_simple(self):
@@ -173,33 +136,67 @@ class TestSanitizer(unittest2.TestCase):
         self.assertEqual(x, html_sanitize(x))
 
     def test_no_exception(self):
-        html_sanitize(test_case)
+        html_sanitize(HTML_SOURCE)
 
     def test_unicode(self):
         html_sanitize("Merci à l'intérêt pour notre produit.nous vous contacterons bientôt. Merci")
 
 
 class TestCleaner(unittest2.TestCase):
+    """ Test the email cleaner function that filter the content of incoming emails """
 
-    def test_gmail(self):
-        # Test1: blahblah
-        new_html = html_email_clean(GMAIL_REPLY_SAN)
-        self.assertNotIn(new_html, 'blockquote')
-        self.assertNotIn(new_html, 'Vive les lapins rapides !')
-        self.assertNotIn(new_html, 'Bert Tartopoils')
+    def test_html_email_clean(self):
+        # Test1: reply through gmail: quote in blockquote, signature --\nAdministrator
+        new_html = html_email_clean(GMAIL_REPLY1_SAN)
+        self.assertNotIn('blockquote', new_html, 'html_email_cleaner did not remove a blockquote')
+        self.assertNotIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+        self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+        self.assertIn('Ok for me', new_html, 'html_email_cleaner erased too much content')
+
+        # Test2: reply through Tunderbird 16.0.2
+        new_html = html_email_clean(THUNDERBIRD_16_REPLY1_SAN)
+        self.assertNotIn('blockquote', new_html, 'html_email_cleaner did not remove a blockquote')
+        self.assertNotIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+        self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+        self.assertNotIn('Grosbedonn', new_html, 'html_email_cleaner did not erase the signature')
+        self.assertIn('Ok for me', new_html, 'html_email_cleaner erased too much content')
+
+        # Test3: text email
+        new_html = html_email_clean(TEXT_MAIL1)
+        self.assertIn('I contact you about our meeting', new_html, 'html_email_cleaner wrongly removed the quoted content')
+        self.assertNotIn('Administrator', new_html, 'html_email_cleaner did not erase the signature')
+
+        # Test4: more complex text email
+        new_html = html_email_clean(TEXT_TPL)
+        self.assertNotIn('quote', new_html, 'html_email_cleaner did not remove correctly plaintext quotes')
 
 
-class TestText2Html(unittest2.TestCase):
+class TestAppendContentToHtml(unittest2.TestCase):
+    """ Test some of our generic utility functions about html """
 
-    def test_text2html(self):
+    def test_plaintext2html(self):
         cases = [
             ("First \nSecond \nThird\n \nParagraph\n\r--\nSignature paragraph", 'div',
              "
First 
Second 
Third
Paragraph
--
Signature paragraph
"),
+            ("First
It should be escaped
\nSignature", False,
+             "
First<p>It should be escaped</p>
Signature
")
         ]
         for content, container_tag, expected in cases:
-            html = text2html(content, container_tag)
+            html = plaintext2html(content, container_tag)
             self.assertEqual(html, expected, 'text2html is broken')
 
+    def test_append_to_html(self):
+        test_samples = [
+            ('some content', '--\nYours truly', True, True, False,
+             'some content\n
--\nYours truly
\n'),
+            ('some content', '--\nYours truly', True, False, False,
+             'some content\n
--
Yours truly
\n'),
+            ('some content', '\n\n
--
\n
Yours truly
\n\n', False, False, False,
+             'some content\n\n\n
--
\n
Yours truly
\n\n\n'),
+        ]
+        for html, content, plaintext_flag, preserve_flag, container_tag, expected in test_samples:
+            self.assertEqual(append_content_to_html(html, content, plaintext_flag, preserve_flag, container_tag), expected, 'append_content_to_html is broken')
+
 
 if __name__ == '__main__':
     unittest2.main()
diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py
index 861f6d82243..020ab2313b4 100644
--- a/openerp/tools/mail.py
+++ b/openerp/tools/mail.py
@@ -121,6 +121,7 @@ def html_email_clean(html):
             be present in the html string. This method therefore takes as input
             html code coming from a sanitized source, like fields.html.
     """
+    html = ustr(html)
     modified_html = ''
 
     # 1.  -> \n, because otherwise the tree is obfuscated
@@ -131,6 +132,7 @@ def html_email_clean(html):
         idx = item.end()
     modified_html += html[idx:]
     html = modified_html
+    # TDE note: seems to have lots of 

 in emails... needs to be checks, could be cleaned
 
     # 2. form a tree, handle (currently ?) pure-text by enclosing them in a pre
     root = lxml.html.fromstring(html)
@@ -138,9 +140,28 @@ def html_email_clean(html):
         html = '
%s
' % html
         root = lxml.html.fromstring(html)
 
+    # 2.5 remove quoted text in nodes
+    quote_tags = re.compile(r'(\n(>)+[^\n\r]*)')
+    for node in root.getiterator():
+        if not node.text:
+            continue
+        idx = 0
+        text = ''
+        for item in re.finditer(quote_tags, node.text):
+            print item
+            text += node.text[idx:item.start()]
+            idx = item.end()
+        text += node.text[idx:]
+        node.text = text
+
     # 3. remove blockquotes
     quotes = [el for el in root.getiterator(tag='blockquote')]
     for node in quotes:
+        # copy the node tail into parent text
+        if node.tail:
+            parent = node.getparent()
+            parent.text = parent.text or '' + node.tail
+        # remove the node
         node.getparent().remove(node)
 
     # 4. strip signatures
@@ -187,9 +208,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'):
     ## download here: http://www.peterbe.com/plog/html2plaintext
 
     html = ustr(html)
-
-    from lxml.etree import tostring, fromstring, HTMLParser
-    tree = fromstring(html, parser=HTMLParser())
+    tree = etree.fromstring(html, parser=etree.HTMLParser())
 
     if body_id is not None:
         source = tree.xpath('//*[@id=%s]' % (body_id,))
@@ -208,7 +227,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'):
             link.text = '%s [%s]' % (link.text, i)
             url_index.append(url)
 
-    html = ustr(tostring(tree, encoding=encoding))
+    html = ustr(etree.tostring(tree, encoding=encoding))
 
     html = html.replace('', '*').replace('', '*')
     html = html.replace('', '*').replace('', '*')
@@ -233,7 +252,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'):
 
     return html
 
-def text2html(text, container_tag='div'):
+def plaintext2html(text, container_tag=False):
     """ Convert plaintext into html. Content of the text is escaped to manage
         html entities, using cgi.escape().
         - all \n,\r are replaced by 

@@ -243,7 +262,7 @@ def text2html(text, container_tag='div'):
         :param string container_tag: container of the html; by default the
             content is embedded into a 

     """
-    text = cgi.escape(text)
+    text = cgi.escape(ustr(text))
 
     # 1. replace \n and \r
     text = text.replace('\n', '
')
@@ -261,7 +280,45 @@ def text2html(text, container_tag='div'):
     # 4. container
     if container_tag:
         final = '<%s>%s' % (container_tag, final, container_tag)
-    return final
+    return ustr(final)
+
+def append_content_to_html(html, content, plaintext=True, preserve=False, container_tag=False):
+    """ Append extra content at the end of an HTML snippet, trying
+        to locate the end of the HTML document (, , or
+        EOF), and converting the provided content in html unless ``plaintext``
+        is False.
+        Content conversion can be done in two ways:
+        - wrapping it into a pre (preserve=True)
+        - use plaintext2html (preserve=False, using container_tag to wrap the
+            whole content)
+        A side-effect of this method is to coerce all HTML tags to
+        lowercase in ``html``, and strip enclosing  or  tags in
+        content if ``plaintext`` is False.
+
+        :param str html: html tagsoup (doesn't have to be XHTML)
+        :param str content: extra content to append
+        :param bool plaintext: whether content is plaintext and should
+            be wrapped in a 
 tag.
+        :param bool preserve: if content is plaintext, wrap it into a 
+            instead of converting it into html
+    """
+    html = ustr(html)
+    if plaintext and preserve:
+        content = u'\n
%s
\n' % ustr(content)
+    elif plaintext:
+        content = '\n%s\n' % plaintext2html(content, container_tag)
+    else:
+        content = re.sub(r'(?i)(||)', '', content)
+        content = u'\n%s\n' % ustr(content)
+    # Force all tags to lowercase
+    html = re.sub(r'(])',
+        lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html)
+    insert_location = html.find('')
+    if insert_location == -1:
+        insert_location = html.find('')
+    if insert_location == -1:
+        return '%s%s' % (html, content)
+    return '%s%s%s' % (html[:insert_location], content, html[insert_location:])
 
 #----------------------------------------------------------
 # Emails
@@ -339,33 +396,3 @@ def email_split(text):
     if not text:
         return []
     return re.findall(r'([^ ,<@]+@[^> ,]+)', text)
-
-def append_content_to_html(html, content, plaintext=True):
-    """Append extra content at the end of an HTML snippet, trying
-       to locate the end of the HTML document (, , or
-       EOF), and wrapping the provided content in a 
 block
-       unless ``plaintext`` is False. A side-effect of this
-       method is to coerce all HTML tags to lowercase in ``html``,
-       and strip enclosing  or  tags in content if
-       ``plaintext`` is False.
-
-       :param str html: html tagsoup (doesn't have to be XHTML)
-       :param str content: extra content to append
-       :param bool plaintext: whether content is plaintext and should
-           be wrapped in a 
 tag.
-    """
-    html = ustr(html)
-    if plaintext:
-        content = u'\n
%s
\n' % ustr(content)
-    else:
-        content = re.sub(r'(?i)(||)', '', content)
-        content = u'\n%s\n' % ustr(content)
-    # Force all tags to lowercase
-    html = re.sub(r'(])',
-        lambda m: '%s%s%s' % (m.group(1), m.group(2).lower(), m.group(3)), html)
-    insert_location = html.find('')
-    if insert_location == -1:
-        insert_location = html.find('')
-    if insert_location == -1:
-        return '%s%s' % (html, content)
-    return '%s%s%s' % (html[:insert_location], content, html[insert_location:])