diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py index 3512014b1e7..f3d327b0281 100755 --- a/openerp/tests/test_mail.py +++ b/openerp/tests/test_mail.py @@ -113,24 +113,35 @@ class TestSanitizer(unittest2.TestCase): class TestCleaner(unittest2.TestCase): """ Test the email cleaner function that filters the content of incoming emails """ - def test_00_html_email_clean_signature(self): + def test_00_basic_text(self): """ html_email_clean test for signatures """ - test_data = [("""This is Sparta!\n--\nAdministrator\n+9988776655""", - ['This is Sparta!'], - ['Administrator', '9988776655']), - ("""

--\nAdministrator

""", - [], - ['--', 'Administrator']), - ("""

This is Sparta!\n---\nAdministrator

""", - ['This is Sparta!'], - ['---', 'Administrator']), - ("""

--
Administrator

""", - [], - []), - ("""

This is Sparta!
--
Administrator

""", - ['This is Sparta!'], - []) - ] + test_data = [ + ( + """This is Sparta!\n--\nAdministrator\n+9988776655""", + ['This is Sparta!'], + ['Administrator', '9988776655'] + ), ( + """

--\nAdministrator

""", + [], + ['--', 'Administrator'] + ), ( + """

This is Sparta!\n---\nAdministrator

""", + ['This is Sparta!'], + ['---', 'Administrator'] + ), ( + """

--
Administrator

""", + [], + [] + ), ( + """

This is Sparta!
--
Administrator

""", + ['This is Sparta!'], + [] + ), ( + """This is Sparta!\n>Ah bon ?\nCertes\n> Chouette !\nClair""", + ['This is Sparta!', 'Certes', 'Clair'], + ['Ah bon', 'Chouette'] + ) + ] for test, in_lst, out_lst in test_data: new_html = html_email_clean(test, remove=True) for text in in_lst: @@ -138,20 +149,7 @@ class TestCleaner(unittest2.TestCase): for text in out_lst: self.assertNotIn(text, new_html, 'html_email_cleaner did not remove unwanted content') - def test_05_html_email_clean_signature(self): - """ html_email_clean test for quotes """ - test_data = [("""This is Sparta!\n>Ah bon ?\nCertes\n> Chouette !\nClair""", - ['This is Sparta!', 'Certes', 'Clair'], - ['Ah bon', 'Chouette']) - ] - for test, in_lst, out_lst in test_data: - new_html = html_email_clean(test, remove=True) - for text in in_lst: - self.assertIn(text, new_html, 'html_email_cleaner wrongly removed content') - for text in out_lst: - self.assertNotIn(text, new_html, 'html_email_cleaner did not remove unwanted content') - - def test_10_html_email_clean_text(self): + def test_10_email_text(self): """ html_email_clean test for text-based emails """ new_html = html_email_clean(test_mail_examples.TEXT_1, remove=True) for ext in test_mail_examples.TEXT_1_IN: @@ -165,7 +163,7 @@ class TestCleaner(unittest2.TestCase): for ext in test_mail_examples.TEXT_2_OUT: self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') - def test_20_html_email_clean_html(self): + def test_20_email_html(self): new_html = html_email_clean(test_mail_examples.HTML_1, remove=True) for ext in test_mail_examples.HTML_1_IN: self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') @@ -178,41 +176,54 @@ class TestCleaner(unittest2.TestCase): for ext in test_mail_examples.HTML_2_OUT: self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') - new_html = html_email_clean(test_mail_examples.HTML_3, remove=False) - for ext in test_mail_examples.HTML_3_IN: - self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') + # --- MAIL ORIGINAL --- -> can't parse this one currently, too much language-dependent + # new_html = html_email_clean(test_mail_examples.HTML_3, remove=False) + # for ext in test_mail_examples.HTML_3_IN: + # self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') # for ext in test_mail_examples.HTML_3_OUT: # self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') - def test_30_html_email_clean_msoffice(self): + def test_30_email_msoffice(self): new_html = html_email_clean(test_mail_examples.MSOFFICE_1, remove=True) for ext in test_mail_examples.MSOFFICE_1_IN: self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') for ext in test_mail_examples.MSOFFICE_1_OUT: self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') - def test_40_html_email_clean_hotmail(self): + new_html = html_email_clean(test_mail_examples.MSOFFICE_2, remove=True) + for ext in test_mail_examples.MSOFFICE_2_IN: + self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') + for ext in test_mail_examples.MSOFFICE_2_OUT: + self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') + + new_html = html_email_clean(test_mail_examples.MSOFFICE_3, remove=True) + for ext in test_mail_examples.MSOFFICE_3_IN: + self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') + for ext in test_mail_examples.MSOFFICE_3_OUT: + self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') + + def test_40_email_hotmail(self): new_html = html_email_clean(test_mail_examples.HOTMAIL_1, remove=True) for ext in test_mail_examples.HOTMAIL_1_IN: self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') for ext in test_mail_examples.HOTMAIL_1_OUT: self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') - def test_50_html_email_clean_gmail(self): + def test_50_email_gmail(self): new_html = html_email_clean(test_mail_examples.GMAIL_1, remove=True) for ext in test_mail_examples.GMAIL_1_IN: self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') for ext in test_mail_examples.GMAIL_1_OUT: self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') - def test_60_html_email_clean_thunderbird(self): + def test_60_email_thunderbird(self): new_html = html_email_clean(test_mail_examples.THUNDERBIRD_1, remove=True) for ext in test_mail_examples.THUNDERBIRD_1_IN: self.assertIn(ext, new_html, 'html_email_cleaner wrongly removed not quoted content') for ext in test_mail_examples.THUNDERBIRD_1_OUT: self.assertNotIn(ext, new_html, 'html_email_cleaner did not erase signature / quoted content') - def test_90_html_email_clean_misc(self): + def test_90_misc(self): # False boolean for text must return empty string new_html = html_email_clean(False) self.assertEqual(new_html, False, 'html_email_cleaner did change a False in an other value.') diff --git a/openerp/tests/test_mail_examples.py b/openerp/tests/test_mail_examples.py index b0a0faf97ca..7631a28e662 100644 --- a/openerp/tests/test_mail_examples.py +++ b/openerp/tests/test_mail_examples.py @@ -176,6 +176,62 @@ XXXXXX HTML_3_OUT = ["Hi,", "My CRM-related question.", "Regards,"] +HTML_4 = """ +
+
Hi Nicholas,
+
+
I'm free now. 00447710085916.
+
+
Regards,
+
Nicholas
+
+ +
+ From: OpenERP Enterprise <sales@openerp.com>
Reply-To: <sales@openerp.com>
Date: Wed, 17 Apr 2013 13:30:47 +0000
To: Microsoft Office User <n.saxlund@babydino.com>
Subject: Re: your OpenERP.com registration
+
+
+
+

Hello Nicholas Saxlund,

+

I noticed you recently registered to our OpenERP Online solution.

+

You indicated that you wish to use OpenERP in your own company. We would like to know more about your your business needs and requirements, and see how we can help you. When would you be available to discuss your project ? +

+

Best regards,

+ 
http://openerp.com
+Belgium: +32.81.81.37.00
+U.S.: +1 (650) 307-6736
+India: +91 (79) 40 500 100
+
+
+ +
""" + +HTML_5 = """
Hi,
+
+I have downloaded OpenERP installer 7.0 and successfully installed the postgresql server and the OpenERP.
+I created a database and started to install module by log in as administrator.
+However, I was not able to install any module due to "OpenERP Server Error" as shown in the attachement.
+Could you please let me know how could I fix this problem?
+
+ Regards,
+Goh Sin Yih
+
+
+________________________________
+ From: OpenERP Enterprise <sales@openerp.com>
+To: sinyih_goh@yahoo.com 
+Sent: Friday, February 8, 2013 12:46 AM
+Subject: Feedback From Your OpenERP Trial
+ 
+
+Hello Goh Sin Yih, 
+Thank you for having tested OpenERP Online. 
+I noticed you started a trial of OpenERP Online (gsy) but you did not decide to keep using it. 
+So, I just wanted to get in touch with you to get your feedback. Can you tell me what kind of application you were you looking for and why you didn't decide to continue with OpenERP? 
+Thanks in advance for providing your feedback, 
+Do not hesitate to contact me if you have any questions, 
+Thanks, 
+
""" + GMAIL_1 = """Hello,

Ok for me. I am replying directly in gmail, without signature.

Kind regards,

Demo.

On Thu, Nov 8, 2012 at 5:29 PM, <dummy@example.com> wrote:
I contact you about our meeting for tomorrow. Here is the schedule I propose:
  • 9 AM: brainstorming about our new amazing business app</span></li>
  • 9.45 AM: summary
  • 10 AM: meeting with Fabien to present our app
Is everything ok for you ?

--
Administrator

@@ -351,4 +407,233 @@ MSOFFICE_1 = """
""" MSOFFICE_1_IN = ['Our requirements are simple. Just looking to replace some spreadsheets for tracking quotes and possibly using the timecard module.'] -MSOFFICE_1_OUT = ['I noticed you recently downloaded OpenERP.', 'Uou mentioned you wish to use OpenERP in your own company.'] +MSOFFICE_1_OUT = ['I noticed you recently downloaded OpenERP.', 'Uou mentioned you wish to use OpenERP in your own company.', 'Belgium: +32.81.81.37.00'] + +MSOFFICE_2 = """ +
+
+

+ Nicolas,

+

+

+ We are currently investigating the possibility of moving away from our current ERP

+

+

+ +

 

+ +

+

+ Thank You

+

+

+ Matt

+

+

+ +

 

+ +

+
+

+ Raoul Petitpoil

+

+

+ Poil Industries

+

+

+ Information Technology

+

+

+ 920 Super Street

+

+

+ Sanchez, Pa 17046 USA

+

+

+ Tel: xxx.xxx

+

+

+ Fax: xxx.xxx

+

+

+ Email: + + raoul@petitpoil.com + + +

+ +

+

+ www.poilindustries.com

+

+

+ www.superproducts.com

+

+
+

+ +

 

+ +

+
+
+

+ + From: + + OpenERP Enterprise [mailto:sales@openerp.com]
Sent: Wednesday, April 17, 2013 1:31 PM
To: Matt Witters
Subject: Re: your OpenERP.com registration

+

+
+
+

+

 

+

Hello Raoul Petitpoil,

+

+

I noticed you recently downloaded OpenERP.

+

+

You indicated that you wish to use OpenERP in your own company. We would like to know more about your your business needs and requirements, and see how we can help you. When would you be available to discuss your project ?

+

+

Best regards,

+

+ 
      
 

+
+ 
--
+ 
Nicolas
+ 
      http://openerp.com
+      

+
+ 
Belgium: +32.81.81.37.00
+ 
U.S.: +1 (650) 307-6736
+ 
India: +91 (79) 40 500 100
+ 
                       
+
+
""" + +MSOFFICE_2_IN = ['We are currently investigating the possibility'] +MSOFFICE_2_OUT = ['I noticed you recently downloaded OpenERP.', 'You indicated that you wish', 'Belgium: +32.81.81.37.00'] + +MSOFFICE_3 = """
+
+

+ Hi Nicolas !

+

+

+ +

 

+ +

+

+ Yes I’d be glad to hear about your offers as we struggle every year with the planning/approving of LOA.

+

+

+ I saw your boss yesterday on tv and immediately wanted to test the interface.

+

+

+ +

 

+ +

+
+

+ + Bien à vous,

+ +

+

+ + Met vriendelijke groeten,

+ +

+

+ + Best regards,

+ +

+

+ + +

 

+ + +

+

+ + R. Petitpoil   
+ + Human Resource Manager

Field Resource s.a n.v.  
Hermesstraat 6A
1930 Zaventem +
+ + ( + + + + + + xxx.xxx   + + +
+ + + 7 + + + + + + +32 2 727.05.91
 + + P +    + + Please consider the environment before printing this email. + + + +

+ +

+
+

+ +

 

+ +

+
+
+

+ + De : + + OpenERP Enterprise [mailto:sales@openerp.com]
Envoyé : jeudi 18 avril 2013 11:31
À : Paul Richard
Objet : Re: your OpenERP.com registration

+

+
+
+

+

 

+

Hello Raoul PETITPOIL,

+

+

I noticed you recently registered to our OpenERP Online solution.

+

+

You indicated that you wish to use OpenERP in your own company. We would like to know more about your your business needs and requirements, and see how we can help you. When would you be available to discuss your project ?

+

+

Best regards,

+

+ 
      
 

+
+ 
--
+ 
Nicolas
+ 
      http://openerp.com
+      

+
+ 
Belgium: +32.81.81.37.00
+ 
U.S.: +1 (650) 307-6736
+ 
India: +91 (79) 40 500 100
+ 
                       
+
+
""" + +MSOFFICE_3_IN = ['I saw your boss yesterday'] +MSOFFICE_3_OUT = ['I noticed you recently downloaded OpenERP.', 'You indicated that you wish', 'Belgium: +32.81.81.37.00'] diff --git a/openerp/tools/mail.py b/openerp/tools/mail.py index da36aed3865..6ab858d1d67 100644 --- a/openerp/tools/mail.py +++ b/openerp/tools/mail.py @@ -76,19 +76,30 @@ def html_sanitize(src): #---------------------------------------------------------- def html_email_clean(html, remove=False, shorten=False, max_length=300): - """ html_email_clean: clean the html - - try to strip email quotes (remove blockquote nodes) - - try to strip signatures - - allows having a shortened version of the html (read more/read less behavior) + """ html_email_clean: clean the html by doing the following steps: - :param string html: sanitized html; tags like html or head should not - be present in the html string. This method therefore takes as input - html code coming from a sanitized source, like fields.html. - :param boolean remove: remove the html code that is unwanted; otherwise - it is only flagged and tagged - :param boolean shorten: shorten the html - :param int max_length: if shortening, maximum number of characters before - shortening + - try to strip email quotes, by removing blockquotes or having some client- + specific heuristics + - try to strip signatures + - shorten the html to a maximum number of characters if requested + + Some specific use case: + + - MsOffice: ``div.style = border-top:solid;`` delimitates the beginning of + a quote; detecting by finding WordSection1 of MsoNormal + - Hotmail: ``hr.stopSpelling`` delimitates the beginning of a quote; detect + Hotmail by funding ``SkyDrivePlaceholder`` + + :param string html: sanitized html; tags like html or head should not + be present in the html string. This method therefore + takes as input html code coming from a sanitized source, + like fields.html. + :param boolean remove: remove the html code that is unwanted; otherwise it + is only flagged and tagged + :param boolean shorten: shorten the html; every excessing content will + be flagged as to remove + :param int max_length: if shortening, maximum number of characters before + shortening """ def _replace_matching_regex(regex, source, replace=''): """ Replace all matching expressions in source by replace """