[MERGE] OPW 577359: mail,fetchmail: more robust handling of incoming mails

- Support inconsistent encoding specifications in
  mail bodies with crashing
- Mark messages as seen on IMAP servers only when they
  have actually been processed
- Properly commit after each incoming message, avoiding
  global rollback in case of later failure (during 
  fetchmail cron task)
- Avoid crashing when importing mail without a proper
  From header

Note: this fix depends on a corresponding server fix
(revno: 4248
rev-id: odo@openerp.com-20120727134402-vafp3wqlwnjsxntk)

bzr revid: odo@openerp.com-20120727150051-e1q0m92tyrazz82f

addons/fetchmail/fetchmail.py

@@ -189,14 +189,15 @@ openerp_mailgate.py -u %(uid)d -p PASSWORD -o %(model)s -d %(dbname)s --host=HOS
                     result, data = imap_server.search(None, '(UNSEEN)')
                     for num in data[0].split():
                         result, data = imap_server.fetch(num, '(RFC822)')
+                        imap_server.store(num, '-FLAGS', '\\Seen')
                         res_id = mail_thread.message_process(cr, uid, server.object_id.model, data[0][1],
                                                              save_original=server.original,
                                                              strip_attachments=(not server.attach),
                                                              context=context)
                         if res_id and server.action_id:
                             action_pool.run(cr, uid, [server.action_id.id], {'active_id': res_id, 'active_ids': [res_id], 'active_model': server.object_id.model})
-                            imap_server.store(num, '+FLAGS', '\\Seen')
-                            cr.commit()
+                        imap_server.store(num, '+FLAGS', '\\Seen')
+                        cr.commit()
                         count += 1
                     logger.info("fetched/processed %s email(s) on %s server %s", count, server.type, server.name)
                 except Exception, e:

addons/mail/mail_message.py

@@ -433,12 +433,12 @@ class mail_message(osv.osv):
                 msg['headers'].update({item[0]: item[1]})
         if not msg_txt.is_multipart() or 'text/plain' in msg.get('content-type', ''):
             encoding = msg_txt.get_content_charset()
-            body = msg_txt.get_payload(decode=True)
+            body = tools.ustr(msg_txt.get_payload(decode=True), encoding, errors='replace')
             if 'text/html' in msg.get('content-type', ''):
                 msg['body_html'] =  body
                 msg['subtype'] = 'html'
                 body = tools.html2plaintext(body)
-            msg['body_text'] = tools.ustr(body, encoding)
+            msg['body_text'] = tools.ustr(body, encoding, errors='replace')
 
         attachments = []
         if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''):
@@ -457,7 +457,7 @@ class mail_message(osv.osv):
                     content = part.get_payload(decode=True)
                     if filename:
                         attachments.append((filename, content))
-                    content = tools.ustr(content, encoding)
+                    content = tools.ustr(content, encoding, errors='replace')
                     if part.get_content_subtype() == 'html':
                         msg['body_html'] = content
                         msg['subtype'] = 'html' # html version prevails
@@ -470,7 +470,7 @@ class mail_message(osv.osv):
                         attachments.append((filename,part.get_payload(decode=True)))
                     else:
                         res = part.get_payload(decode=True)
-                        body += tools.ustr(res, encoding)
+                        body += tools.ustr(res, encoding, errors='replace')
 
             msg['body_text'] = body
         msg['attachments'] = attachments

addons/mail/mail_thread.py

@@ -461,7 +461,8 @@ class mail_thread(osv.osv):
             'partner_id': False
         }
         if email:
-            email = to_email(email)[0]
+            email = to_email(email) and to_email(email)[0]
+        if email:
             address_ids = address_pool.search(cr, uid, [('email', '=', email)])
             if address_ids:
                 address = address_pool.browse(cr, uid, address_ids[0])