From 2cfb9451d355b7ee8605165ef3057d3f42c6b2b5 Mon Sep 17 00:00:00 2001 From: "Harry (OpenERP)" Date: Tue, 5 Apr 2011 17:29:20 +0530 Subject: [PATCH] [REVIEW] mail: * email.message : add new method parse_message to parse message string into dict object * email.thread : refector process_email, message_new, message_update bzr revid: hmo@tinyerp.com-20110405115920-10yxztp0p7vvhg2x --- addons/mail/email_message.py | 119 +++++++++++ addons/mail/email_thread.py | 282 +++++++++----------------- addons/thunderbird/partner/partner.py | 91 +-------- 3 files changed, 222 insertions(+), 270 deletions(-) diff --git a/addons/mail/email_message.py b/addons/mail/email_message.py index b7a3dbfea63..c6faf9c0bd9 100644 --- a/addons/mail/email_message.py +++ b/addons/mail/email_message.py @@ -26,6 +26,10 @@ import tools import netsvc import base64 import time +import logging +import re +import email +from email.header import decode_header #import binascii #import email #from email.header import decode_header @@ -57,6 +61,8 @@ import time #] LOGGER = netsvc.Logger() +_logger = logging.getLogger('mail') + def format_date_tz(date, tz=None): if not date: return 'n/a' @@ -272,6 +278,119 @@ class email_message(osv.osv): return False return res + def _decode_header(self, text): + """Returns unicode() string conversion of the the given encoded smtp header""" + if text: + text = decode_header(text.replace('\r', '')) + return ''.join([tools.ustr(x[0], x[1]) for x in text]) + + def to_email(self,text): + return re.findall(r'([^ ,<@]+@[^> ,]+)',text) + + def parse_message(self, message): + """Return Dictionary Object after parse EML Message String + @param message: email.message.Message object or string or unicode object + """ + if isinstance(message, str): + msg_txt = email.message_from_string(message) + + # Warning: message_from_string doesn't always work correctly on unicode, + # we must use utf-8 strings here :-( + if isinstance(message, unicode): + message = message.encode('utf-8') + msg_txt = email.message_from_string(message) + + msg_txt = message + message_id = msg_txt.get('message-id', False) + msg = {} + + if not message_id: + # Very unusual situation, be we should be fault-tolerant here + message_id = time.time() + msg_txt['message-id'] = message_id + _logger.info('Parsing Message without message-id, generating a random one: %s', message_id) + + + fields = msg_txt.keys() + msg['id'] = message_id + msg['message-id'] = message_id + + if 'Subject' in fields: + msg['subject'] = self._decode_header(msg_txt.get('Subject')) + + if 'Content-Type' in fields: + msg['content-type'] = msg_txt.get('Content-Type') + + if 'From' in fields: + msg['from'] = self._decode_header(msg_txt.get('From') or msg_txt.get_unixfrom()) + + if 'Delivered-To' in fields: + msg['to'] = self._decode_header(msg_txt.get('Delivered-To')) + + if 'CC' in fields: + msg['cc'] = self._decode_header(msg_txt.get('CC')) + + if 'Reply-to' in fields: + msg['reply'] = self._decode_header(msg_txt.get('Reply-To')) + + if 'Date' in fields: + msg['date'] = self._decode_header(msg_txt.get('Date')) + + if 'Content-Transfer-Encoding' in fields: + msg['encoding'] = msg_txt.get('Content-Transfer-Encoding') + + if 'References' in fields: + msg['references'] = msg_txt.get('References') + + if 'In-Reply-To' in fields: + msg['in-reply-to'] = msg_txt.get('In-Reply-To') + + if 'X-Priority' in fields: + msg['priority'] = msg_txt.get('X-Priority', '3 (Normal)').split(' ')[0] #TOFIX: + + if not msg_txt.is_multipart() or 'text/plain' in msg.get('content-type', ''): + encoding = msg_txt.get_content_charset() + body = msg_txt.get_payload(decode=True) + if 'text/html' in msg.get('content-type', ''): + body = tools.html2plaintext(body) + msg['body'] = tools.ustr(body, encoding) + + attachments = {} + has_plain_text = False + if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''): + body = "" + for part in msg_txt.walk(): + if part.get_content_maintype() == 'multipart': + continue + + encoding = part.get_content_charset() + filename = part.get_filename() + if part.get_content_maintype()=='text': + content = part.get_payload(decode=True) + if filename: + attachments[filename] = content + elif not has_plain_text: + # main content parts should have 'text' maintype + # and no filename. we ignore the html part if + # there is already a plaintext part without filename, + # because presumably these are alternatives. + content = tools.ustr(content, encoding) + if part.get_content_subtype() == 'html': + body = tools.ustr(tools.html2plaintext(content)) + elif part.get_content_subtype() == 'plain': + body = content + has_plain_text = True + elif part.get_content_maintype() in ('application', 'image'): + if filename : + attachments[filename] = part.get_payload(decode=True) + else: + res = part.get_payload(decode=True) + body += tools.ustr(res, encoding) + + msg['body'] = body + msg['attachments'] = attachments + return msg + def send_email(self, cr, uid, ids, auto_commit=False, context=None): """ send email message diff --git a/addons/mail/email_thread.py b/addons/mail/email_thread.py index e48ff9d27df..aaf6f46b951 100644 --- a/addons/mail/email_thread.py +++ b/addons/mail/email_thread.py @@ -24,10 +24,10 @@ import time import tools import binascii import email -from email.header import decode_header + from email.utils import parsedate import base64 -import re + from tools.translate import _ import logging import xmlrpclib @@ -60,16 +60,74 @@ class email_thread(osv.osv): default.update({ 'message_ids': [], - 'date_closed': False, - 'date_open': False }) return super(email_thread, self).copy(cr, uid, id, default, context=context) def message_new(self, cr, uid, msg, context): - raise Exception, _('Method is not implemented') + """ + Called by process_email() to create a new record + corresponding to an incoming message for a new thread. + @param msg: Dictionary Object to contain email message data + """ + model_pool = self.pool.get(self._name) + fields = model_pool.fields_get(cr, uid, context=context) + data = model_pool.default_get(cr, uid, fields, context=context) + res_id = model_pool.create(cr, uid, data, context=context) - def message_update(self, cr, uid, ids, vals={}, msg="", default_act='pending', context=None): - raise Exception, _('Method is not implemented') + for attachment in msg.get('attachments', []): + data_attach = { + 'name': attachment, + 'datas': binascii.b2a_base64(str(attachments.get(attachment))), + 'datas_fname': attachment, + 'description': _('Mail attachment'), + 'res_model': self._name, + 'res_id': res_id, + } + att_ids.append(self.pool.get('ir.attachment').create(cr, uid, data_attach)) + + model_pool.history(cr, uid, res_id, _('receive'), history=True, + subject = msg.get('subject'), + email = msg.get('to'), + details = msg.get('body'), + email_from = msg.get('from'), + email_cc = msg.get('cc'), + message_id = msg.get('message-id'), + references = msg.get('references', False) or msg.get('in-reply-to', False), + attach = att_ids.items(), + email_date = msg.get('date'), + context = context) + return res_id + + def message_update(self, cr, uid, ids, msg, vals={}, default_act='pending', context=None): + """ + Called by process_email() to add a new incoming message for an existing thread + @param msg: Dictionary Object to contain email message data + """ + model_pool = self.pool.get(self._name) + for res_id in ids: + for attachment in msg.get('attachments', []): + data_attach = { + 'name': attachment, + 'datas': binascii.b2a_base64(str(attachments.get(attachment))), + 'datas_fname': attachment, + 'description': _('Mail attachment'), + 'res_model': self._name, + 'res_id': res_id, + } + att_ids.append(self.pool.get('ir.attachment').create(cr, uid, data_attach)) + + model_pool.history(cr, uid, res_id, _('receive'), history=True, + subject = msg.get('subject'), + email = msg.get('to'), + details = msg.get('body'), + email_from = msg.get('from'), + email_cc = msg.get('cc'), + message_id = msg.get('message-id'), + references = msg.get('references', False) or msg.get('in-reply-to', False), + attach = att_ids.items(), + email_date = msg.get('date'), + context = context) + return True def thread_followers(self, cr, uid, ids, context=None): """ Get a list of emails of the people following this thread @@ -164,36 +222,31 @@ class email_thread(osv.osv): 'partner_id': partner_id, 'references': references, 'message_id': message_id, - 'attachment_ids': [(6, 0, attachments)] + 'attachment_ids': [(6, 0, attachments)], + 'state' : 'received', } obj.create(cr, uid, data, context=context) return True - def _decode_header(self, text): - """Returns unicode() string conversion of the the given encoded smtp header""" - if text: - text = decode_header(text.replace('\r', '')) - return ''.join([tools.ustr(x[0], x[1]) for x in text]) - - def to_email(self,text): - return re.findall(r'([^ ,<@]+@[^> ,]+)',text) + def email_forward(self, cr, uid, model, res_ids, msg, email_error=False, context=None): """Sends an email to all people following the thread @param res_id: Id of the record of OpenObject model created from the email message - @param msg: email.message.Message to forward + @param msg: email.message.Message object to forward @param email_error: Default Email address in case of any Problem """ model_pool = self.pool.get(model) smtp_server_obj = self.pool.get('ir.mail_server') - + email_message_obj = self.pool.get('email.message') + _decode_header = email_message_obj._decode_header for res in model_pool.browse(cr, uid, res_ids, context=context): thread_followers = model_pool.thread_followers(cr, uid, [res.id])[res.id] - message_followers_emails = self.to_email(','.join(filter(None, thread_followers))) - message_recipients = self.to_email(','.join(filter(None, - [self._decode_header(msg['from']), - self._decode_header(msg['to']), - self._decode_header(msg['cc'])]))) + message_followers_emails = email_message_obj.to_email(','.join(filter(None, thread_followers))) + message_recipients = email_message_obj.to_email(','.join(filter(None, + [_decode_header(msg['from']), + _decode_header(msg['to']), + _decode_header(msg['cc'])]))) message_forward = [i for i in message_followers_emails if (i and (i not in message_recipients))] if message_forward: @@ -205,13 +258,14 @@ class email_thread(osv.osv): smtp_from = self.to_email(msg['from']) msg['from'] = smtp_from msg['to'] = message_forward - msg['Message-Id'] = tools.generate_tracking_message_id(res.id) + msg['message-id'] = tools.generate_tracking_message_id(res.id) if not smtp_server_obj.send_email(cr, uid, msg) and email_error: subj = msg['subject'] del msg['subject'], msg['to'], msg['cc'], msg['bcc'] - msg['subject'] = '[OpenERP-Forward-Failed] %s' % subj - msg['To'] = email_error + msg['subject'] = _('[OpenERP-Forward-Failed] %s') % subj + msg['to'] = email_error smtp_server_obj.send_email(cr, uid, msg) + return True def process_email(self, cr, uid, model, message, custom_values=None, attach=True, context=None): """This function Processes email and create record for given OpenERP model @@ -236,136 +290,26 @@ class email_thread(osv.osv): custom_values = {} model_pool = self.pool.get(model) + email_message_pool = self.pool.get('email.message') res_id = False - # Create New Record into particular model - def create_record(msg): - att_ids = [] - if hasattr(model_pool, 'message_new'): - res_id = model_pool.message_new(cr, uid, msg, context=context) - if custom_values: - model_pool.write(cr, uid, [res_id], custom_values, context=context) - else: - data = { - 'subject': msg.get('subject'), - 'email_from': msg.get('from'), - 'email_cc': msg.get('cc'), - 'user_id': False, - 'body': msg.get('body'), - #'state' : 'draft', - } - data.update(self.get_partner(cr, uid, msg.get('from'), context=context)) - res_id = model_pool.create(cr, uid, data, context=context) - - if attach: - for attachment in msg.get('attachments', []): - data_attach = { - 'name': attachment, - 'datas': binascii.b2a_base64(str(attachments.get(attachment))), - 'datas_fname': attachment, - 'description': 'Mail attachment', - 'res_model': model, - 'res_id': res_id, - } - att_ids.append(self.pool.get('ir.attachment').create(cr, uid, data_attach)) - - return res_id, att_ids + # Parse Message # Warning: message_from_string doesn't always work correctly on unicode, # we must use utf-8 strings here :-( if isinstance(message, unicode): message = message.encode('utf-8') msg_txt = email.message_from_string(message) - message_id = msg_txt.get('message-id', False) - msg = {} + msg = email_message_pool.parse_message(msg_txt) - if not message_id: - # Very unusual situation, be we should be fault-tolerant here - message_id = time.time() - msg_txt['message-id'] = message_id - _logger.info('Message without message-id, generating a random one: %s', message_id) + # Create New Record into particular model + def create_record(msg): + new_res_id = model_pool.message_new(cr, uid, msg, context=context) + if custom_values: + model_pool.write(cr, uid, [res_id], custom_values, context=context) + return new_res_id - fields = msg_txt.keys() - msg['id'] = message_id - msg['message-id'] = message_id - - if 'Subject' in fields: - msg['subject'] = self._decode_header(msg_txt.get('Subject')) - - if 'Content-Type' in fields: - msg['content-type'] = msg_txt.get('Content-Type') - - if 'From' in fields: - msg['from'] = self._decode_header(msg_txt.get('From') or msg_txt.get_unixfrom()) - - if 'Delivered-To' in fields: - msg['to'] = self._decode_header(msg_txt.get('Delivered-To')) - - if 'CC' in fields: - msg['cc'] = self._decode_header(msg_txt.get('CC')) - - if 'Reply-to' in fields: - msg['reply'] = self._decode_header(msg_txt.get('Reply-To')) - - if 'Date' in fields: - msg['date'] = self._decode_header(msg_txt.get('Date')) - - if 'Content-Transfer-Encoding' in fields: - msg['encoding'] = msg_txt.get('Content-Transfer-Encoding') - - if 'References' in fields: - msg['references'] = msg_txt.get('References') - - if 'In-Reply-To' in fields: - msg['in-reply-to'] = msg_txt.get('In-Reply-To') - - if 'X-Priority' in fields: - msg['priority'] = msg_txt.get('X-Priority', '3 (Normal)').split(' ')[0] - - if not msg_txt.is_multipart() or 'text/plain' in msg.get('Content-Type', ''): - encoding = msg_txt.get_content_charset() - body = msg_txt.get_payload(decode=True) - if 'text/html' in msg_txt.get('Content-Type', ''): - body = tools.html2plaintext(body) - msg['body'] = tools.ustr(body, encoding) - - attachments = {} - has_plain_text = False - if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''): - body = "" - for part in msg_txt.walk(): - if part.get_content_maintype() == 'multipart': - continue - - encoding = part.get_content_charset() - filename = part.get_filename() - if part.get_content_maintype()=='text': - content = part.get_payload(decode=True) - if filename: - attachments[filename] = content - elif not has_plain_text: - # main content parts should have 'text' maintype - # and no filename. we ignore the html part if - # there is already a plaintext part without filename, - # because presumably these are alternatives. - content = tools.ustr(content, encoding) - if part.get_content_subtype() == 'html': - body = tools.ustr(tools.html2plaintext(content)) - elif part.get_content_subtype() == 'plain': - body = content - has_plain_text = True - elif part.get_content_maintype() in ('application', 'image'): - if filename : - attachments[filename] = part.get_payload(decode=True) - else: - res = part.get_payload(decode=True) - body += tools.ustr(res, encoding) - - msg['body'] = body - msg['attachments'] = attachments - res_ids = [] - attachment_ids = [] - new_res_id = False + res_id = False if msg.get('references') or msg.get('in-reply-to'): references = msg.get('references') or msg.get('in-reply-to') if '\r\n' in references: @@ -374,45 +318,25 @@ class email_thread(osv.osv): references = references.split(' ') for ref in references: ref = ref.strip() - res_id = tools.misc.reference_re.search(ref) + res_id = tools.reference_re.search(ref) if res_id: res_id = res_id.group(1) else: - res_id = tools.misc.res_re.search(msg['subject']) + res_id = tools.res_re.search(msg['subject']) if res_id: res_id = res_id.group(1) if res_id: res_id = int(res_id) - model_pool = self.pool.get(model) if model_pool.exists(cr, uid, res_id): - res_ids.append(res_id) - if hasattr(model_pool, 'message_update'): - model_pool.message_update(cr, uid, [res_id], {}, msg, context=context) - else: - raise NotImplementedError('model %s does not support updating records, mailgate API method message_update() is missing'%model) + model_pool.message_update(cr, uid, [res_id], {}, msg, context=context) + - if not len(res_ids): - new_res_id, attachment_ids = create_record(msg) - res_ids = [new_res_id] + if not res_id: + res_id = create_record(msg) - # Store messages - context.update({'model' : model}) - if hasattr(model_pool, 'history'): - model_pool.history(cr, uid, res_ids, _('receive'), history=True, - subject = msg.get('subject'), - email = msg.get('to'), - details = msg.get('body'), - email_from = msg.get('from'), - email_cc = msg.get('cc'), - message_id = msg.get('message-id'), - references = msg.get('references', False) or msg.get('in-reply-to', False), - attach = attachments.items(), - email_date = msg.get('date'), - context = context) - else: - self.history(cr, uid, model, res_ids, msg, attachment_ids, context=context) - self.email_forward(cr, uid, model, res_ids, msg_txt) - return new_res_id + #To forward the email to other followers + self.email_forward(cr, uid, model, [res_id], msg_txt) + return res_id def get_partner(self, cr, uid, from_email, context=None): """This function returns partner Id based on email passed @@ -437,12 +361,4 @@ class email_thread(osv.osv): email_thread() - -def format_date_tz(date, tz=None): - if not date: - return 'n/a' - format = tools.DEFAULT_SERVER_DATETIME_FORMAT - return tools.server_to_local_timestamp(date, format, format, tz) - - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/addons/thunderbird/partner/partner.py b/addons/thunderbird/partner/partner.py index b54339dae20..3d0f2937a1f 100644 --- a/addons/thunderbird/partner/partner.py +++ b/addons/thunderbird/partner/partner.py @@ -29,7 +29,8 @@ class email_thread(osv.osv): def history_message(self, cr, uid, model, res_id, message, context=None): #@param message: string of mail which is read from EML File attachment_pool = self.pool.get('ir.attachment') - msg = self.parse_message(message) + email_message_pool = self.pool.get('email.message') + msg = email_message_pool.parse_message(message) attachments = msg.get('attachments', []) att_ids = [] for attachment in attachments: @@ -43,90 +44,6 @@ class email_thread(osv.osv): } att_ids.append(attachment_pool.create(cr, uid, data_attach)) return self.history(cr, uid, model, res_id, msg, att_ids) - - def parse_message(self, message): - #TOCHECK: put this function in mailgateway module - if isinstance(message, unicode): - message = message.encode('utf-8') - msg_txt = email.message_from_string(message) - message_id = msg_txt.get('message-id', False) - msg = {} - fields = msg_txt.keys() - msg['id'] = message_id - msg['message-id'] = message_id - if 'Subject' in fields: - msg['subject'] = self._decode_header(msg_txt.get('Subject')) - - if 'Content-Type' in fields: - msg['content-type'] = msg_txt.get('Content-Type') - - if 'From' in fields: - msg['from'] = self._decode_header(msg_txt.get('From')) - - if 'Delivered-To' in fields: - msg['to'] = self._decode_header(msg_txt.get('Delivered-To')) - - if 'CC' in fields: - msg['cc'] = self._decode_header(msg_txt.get('CC')) - - if 'Reply-to' in fields: - msg['reply'] = self._decode_header(msg_txt.get('Reply-To')) - - if 'Date' in fields: - msg['date'] = self._decode_header(msg_txt.get('Date')) - - if 'Content-Transfer-Encoding' in fields: - msg['encoding'] = msg_txt.get('Content-Transfer-Encoding') - - if 'References' in fields: - msg['references'] = msg_txt.get('References') - - if 'In-Reply-To' in fields: - msg['in-reply-to'] = msg_txt.get('In-Reply-To') - - if 'X-Priority' in fields: - msg['priority'] = msg_txt.get('X-Priority', '3 (Normal)').split(' ')[0] - - if not msg_txt.is_multipart() or 'text/plain' in msg.get('Content-Type', ''): - encoding = msg_txt.get_content_charset() - body = msg_txt.get_payload(decode=True) - msg['body'] = tools.ustr(body, encoding) - - attachments = {} - has_plain_text = False - if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', ''): - body = "" - for part in msg_txt.walk(): - if part.get_content_maintype() == 'multipart': - continue - - encoding = part.get_content_charset() - filename = part.get_filename() - if part.get_content_maintype()=='text': - content = part.get_payload(decode=True) - if filename: - attachments[filename] = content - elif not has_plain_text: - # main content parts should have 'text' maintype - # and no filename. we ignore the html part if - # there is already a plaintext part without filename, - # because presumably these are alternatives. - content = tools.ustr(content, encoding) - if part.get_content_subtype() == 'html': - body = tools.ustr(tools.html2plaintext(content)) - elif part.get_content_subtype() == 'plain': - body = content - has_plain_text = True - elif part.get_content_maintype() in ('application', 'image'): - if filename : - attachments[filename] = part.get_payload(decode=True) - else: - res = part.get_payload(decode=True) - body += tools.ustr(res, encoding) - - msg['body'] = body - msg['attachments'] = attachments - return msg email_thread() class thunderbird_partner(osv.osv_memory): @@ -149,7 +66,7 @@ class thunderbird_partner(osv.osv_memory): ref_ids = str(dictcreate.get('ref_ids')).split(';') msg = dictcreate.get('message') mail = msg - msg = self.pool.get('email.thread').parse_message(msg) + msg = self.pool.get('email.message').parse_message(msg) thread_pool = self.pool.get('email.thread') message_id = msg.get('message-id', False) msg_pool = self.pool.get('email.message') @@ -207,7 +124,7 @@ class thunderbird_partner(osv.osv_memory): references = [] dictcreate = dict(message) msg = dictcreate.get('message') - msg = self.pool.get('email.thread').parse_message(msg) + msg = self.pool.get('email.message').parse_message(msg) message_id = msg.get('message-id') refs = msg.get('references',False) references = False