pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP]: mail_gateway: * Made compatible with fetchmail,

Browse Source 
* Added common class and script that are used for mail processing,
* Optimised mailgateway script and used common functions

bzr revid: rpa@tinyerp.com-20100611122228-7fjko8endi5d1xqr
This commit is contained in:
rpa (Open ERP)
2010-06-11 17:52:28 +05:30
parent 30e69f339e
commit cd19a60f10
2 changed files with 268 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/mail_gateway/mail_gateway.py
+238
View File
@@ -22,6 +22,15 @@
from osv import osv, fields
import time
import base64
import re
import tools
import binascii
import email
from email.header import decode_header
import netsvc
logger = netsvc.Logger()
class one2many_domain(fields.one2many):
def set(self, cr, obj, id, field, values, user=None, context=None):
@@ -154,4 +163,233 @@ class mailgate_message(osv.osv):
mailgate_message()
class mailgate_tool(osv.osv):
_name = 'email.server.tools'
_description = "Email Tools"
_auto = False
def to_email(self, cr, uid, text):
_email = re.compile(r'.*<.*@.*\..*>', re.UNICODE)
def record(path):
eml = path.group()
index = eml.index('<')
eml = eml[index:-1].replace('<', '').replace('>', '')
return eml
bits = _email.sub(record, text)
return bits
def history(self, cr, uid, model, new_id, msg, attach, server_id=None, server_type=None):
try:
thread_id = self.pool.get(model).read(cr, uid, new_id, ['thread_id'])['thread_id'][0]
except Exception, e:
thread_id = None
msg_data = {
'name': msg.get('subject', 'No subject'),
'date': msg.get('date') , # or time.strftime('%Y-%m-%d %H:%M:%S')??
'description': msg.get('body', msg.get('from')),
'history': True,
'model': model,
'email_cc': msg.get('cc'),
'email_from': msg.get('from'),
'email_to': msg.get('to'),
'message_id': msg.get('message-id'),
'ref_id': msg.get('references', msg.get('id')),
'res_id': new_id,
'server_id': server_id,
'thread_id': thread_id,
'type': server_type,
'user_id': uid,
'attachment_ids': [(6, 0, attach)]
}
msg_id = self.pool.get('mailgate.message').create(cr, uid, msg_data)
return True
def process_email(self, cr, uid, model, message, attach=True, server_id=None, server_type=None, context=None):
if not context:
context = {}
context.update({
'server_id': server_id
})
res_id = False
def create_record(msg):
model_pool = self.pool.get(model)
if hasattr(model_pool, 'message_new'):
res_id = model_pool.message_new(cr, uid, msg, context)
else:
data = {
'name': msg.get('subject'),
'email_from': msg.get('from'),
'email_cc': msg.get('cc'),
'user_id': False,
'description': msg.get('body'),
'state' : 'draft',
}
data.update(self.get_partner(cr, uid, msg.get('from'), context=context))
res_id = model_pool.create(cr, uid, data, context=context)
logger.notifyChannel('imap', netsvc.LOG_WARNING, 'method def message_new is not define in model %s. Using default method' % (model_pool._name))
att_ids = []
if attach:
for attachment in msg.get('attachments', []):
data_attach = {
'name': attachment,
'datas': binascii.b2a_base64(str(attachments.get(attachment))),
'datas_fname': attachment,
'description': 'Mail attachment',
'res_model': model,
'res_id': res_id,
}
att_ids.append(self.pool.get('ir.attachment').create(cr, uid, data_attach))
if hasattr(model_pool, 'history'):
model_pool.history(cr, uid, [res_id], 'Receive', True, msg.get('to'), msg.get('body'), msg.get('from'), False, {'model' : model})
else:
self.history(cr, uid, model, res_id, msg, att_ids, server_id=server_id, server_type=server_type)
return res_id
history_pool = self.pool.get('mailgate.message')
msg_txt = email.message_from_string(message)
message_id = msg_txt.get('Message-ID', False)
msg = {}
if not message_id:
return False
fields = msg_txt.keys()
msg['id'] = message_id
msg['message-id'] = message_id
def _decode_header(txt):
txt = txt.replace('\r', '')
return ' '.join(map(lambda (x, y): unicode(x, y or 'ascii'), decode_header(txt)))
if 'Subject' in fields:
msg['subject'] = _decode_header(msg_txt.get('Subject'))
if 'Content-Type' in fields:
msg['content-type'] = msg_txt.get('Content-Type')
if 'From' in fields:
msg['from'] = _decode_header(msg_txt.get('From'))
if 'Delivered-To' in fields:
msg['to'] = _decode_header(msg_txt.get('Delivered-To'))
if 'Cc' in fields:
msg['cc'] = _decode_header(msg_txt.get('Cc'))
if 'Reply-To' in fields:
msg['reply'] = _decode_header(msg_txt.get('Reply-To'))
if 'Date' in fields:
msg['date'] = msg_txt.get('Date')
if 'Content-Transfer-Encoding' in fields:
msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
if 'References' in fields:
msg['references'] = msg_txt.get('References')
if 'X-openerp-caseid' in fields:
msg['caseid'] = msg_txt.get('X-openerp-caseid')
if 'X-Priority' in fields:
msg['priority'] = msg_txt.get('X-priority', '3 (Normal)').split(' ')[0]
if not msg_txt.is_multipart() or 'text/plain' in msg.get('content-type', None):
encoding = msg_txt.get_content_charset()
msg['body'] = msg_txt.get_payload(decode=True)
if encoding:
msg['body'] = msg['body'].decode(encoding).encode('utf-8')
attachments = {}
if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', None):
body = ""
counter = 1
for part in msg_txt.walk():
if part.get_content_maintype() == 'multipart':
continue
encoding = part.get_content_charset()
if part.get_content_maintype()=='text':
content = part.get_payload(decode=True)
filename = part.get_filename()
if filename :
attachments[filename] = content
else:
if encoding:
content = unicode(content, encoding)
if part.get_content_subtype() == 'html':
body = tools.html2plaintext(content)
elif part.get_content_subtype() == 'plain':
body = content
elif part.get_content_maintype()=='application' or part.get_content_maintype()=='image' or part.get_content_maintype()=='text':
filename = part.get_filename();
if filename :
attachments[filename] = part.get_payload(decode=True)
else:
res = part.get_payload(decode=True)
if encoding:
res = res.decode(encoding).encode('utf-8')
body += res
msg['body'] = body
msg['attachments'] = attachments
if msg.get('references', False):
id = False
ref = msg.get('references')
if '\r\n' in ref:
ref = msg.get('references').split('\r\n')
else:
ref = msg.get('references').split(' ')
if ref:
hids = history_pool.search(cr, uid, [('name', '=', ref[0].strip())])
if hids:
id = hids[0]
history = history_pool.browse(cr, uid, id)
model_pool = self.pool.get(model)
context.update({
'references_id':ref[0]
})
vals = {}
if hasattr(model_pool, 'message_update'):
model_pool.message_update(cr, uid, [history.res_id], vals, msg, context=context)
else:
logger.notifyChannel('imap', netsvc.LOG_WARNING, 'method def message_update is not define in model %s' % (model_pool._name))
return False
else:
res_id = create_record(msg)
else:
res_id = create_record(msg)
return res_id
def get_partner(self, cr, uid, from_email, context=None):
"""This function returns partner Id based on email passed
@param self: The object pointer
@param cr: the current row, from the database cursor,
@param uid: the current user’s ID for security checks
@param from_email: email address based on that function will search for the correct
"""
res = {
'partner_address_id': False,
'partner_id': False
}
from_email = self.to_email(cr, uid, from_email)
address_ids = self.pool.get('res.partner.address').search(cr, uid, [('email', '=', from_email)])
if address_ids:
address = self.pool.get('res.partner.address').browse(cr, uid, address_ids[0])
res['partner_address_id'] = address_ids[0]
res['partner_id'] = address.partner_id.id
return res
mailgate_tool()
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
addons/mail_gateway/scripts/openerp_mailgate/openerp_mailgate.py
+30 -230
View File
@@ -20,15 +20,12 @@
#
###########################################################################################
from email.header import decode_header
import email as EMAIL
import re
import smtplib
import email, mimetypes
from email.Header import decode_header
from email.MIMEText import MIMEText
import xmlrpclib
import os
import binascii
import time, socket
import xmlrpclib
email_re = re.compile(r"([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6})")
@@ -44,84 +41,9 @@ priorities = {
'5': '5 (Lowest)',
}
def html2plaintext(html, body_id=None, encoding='utf-8'):
## (c) Fry-IT, www.fry-it.com, 2007
## <peter@fry-it.com>
## download here: http://www.peterbe.com/plog/html2plaintext
""" from an HTML text, convert the HTML to plain text.
If @body_id is provided then this is the tag where the
body (not necessarily <body>) starts.
"""
try:
from BeautifulSoup import BeautifulSoup, SoupStrainer, Comment
except:
return html
urls = []
if body_id is not None:
strainer = SoupStrainer(id=body_id)
else:
strainer = SoupStrainer('body')
soup = BeautifulSoup(html, parseOnlyThese=strainer, fromEncoding=encoding)
for link in soup.findAll('a'):
title = unicode(link)
for url in [x[1] for x in link.attrs if x[0]=='href']:
urls.append(dict(url=url, tag=unicode(link), title=title))
html = unicode(soup)
url_index = []
i = 0
for d in urls:
if d['title'] == d['url'] or 'http://'+d['title'] == d['url']:
html = html.replace(d['tag'], d['url'])
else:
i += 1
html = html.replace(d['tag'], '%s [%s]' % (d['title'], i))
url_index.append(d['url'])
html = html.replace('<strong>', '*').replace('</strong>', '*')
html = html.replace('<b>', '*').replace('</b>', '*')
html = html.replace('<h3>', '*').replace('</h3>', '*')
html = html.replace('<h2>', '**').replace('</h2>', '**')
html = html.replace('<h1>', '**').replace('</h1>', '**')
html = html.replace('<em>', '/').replace('</em>', '/')
# the only line breaks we respect is those of ending tags and
# breaks
html = html.replace('\n', ' ')
html = html.replace('<br>', '\n')
html = html.replace('<tr>', '\n')
html = html.replace('</p>', '\n\n')
html = re.sub('<br\s*/>', '\n', html)
html = html.replace(' ' * 2, ' ')
# for all other tags we failed to clean up, just remove then and
# complain about them on the stderr
def desperate_fixer(g):
#print >>sys.stderr, "failed to clean up %s" % str(g.group())
return ' '
html = re.sub('<.*?>', desperate_fixer, html)
# lstrip all lines
html = '\n'.join([x.lstrip() for x in html.splitlines()])
for i, url in enumerate(url_index):
if i == 0:
html += '\n\n'
html += '[%s] %s\n' % (i+1, url)
return html
class rpc_proxy(object):
def __init__(self, uid, passwd, host='localhost', port=8069, path='object', dbname='terp'):
self.rpc = xmlrpclib.ServerProxy('http://%s:%s/xmlrpc/%s' % (host, port, path))
def __init__(self, uid, passwd, host='localhost', port=8069, path='object', dbname='terp'):
self.rpc = xmlrpclib.ServerProxy('http://%s:%s/xmlrpc/%s' % (host, port, path), allow_none=True)
self.user_id = uid
self.passwd = passwd
self.dbname = dbname
@@ -149,162 +71,32 @@ class email_parser(object):
def email_get(self, email_from):
res = email_re.search(email_from)
return res and res.group(1)
def partner_get(self, email):
mail = self.email_get(email)
adr_ids = self.rpc('res.partner.address', 'search', [('email', '=', mail)])
if not adr_ids:
return {}
adr = self.rpc('res.partner.address', 'read', adr_ids, ['partner_id'])
return {
'partner_address_id': adr[0]['id'],
'partner_id': adr[0].get('partner_id', False) and adr[0]['partner_id'][0] or False
}
return self.rpc('email.server.tools', 'to_email', email_from)
def _to_decode(self, s, charsets):
for charset in charsets:
if charset:
try:
return s.decode(charset)
except UnicodeError:
for charset in charsets:
if charset:
try:
return s.decode(charset)
except UnicodeError:
pass
return s.decode('latin1')
return s.decode('latin1')
def _decode_header(self, s):
from email.Header import decode_header
s = decode_header(s.replace('\r', ''))
if s:
s = decode_header(s.replace('\r', ''))
return ''.join(map(lambda x:self._to_decode(x[0], [x[1]]), s or []))
def history(self, model, new_id, msg_id, ref_id, subject, msg_to, msg_from, body, attach):
try:
thread_id = self.rpc(model, 'read', new_id, ['thread_id'])['thread_id'][0]
except Exception, e:
thread_id = None
msg_data = {
'name': subject,
'history': True,
'model': model,
'res_id': new_id,
'thread_id': thread_id,
'message_id': msg_id,
'ref_id': ref_id or '',
'user_id': self.rpc.user_id,
'date': time.strftime('%Y-%m-%d %H:%M:%S'),
'email_from': msg_from,
'email_to': msg_to,
'description': body,
'attachment_ids': [(6, 0, attach)]
}
msg_id = self.rpc('mailgate.message', 'create', msg_data)
return True
def msg_new(self, msg):
message = self.msg_body_get(msg)
msg_subject = self._decode_header(msg['Subject'])
msg_from = self._decode_header(msg['From'])
msg_to = self._decode_header(msg['To'])
msg_cc = self._decode_header(msg['Cc'] or '')
data = {
'name': msg_subject,
'email_from': msg_from,
'email_cc': msg_cc,
'user_id': False,
'description': message['body'],
'state' : 'draft',
}
data.update(self.partner_get(msg_from))
try:
att_ids = []
new_id = self.rpc(self.model, 'create', data)
attachments = message['attachment']
for attach in attachments or []:
data_attach = {
'name': str(attach),
'datas': binascii.b2a_base64(str(attachments[attach])),
'datas_fname': str(attach),
'description': 'Mail attachment',
'res_model': self.model,
'res_id': new_id
}
att_ids.append(self.rpc('ir.attachment', 'create', data_attach))
try:
self.rpc(self.model, 'history', [new_id], 'Receive', True, msg_to, message['body'], msg_from, False, {'model' : self.model})
except Exception, e:
self.history(self.model, new_id, msg['Message-Id'], msg['References'], msg_subject, msg_to, msg_from, message['body'], att_ids)
except Exception, e:
if getattr(e, 'faultCode', '') and 'AccessError' in e.faultCode:
e = '\n\nThe Specified user does not have an access to the Model.'
print e
return new_id
# #change the return type format to dictionary
# {
# 'body':'body part',
# 'attachment':{
# 'file_name':'file data',
# 'file_name':'file data',
# 'file_name':'file data',
# }
# }
def msg_body_get(self, msg):
message = {};
message['body'] = '';
message['attachment'] = {};
attachment = message['attachment'];
counter = 1;
def replace(match):
return ''
for part in msg.walk():
if part.get_content_maintype() == 'multipart':
continue
if part.get_content_maintype()=='text':
buf = part.get_payload(decode=True)
if buf:
txt = self._to_decode(buf, part.get_charsets())
txt = re.sub("<(\w)>", replace, txt)
txt = re.sub("<\/(\w)>", replace, txt)
if txt and part.get_content_subtype() == 'plain':
message['body'] += txt
elif txt and part.get_content_subtype() == 'html':
message['body'] += html2plaintext(txt)
filename = part.get_filename();
if filename :
attachment[filename] = part.get_payload(decode=True);
elif part.get_content_maintype()=='application' or part.get_content_maintype()=='image' or part.get_content_maintype()=='text':
filename = part.get_filename();
if filename :
attachment[filename] = part.get_payload(decode=True);
else:
filename = 'attach_file'+str(counter);
counter += 1;
attachment[filename] = part.get_payload(decode=True);
#end if
#end if
message['attachment'] = attachment
#end for
return message
def msg_send(self, msg, emails, priority=None):
if not len(emails):
return False
del msg['To']
msg['To'] = emails[0]
msg['Subject'] = 'OpenERP Record-id:' + msg['Subject']
if len(emails)>1:
if 'Cc' in msg:
del msg['Cc']
msg['Cc'] = ','.join(emails[1:])
del msg['Reply-To']
msg['Reply-To'] = self.email
if self.smtp_user and self.smtp_password:
@@ -319,17 +111,25 @@ class email_parser(object):
return False
def parse(self, msg):
#TODO: Something with Message hierarchy
res_id = self.msg_new(msg)
subject = self._decode_header(msg['subject'])
def parse(self, message):
try:
res_id = self.rpc('email.server.tools', 'process_email', self.model, message)
except Exception, e:
res_id = False
msg = EMAIL.message_from_string(str(message))
subject = self._decode_header(msg['Subject'])
if msg.get('Subject', ''):
del msg['Subject']
msg['Subject'] = '['+str(res_id)+'] '+subject
# msg['Message-Id'] = '<'+str(time.time())+'-openerpcrm-'+str(res_id)+'@'+socket.gethostname()+'>'
#Changed for sending reply mail
if res_id:
msg['Subject'] = '['+str(res_id)+'] '+subject
msg['Message-Id'] = '<'+str(time.time())+'-openerpcrm-'+str(res_id)+'@'+socket.gethostname()+'>'
mm = [self._decode_header(msg['From']), self._decode_header(msg['To'])]+self._decode_header(msg.get('Cc', '')).split(',')
msg_mails = map(self.email_get, filter(None, mm))
try:
self.msg_send(msg, msg_mails)
except Exception, e:
@@ -364,7 +164,7 @@ if __name__ == '__main__':
(options, args) = parser.parse_args()
parser = email_parser(options.userid, options.password, options.model, options.email, options.default, dbname=options.dbname, host=options.host, port=options.port, smtp_server=options.smtp_server, smtp_port=options.smtp_port, smtp_ssl=options.smtp_ssl, smtp_user=options.smtp_user, smtp_password=options.smtp_password)
msg_txt = email.message_from_file(sys.stdin)
msg_txt = sys.stdin.read().decode('utf8')
parser.parse(msg_txt)