From cd19a60f107796d719b7f669bc6b689bcceafd0d Mon Sep 17 00:00:00 2001
From: "rpa (Open ERP)" <rpa@tinyerp.com>
Date: Fri, 11 Jun 2010 17:52:28 +0530
Subject: [PATCH] [IMP]: mail_gateway: * Made compatible with fetchmail, *
 Added common class and script that are used for mail processing, * Optimised
 mailgateway script and used common functions

bzr revid: rpa@tinyerp.com-20100611122228-7fjko8endi5d1xqr
---
 addons/mail_gateway/mail_gateway.py           | 238 ++++++++++++++++
 .../openerp_mailgate/openerp_mailgate.py      | 260 ++----------------
 2 files changed, 268 insertions(+), 230 deletions(-)

diff --git a/addons/mail_gateway/mail_gateway.py b/addons/mail_gateway/mail_gateway.py
index 03c40b9cefb..f57caf6b879 100644
--- a/addons/mail_gateway/mail_gateway.py
+++ b/addons/mail_gateway/mail_gateway.py
@@ -22,6 +22,15 @@
 from osv import osv, fields
 import time
 import  base64
+import re
+import tools
+import binascii
+
+import email
+from email.header import decode_header
+import netsvc
+
+logger = netsvc.Logger()
 
 class one2many_domain(fields.one2many):
     def set(self, cr, obj, id, field, values, user=None, context=None):
@@ -154,4 +163,233 @@ class mailgate_message(osv.osv):
 mailgate_message()
 
 
+class mailgate_tool(osv.osv):
+
+    _name = 'email.server.tools'
+    _description = "Email Tools"
+    _auto = False
+
+    def to_email(self, cr, uid, text):
+        _email = re.compile(r'.*<.*@.*\..*>', re.UNICODE)
+        def record(path):
+            eml = path.group()
+            index = eml.index('<')
+            eml = eml[index:-1].replace('<', '').replace('>', '')
+            return eml
+
+        bits = _email.sub(record, text)
+        return bits
+    
+    def history(self, cr, uid, model, new_id, msg, attach, server_id=None, server_type=None):
+        try:
+            thread_id = self.pool.get(model).read(cr, uid, new_id, ['thread_id'])['thread_id'][0]
+        except Exception, e:
+            thread_id = None
+        msg_data = {
+                    'name': msg.get('subject', 'No subject'), 
+                    'date': msg.get('date') , # or time.strftime('%Y-%m-%d %H:%M:%S')??
+                    'description': msg.get('body', msg.get('from')), 
+                    'history': True,
+                    'model': model, 
+                    'email_cc': msg.get('cc'), 
+                    'email_from': msg.get('from'), 
+                    'email_to': msg.get('to'), 
+                    'message_id': msg.get('message-id'), 
+                    'ref_id': msg.get('references', msg.get('id')), 
+                    'res_id': new_id, 
+                    'server_id': server_id, 
+                    'thread_id': thread_id, 
+                    'type': server_type, 
+                    'user_id': uid, 
+                    'attachment_ids': [(6, 0, attach)]
+                    }
+        msg_id = self.pool.get('mailgate.message').create(cr, uid, msg_data)
+        return True
+
+    def process_email(self, cr, uid, model, message, attach=True, server_id=None, server_type=None, context=None):
+        if not context:
+            context = {}
+        context.update({
+            'server_id': server_id
+        })
+        res_id = False
+        def create_record(msg):
+            model_pool = self.pool.get(model)
+            if hasattr(model_pool, 'message_new'):
+                res_id = model_pool.message_new(cr, uid, msg, context)
+            else:
+                data = {
+                    'name': msg.get('subject'), 
+                    'email_from': msg.get('from'), 
+                    'email_cc': msg.get('cc'), 
+                    'user_id': False, 
+                    'description': msg.get('body'), 
+                    'state' : 'draft', 
+                }
+                data.update(self.get_partner(cr, uid, msg.get('from'), context=context))
+                res_id = model_pool.create(cr, uid, data, context=context)
+                logger.notifyChannel('imap', netsvc.LOG_WARNING, 'method def message_new is not define in model %s. Using default method' % (model_pool._name))
+            att_ids = []
+            if attach:
+                for attachment in msg.get('attachments', []):
+                    data_attach = {
+                        'name': attachment, 
+                        'datas': binascii.b2a_base64(str(attachments.get(attachment))), 
+                        'datas_fname': attachment, 
+                        'description': 'Mail attachment', 
+                        'res_model': model, 
+                        'res_id': res_id, 
+                    }
+                    att_ids.append(self.pool.get('ir.attachment').create(cr, uid, data_attach))
+
+            if hasattr(model_pool, 'history'):
+                model_pool.history(cr, uid, [res_id], 'Receive', True, msg.get('to'), msg.get('body'), msg.get('from'), False, {'model' : model})
+            else:
+                self.history(cr, uid, model, res_id, msg, att_ids, server_id=server_id, server_type=server_type)
+            
+            return res_id
+
+        history_pool = self.pool.get('mailgate.message')
+        msg_txt = email.message_from_string(message)
+        message_id = msg_txt.get('Message-ID', False)
+
+        msg = {}
+        if not message_id:
+            return False
+
+        fields = msg_txt.keys()
+        msg['id'] = message_id
+        msg['message-id'] = message_id
+
+        def _decode_header(txt):
+            txt = txt.replace('\r', '')
+            return ' '.join(map(lambda (x, y): unicode(x, y or 'ascii'), decode_header(txt)))
+
+        if 'Subject' in fields:
+            msg['subject'] = _decode_header(msg_txt.get('Subject'))
+
+        if 'Content-Type' in fields:
+            msg['content-type'] = msg_txt.get('Content-Type')
+
+        if 'From' in fields:
+            msg['from'] = _decode_header(msg_txt.get('From'))
+
+        if 'Delivered-To' in fields:
+            msg['to'] = _decode_header(msg_txt.get('Delivered-To'))
+
+        if 'Cc' in fields:
+            msg['cc'] = _decode_header(msg_txt.get('Cc'))
+
+        if 'Reply-To' in fields:
+            msg['reply'] = _decode_header(msg_txt.get('Reply-To'))
+
+        if 'Date' in fields:
+            msg['date'] = msg_txt.get('Date')
+
+        if 'Content-Transfer-Encoding' in fields:
+            msg['encoding'] = msg_txt.get('Content-Transfer-Encoding')
+
+        if 'References' in fields:
+            msg['references'] = msg_txt.get('References')
+
+        if 'X-openerp-caseid' in fields:
+            msg['caseid'] = msg_txt.get('X-openerp-caseid')
+
+        if 'X-Priority' in fields:
+            msg['priority'] = msg_txt.get('X-priority', '3 (Normal)').split(' ')[0]
+
+        if not msg_txt.is_multipart() or 'text/plain' in msg.get('content-type', None):
+            encoding = msg_txt.get_content_charset()
+            msg['body'] = msg_txt.get_payload(decode=True)
+            if encoding:
+                msg['body'] = msg['body'].decode(encoding).encode('utf-8')
+
+        attachments = {}
+        if msg_txt.is_multipart() or 'multipart/alternative' in msg.get('content-type', None):
+            body = ""
+            counter = 1
+            for part in msg_txt.walk():
+                if part.get_content_maintype() == 'multipart':
+                    continue
+
+                encoding = part.get_content_charset()
+
+                if part.get_content_maintype()=='text':
+                    content = part.get_payload(decode=True)
+                    filename = part.get_filename()
+                    if filename :
+                        attachments[filename] = content
+                    else:
+                        if encoding:
+                            content = unicode(content, encoding)
+                        if part.get_content_subtype() == 'html':
+                            body = tools.html2plaintext(content)
+                        elif part.get_content_subtype() == 'plain':
+                            body = content
+                elif part.get_content_maintype()=='application' or part.get_content_maintype()=='image' or part.get_content_maintype()=='text':
+                    filename = part.get_filename();
+                    if filename :
+                        attachments[filename] = part.get_payload(decode=True)
+                    else:
+                        res = part.get_payload(decode=True)
+                        if encoding:
+                            res = res.decode(encoding).encode('utf-8')
+
+                        body += res
+
+            msg['body'] = body
+            msg['attachments'] = attachments
+
+        if msg.get('references', False):
+            id = False
+            ref = msg.get('references')
+            if '\r\n' in ref:
+                ref = msg.get('references').split('\r\n')
+            else:
+                ref = msg.get('references').split(' ')
+            if ref:
+                hids = history_pool.search(cr, uid, [('name', '=', ref[0].strip())])
+                if hids:
+                    id = hids[0]
+                    history = history_pool.browse(cr, uid, id)
+                    model_pool = self.pool.get(model)
+                    context.update({
+                        'references_id':ref[0]
+                    })
+                    vals = {}
+                    if hasattr(model_pool, 'message_update'):
+                        model_pool.message_update(cr, uid, [history.res_id], vals, msg, context=context)
+                    else:
+                        logger.notifyChannel('imap', netsvc.LOG_WARNING, 'method def message_update is not define in model %s' % (model_pool._name))
+                        return False
+                else:
+                    res_id = create_record(msg)
+
+        else:
+            res_id = create_record(msg)
+
+        return res_id
+
+    def get_partner(self, cr, uid, from_email, context=None):
+        """This function returns partner Id based on email passed
+        @param self: The object pointer
+        @param cr: the current row, from the database cursor,
+        @param uid: the current user’s ID for security checks
+        @param from_email: email address based on that function will search for the correct
+        """
+        res = {
+            'partner_address_id': False, 
+            'partner_id': False
+        }
+        from_email = self.to_email(cr, uid, from_email)
+        address_ids = self.pool.get('res.partner.address').search(cr, uid, [('email', '=', from_email)])
+        if address_ids:
+            address = self.pool.get('res.partner.address').browse(cr, uid, address_ids[0])
+            res['partner_address_id'] = address_ids[0]
+            res['partner_id'] = address.partner_id.id
+
+        return res
+
+mailgate_tool()
+
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
diff --git a/addons/mail_gateway/scripts/openerp_mailgate/openerp_mailgate.py b/addons/mail_gateway/scripts/openerp_mailgate/openerp_mailgate.py
index de49ddfaccb..f77fccadaee 100755
--- a/addons/mail_gateway/scripts/openerp_mailgate/openerp_mailgate.py
+++ b/addons/mail_gateway/scripts/openerp_mailgate/openerp_mailgate.py
@@ -20,15 +20,12 @@
 #
 ###########################################################################################
 
+from email.header import decode_header
+import email as EMAIL
 import re
 import smtplib
-import email, mimetypes
-from email.Header import decode_header
-from email.MIMEText import MIMEText
-import xmlrpclib
-import os
-import binascii
 import time, socket
+import xmlrpclib
 
 
 email_re = re.compile(r"([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6})")
@@ -44,84 +41,9 @@ priorities = {
     '5': '5 (Lowest)', 
 }
 
-def html2plaintext(html, body_id=None, encoding='utf-8'):
-    ## (c) Fry-IT, www.fry-it.com, 2007
-    ## <peter@fry-it.com>
-    ## download here: http://www.peterbe.com/plog/html2plaintext
-    
-    
-    """ from an HTML text, convert the HTML to plain text.
-    If @body_id is provided then this is the tag where the 
-    body (not necessarily <body>) starts.
-    """
-    try:
-        from BeautifulSoup import BeautifulSoup, SoupStrainer, Comment
-    except:
-        return html
-            
-    urls = []
-    if body_id is not None:
-        strainer = SoupStrainer(id=body_id)
-    else:
-        strainer = SoupStrainer('body')
-    
-    soup = BeautifulSoup(html, parseOnlyThese=strainer, fromEncoding=encoding)
-    for link in soup.findAll('a'):
-        title = unicode(link)
-        for url in [x[1] for x in link.attrs if x[0]=='href']:
-            urls.append(dict(url=url, tag=unicode(link), title=title))
-
-    html = unicode(soup)
-            
-    url_index = []
-    i = 0
-    for d in urls:
-        if d['title'] == d['url'] or 'http://'+d['title'] == d['url']:
-            html = html.replace(d['tag'], d['url'])
-        else:
-            i += 1
-            html = html.replace(d['tag'], '%s [%s]' % (d['title'], i))
-            url_index.append(d['url'])
-
-    html = html.replace('<strong>', '*').replace('</strong>', '*')
-    html = html.replace('<b>', '*').replace('</b>', '*')
-    html = html.replace('<h3>', '*').replace('</h3>', '*')
-    html = html.replace('<h2>', '**').replace('</h2>', '**')
-    html = html.replace('<h1>', '**').replace('</h1>', '**')
-    html = html.replace('<em>', '/').replace('</em>', '/')
-    
-
-    # the only line breaks we respect is those of ending tags and 
-    # breaks
-    
-    html = html.replace('\n', ' ')
-    html = html.replace('<br>', '\n')
-    html = html.replace('<tr>', '\n')
-    html = html.replace('</p>', '\n\n')
-    html = re.sub('<br\s*/>', '\n', html)
-    html = html.replace(' ' * 2, ' ')
-
-
-    # for all other tags we failed to clean up, just remove then and 
-    # complain about them on the stderr
-    def desperate_fixer(g):
-        #print >>sys.stderr, "failed to clean up %s" % str(g.group())
-        return ' '
-
-    html = re.sub('<.*?>', desperate_fixer, html)
-
-    # lstrip all lines
-    html = '\n'.join([x.lstrip() for x in html.splitlines()])
-
-    for i, url in enumerate(url_index):
-        if i == 0:
-            html += '\n\n'
-        html += '[%s] %s\n' % (i+1, url)       
-    return html
-
 class rpc_proxy(object):
-    def __init__(self, uid, passwd, host='localhost', port=8069, path='object', dbname='terp'):        
-        self.rpc = xmlrpclib.ServerProxy('http://%s:%s/xmlrpc/%s' % (host, port, path))
+    def __init__(self, uid, passwd, host='localhost', port=8069, path='object', dbname='terp'):
+        self.rpc = xmlrpclib.ServerProxy('http://%s:%s/xmlrpc/%s' % (host, port, path), allow_none=True)
         self.user_id = uid
         self.passwd = passwd
         self.dbname = dbname
@@ -149,162 +71,32 @@ class email_parser(object):
         
         
     def email_get(self, email_from):
-        res = email_re.search(email_from)
-        return res and res.group(1)
-
-    def partner_get(self, email):
-        mail = self.email_get(email)
-        adr_ids = self.rpc('res.partner.address', 'search', [('email', '=', mail)])
-        if not adr_ids:
-            return {}
-        adr = self.rpc('res.partner.address', 'read', adr_ids, ['partner_id'])
-        return {
-            'partner_address_id': adr[0]['id'], 
-            'partner_id': adr[0].get('partner_id', False) and adr[0]['partner_id'][0] or False
-        }
+        return self.rpc('email.server.tools', 'to_email', email_from)
 
     def _to_decode(self, s, charsets):
-       for charset in charsets:
-           if charset:
-               try:
-                   return s.decode(charset)
-               except UnicodeError:
+        for charset in charsets:
+            if charset:
+                try:
+                    return s.decode(charset)
+                except UnicodeError:
                     pass
-       return s.decode('latin1')
+        return s.decode('latin1')
 
     def _decode_header(self, s):
-        from email.Header import decode_header
-        s = decode_header(s.replace('\r', '')) 
+        if s:
+            s = decode_header(s.replace('\r', '')) 
         return ''.join(map(lambda x:self._to_decode(x[0], [x[1]]), s or []))
-    
-    def history(self, model, new_id, msg_id, ref_id, subject, msg_to, msg_from, body, attach):
-        try:
-            thread_id = self.rpc(model, 'read', new_id, ['thread_id'])['thread_id'][0]
-        except Exception, e:
-            thread_id = None
-        msg_data = {
-                    'name': subject, 
-                    'history': True, 
-                    'model': model, 
-                    'res_id': new_id, 
-                    'thread_id': thread_id, 
-                    'message_id': msg_id, 
-                    'ref_id': ref_id or '', 
-                    'user_id': self.rpc.user_id, 
-                    'date': time.strftime('%Y-%m-%d %H:%M:%S'), 
-                    'email_from': msg_from, 
-                    'email_to': msg_to, 
-                    'description': body, 
-                    'attachment_ids': [(6, 0, attach)]
-        }
-        msg_id = self.rpc('mailgate.message', 'create', msg_data)
-        return True
-    
-    def msg_new(self, msg):
-        message = self.msg_body_get(msg)
-        msg_subject = self._decode_header(msg['Subject'])
-        msg_from = self._decode_header(msg['From'])
-        msg_to = self._decode_header(msg['To'])
-        msg_cc = self._decode_header(msg['Cc'] or '')
-        
-        data = {
-            'name': msg_subject, 
-            'email_from': msg_from, 
-            'email_cc': msg_cc,             
-            'user_id': False, 
-            'description': message['body'], 
-            'state' : 'draft',
-        }
-        data.update(self.partner_get(msg_from))
-
-        try:
-            att_ids = []
-            new_id = self.rpc(self.model, 'create', data)
-            attachments = message['attachment']        
-            for attach in attachments or []:
-                data_attach = {
-                    'name': str(attach), 
-                    'datas': binascii.b2a_base64(str(attachments[attach])), 
-                    'datas_fname': str(attach), 
-                    'description': 'Mail attachment', 
-                    'res_model': self.model, 
-                    'res_id': new_id
-                }
-                att_ids.append(self.rpc('ir.attachment', 'create', data_attach))
-            try:
-                self.rpc(self.model, 'history', [new_id], 'Receive', True, msg_to, message['body'], msg_from, False, {'model' : self.model})
-            except Exception, e:
-                self.history(self.model, new_id, msg['Message-Id'], msg['References'], msg_subject, msg_to, msg_from, message['body'], att_ids)
-        except Exception, e:
-            if getattr(e, 'faultCode', '') and 'AccessError' in e.faultCode:
-                e = '\n\nThe Specified user does not have an access to the Model.'
-            print e
-        
-
-        return new_id
-
-#   #change the return type format to dictionary
-#   {
-#       'body':'body part',
-#       'attachment':{
-#                       'file_name':'file data',
-#                       'file_name':'file data',
-#                       'file_name':'file data',
-#                   }
-#   }
-
-    def msg_body_get(self, msg):
-        message = {};
-        message['body'] = '';
-        message['attachment'] = {};
-        attachment = message['attachment'];
-        counter = 1;
-        def replace(match):
-            return ''
-
-        for part in msg.walk():
-            if part.get_content_maintype() == 'multipart':
-                continue
-
-            if part.get_content_maintype()=='text':
-                buf = part.get_payload(decode=True)
-                if buf:
-                    txt = self._to_decode(buf, part.get_charsets())
-                    txt = re.sub("<(\w)>", replace, txt)
-                    txt = re.sub("<\/(\w)>", replace, txt)
-                if txt and part.get_content_subtype() == 'plain':
-                    message['body'] += txt 
-                elif txt and part.get_content_subtype() == 'html':                                                               
-                    message['body'] += html2plaintext(txt)  
-                
-                filename = part.get_filename();
-                if filename :
-                    attachment[filename] = part.get_payload(decode=True);
-                    
-            elif part.get_content_maintype()=='application' or part.get_content_maintype()=='image' or part.get_content_maintype()=='text':
-                filename = part.get_filename();
-                if filename :
-                    attachment[filename] = part.get_payload(decode=True);
-                else:
-                    filename = 'attach_file'+str(counter);
-                    counter += 1;
-                    attachment[filename] = part.get_payload(decode=True);
-                #end if
-            #end if
-            message['attachment'] = attachment
-        #end for        
-        return message
 
     def msg_send(self, msg, emails, priority=None):
         if not len(emails):
             return False
         del msg['To']
         msg['To'] = emails[0]
-        msg['Subject'] = 'OpenERP Record-id:' + msg['Subject']
         if len(emails)>1:
             if 'Cc' in msg:
                 del msg['Cc']
             msg['Cc'] = ','.join(emails[1:])
+
         del msg['Reply-To']
         msg['Reply-To'] = self.email
         if self.smtp_user and self.smtp_password:
@@ -319,17 +111,25 @@ class email_parser(object):
         return False
 
 
-    def parse(self, msg):
-        #TODO: Something with Message hierarchy 
-        res_id = self.msg_new(msg)
-        subject = self._decode_header(msg['subject'])
+    def parse(self, message):
+        try:
+            res_id = self.rpc('email.server.tools', 'process_email', self.model, message)
+        except Exception, e:
+            res_id = False
+
+        msg = EMAIL.message_from_string(str(message))
+        subject = self._decode_header(msg['Subject'])
         if msg.get('Subject', ''):
             del msg['Subject']
-        msg['Subject'] = '['+str(res_id)+'] '+subject
-#        msg['Message-Id'] = '<'+str(time.time())+'-openerpcrm-'+str(res_id)+'@'+socket.gethostname()+'>'
+
+       #Changed for sending reply mail
+        if res_id:
+            msg['Subject'] = '['+str(res_id)+'] '+subject
+            msg['Message-Id'] = '<'+str(time.time())+'-openerpcrm-'+str(res_id)+'@'+socket.gethostname()+'>'
 
         mm = [self._decode_header(msg['From']), self._decode_header(msg['To'])]+self._decode_header(msg.get('Cc', '')).split(',')
         msg_mails = map(self.email_get, filter(None, mm))
+
         try:
             self.msg_send(msg, msg_mails)
         except Exception, e:
@@ -364,7 +164,7 @@ if __name__ == '__main__':
     (options, args) = parser.parse_args()
     parser = email_parser(options.userid, options.password, options.model, options.email, options.default, dbname=options.dbname, host=options.host, port=options.port, smtp_server=options.smtp_server, smtp_port=options.smtp_port, smtp_ssl=options.smtp_ssl, smtp_user=options.smtp_user, smtp_password=options.smtp_password)
 
-    msg_txt = email.message_from_file(sys.stdin)
+    msg_txt = sys.stdin.read().decode('utf8')
 
     parser.parse(msg_txt)