[IMP] ir.attachment : move the fields index_content (from document), mimetype and datas_checksum (from website) to base. In base, the mimetype is computed with the Magic python lib (if installed, else with a custom function using magic number), and the index_content is an implementation of strings unix command. In document, index_content is still computed with the indexer of document module.

addons/document/document.py

@@ -47,6 +47,15 @@ _logger = logging.getLogger(__name__)
 class document_file(osv.osv):
     _inherit = 'ir.attachment'
 
+    def _index(self, cr, uid, bin_data, datas_fname, mimetype):
+        """
+            Override the parent method to use the indexer of document module to find the index_content.
+            It is called from _set_data, when the data changed.
+        """
+        mime, icont = cntIndex.doIndex(bin_data, datas_fname,  mimetype or None, None)
+        icont_u = ustr(icont)
+        return icont_u
+
     _columns = {
         # Columns from ir.attachment:
         'write_date': fields.datetime('Date Modified', readonly=True),
@@ -54,9 +63,7 @@ class document_file(osv.osv):
         # Fields of document:
         'user_id': fields.many2one('res.users', 'Owner', select=1),
         'parent_id': fields.many2one('document.directory', 'Directory', select=1, change_default=True),
-        'index_content': fields.text('Indexed Content'),
         'partner_id':fields.many2one('res.partner', 'Partner', select=1),
-        'file_type': fields.char('Content Type'),
     }
     _order = "id desc"
 
@@ -74,7 +81,7 @@ class document_file(osv.osv):
             ids = [ids]
 
         super(document_file, self).check(cr, uid, ids, mode, context=context, values=values)
-        
+
         if ids:
             self.pool.get('ir.model.access').check(cr, uid, 'document.directory', mode)
 
@@ -124,22 +131,13 @@ class document_file(osv.osv):
         # take partner from uid
         if vals.get('res_id', False) and vals.get('res_model', False) and not vals.get('partner_id', False):
             vals['partner_id'] = self.__get_partner_id(cr, uid, vals['res_model'], vals['res_id'], context)
-        if vals.get('datas', False):
-            vals['file_type'], vals['index_content'] = self._index(cr, uid, vals['datas'].decode('base64'), vals.get('datas_fname', False), None)
         return super(document_file, self).create(cr, uid, vals, context)
 
     def write(self, cr, uid, ids, vals, context=None):
         if context is None:
             context = {}
-        if vals.get('datas', False):
-            vals['file_type'], vals['index_content'] = self._index(cr, uid, vals['datas'].decode('base64'), vals.get('datas_fname', False), None)
         return super(document_file, self).write(cr, uid, ids, vals, context)
 
-    def _index(self, cr, uid, data, datas_fname, file_type):
-        mime, icont = cntIndex.doIndex(data, datas_fname,  file_type or None, None)
-        icont_u = ustr(icont)
-        return mime, icont_u
-
     def __get_partner_id(self, cr, uid, res_model, res_id, context=None):
         """ A helper to retrieve the associated partner from any res_model+id
             It is a hack that will try to discover if the mentioned record is
@@ -395,7 +393,7 @@ class document_directory_content(osv.osv):
         'suffix': fields.char('Suffix', size=16),
         'report_id': fields.many2one('ir.actions.report.xml', 'Report'),
         'extension': fields.selection(_extension_get, 'Document Type', required=True, size=4),
-        'include_name': fields.boolean('Include Record Name', 
+        'include_name': fields.boolean('Include Record Name',
                 help="Check this field if you want that the name of the file to contain the record name." \
                     "\nIf set, the directory will have to be a resource one."),
         'directory_id': fields.many2one('document.directory', 'Directory'),
@@ -405,17 +403,17 @@ class document_directory_content(osv.osv):
         'sequence': lambda *args: 1,
         'include_name': lambda *args: 1,
     }
-    
+
     def _file_get(self, cr, node, nodename, content, context=None):
         """ return the nodes of a <node> parent having a <content> content
             The return value MUST be false or a list of node_class objects.
         """
-    
+
         # TODO: respect the context!
         model = node.res_model
         if content.include_name and not model:
             return False
-        
+
         res2 = []
         tname = ''
         if content.include_name:
@@ -447,7 +445,7 @@ class document_directory_content(osv.osv):
         if node.extension != '.pdf':
             raise Exception("Invalid content: %s" % node.extension)
         return True
-    
+
     def process_read(self, cr, uid, node, context=None):
         if node.extension != '.pdf':
             raise Exception("Invalid content: %s" % node.extension)
@@ -546,11 +544,11 @@ class document_storage(osv.osv):
         # 2nd phase: store the metadata
         try:
             icont = ''
-            mime = ira.file_type
+            mime = ira.mimetype
             if not mime:
                 mime = ""
             try:
-                mime, icont = cntIndex.doIndex(data, ira.datas_fname, ira.file_type or None, fname)
+                mime, icont = cntIndex.doIndex(data, ira.datas_fname, ira.mimetype or None, fname)
             except Exception:
                 _logger.debug('Cannot index file.', exc_info=True)
                 pass
@@ -561,8 +559,8 @@ class document_storage(osv.osv):
             # a hack: /assume/ that the calling write operation will not try
             # to write the fname and size, and update them in the db concurrently.
             # We cannot use a write() here, because we are already in one.
-            cr.execute('UPDATE ir_attachment SET file_size = %s, index_content = %s, file_type = %s WHERE id = %s', (filesize, icont_u, mime, file_node.file_id))
-            self.pool.get('ir.attachment').invalidate_cache(cr, uid, ['file_size', 'index_content', 'file_type'], [file_node.file_id], context=context)
+            cr.execute('UPDATE ir_attachment SET file_size = %s, index_content = %s, mimetype = %s WHERE id = %s', (filesize, icont_u, mime, file_node.file_id))
+            self.pool.get('ir.attachment').invalidate_cache(cr, uid, ['file_size', 'index_content', 'mimetype'], [file_node.file_id], context=context)
             file_node.content_length = filesize
             file_node.content_type = mime
             return True
@@ -1650,8 +1648,8 @@ class node_file(node_class):
         super(node_file,self).__init__(path, parent,context)
         self.file_id = fil.id
         #todo: more info from ir_attachment
-        if fil.file_type and '/' in fil.file_type:
-            self.mimetype = str(fil.file_type)
+        if fil.mimetype and '/' in fil.mimetype:
+            self.mimetype = str(fil.mimetype)
         self.create_date = fil.create_date
         self.write_date = fil.write_date or fil.create_date
         self.content_length = fil.file_size

addons/document/document_view.xml

@@ -179,15 +179,10 @@
             <xpath expr="//div[@name='creation_div']" position='after'>
                 <label for="write_uid" string="Modification"/>
                 <div>
-                    <field name="write_uid" readonly="1" class="oe_inline"/> on 
+                    <field name="write_uid" readonly="1" class="oe_inline"/> on
                     <field name="write_date" readonly="1" class="oe_inline"/>
                 </div>
             </xpath>
-            <xpath expr="//group[@name='description_group']" position='after'>
-                <group string="Indexed Content" colspan="4">
-                    <field name="index_content" nolabel="1"/>
-                </group>
-            </xpath>
         </field>
     </record>
 
@@ -246,7 +241,7 @@
         <field name="view_mode">kanban,tree,form</field>
         <field name="help" type="html">
           <p class="oe_view_nocontent_create">
-            Click to create a new document. 
+            Click to create a new document.
           </p><p>
             The Documents repository gives you access to all attachments, such
             as mails, project documents, invoices etc.

addons/website/models/website.py

@@ -698,8 +698,11 @@ class website_menu(osv.osv):
             self.write(cr, uid, [menu['id']], menu, context=context)
         return True
 
+
 class ir_attachment(osv.osv):
+
     _inherit = "ir.attachment"
+
     def _website_url_get(self, cr, uid, ids, name, arg, context=None):
         result = {}
         for attach in self.browse(cr, uid, ids, context=context):
@@ -708,21 +711,6 @@ class ir_attachment(osv.osv):
             else:
                 result[attach.id] = self.pool['website'].image_url(cr, uid, attach, 'datas')
         return result
-    def _datas_checksum(self, cr, uid, ids, name, arg, context=None):
-        result = dict.fromkeys(ids, False)
-        attachments = self.read(cr, uid, ids, ['res_model'], context=context)
-        view_attachment_ids = [attachment['id'] for attachment in attachments if attachment['res_model'] == 'ir.ui.view']
-        for attach in self.read(cr, uid, view_attachment_ids, ['res_model', 'res_id', 'type', 'datas'], context=context):
-            result[attach['id']] = self._compute_checksum(attach)
-        return result
-
-    def _compute_checksum(self, attachment_dict):
-        if attachment_dict.get('res_model') == 'ir.ui.view'\
-                and not attachment_dict.get('res_id') and not attachment_dict.get('url')\
-                and attachment_dict.get('type', 'binary') == 'binary'\
-                and attachment_dict.get('datas'):
-            return hashlib.new('sha1', attachment_dict['datas']).hexdigest()
-        return None
 
     def _datas_big(self, cr, uid, ids, name, arg, context=None):
         result = dict.fromkeys(ids, False)
@@ -739,32 +727,11 @@ class ir_attachment(osv.osv):
         return result
 
     _columns = {
-        'datas_checksum': fields.function(_datas_checksum, size=40,
-              string="Datas checksum", type='char', store=True, select=True),
         'website_url': fields.function(_website_url_get, string="Attachment URL", type='char'),
         'datas_big': fields.function (_datas_big, type='binary', store=True,
                                       string="Resized file content"),
-        'mimetype': fields.char('Mime Type', readonly=True),
     }
 
-    def _add_mimetype_if_needed(self, values):
-        if values.get('datas_fname'):
-            values['mimetype'] = mimetypes.guess_type(values.get('datas_fname'))[0] or 'application/octet-stream'
-
-    def create(self, cr, uid, values, context=None):
-        chk = self._compute_checksum(values)
-        if chk:
-            match = self.search(cr, uid, [('datas_checksum', '=', chk)], context=context)
-            if match:
-                return match[0]
-        self._add_mimetype_if_needed(values)
-        return super(ir_attachment, self).create(
-            cr, uid, values, context=context)
-
-    def write(self, cr, uid, ids, values, context=None):
-        self._add_mimetype_if_needed(values)
-        return super(ir_attachment, self).write(cr, uid, ids, values, context=context)
-
     def try_remove(self, cr, uid, ids, context=None):
         """ Removes a web-based image attachment if it is used by no view
         (template)

openerp/addons/base/ir/ir_attachment.py

@@ -32,20 +32,56 @@ from openerp.osv import fields,osv
 from openerp import SUPERUSER_ID
 from openerp.osv.orm import except_orm
 from openerp.tools.translate import _
+from openerp.tools.misc import ustr
 
 _logger = logging.getLogger(__name__)
 
+
+# optional python-magic import (https://github.com/ahupp/python-magic)
+try:
+    import magic
+except ImportError:
+    magic = None
+
+def guess_mimetype(bin_data):
+    if magic:
+        # if binary not identified or error, will return 'application/octet-stream'
+        return magic.from_buffer(bin_data, mime=True)
+    else:
+        # else, guess the type using the magic number of file hex signature (like magic, but more limited)
+        # see http://www.filesignatures.net/ for file signatures
+        mapping = {
+            # pdf
+            'application/pdf' : ['%PDF'],
+             # jpg, jpeg, png, gif
+            'image/jpeg' : ['\xFF\xD8\xFF\xE0', '\xFF\xD8\xFF\xE2', '\xFF\xD8\xFF\xE3', '\xFF\xD8\xFF\xE1'],
+            'image/png' : ['\x89\x50\x4E\x47\x0D\x0A\x1A\x0A'],
+            'image/gif' : ['GIF87a', '\x47\x49\x46\x38\x37\x61', 'GIF89a', '\x47\x49\x46\x38\x39\x61'],
+            # zip, but will include jar, odt, ods, odp, docx, xlsx, pptx, apk
+            'application/zip' : ['PK'],
+            # doc, xls
+            'application/msword' : ['\xCF\x11\xE0\xA1\xB1\x1A\xE1\x00', '\xEC\xA5\xC1\x00', '\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x0D\x44\x4F\x43'],
+            'application/vnd.ms-excel' : ['\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1','\x09\x08\x10\x00\x00\x06\x05\x00','\xFD\xFF\xFF\xFF\x10','\xFD\xFF\xFF\xFF\x1F','\xFD\xFF\xFF\xFF\x23','\xFD\xFF\xFF\xFF\x28','\xFD\xFF\xFF\xFF\x29'],
+        }
+        for mimetype in mapping.keys():
+            for signature in mapping[mimetype]:
+                if bin_data.startswith(signature):
+                    return mimetype
+    return 'application/octet-stream'
+
+
+
 class ir_attachment(osv.osv):
     """Attachments are used to link binary files or url to any openerp document.
 
     External attachment storage
     ---------------------------
-    
+
     The 'data' function field (_data_get,data_set) is implemented using
     _file_read, _file_write and _file_delete which can be overridden to
     implement other storage engines, shuch methods should check for other
     location pseudo uri (example: hdfs://hadoppserver)
-    
+
     The default implementation is the file:dirname location that stores files
     on the local filesystem using name based on their sha1 hash
     """
@@ -62,7 +98,7 @@ class ir_attachment(osv.osv):
                 if res_name:
                     field = self._columns.get('res_name',False)
                     if field and len(res_name) > field.size:
-                        res_name = res_name[:30] + '...' 
+                        res_name = res_name[:30] + '...'
                 data[attachment.id] = res_name or False
             else:
                 data[attachment.id] = False
@@ -98,9 +134,7 @@ class ir_attachment(osv.osv):
         path = path.strip('/\\')
         return os.path.join(self._filestore(cr, uid), path)
 
-    def _get_path(self, cr, uid, bin_data):
-        sha = hashlib.sha1(bin_data).hexdigest()
-
+    def _get_path(self, cr, uid, bin_data, sha):
         # retro compatibility
         fname = sha[:3] + '/' + sha
         full_path = self._full_path(cr, uid, fname)
@@ -128,9 +162,9 @@ class ir_attachment(osv.osv):
             _logger.exception("_read_file reading %s", full_path)
         return r
 
-    def _file_write(self, cr, uid, value):
+    def _file_write(self, cr, uid, value, checksum):
         bin_value = value.decode('base64')
-        fname, full_path = self._get_path(cr, uid, bin_value)
+        fname, full_path = self._get_path(cr, uid, bin_value, checksum)
         if not os.path.exists(full_path):
             try:
                 with open(full_path, 'wb') as fp:
@@ -166,19 +200,39 @@ class ir_attachment(osv.osv):
     def _data_set(self, cr, uid, id, name, value, arg, context=None):
         # We dont handle setting data to null
         if not value:
+            # reset computed fields
+            super(ir_attachment, self).write(cr, SUPERUSER_ID, [id], {'file_size' : 0, 'checksum' : None, 'mimetype' : None, 'index_content' : None}, context=context)
             return True
         if context is None:
             context = {}
-        location = self._storage(cr, uid, context)
-        file_size = len(value.decode('base64'))
+        # browse the attachment and get the file to delete
         attach = self.browse(cr, uid, id, context=context)
         fname_to_delete = attach.store_fname
+        location = self._storage(cr, uid, context)
+        # compute the field depending of datas
+        bin_data = value.decode('base64')
+        mimetype = self._compute_mimetype(bin_data)
+        checksum = self._compute_checksum(bin_data)
+        vals = {
+            'file_size': len(bin_data),
+            'checksum' : checksum,
+            'mimetype' : mimetype,
+            'index_content' : self._index(cr, SUPERUSER_ID, bin_data, attach.datas_fname, mimetype),
+        }
         if location != 'db':
-            fname = self._file_write(cr, uid, value)
-            # SUPERUSER_ID as probably don't have write access, trigger during create
-            super(ir_attachment, self).write(cr, SUPERUSER_ID, [id], {'store_fname': fname, 'file_size': file_size, 'db_datas': False}, context=context)
+            # create the file
+            fname = self._file_write(cr, uid, value, checksum)
+            vals.update({
+                'store_fname': fname,
+                'db_datas': False
+            })
         else:
-            super(ir_attachment, self).write(cr, SUPERUSER_ID, [id], {'db_datas': value, 'file_size': file_size, 'store_fname': False}, context=context)
+            vals.update({
+                'store_fname': False,
+                'db_datas': value
+            })
+        # SUPERUSER_ID as probably don't have write access, trigger during create
+        super(ir_attachment, self).write(cr, SUPERUSER_ID, [id], vals, context=context)
 
         # After de-referencing the file in the database, check whether we need
         # to garbage-collect it on the filesystem
@@ -186,6 +240,36 @@ class ir_attachment(osv.osv):
             self._file_delete(cr, uid, fname_to_delete)
         return True
 
+    def _compute_checksum(self, bin_data):
+        """ compute the checksum for the given datas
+            :param bin_data : datas in its binary form
+        """
+        if bin_data:
+            return hashlib.sha1(bin_data).hexdigest()
+        return False
+
+    def _compute_mimetype(self, bin_data):
+        """ compute the mimetype of the given datas
+            :param bin_data : binary data
+            :return mime : string indicating the mimetype, or application/octet-stream by default
+        """
+        return guess_mimetype(bin_data)
+
+    def _index(self, cr, uid, bin_data, datas_fname, file_type):
+        """ compute the index content of the given filename, or binary data.
+            This is a python implementation of the unix command 'strings'.
+            :param bin_data : datas in binary form
+            :return index_content : string containing all the printable character of the binary data
+        """
+        index_content = False
+        if file_type:
+            index_content = file_type.split('/')[0]
+            if index_content == 'text': # compute index_content only for text type
+                words = re.findall("[^\x00-\x1F\x7F-\xFF]{4,}", bin_data)
+                index_content = ustr("\n".join(words))
+        return index_content
+
+
     _name = 'ir.attachment'
     _columns = {
         'name': fields.char('Attachment Name', required=True),
@@ -204,12 +288,17 @@ class ir_attachment(osv.osv):
         'datas': fields.function(_data_get, fnct_inv=_data_set, string='File Content', type="binary", nodrop=True),
         'store_fname': fields.char('Stored Filename'),
         'db_datas': fields.binary('Database Data'),
-        'file_size': fields.integer('File Size'),
+        # computed fields depending on datas
+        'file_size': fields.integer('File Size', readonly=True),
+        'checksum': fields.char("Checksum/SHA1", size=40, select=True, readonly=True),
+        'mimetype': fields.char('Mime Type', readonly=True),
+        'index_content': fields.text('Indexed Content', readonly=True),
     }
 
     _defaults = {
         'type': 'binary',
         'file_size': 0,
+        'mimetype' : False,
         'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c),
     }
 
@@ -319,8 +408,9 @@ class ir_attachment(osv.osv):
         if isinstance(ids, (int, long)):
             ids = [ids]
         self.check(cr, uid, ids, 'write', context=context, values=vals)
-        if 'file_size' in vals:
-            del vals['file_size']
+        # remove computed field depending of datas
+        for field in ['file_size', 'checksum', 'mimetype']:
+            vals.pop(field, False)
         return super(ir_attachment, self).write(cr, uid, ids, vals, context)
 
     def copy(self, cr, uid, id, default=None, context=None):
@@ -346,9 +436,10 @@ class ir_attachment(osv.osv):
         return res
 
     def create(self, cr, uid, values, context=None):
+        # remove computed field depending of datas
+        for field in ['file_size', 'checksum', 'mimetype']:
+            values.pop(field, False)
         self.check(cr, uid, [], mode='write', context=context, values=values)
-        if 'file_size' in values:
-            del values['file_size']
         return super(ir_attachment, self).create(cr, uid, values, context)
 
     def action_get(self, cr, uid, context=None):

openerp/addons/base/ir/ir_attachment_view.xml

@@ -17,6 +17,7 @@
                             <field name="datas" filename="datas_fname" attrs="{'invisible':[('type','=','url')]}"/>
                             <field name="datas_fname" invisible="1" attrs="{'invisible':[('type','=','url')]}" class="oe_inline oe_right"/>
                             <field name="url" widget="url" attrs="{'invisible':[('type','=','binary')]}"/>
+                            <field name="mimetype"/>
                         </group>
                         <group string="Attached To">
                             <field name="res_model"/>
@@ -27,13 +28,16 @@
                         <group groups="base.group_no_one" string="History">
                             <label for="create_uid" string="Creation"/>
                             <div name="creation_div">
-                                <field name="create_uid" readonly="1" class="oe_inline"/> on 
+                                <field name="create_uid" readonly="1" class="oe_inline"/> on
                                 <field name="create_date" readonly="1" class="oe_inline"/>
                             </div>
                         </group>
                         <group name="description_group" string="Description" colspan="4">
                             <field name="description" nolabel="1"/>
                         </group>
+                        <group string="Indexed Content" colspan="4">
+                            <field name="index_content" nolabel="1"/>
+                        </group>
                     </group>
                   </sheet>
                 </form>
@@ -67,7 +71,7 @@
                         string="Binary"
                         domain="[('type','=','binary')]"/>
                     <separator/>
-                    <filter name="my_documents_filter" 
+                    <filter name="my_documents_filter"
                         string="My Document(s)"
                         icon="terp-personal"
                         domain="[('create_uid','=',uid)]"