[IMP] move guess_mimetype to tools and improve it

* move guess_mimetype from ir_attachment to a misc submodule so it can
  be used outside of attachments (in base_import in this case)
* fix the broken detection of zip and OpenDocument files (would all be
  detected as msword files)
* correctly implement detection of OOXML files (including returning the
  right MIME type)
* attempt to correctly detect OLECF Excel files (.xls)
* add PPT detection (untested)
* remove duplicate patterns in GIF detection

openerp/addons/base/ir/ir_attachment.py

@@ -8,61 +8,15 @@ import mimetypes
 import os
 import re
 
-from openerp import tools
-from openerp.tools.translate import _
+from openerp import SUPERUSER_ID, tools
 from openerp.exceptions import AccessError
 from openerp.osv import fields,osv
-from openerp import SUPERUSER_ID
-from openerp.exceptions import UserError
 from openerp.tools.translate import _
 from openerp.tools.misc import ustr
+from openerp.tools.mimetypes import guess_mimetype
 
 _logger = logging.getLogger(__name__)
 
-try:
-    import magic
-except ImportError:
-    magic = None
-
-# We define our own guess_mimetype implementation and if magic is available we
-# use it instead.
-
-# However there are 2 python libs named 'magic' with incompatible api.
-# magic from pypi https://pypi.python.org/pypi/python-magic/
-# magic from file(1) https://packages.debian.org/squeeze/python-magic
-
-def guess_mimetype(bin_data):
-    # by default, guess the type using the magic number of file hex signature (like magic, but more limited)
-    # see http://www.filesignatures.net/ for file signatures
-    mapping = (
-        # pdf
-        ('application/pdf', ['%PDF']),
-        # jpg, jpeg, png, gif
-        ('image/jpeg', ['\xFF\xD8\xFF\xE0', '\xFF\xD8\xFF\xE2', '\xFF\xD8\xFF\xE3', '\xFF\xD8\xFF\xE1']),
-        ('image/png', ['\x89\x50\x4E\x47\x0D\x0A\x1A\x0A']),
-        ('image/gif', ['GIF87a', '\x47\x49\x46\x38\x37\x61', 'GIF89a', '\x47\x49\x46\x38\x39\x61']),
-        # docx, odt, doc, xls
-        ('application/msword', ['PK\x03\x04', '\xCF\x11\xE0\xA1\xB1\x1A\xE1\x00', '\xEC\xA5\xC1\x00', '\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x0D\x44\x4F\x43']),
-        # zip, but will include jar, odt, ods, odp, docx, xlsx, pptx, apk
-        ('application/zip', ['PK']),
-        ('application/vnd.ms-excel', ['\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x09\x08\x10\x00\x00\x06\x05\x00', '\xFD\xFF\xFF\xFF\x10', '\xFD\xFF\xFF\xFF\x1F', '\xFD\xFF\xFF\xFF\x23', '\xFD\xFF\xFF\xFF\x28', '\xFD\xFF\xFF\xFF\x29']),
-    )
-    for mimetype, signatures in mapping:
-        for signature in signatures:
-            if bin_data.startswith(signature):
-                return mimetype
-    return 'application/octet-stream'
-
-# if available adapt magic from pypi
-if hasattr(magic,'from_buffer'):
-    guess_mimetype = lambda bin_data: magic.from_buffer(bin_data, mime=True)
-# or if available adapt magic from file(1)
-elif hasattr(magic,'open'):
-    ms = magic.open(magic.MAGIC_MIME_TYPE)
-    ms.load()
-    guess_mimetype = ms.buffer
-
-
 class ir_attachment(osv.osv):
     """Attachments are used to link binary files or url to any openerp document.
 

openerp/addons/test_mimetypes/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

openerp/addons/test_mimetypes/__openerp__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+{
+    'name': 'test mimetypes-guessing',
+    'version': '0.1',
+    'category': 'Tests',
+    'description': """A module to generate exceptions.""",
+}

openerp/addons/test_mimetypes/tests/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+from . import test_guess_mimetypes

openerp/addons/test_mimetypes/tests/test_guess_mimetypes.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+import os.path
+import unittest
+
+from openerp.tools.mimetypes import guess_mimetype
+
+def contents(extension):
+    with open(os.path.join(
+        os.path.dirname(__file__),
+        'testfiles',
+        'case.{}'.format(extension)
+    ), 'rb') as f:
+        return f.read()
+
+class TestMimeGuessing(unittest.TestCase):
+    def test_doc(self):
+        self.assertEqual(
+            guess_mimetype(contents('doc')),
+            'application/msword'
+        )
+    def test_xls(self):
+        self.assertEqual(
+            guess_mimetype(contents('xls')),
+            'application/vnd.ms-excel'
+        )
+    def test_docx(self):
+        self.assertEqual(
+            guess_mimetype(contents('docx')),
+            'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+        )
+    def test_xlsx(self):
+        self.assertEqual(
+            guess_mimetype(contents('xlsx')),
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+        )
+    def test_odt(self):
+        self.assertEqual(
+            guess_mimetype(contents('odt')),
+            'application/vnd.oasis.opendocument.text'
+        )
+    def test_ods(self):
+        self.assertEqual(
+            guess_mimetype(contents('ods')),
+            'application/vnd.oasis.opendocument.spreadsheet'
+        )
+
+    def test_zip(self):
+        self.assertEqual(
+            guess_mimetype(contents('zip')),
+            'application/zip'
+        )
+
+    def test_gif(self):
+        self.assertEqual(
+            guess_mimetype(contents('gif')),
+            'image/gif'
+        )
+    def test_jpeg(self):
+        self.assertEqual(
+            guess_mimetype(contents('jpg')),
+            'image/jpeg'
+        )
+
+    def test_unknown(self):
+        self.assertEqual(
+            guess_mimetype(contents('csv')),
+            'application/octet-stream'
+        )

openerp/addons/test_mimetypes/tests/testfiles/case.csv

@@ -0,0 +1,3 @@
+a,b,c
+1,2,3
+4,5,6

openerp/tools/mimetypes.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""
+Mimetypes-related utilities
+
+# TODO: reexport stdlib mimetypes?
+"""
+import collections
+import io
+import logging
+import re
+import zipfile
+
+__all__ = ['guess_mimetype']
+
+_logger = logging.getLogger(__name__)
+
+# We define our own guess_mimetype implementation and if magic is available we
+# use it instead.
+
+# discriminants for zip-based file formats
+_ooxml_dirs = {
+    'word/': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    'pt/': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    'xl/': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+}
+def _check_ooxml(data):
+    with io.BytesIO(data) as f, zipfile.ZipFile(f) as z:
+        filenames = z.namelist()
+        # OOXML documents should have a [Content_Types].xml file for early
+        # check that we're interested in this thing at all
+        if '[Content_Types].xml' not in filenames:
+            return False
+
+        # then there is a directory whose name denotes the type of the file:
+        # word, pt (powerpoint) or xl (excel)
+        for dirname, mime in _ooxml_dirs.iteritems():
+            if any(entry.startswith(dirname) for entry in filenames):
+                return mime
+
+        return False
+
+
+# checks that a string looks kinda sorta like a mimetype
+_mime_validator = re.compile(r"""
+    [\w-]+ # type-name
+    / # subtype separator
+    [\w-]+ # registration facet or subtype
+    (?:\.[\w-]+)* # optional faceted name
+    (?:\+[\w-]+)? # optional structured syntax specifier
+""", re.VERBOSE)
+def _check_open_container_format(data):
+    with io.BytesIO(data) as f, zipfile.ZipFile(f) as z:
+        # OCF zip files must contain a ``mimetype`` entry
+        if 'mimetype' not in z.namelist():
+            return False
+
+        # it holds the exact mimetype for the file
+        marcel = z.read('mimetype')
+        # check that it's not too long (RFC6838 § 4.2 restricts type and
+        # subtype to 127 characters each + separator, strongly recommends
+        # limiting them to 64 but does not require it) and that it looks a lot
+        # like a valid mime type
+        if len(marcel) < 256 and _mime_validator.match(marcel):
+            return marcel
+
+        return False
+
+_xls_pattern = re.compile("""
+    \x09\x08\x10\x00\x00\x06\x05\x00
+  | \xFD\xFF\xFF\xFF(\x10|\x1F|\x20|"|\\#|\\(|\\))
+""", re.VERBOSE)
+_ppt_pattern = re.compile("""
+    \x00\x6E\x1E\xF0
+  | \x0F\x00\xE8\x03
+  | \xA0\x46\x1D\xF0
+  | \xFD\xFF\xFF\xFF(\x0E|\x1C|\x43)\x00\x00\x00
+""", re.VERBOSE)
+def _check_olecf(data):
+    """ Pre-OOXML Office formats are OLE Compound Files which all use the same
+    file signature ("magic bytes") and should have a subheader at offset 512
+    (0x200).
+
+    Subheaders taken from http://www.garykessler.net/library/file_sigs.html
+    according to which Mac office files *may* have different subheaders. We'll
+    ignore that.
+    """
+    offset = 0x200
+    if data.startswith('\xEC\xA5\xC1\x00', offset):
+        return 'application/msword'
+    # the _xls_pattern stuff doesn't seem to work correctly (the test file
+    # only has a bunch of \xf* at offset 0x200), that apparently works
+    elif 'Microsoft Excel' in data:
+        return 'application/vnd.ms-excel'
+    elif _ppt_pattern.match(data, offset):
+        return 'application/vnd.ms-powerpoint'
+    return False
+
+# for "master" formats with many subformats, discriminants is a list of
+# functions, tried in order and the first non-falsy value returned is the
+# selected mime type. If all functions return falsy values, the master
+# mimetype is returned.
+_Entry = collections.namedtuple('_Entry', ['mimetype', 'signatures', 'discriminants'])
+_mime_mappings = (
+    # pdf
+    _Entry('application/pdf', ['%PDF'], []),
+    # jpg, jpeg, png, gif
+    _Entry('image/jpeg', ['\xFF\xD8\xFF\xE0', '\xFF\xD8\xFF\xE2', '\xFF\xD8\xFF\xE3', '\xFF\xD8\xFF\xE1'], []),
+    _Entry('image/png', ['\x89PNG\r\n\x1A\n'], []),
+    _Entry('image/gif', ['GIF87a', 'GIF89a'], []),
+    # OLECF files in general (Word, Excel, PPT, default to word because why not?)
+    _Entry('application/msword', ['\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x0D\x44\x4F\x43'], [
+        _check_olecf
+    ]),
+    # zip, but will include jar, odt, ods, odp, docx, xlsx, pptx, apk
+    _Entry('application/zip', ['PK\x03\x04'], [_check_ooxml, _check_open_container_format]),
+)
+def guess_mimetype(bin_data):
+    """ Attempts to guess the mime type of the provided binary data, similar
+    to but significantly more limited than libmagic
+
+    :param str bin_data: binary data to try and guess a mime type for
+    :returns: matched mimetype or ``application/octet-stream`` if none matched
+    """
+    # by default, guess the type using the magic number of file hex signature (like magic, but more limited)
+    # see http://www.filesignatures.net/ for file signatures
+    for entry in _mime_mappings:
+        for signature in entry.signatures:
+            if bin_data.startswith(signature):
+                for discriminant in entry.discriminants:
+                    try:
+                        guess = discriminant(bin_data)
+                        if guess: return guess
+                    except Exception:
+                        # log-and-next
+                        _logger.getChild('guess_mimetype').warn(
+                            "Sub-checker '%s' of type '%s' failed",
+                            discriminant.__name__, entry.mimetype,
+                            exc_info=True
+                        )
+                # if no discriminant or no discriminant matches, return
+                # primary mime type
+                return entry.mimetype
+    return 'application/octet-stream'
+
+
+try:
+    import magic
+except ImportError:
+    magic = None
+else:
+    # There are 2 python libs named 'magic' with incompatible api.
+
+    # magic from pypi https://pypi.python.org/pypi/python-magic/
+    if hasattr(magic,'from_buffer'):
+        guess_mimetype = lambda bin_data: magic.from_buffer(bin_data, mime=True)
+    # magic from file(1) https://packages.debian.org/squeeze/python-magic
+    elif hasattr(magic,'open'):
+        ms = magic.open(magic.MAGIC_MIME_TYPE)
+        ms.load()
+        guess_mimetype = ms.buffer