diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index c0bd7198a1a..0a06de58390 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -8,61 +8,15 @@ import mimetypes import os import re -from openerp import tools -from openerp.tools.translate import _ +from openerp import SUPERUSER_ID, tools from openerp.exceptions import AccessError from openerp.osv import fields,osv -from openerp import SUPERUSER_ID -from openerp.exceptions import UserError from openerp.tools.translate import _ from openerp.tools.misc import ustr +from openerp.tools.mimetypes import guess_mimetype _logger = logging.getLogger(__name__) -try: - import magic -except ImportError: - magic = None - -# We define our own guess_mimetype implementation and if magic is available we -# use it instead. - -# However there are 2 python libs named 'magic' with incompatible api. -# magic from pypi https://pypi.python.org/pypi/python-magic/ -# magic from file(1) https://packages.debian.org/squeeze/python-magic - -def guess_mimetype(bin_data): - # by default, guess the type using the magic number of file hex signature (like magic, but more limited) - # see http://www.filesignatures.net/ for file signatures - mapping = ( - # pdf - ('application/pdf', ['%PDF']), - # jpg, jpeg, png, gif - ('image/jpeg', ['\xFF\xD8\xFF\xE0', '\xFF\xD8\xFF\xE2', '\xFF\xD8\xFF\xE3', '\xFF\xD8\xFF\xE1']), - ('image/png', ['\x89\x50\x4E\x47\x0D\x0A\x1A\x0A']), - ('image/gif', ['GIF87a', '\x47\x49\x46\x38\x37\x61', 'GIF89a', '\x47\x49\x46\x38\x39\x61']), - # docx, odt, doc, xls - ('application/msword', ['PK\x03\x04', '\xCF\x11\xE0\xA1\xB1\x1A\xE1\x00', '\xEC\xA5\xC1\x00', '\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x0D\x44\x4F\x43']), - # zip, but will include jar, odt, ods, odp, docx, xlsx, pptx, apk - ('application/zip', ['PK']), - ('application/vnd.ms-excel', ['\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x09\x08\x10\x00\x00\x06\x05\x00', '\xFD\xFF\xFF\xFF\x10', '\xFD\xFF\xFF\xFF\x1F', '\xFD\xFF\xFF\xFF\x23', '\xFD\xFF\xFF\xFF\x28', '\xFD\xFF\xFF\xFF\x29']), - ) - for mimetype, signatures in mapping: - for signature in signatures: - if bin_data.startswith(signature): - return mimetype - return 'application/octet-stream' - -# if available adapt magic from pypi -if hasattr(magic,'from_buffer'): - guess_mimetype = lambda bin_data: magic.from_buffer(bin_data, mime=True) -# or if available adapt magic from file(1) -elif hasattr(magic,'open'): - ms = magic.open(magic.MAGIC_MIME_TYPE) - ms.load() - guess_mimetype = ms.buffer - - class ir_attachment(osv.osv): """Attachments are used to link binary files or url to any openerp document. diff --git a/openerp/addons/test_mimetypes/__init__.py b/openerp/addons/test_mimetypes/__init__.py new file mode 100644 index 00000000000..40a96afc6ff --- /dev/null +++ b/openerp/addons/test_mimetypes/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/openerp/addons/test_mimetypes/__openerp__.py b/openerp/addons/test_mimetypes/__openerp__.py new file mode 100644 index 00000000000..dddcd3f57db --- /dev/null +++ b/openerp/addons/test_mimetypes/__openerp__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'test mimetypes-guessing', + 'version': '0.1', + 'category': 'Tests', + 'description': """A module to generate exceptions.""", +} diff --git a/openerp/addons/test_mimetypes/tests/__init__.py b/openerp/addons/test_mimetypes/tests/__init__.py new file mode 100644 index 00000000000..29d67e5c463 --- /dev/null +++ b/openerp/addons/test_mimetypes/tests/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from . import test_guess_mimetypes diff --git a/openerp/addons/test_mimetypes/tests/test_guess_mimetypes.py b/openerp/addons/test_mimetypes/tests/test_guess_mimetypes.py new file mode 100644 index 00000000000..f6d3af053d5 --- /dev/null +++ b/openerp/addons/test_mimetypes/tests/test_guess_mimetypes.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import os.path +import unittest + +from openerp.tools.mimetypes import guess_mimetype + +def contents(extension): + with open(os.path.join( + os.path.dirname(__file__), + 'testfiles', + 'case.{}'.format(extension) + ), 'rb') as f: + return f.read() + +class TestMimeGuessing(unittest.TestCase): + def test_doc(self): + self.assertEqual( + guess_mimetype(contents('doc')), + 'application/msword' + ) + def test_xls(self): + self.assertEqual( + guess_mimetype(contents('xls')), + 'application/vnd.ms-excel' + ) + def test_docx(self): + self.assertEqual( + guess_mimetype(contents('docx')), + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) + def test_xlsx(self): + self.assertEqual( + guess_mimetype(contents('xlsx')), + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + ) + def test_odt(self): + self.assertEqual( + guess_mimetype(contents('odt')), + 'application/vnd.oasis.opendocument.text' + ) + def test_ods(self): + self.assertEqual( + guess_mimetype(contents('ods')), + 'application/vnd.oasis.opendocument.spreadsheet' + ) + + def test_zip(self): + self.assertEqual( + guess_mimetype(contents('zip')), + 'application/zip' + ) + + def test_gif(self): + self.assertEqual( + guess_mimetype(contents('gif')), + 'image/gif' + ) + def test_jpeg(self): + self.assertEqual( + guess_mimetype(contents('jpg')), + 'image/jpeg' + ) + + def test_unknown(self): + self.assertEqual( + guess_mimetype(contents('csv')), + 'application/octet-stream' + ) diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.csv b/openerp/addons/test_mimetypes/tests/testfiles/case.csv new file mode 100644 index 00000000000..88700c7147b --- /dev/null +++ b/openerp/addons/test_mimetypes/tests/testfiles/case.csv @@ -0,0 +1,3 @@ +a,b,c +1,2,3 +4,5,6 diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.doc b/openerp/addons/test_mimetypes/tests/testfiles/case.doc new file mode 100644 index 00000000000..5eb9fdc05e1 Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.doc differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.docx b/openerp/addons/test_mimetypes/tests/testfiles/case.docx new file mode 100644 index 00000000000..a788e4ba804 Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.docx differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.gif b/openerp/addons/test_mimetypes/tests/testfiles/case.gif new file mode 100644 index 00000000000..551a467849a Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.gif differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.jpg b/openerp/addons/test_mimetypes/tests/testfiles/case.jpg new file mode 100644 index 00000000000..4f93306d298 Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.jpg differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.ods b/openerp/addons/test_mimetypes/tests/testfiles/case.ods new file mode 100644 index 00000000000..487d28080d7 Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.ods differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.odt b/openerp/addons/test_mimetypes/tests/testfiles/case.odt new file mode 100644 index 00000000000..24f1f0b502a Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.odt differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.xls b/openerp/addons/test_mimetypes/tests/testfiles/case.xls new file mode 100644 index 00000000000..e1409586064 Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.xls differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.xlsx b/openerp/addons/test_mimetypes/tests/testfiles/case.xlsx new file mode 100644 index 00000000000..c549c7375f0 Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.xlsx differ diff --git a/openerp/addons/test_mimetypes/tests/testfiles/case.zip b/openerp/addons/test_mimetypes/tests/testfiles/case.zip new file mode 100644 index 00000000000..3f153366429 Binary files /dev/null and b/openerp/addons/test_mimetypes/tests/testfiles/case.zip differ diff --git a/openerp/tools/mimetypes.py b/openerp/tools/mimetypes.py new file mode 100644 index 00000000000..cf5a9debc90 --- /dev/null +++ b/openerp/tools/mimetypes.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +""" +Mimetypes-related utilities + +# TODO: reexport stdlib mimetypes? +""" +import collections +import io +import logging +import re +import zipfile + +__all__ = ['guess_mimetype'] + +_logger = logging.getLogger(__name__) + +# We define our own guess_mimetype implementation and if magic is available we +# use it instead. + +# discriminants for zip-based file formats +_ooxml_dirs = { + 'word/': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'pt/': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'xl/': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +} +def _check_ooxml(data): + with io.BytesIO(data) as f, zipfile.ZipFile(f) as z: + filenames = z.namelist() + # OOXML documents should have a [Content_Types].xml file for early + # check that we're interested in this thing at all + if '[Content_Types].xml' not in filenames: + return False + + # then there is a directory whose name denotes the type of the file: + # word, pt (powerpoint) or xl (excel) + for dirname, mime in _ooxml_dirs.iteritems(): + if any(entry.startswith(dirname) for entry in filenames): + return mime + + return False + + +# checks that a string looks kinda sorta like a mimetype +_mime_validator = re.compile(r""" + [\w-]+ # type-name + / # subtype separator + [\w-]+ # registration facet or subtype + (?:\.[\w-]+)* # optional faceted name + (?:\+[\w-]+)? # optional structured syntax specifier +""", re.VERBOSE) +def _check_open_container_format(data): + with io.BytesIO(data) as f, zipfile.ZipFile(f) as z: + # OCF zip files must contain a ``mimetype`` entry + if 'mimetype' not in z.namelist(): + return False + + # it holds the exact mimetype for the file + marcel = z.read('mimetype') + # check that it's not too long (RFC6838 ยง 4.2 restricts type and + # subtype to 127 characters each + separator, strongly recommends + # limiting them to 64 but does not require it) and that it looks a lot + # like a valid mime type + if len(marcel) < 256 and _mime_validator.match(marcel): + return marcel + + return False + +_xls_pattern = re.compile(""" + \x09\x08\x10\x00\x00\x06\x05\x00 + | \xFD\xFF\xFF\xFF(\x10|\x1F|\x20|"|\\#|\\(|\\)) +""", re.VERBOSE) +_ppt_pattern = re.compile(""" + \x00\x6E\x1E\xF0 + | \x0F\x00\xE8\x03 + | \xA0\x46\x1D\xF0 + | \xFD\xFF\xFF\xFF(\x0E|\x1C|\x43)\x00\x00\x00 +""", re.VERBOSE) +def _check_olecf(data): + """ Pre-OOXML Office formats are OLE Compound Files which all use the same + file signature ("magic bytes") and should have a subheader at offset 512 + (0x200). + + Subheaders taken from http://www.garykessler.net/library/file_sigs.html + according to which Mac office files *may* have different subheaders. We'll + ignore that. + """ + offset = 0x200 + if data.startswith('\xEC\xA5\xC1\x00', offset): + return 'application/msword' + # the _xls_pattern stuff doesn't seem to work correctly (the test file + # only has a bunch of \xf* at offset 0x200), that apparently works + elif 'Microsoft Excel' in data: + return 'application/vnd.ms-excel' + elif _ppt_pattern.match(data, offset): + return 'application/vnd.ms-powerpoint' + return False + +# for "master" formats with many subformats, discriminants is a list of +# functions, tried in order and the first non-falsy value returned is the +# selected mime type. If all functions return falsy values, the master +# mimetype is returned. +_Entry = collections.namedtuple('_Entry', ['mimetype', 'signatures', 'discriminants']) +_mime_mappings = ( + # pdf + _Entry('application/pdf', ['%PDF'], []), + # jpg, jpeg, png, gif + _Entry('image/jpeg', ['\xFF\xD8\xFF\xE0', '\xFF\xD8\xFF\xE2', '\xFF\xD8\xFF\xE3', '\xFF\xD8\xFF\xE1'], []), + _Entry('image/png', ['\x89PNG\r\n\x1A\n'], []), + _Entry('image/gif', ['GIF87a', 'GIF89a'], []), + # OLECF files in general (Word, Excel, PPT, default to word because why not?) + _Entry('application/msword', ['\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1', '\x0D\x44\x4F\x43'], [ + _check_olecf + ]), + # zip, but will include jar, odt, ods, odp, docx, xlsx, pptx, apk + _Entry('application/zip', ['PK\x03\x04'], [_check_ooxml, _check_open_container_format]), +) +def guess_mimetype(bin_data): + """ Attempts to guess the mime type of the provided binary data, similar + to but significantly more limited than libmagic + + :param str bin_data: binary data to try and guess a mime type for + :returns: matched mimetype or ``application/octet-stream`` if none matched + """ + # by default, guess the type using the magic number of file hex signature (like magic, but more limited) + # see http://www.filesignatures.net/ for file signatures + for entry in _mime_mappings: + for signature in entry.signatures: + if bin_data.startswith(signature): + for discriminant in entry.discriminants: + try: + guess = discriminant(bin_data) + if guess: return guess + except Exception: + # log-and-next + _logger.getChild('guess_mimetype').warn( + "Sub-checker '%s' of type '%s' failed", + discriminant.__name__, entry.mimetype, + exc_info=True + ) + # if no discriminant or no discriminant matches, return + # primary mime type + return entry.mimetype + return 'application/octet-stream' + + +try: + import magic +except ImportError: + magic = None +else: + # There are 2 python libs named 'magic' with incompatible api. + + # magic from pypi https://pypi.python.org/pypi/python-magic/ + if hasattr(magic,'from_buffer'): + guess_mimetype = lambda bin_data: magic.from_buffer(bin_data, mime=True) + # magic from file(1) https://packages.debian.org/squeeze/python-magic + elif hasattr(magic,'open'): + ms = magic.open(magic.MAGIC_MIME_TYPE) + ms.load() + guess_mimetype = ms.buffer