[IMP] base: auto resize attachment to 1080p

After some analyze on a lot of customer databases, seems like most of the time their are performance probleme, and big store, it is due to a lot of big file uploaded without reason. E.g. barcode, photo, ... a small one will be enough. Now, we decided (in stable) to auto resize these pictures to 1920x1920px by default and compress it with a quality of 80 when the source is bigger. You can bypass this behaviour in your specific use case, using a context key: 'image_no_postprocess' set to True. You can disable the resize (and quality implicitely) using an icp: 'base.image_autoresize_max_px' set to '0'. You can change the default resize (1920x1920) format using an icp: 'base.image_autoresize_max_px' set to '<width>x<height>' (e.g. '1024x768') You can change the default quality (80) using an icp: 'base.image_autoresize_quality' with a value between 0 and 100 where 0 skip it. You can change the type of file that will be post process using icp: 'base.image_autoresize_extensions' (subtype of the mimetype comma separated). Api of image has not be changed in this commit, only refactored to allow to work with image directly without the need to encode/Decode in base64 the raw. We decide to keep 1920x1920 by default instead of 1080p to avoid to resize portrait picture in 1080px and stay consistent with field image_1920 that return a 1920px image for width or height whatever the orientation. + fix some lint diff for ci style in master closes odoo/odoo#78556 X-original-commit: d9ce0507960f247e1187baf7bd8399f90be237aa Signed-off-by: Jérémy Kersten (jke) <jke@openerp.com>
2021-10-22 17:34:29 +00:00
parent 74ff542278
commit 300954d8c4
5 changed files with 236 additions and 40 deletions
@@ -43,4 +43,4 @@ class Website(models.Model):
            image.crop_resize(square_size, square_size)
            image.image = image.image.resize((512, 512))
            image.operationsCount += 1
-            website.app_icon = image.image_base64(output_format='PNG')
+            website.app_icon = image.image_quality_base64(output_format='PNG')
@@ -2,17 +2,20 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import base64
 import hashlib
+import io
 import itertools
 import logging
 import mimetypes
 import os
 import re
-from collections import defaultdict
 import uuid

+from collections import defaultdict
+from PIL import Image
+
 from odoo import api, fields, models, tools, _
-from odoo.exceptions import AccessError, ValidationError, MissingError, UserError
-from odoo.tools import config, human_size, ustr, html_escape
+from odoo.exceptions import AccessError, ValidationError, UserError
+from odoo.tools import config, human_size, ImageProcess, str2bool
 from odoo.tools.mimetypes import guess_mimetype
 from odoo.osv import expression

@@ -294,6 +297,42 @@ class IrAttachment(models.Model):
                mimetype = guess_mimetype(raw)
        return mimetype or 'application/octet-stream'

+    def _postprocess_contents(self, values):
+        ICP = self.env['ir.config_parameter'].sudo().get_param
+        supported_subtype = ICP('base.image_autoresize_extensions', 'png,jpeg,gif,bmp,tif').split(',')
+
+        mimetype = values['mimetype'] = self._compute_mimetype(values)
+        _type, _subtype = mimetype.split('/')
+        is_image_resizable = _type == 'image' and _subtype in supported_subtype
+        if is_image_resizable and (values.get('datas') or values.get('raw')):
+            is_raw = values.get('raw')
+
+            # Can be set to 0 to skip the resize
+            max_resolution = ICP('base.image_autoresize_max_px', '1920x1920')
+            if str2bool(max_resolution, True):
+                try:
+                    img = fn_quality = False
+                    if is_raw:
+                        img = ImageProcess(False, verify_resolution=False)
+                        img.image = Image.open(io.BytesIO(values['raw']))
+                        img.original_format = (img.image.format or '').upper()
+                        fn_quality = img.image_quality
+                    else:  # datas
+                        img = ImageProcess(values['datas'], verify_resolution=False)
+                        fn_quality = img.image_quality_base64
+
+                    w, h = img.image.size
+                    nw, nh = map(int, max_resolution.split('x'))
+                    if w > nw or h > nh:
+                        img.resize(nw, nh)
+                        quality = int(ICP('base.image_autoresize_quality', 80))
+                        values[is_raw and 'raw' or 'datas'] = fn_quality(quality=quality)
+                except UserError as e:
+                    # Catch error during test where we provide fake image
+                    # raise UserError(_("This file could not be decoded as an image file. Please try with a different file."))
+                    _logger.info('Post processing ignored : %s', e)
+        return values
+
    def _check_contents(self, values):
        mimetype = values['mimetype'] = self._compute_mimetype(values)
        xml_like = 'ht' in mimetype or ( # hta, html, xhtml, etc.
@@ -304,6 +343,8 @@ class IrAttachment(models.Model):
            self.env.context.get('attachments_mime_plainxml')))
        if force_text:
            values['mimetype'] = 'text/plain'
+        if not self.env.context.get('image_no_postprocess'):
+            values = self._postprocess_contents(values)
        return values

    @api.model
@@ -2,10 +2,14 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import base64
 import hashlib
+import io
 import os

+from PIL import Image
+
 from odoo.exceptions import AccessError
 from odoo.tests.common import TransactionCase
+from odoo.tools import image_to_base64, human_size

 HASH_SPLIT = 2      # FIXME: testing implementations detail is not a good idea

@@ -108,6 +112,7 @@ class TestIrAttachment(TransactionCase):
        """
        Tests the consistency of documents' mimetypes
        """
+
        Attachment = self.Attachment.with_user(self.env.ref('base.user_demo').id)
        a2 = Attachment.create({'name': 'a2', 'datas': self.blob1_b64, 'mimetype': 'image/png'})
        self.assertEqual(a2.mimetype, 'image/png', "the new mimetype should be the one given on write")
@@ -136,3 +141,100 @@ class TestIrAttachment(TransactionCase):
        document = self.Attachment.create({'name': 'document', 'datas': self.blob1_b64})
        document.write({'datas': self.blob1_b64, 'mimetype': 'text/xml'})
        self.assertEqual(document.mimetype, 'text/xml', "XML mimetype should not be forced to text, for admin user")
+
+    def test_10_image_autoresize(self):
+        Attachment = self.env['ir.attachment']
+        img_bin = io.BytesIO()
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+        with Image.open(os.path.join(dir_path, 'odoo.jpg'), 'r') as logo:
+            img = Image.new('RGB', (4000, 2000), '#4169E1')
+            img.paste(logo)
+            img.save(img_bin, 'JPEG')
+
+        img_encoded = image_to_base64(img, 'JPEG')
+        img_bin = img_bin.getvalue()
+
+        # we don't used bin_size in context, because on write, the cached value is the data and not
+        # the size, so we need on each write to invalidate cache if we really want to get the size.
+        def bin_size(value):
+            try:
+                value = base64.b64decode(value.decode())
+            except UnicodeDecodeError:
+                pass
+
+            return human_size(len(value))
+
+        ####################################
+        ### test create/write on 'datas'
+        ####################################
+
+        fullsize = '124.99 Kb'
+
+        attach = Attachment.with_context(image_no_postprocess=True).create({
+            'name': 'image',
+            'datas': img_encoded,
+        })
+        self.assertEqual(bin_size(attach.datas), fullsize)  # no resize, no compression
+
+        attach = attach.with_context(image_no_postprocess=False)
+        attach.datas = img_encoded
+        self.assertEqual(bin_size(attach.datas), '12.06 Kb')  # default resize + default compression
+
+        # resize + default quality (80)
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '1024x768')
+        attach.datas = img_encoded
+        self.assertEqual(bin_size(attach.datas), '3.71 Kb')
+
+        # resize + quality 50
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '50')
+        attach.datas = img_encoded
+        self.assertEqual(bin_size(attach.datas), '3.57 Kb')
+
+        # no resize + no quality implicit
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '0')
+        attach.datas = img_encoded
+        self.assertEqual(bin_size(attach.datas), fullsize)
+
+        # Check that we only compress quality when we resize. We avoid to compress again during a new write.
+        # no resize + quality -> should have no effect
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '10000x10000')
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '50')
+        attach.datas = img_encoded
+        self.assertEqual(bin_size(attach.datas), fullsize)
+
+        ####################################
+        ### test create/write on 'raw'
+        ####################################
+
+        # reset default ~ delete
+        self.env['ir.config_parameter'].search([('key', 'ilike', 'base.image_autoresize%')]).unlink()
+
+        attach = Attachment.with_context(image_no_postprocess=True).create({
+            'name': 'image',
+            'raw': img_bin,
+        })
+        self.assertEqual(bin_size(attach.raw), fullsize)  # no resize, no compression
+
+        attach = attach.with_context(image_no_postprocess=False)
+        attach.raw = img_bin
+        self.assertEqual(bin_size(attach.raw), '12.06 Kb')  # default resize + default compression
+
+        # resize + default quality (80)
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '1024x768')
+        attach.raw = img_bin
+        self.assertEqual(bin_size(attach.raw), '3.71 Kb')
+
+        # resize + no quality
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '0')
+        attach.raw = img_bin
+        self.assertIn('4.0', bin_size(attach.raw))  # '4.09 Kb' or '4.08 Kb'
+
+        # resize + quality 50
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '50')
+        attach.raw = img_bin
+        self.assertEqual(bin_size(attach.raw), '3.57 Kb')
+
+        # no resize + no quality implicit
+        self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '0')
+        attach.raw = img_bin
+        self.assertEqual(bin_size(attach.raw), fullsize)
@@ -2162,7 +2162,7 @@ class TestFields(TransactionCaseWithUserDemo):
        self.assertEqual(Image.open(io.BytesIO(base64.b64decode(record.image_256))).size, (128, 256))

        # test create inverse no store
-        record = self.env['test_new_api.model_image'].create({
+        record = self.env['test_new_api.model_image'].with_context(image_no_postprocess=True).create({
            'name': 'image',
            'image_256': image_w,
        })
@@ -88,7 +88,68 @@ class ImageProcess():
            if verify_resolution and w * h > IMAGE_MAX_RESOLUTION:
                raise ValueError(_("Image size excessive, uploaded images must be smaller than %s million pixels.", str(IMAGE_MAX_RESOLUTION / 10e6)))

-    def image_base64(self, quality=0, output_format=''):
+    def image_quality(self, quality=0, output_format=''):
+        """Return the image resulting of all the image processing
+        operations that have been applied previously.
+
+        Return False if the initialized `image` was falsy, and return
+        the initialized `image` without change if it was SVG.
+
+        Also return the initialized `image` if no operations have been applied
+        and the `output_format` is the same as the original format and the
+        quality is not specified.
+
+        :param quality: quality setting to apply. Default to 0.
+            - for JPEG: 1 is worse, 95 is best. Values above 95 should be
+                avoided. Falsy values will fallback to 95, but only if the image
+                was changed, otherwise the original image is returned.
+            - for PNG: set falsy to prevent conversion to a WEB palette.
+            - for other formats: no effect.
+        :type quality: int
+
+        :param output_format: the output format. Can be PNG, JPEG, GIF, or ICO.
+            Default to the format of the original image. BMP is converted to
+            PNG, other formats than those mentioned above are converted to JPEG.
+        :type output_format: string
+
+        :return: image
+        :rtype: bytes or False
+        """
+        if not self.image:
+            return self.image
+
+        output_image = self.image
+
+        output_format = output_format.upper() or self.original_format
+        if output_format == 'BMP':
+            output_format = 'PNG'
+        elif output_format not in ['PNG', 'JPEG', 'GIF', 'ICO']:
+            output_format = 'JPEG'
+
+        if not self.operationsCount and output_format == self.original_format and not quality:
+            return self.image
+
+        opt = {'output_format': output_format}
+
+        if output_format == 'PNG':
+            opt['optimize'] = True
+            if quality:
+                if output_image.mode != 'P':
+                    # Floyd Steinberg dithering by default
+                    output_image = output_image.convert('RGBA').convert('P', palette=Image.WEB, colors=256)
+        if output_format == 'JPEG':
+            opt['optimize'] = True
+            opt['quality'] = quality or 95
+        if output_format == 'GIF':
+            opt['optimize'] = True
+            opt['save_all'] = True
+
+        if output_image.mode not in ["1", "L", "P", "RGB", "RGBA"] or (output_format == 'JPEG' and output_image.mode == 'RGBA'):
+            output_image = output_image.convert("RGB")
+
+        return image_apply_opt(output_image, **opt)
+
+    def image_quality_base64(self, quality=0, output_format=''):
        """Return the base64 encoded image resulting of all the image processing
        operations that have been applied previously.

@@ -115,39 +176,15 @@ class ImageProcess():
        :return: image base64 encoded or False
        :rtype: bytes or False
        """
-        output_image = self.image

-        if not output_image:
+        if not self.image:
            return self.base64_source

-        output_format = output_format.upper() or self.original_format
-        if output_format == 'BMP':
-            output_format = 'PNG'
-        elif output_format not in ['PNG', 'JPEG', 'GIF', 'ICO']:
-            output_format = 'JPEG'
+        stream = self.image_quality(quality=quality, output_format=output_format)

-        if not self.operationsCount and output_format == self.original_format and not quality:
-            return self.base64_source
-
-        opt = {'format': output_format}
-
-        if output_format == 'PNG':
-            opt['optimize'] = True
-            if quality:
-                if output_image.mode != 'P':
-                    # Floyd Steinberg dithering by default
-                    output_image = output_image.convert('RGBA').convert('P', palette=Image.WEB, colors=256)
-        if output_format == 'JPEG':
-            opt['optimize'] = True
-            opt['quality'] = quality or 95
-        if output_format == 'GIF':
-            opt['optimize'] = True
-            opt['save_all'] = True
-
-        if output_image.mode not in ["1", "L", "P", "RGB", "RGBA"] or (output_format == 'JPEG' and output_image.mode == 'RGBA'):
-            output_image = output_image.convert("RGB")
-
-        return image_to_base64(output_image, **opt)
+        if stream != self.image:
+            return base64.b64encode(stream)
+        return self.base64_source

    def resize(self, max_width=0, max_height=0):
        """Resize the image.
@@ -285,7 +322,7 @@ def image_process(base64_source, size=(0, 0), verify_resolution=False, quality=0
            image.resize(max_width=size[0], max_height=size[1])
    if colorize:
        image.colorize()
-    return image.image_base64(quality=quality, output_format=output_format)
+    return image.image_quality_base64(quality=quality, output_format=output_format)


 # ----------------------------------------
@@ -406,7 +443,24 @@ def base64_to_image(base64_source):
        raise UserError(_("This file could not be decoded as an image file. Please try with a different file."))


-def image_to_base64(image, format, **params):
+def image_apply_opt(image, output_format, **params):
+    """Return the given PIL `image` using `params`.
+
+    :param image: the PIL image
+    :type image: PIL.Image
+
+    :param params: params to expand when calling PIL.Image.save()
+    :type params: dict
+
+    :return: the image formatted
+    :rtype: bytes
+    """
+    stream = io.BytesIO()
+    image.save(stream, format=output_format, **params)
+    return stream.getvalue()
+
+
+def image_to_base64(image, output_format, **params):
    """Return a base64_image from the given PIL `image` using `params`.

    :param image: the PIL image
@@ -418,9 +472,8 @@ def image_to_base64(image, format, **params):
    :return: the image base64 encoded
    :rtype: bytes
    """
-    stream = io.BytesIO()
-    image.save(stream, format=format, **params)
-    return base64.b64encode(stream.getvalue())
+    stream = image_apply_opt(image, output_format, **params)
+    return base64.b64encode(stream)


 def is_image_size_above(base64_source_1, base64_source_2):