From 300954d8c4df2e78743e46cf7920ecb064fafa81 Mon Sep 17 00:00:00 2001 From: Jeremy Kersten Date: Thu, 7 Oct 2021 11:19:12 +0000 Subject: [PATCH] [IMP] base: auto resize attachment to 1080p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After some analyze on a lot of customer databases, seems like most of the time their are performance probleme, and big store, it is due to a lot of big file uploaded without reason. E.g. barcode, photo, ... a small one will be enough. Now, we decided (in stable) to auto resize these pictures to 1920x1920px by default and compress it with a quality of 80 when the source is bigger. You can bypass this behaviour in your specific use case, using a context key: 'image_no_postprocess' set to True. You can disable the resize (and quality implicitely) using an icp: 'base.image_autoresize_max_px' set to '0'. You can change the default resize (1920x1920) format using an icp: 'base.image_autoresize_max_px' set to 'x' (e.g. '1024x768') You can change the default quality (80) using an icp: 'base.image_autoresize_quality' with a value between 0 and 100 where 0 skip it. You can change the type of file that will be post process using icp: 'base.image_autoresize_extensions' (subtype of the mimetype comma separated). Api of image has not be changed in this commit, only refactored to allow to work with image directly without the need to encode/Decode in base64 the raw. We decide to keep 1920x1920 by default instead of 1080p to avoid to resize portrait picture in 1080px and stay consistent with field image_1920 that return a 1920px image for width or height whatever the orientation. + fix some lint diff for ci style in master closes odoo/odoo#78556 X-original-commit: d9ce0507960f247e1187baf7bd8399f90be237aa Signed-off-by: Jérémy Kersten (jke) --- addons/website_event_track/models/website.py | 2 +- odoo/addons/base/models/ir_attachment.py | 47 ++++++- odoo/addons/base/tests/test_ir_attachment.py | 102 +++++++++++++++ .../test_new_api/tests/test_new_fields.py | 2 +- odoo/tools/image.py | 123 +++++++++++++----- 5 files changed, 236 insertions(+), 40 deletions(-) diff --git a/addons/website_event_track/models/website.py b/addons/website_event_track/models/website.py index f6f21d5ed62..64098f13350 100644 --- a/addons/website_event_track/models/website.py +++ b/addons/website_event_track/models/website.py @@ -43,4 +43,4 @@ class Website(models.Model): image.crop_resize(square_size, square_size) image.image = image.image.resize((512, 512)) image.operationsCount += 1 - website.app_icon = image.image_base64(output_format='PNG') + website.app_icon = image.image_quality_base64(output_format='PNG') diff --git a/odoo/addons/base/models/ir_attachment.py b/odoo/addons/base/models/ir_attachment.py index 4feb3e43f0a..6b0a6d5392f 100644 --- a/odoo/addons/base/models/ir_attachment.py +++ b/odoo/addons/base/models/ir_attachment.py @@ -2,17 +2,20 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 import hashlib +import io import itertools import logging import mimetypes import os import re -from collections import defaultdict import uuid +from collections import defaultdict +from PIL import Image + from odoo import api, fields, models, tools, _ -from odoo.exceptions import AccessError, ValidationError, MissingError, UserError -from odoo.tools import config, human_size, ustr, html_escape +from odoo.exceptions import AccessError, ValidationError, UserError +from odoo.tools import config, human_size, ImageProcess, str2bool from odoo.tools.mimetypes import guess_mimetype from odoo.osv import expression @@ -294,6 +297,42 @@ class IrAttachment(models.Model): mimetype = guess_mimetype(raw) return mimetype or 'application/octet-stream' + def _postprocess_contents(self, values): + ICP = self.env['ir.config_parameter'].sudo().get_param + supported_subtype = ICP('base.image_autoresize_extensions', 'png,jpeg,gif,bmp,tif').split(',') + + mimetype = values['mimetype'] = self._compute_mimetype(values) + _type, _subtype = mimetype.split('/') + is_image_resizable = _type == 'image' and _subtype in supported_subtype + if is_image_resizable and (values.get('datas') or values.get('raw')): + is_raw = values.get('raw') + + # Can be set to 0 to skip the resize + max_resolution = ICP('base.image_autoresize_max_px', '1920x1920') + if str2bool(max_resolution, True): + try: + img = fn_quality = False + if is_raw: + img = ImageProcess(False, verify_resolution=False) + img.image = Image.open(io.BytesIO(values['raw'])) + img.original_format = (img.image.format or '').upper() + fn_quality = img.image_quality + else: # datas + img = ImageProcess(values['datas'], verify_resolution=False) + fn_quality = img.image_quality_base64 + + w, h = img.image.size + nw, nh = map(int, max_resolution.split('x')) + if w > nw or h > nh: + img.resize(nw, nh) + quality = int(ICP('base.image_autoresize_quality', 80)) + values[is_raw and 'raw' or 'datas'] = fn_quality(quality=quality) + except UserError as e: + # Catch error during test where we provide fake image + # raise UserError(_("This file could not be decoded as an image file. Please try with a different file.")) + _logger.info('Post processing ignored : %s', e) + return values + def _check_contents(self, values): mimetype = values['mimetype'] = self._compute_mimetype(values) xml_like = 'ht' in mimetype or ( # hta, html, xhtml, etc. @@ -304,6 +343,8 @@ class IrAttachment(models.Model): self.env.context.get('attachments_mime_plainxml'))) if force_text: values['mimetype'] = 'text/plain' + if not self.env.context.get('image_no_postprocess'): + values = self._postprocess_contents(values) return values @api.model diff --git a/odoo/addons/base/tests/test_ir_attachment.py b/odoo/addons/base/tests/test_ir_attachment.py index 94bf2e449a2..0a35faaead8 100644 --- a/odoo/addons/base/tests/test_ir_attachment.py +++ b/odoo/addons/base/tests/test_ir_attachment.py @@ -2,10 +2,14 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 import hashlib +import io import os +from PIL import Image + from odoo.exceptions import AccessError from odoo.tests.common import TransactionCase +from odoo.tools import image_to_base64, human_size HASH_SPLIT = 2 # FIXME: testing implementations detail is not a good idea @@ -108,6 +112,7 @@ class TestIrAttachment(TransactionCase): """ Tests the consistency of documents' mimetypes """ + Attachment = self.Attachment.with_user(self.env.ref('base.user_demo').id) a2 = Attachment.create({'name': 'a2', 'datas': self.blob1_b64, 'mimetype': 'image/png'}) self.assertEqual(a2.mimetype, 'image/png', "the new mimetype should be the one given on write") @@ -136,3 +141,100 @@ class TestIrAttachment(TransactionCase): document = self.Attachment.create({'name': 'document', 'datas': self.blob1_b64}) document.write({'datas': self.blob1_b64, 'mimetype': 'text/xml'}) self.assertEqual(document.mimetype, 'text/xml', "XML mimetype should not be forced to text, for admin user") + + def test_10_image_autoresize(self): + Attachment = self.env['ir.attachment'] + img_bin = io.BytesIO() + dir_path = os.path.dirname(os.path.realpath(__file__)) + with Image.open(os.path.join(dir_path, 'odoo.jpg'), 'r') as logo: + img = Image.new('RGB', (4000, 2000), '#4169E1') + img.paste(logo) + img.save(img_bin, 'JPEG') + + img_encoded = image_to_base64(img, 'JPEG') + img_bin = img_bin.getvalue() + + # we don't used bin_size in context, because on write, the cached value is the data and not + # the size, so we need on each write to invalidate cache if we really want to get the size. + def bin_size(value): + try: + value = base64.b64decode(value.decode()) + except UnicodeDecodeError: + pass + + return human_size(len(value)) + + #################################### + ### test create/write on 'datas' + #################################### + + fullsize = '124.99 Kb' + + attach = Attachment.with_context(image_no_postprocess=True).create({ + 'name': 'image', + 'datas': img_encoded, + }) + self.assertEqual(bin_size(attach.datas), fullsize) # no resize, no compression + + attach = attach.with_context(image_no_postprocess=False) + attach.datas = img_encoded + self.assertEqual(bin_size(attach.datas), '12.06 Kb') # default resize + default compression + + # resize + default quality (80) + self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '1024x768') + attach.datas = img_encoded + self.assertEqual(bin_size(attach.datas), '3.71 Kb') + + # resize + quality 50 + self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '50') + attach.datas = img_encoded + self.assertEqual(bin_size(attach.datas), '3.57 Kb') + + # no resize + no quality implicit + self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '0') + attach.datas = img_encoded + self.assertEqual(bin_size(attach.datas), fullsize) + + # Check that we only compress quality when we resize. We avoid to compress again during a new write. + # no resize + quality -> should have no effect + self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '10000x10000') + self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '50') + attach.datas = img_encoded + self.assertEqual(bin_size(attach.datas), fullsize) + + #################################### + ### test create/write on 'raw' + #################################### + + # reset default ~ delete + self.env['ir.config_parameter'].search([('key', 'ilike', 'base.image_autoresize%')]).unlink() + + attach = Attachment.with_context(image_no_postprocess=True).create({ + 'name': 'image', + 'raw': img_bin, + }) + self.assertEqual(bin_size(attach.raw), fullsize) # no resize, no compression + + attach = attach.with_context(image_no_postprocess=False) + attach.raw = img_bin + self.assertEqual(bin_size(attach.raw), '12.06 Kb') # default resize + default compression + + # resize + default quality (80) + self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '1024x768') + attach.raw = img_bin + self.assertEqual(bin_size(attach.raw), '3.71 Kb') + + # resize + no quality + self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '0') + attach.raw = img_bin + self.assertIn('4.0', bin_size(attach.raw)) # '4.09 Kb' or '4.08 Kb' + + # resize + quality 50 + self.env['ir.config_parameter'].set_param('base.image_autoresize_quality', '50') + attach.raw = img_bin + self.assertEqual(bin_size(attach.raw), '3.57 Kb') + + # no resize + no quality implicit + self.env['ir.config_parameter'].set_param('base.image_autoresize_max_px', '0') + attach.raw = img_bin + self.assertEqual(bin_size(attach.raw), fullsize) diff --git a/odoo/addons/test_new_api/tests/test_new_fields.py b/odoo/addons/test_new_api/tests/test_new_fields.py index 2e4db8cf336..820f95497db 100644 --- a/odoo/addons/test_new_api/tests/test_new_fields.py +++ b/odoo/addons/test_new_api/tests/test_new_fields.py @@ -2162,7 +2162,7 @@ class TestFields(TransactionCaseWithUserDemo): self.assertEqual(Image.open(io.BytesIO(base64.b64decode(record.image_256))).size, (128, 256)) # test create inverse no store - record = self.env['test_new_api.model_image'].create({ + record = self.env['test_new_api.model_image'].with_context(image_no_postprocess=True).create({ 'name': 'image', 'image_256': image_w, }) diff --git a/odoo/tools/image.py b/odoo/tools/image.py index ff0c8f1749b..047c3784d5e 100644 --- a/odoo/tools/image.py +++ b/odoo/tools/image.py @@ -88,7 +88,68 @@ class ImageProcess(): if verify_resolution and w * h > IMAGE_MAX_RESOLUTION: raise ValueError(_("Image size excessive, uploaded images must be smaller than %s million pixels.", str(IMAGE_MAX_RESOLUTION / 10e6))) - def image_base64(self, quality=0, output_format=''): + def image_quality(self, quality=0, output_format=''): + """Return the image resulting of all the image processing + operations that have been applied previously. + + Return False if the initialized `image` was falsy, and return + the initialized `image` without change if it was SVG. + + Also return the initialized `image` if no operations have been applied + and the `output_format` is the same as the original format and the + quality is not specified. + + :param quality: quality setting to apply. Default to 0. + - for JPEG: 1 is worse, 95 is best. Values above 95 should be + avoided. Falsy values will fallback to 95, but only if the image + was changed, otherwise the original image is returned. + - for PNG: set falsy to prevent conversion to a WEB palette. + - for other formats: no effect. + :type quality: int + + :param output_format: the output format. Can be PNG, JPEG, GIF, or ICO. + Default to the format of the original image. BMP is converted to + PNG, other formats than those mentioned above are converted to JPEG. + :type output_format: string + + :return: image + :rtype: bytes or False + """ + if not self.image: + return self.image + + output_image = self.image + + output_format = output_format.upper() or self.original_format + if output_format == 'BMP': + output_format = 'PNG' + elif output_format not in ['PNG', 'JPEG', 'GIF', 'ICO']: + output_format = 'JPEG' + + if not self.operationsCount and output_format == self.original_format and not quality: + return self.image + + opt = {'output_format': output_format} + + if output_format == 'PNG': + opt['optimize'] = True + if quality: + if output_image.mode != 'P': + # Floyd Steinberg dithering by default + output_image = output_image.convert('RGBA').convert('P', palette=Image.WEB, colors=256) + if output_format == 'JPEG': + opt['optimize'] = True + opt['quality'] = quality or 95 + if output_format == 'GIF': + opt['optimize'] = True + opt['save_all'] = True + + if output_image.mode not in ["1", "L", "P", "RGB", "RGBA"] or (output_format == 'JPEG' and output_image.mode == 'RGBA'): + output_image = output_image.convert("RGB") + + return image_apply_opt(output_image, **opt) + + def image_quality_base64(self, quality=0, output_format=''): """Return the base64 encoded image resulting of all the image processing operations that have been applied previously. @@ -115,39 +176,15 @@ class ImageProcess(): :return: image base64 encoded or False :rtype: bytes or False """ - output_image = self.image - if not output_image: + if not self.image: return self.base64_source - output_format = output_format.upper() or self.original_format - if output_format == 'BMP': - output_format = 'PNG' - elif output_format not in ['PNG', 'JPEG', 'GIF', 'ICO']: - output_format = 'JPEG' + stream = self.image_quality(quality=quality, output_format=output_format) - if not self.operationsCount and output_format == self.original_format and not quality: - return self.base64_source - - opt = {'format': output_format} - - if output_format == 'PNG': - opt['optimize'] = True - if quality: - if output_image.mode != 'P': - # Floyd Steinberg dithering by default - output_image = output_image.convert('RGBA').convert('P', palette=Image.WEB, colors=256) - if output_format == 'JPEG': - opt['optimize'] = True - opt['quality'] = quality or 95 - if output_format == 'GIF': - opt['optimize'] = True - opt['save_all'] = True - - if output_image.mode not in ["1", "L", "P", "RGB", "RGBA"] or (output_format == 'JPEG' and output_image.mode == 'RGBA'): - output_image = output_image.convert("RGB") - - return image_to_base64(output_image, **opt) + if stream != self.image: + return base64.b64encode(stream) + return self.base64_source def resize(self, max_width=0, max_height=0): """Resize the image. @@ -285,7 +322,7 @@ def image_process(base64_source, size=(0, 0), verify_resolution=False, quality=0 image.resize(max_width=size[0], max_height=size[1]) if colorize: image.colorize() - return image.image_base64(quality=quality, output_format=output_format) + return image.image_quality_base64(quality=quality, output_format=output_format) # ---------------------------------------- @@ -406,7 +443,24 @@ def base64_to_image(base64_source): raise UserError(_("This file could not be decoded as an image file. Please try with a different file.")) -def image_to_base64(image, format, **params): +def image_apply_opt(image, output_format, **params): + """Return the given PIL `image` using `params`. + + :param image: the PIL image + :type image: PIL.Image + + :param params: params to expand when calling PIL.Image.save() + :type params: dict + + :return: the image formatted + :rtype: bytes + """ + stream = io.BytesIO() + image.save(stream, format=output_format, **params) + return stream.getvalue() + + +def image_to_base64(image, output_format, **params): """Return a base64_image from the given PIL `image` using `params`. :param image: the PIL image @@ -418,9 +472,8 @@ def image_to_base64(image, format, **params): :return: the image base64 encoded :rtype: bytes """ - stream = io.BytesIO() - image.save(stream, format=format, **params) - return base64.b64encode(stream.getvalue()) + stream = image_apply_opt(image, output_format, **params) + return base64.b64encode(stream) def is_image_size_above(base64_source_1, base64_source_2):