From 3db7956b4c43ee580de91a91332ec8c845b2cd66 Mon Sep 17 00:00:00 2001 From: Olivier Dony Date: Thu, 27 Oct 2022 10:53:06 +0000 Subject: [PATCH] [IMP] assets: switch to optimized rjsmin minification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C implementation of the JS minification gives speedups between 6 and 55 times faster than the regex-based Python port, depending on how compressed the input it (which is what our default implementation does). This is measurable when generating compiled assets bundle from scratch, e.g. after installing/updating modules or source code. As an illustration, the minification of a 2MB JS bundle can be 50x faster: ```py import rjsmin from odoo.addons.base.models.assetsbundle import rjsmin as rjsm js_source = open("web.assets_common_lazy.js").read() # 2MB JS %timeit rjsm(js_source) # -> 339 ms ± 495 µs per loop (mean ± std. dev. of 7 runs, 1 loop each) %timeit rjsmin.jsmin(js_source) # -> 6.88 ms ± 213 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) ``` It's also a drop-in replacement, as long as you rjsmin 1.1.0 or better is available (to support format strings properly, a.o.). See also the documentation of rjsmin: http://opensource.perlig.de/rjsmin/ closes odoo/odoo#104283 Signed-off-by: Xavier Dollé (xdo) --- debian/control | 1 + odoo/addons/base/models/assetsbundle.py | 61 +------------------------ odoo/netsvc.py | 2 + requirements.txt | 1 + setup.py | 1 + setup/package.dfdebian | 1 + setup/package.dffedora | 1 + 7 files changed, 9 insertions(+), 59 deletions(-) diff --git a/debian/control b/debian/control index 1303c669200..164cc3c7aad 100644 --- a/debian/control +++ b/debian/control @@ -44,6 +44,7 @@ Depends: python3-pydot, python3-openssl, python3-pypdf2, + python3-rjsmin, python3-qrcode, python3-renderpm, python3-reportlab, diff --git a/odoo/addons/base/models/assetsbundle.py b/odoo/addons/base/models/assetsbundle.py index 56d30af95d5..6359bcaf09a 100644 --- a/odoo/addons/base/models/assetsbundle.py +++ b/odoo/addons/base/models/assetsbundle.py @@ -24,6 +24,8 @@ except ImportError: # `sassc` executable in the path. libsass = None +from rjsmin import jsmin as rjsmin + from odoo import release, SUPERUSER_ID, _ from odoo.http import request from odoo.modules.module import get_resource_path @@ -39,65 +41,6 @@ EXTENSIONS = (".js", ".css", ".scss", ".sass", ".less", ".xml") class CompileError(RuntimeError): pass -def rjsmin(script): - """ Minify js with a clever regex. - Taken from http://opensource.perlig.de/rjsmin (version 1.1.0) - Apache License, Version 2.0 """ - def subber(match): - """ Substitution callback """ - groups = match.groups() - return ( - groups[0] or - groups[1] or - (groups[3] and (groups[2] + '\n')) or - groups[2] or - (groups[5] and "%s%s%s" % ( - groups[4] and '\n' or '', - groups[5], - groups[6] and '\n' or '', - )) or - (groups[7] and '\n') or - (groups[8] and ' ') or - (groups[9] and ' ') or - (groups[10] and ' ') or - '' - ) - - result = re.sub( - r'([^\047"\140/\000-\040]+)|((?:(?:\047[^\047\\\r\n]*(?:\\(?:[^' - r'\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)|(?:"[^"\\\r\n]*(?:\\(?:[^' - r'\r\n]|\r?\n|\r)[^"\\\r\n]*)*")|(?:\140[^\140\\]*(?:\\(?:[^\r\n' - r']|\r?\n|\r)[^\140\\]*)*\140))[^\047"\140/\000-\040]*)|(?<=[(,=' - r':\[!&|?{};\r\n+*-])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*' - r'\*+(?:[^/*][^*]*\*+)*/))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-' - r'\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)*(' - r'(?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(?:\\[^\r\n]|(?:\[[^\\\]\r\n]*' - r'(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/\\\[\r\n]*)*/))((?:[\000-\011' - r'\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*(?:(?:(' - r'?://[^\r\n]*)?[\r\n])(?:[\000-\011\013\014\016-\040]|(?:/\*[^*' - r']*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000-\040&)+,.:;=?\]|}-]))?|' - r'(?<=[\000-#%-,./:-@\[-^\140{-~-]return)(?:[\000-\011\013\014\0' - r'16-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*(?:((?:(?://[^\r' - r'\n]*)?[\r\n]))(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?' - r':[^/*][^*]*\*+)*/))*)*((?:/(?![\r\n/*])[^/\\\[\r\n]*(?:(?:\\[^' - r'\r\n]|(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\]))[^/\\\[\r' - r'\n]*)*/))((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/' - r'*][^*]*\*+)*/))*(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-\011\013' - r'\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000' - r'-\040&)+,.:;=?\]|}-]))?|(?<=[^\000-!#%&(*,./:-@\[\\^{|~])(?:[' - r'\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)' - r')*(?:((?:(?://[^\r\n]*)?[\r\n]))(?:[\000-\011\013\014\016-\040' - r']|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+(?=[^\000-\040"#%-\047' - r')*,./:-@\\-^\140|-~])|(?<=[^\000-#%-,./:-@\[-^\140{-~-])((?:[' - r'\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)' - r'))+(?=[^\000-#%-,./:-@\[-^\140{-~-])|(?<=\+)((?:[\000-\011\013' - r'\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)))+(?=\+)|(?<' - r'=-)((?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]' - r'*\*+)*/)))+(?=-)|(?:[\000-\011\013\014\016-\040]|(?:/\*[^*]*\*' - r'+(?:[^/*][^*]*\*+)*/))+|(?:(?:(?://[^\r\n]*)?[\r\n])(?:[\000-' - r'\011\013\014\016-\040]|(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/))*)+', subber, '\n%s\n' % script - ).strip() - return result class AssetError(Exception): pass diff --git a/odoo/netsvc.py b/odoo/netsvc.py index 776a6f49b6c..687a1899d83 100644 --- a/odoo/netsvc.py +++ b/odoo/netsvc.py @@ -166,6 +166,8 @@ def init_logger(): ]: warnings.filterwarnings('ignore', category=DeprecationWarning, module=module) + # rsjmin triggers this with Python 3.10+ (that warning comes from the C code and has no `module`) + warnings.filterwarnings('ignore', r'^PyUnicode_FromUnicode\(NULL, size\) is deprecated', category=DeprecationWarning) # the SVG guesser thing always compares str and bytes, ignore it warnings.filterwarnings('ignore', category=BytesWarning, module='odoo.tools.image') # reportlab does a bunch of bytes/str mixing in a hashmap diff --git a/requirements.txt b/requirements.txt index 8934f148cc8..5c7d36f101a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,6 +50,7 @@ qrcode==6.1 reportlab==3.5.59 ; python_version <= '3.10' # version < 3.5.54 are not compatible with Pillow 8.1.2 and 3.5.59 is bullseye reportlab==3.6.12 ; python_version > '3.10' requests==2.25.1 # versions < 2.25 aren't compatible w/ urllib3 1.26. Bullseye = 2.25.1. min version = 2.22.0 (Focal) +rjsmin==1.1.0 urllib3==1.26.5 # indirect / min version = 1.25.8 (Focal with security backports) vobject==0.9.6.1 Werkzeug==0.16.1 ; python_version <= '3.9' diff --git a/setup.py b/setup.py index 2cff960618d..232b017e460 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ setup( 'pyusb >= 1.0.0b1', 'qrcode', 'reportlab', # windows binary pypi.python.org/pypi/reportlab + 'rjsmin', 'requests', 'urllib3', 'vobject', diff --git a/setup/package.dfdebian b/setup/package.dfdebian index 7850130a385..f4977055b6b 100644 --- a/setup/package.dfdebian +++ b/setup/package.dfdebian @@ -41,6 +41,7 @@ RUN apt-get update -qq && \ python3-pydot \ python3-openssl \ python3-pypdf2 \ + python3-rjsmin \ python3-qrcode \ python3-renderpm \ python3-reportlab \ diff --git a/setup/package.dffedora b/setup/package.dffedora index bed19777cbd..ecd55c376e4 100644 --- a/setup/package.dffedora +++ b/setup/package.dffedora @@ -47,6 +47,7 @@ RUN dnf update -d 0 -e 0 -y && \ python3-qrcode \ python3-reportlab \ python3-requests \ + python3-rjsmin \ python3-six \ python3-stdnum \ python3-vobject \