From b58967d8d140cf2f6ec0d1ed159041d8857e971b Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Wed, 10 Aug 2022 13:20:49 +0000 Subject: [PATCH] [IMP] loading: pregenerate assets bundles after install During tests on runbot, the main reason tours are slow to start is because the first loading of "/web" need to generate assets bundles. Generation can take up to 5 seconds time the number of tour. Before this, the first requests to /web takes around 7 seconds on runbot and the next ones less than 1 second. After this commit, the first request to /web takes around 1.5 seconds Note that the main difficulty is to choose when to generate the assets. (With optimisations from following commits) the generation time is arround 30 seconds (21 css + 12 js) the first time, for all modules. If the attachements exists the generation time is arround 3 seconds (2.5 js + 0.5 css) mainly because of globs to find usefull files. In practice for runbot the ideal would be to generate them at the end of the install so that it is shared for all post_install builds. Doint it at the end of an install is not wanted in all cases, saas pregenerated templates and upgrade may avoid doing that. Doing it in a special runbot step (subcommand) is not possible for niglty execution (no split, in one go). This needs to be in the code. It is also not practical for devs wanting to have the same behaviour on a local machine. A solution to make the test conditionnal at install is to base the condion on an existing test_module. test_assetsbundle is a good candidate here. On runbot, the at_install step (befor split) will automatically generate assets with this solution. Assets are also generated before post_install tests. This should be fast if they already exists and ensure that assets are up to date after modifying sources locally or after downloading a database on runbot. It should be fast enough for small database and may be even faster in the future for small diffs. Part-of: odoo/odoo#99176 --- odoo/addons/base/models/ir_qweb.py | 37 ++++++++++++++++++- odoo/addons/test_assetsbundle/__init__.py | 1 + .../test_assetsbundle/models/__init__.py | 1 + .../test_assetsbundle/models/ir_qweb.py | 11 ++++++ odoo/modules/loading.py | 8 ++-- odoo/service/server.py | 7 +++- 6 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 odoo/addons/test_assetsbundle/models/__init__.py create mode 100644 odoo/addons/test_assetsbundle/models/ir_qweb.py diff --git a/odoo/addons/base/models/ir_qweb.py b/odoo/addons/base/models/ir_qweb.py index 1f7782a1ba4..274f88bf1b5 100644 --- a/odoo/addons/base/models/ir_qweb.py +++ b/odoo/addons/base/models/ir_qweb.py @@ -365,6 +365,7 @@ import logging import math import re import textwrap +import time import token import tokenize import traceback @@ -381,7 +382,7 @@ from odoo import api, models, tools from odoo.tools import config, safe_eval, pycompat, SUPPORTED_DEBUGGER from odoo.tools.safe_eval import assert_valid_codeobj, _BUILTINS, to_opcodes, _EXPR_OPCODES, _BLACKLIST from odoo.tools.json import scriptsafe -from odoo.tools.misc import get_lang +from odoo.tools.misc import get_lang, str2bool from odoo.tools.image import image_data_uri from odoo.http import request from odoo.modules.module import get_resource_path @@ -2508,6 +2509,40 @@ class IrQWeb(models.AbstractModel): asset_nodes = self._get_asset_nodes(bundle, js=False, debug=debug) return [node[1]['href'] for node in asset_nodes if node[0] == 'link'] + def _pregenerate_assets_bundles(self): + """ + Pregenerates all assets that may be used in web pages to speedup first loading. + This may is mainly usefull for tests. + + The current version is looking for all t-call-assets in view to generate the minimal + set of bundles to generate. + + Current version only generate assets without extra, not taking care of rtl. + """ + _logger.runbot('Pregenerating assets bundles') + + views = self.env['ir.ui.view'].search([('type', '=', 'qweb'), ('arch_db', 'like', 't-call-assets')]) + js_bundles = set() + css_bundles = set() + for view in views: + for call_asset in etree.fromstring(view.arch_db).xpath("//*[@t-call-assets]"): + asset = call_asset.get('t-call-assets') + js = str2bool(call_asset.get('t-js', 'True')) + css = str2bool(call_asset.get('t-css', 'True')) + if js: + js_bundles.add(asset) + if css: + css_bundles.add(asset) + + start = time.time() + for bundle in sorted(js_bundles): + self._generate_asset_nodes(bundle, css=False, js=True) + _logger.info('JS Assets bundles generated in %s seconds', time.time()-start) + start = time.time() + for bundle in sorted(css_bundles): + self._generate_asset_nodes(bundle, css=True, js=False) + _logger.info('CSS Assets bundles generated in %s seconds', time.time()-start) + def render(template_name, values, load, **options): """ Rendering of a qweb template without database and outside the registry. (Widget, field, or asset rendering is not implemented.) diff --git a/odoo/addons/test_assetsbundle/__init__.py b/odoo/addons/test_assetsbundle/__init__.py index e046e49fbe2..91c5580fed3 100644 --- a/odoo/addons/test_assetsbundle/__init__.py +++ b/odoo/addons/test_assetsbundle/__init__.py @@ -1 +1,2 @@ from . import controllers +from . import models diff --git a/odoo/addons/test_assetsbundle/models/__init__.py b/odoo/addons/test_assetsbundle/models/__init__.py new file mode 100644 index 00000000000..33ac73eece1 --- /dev/null +++ b/odoo/addons/test_assetsbundle/models/__init__.py @@ -0,0 +1 @@ +from . import ir_qweb diff --git a/odoo/addons/test_assetsbundle/models/ir_qweb.py b/odoo/addons/test_assetsbundle/models/ir_qweb.py new file mode 100644 index 00000000000..4360810b76a --- /dev/null +++ b/odoo/addons/test_assetsbundle/models/ir_qweb.py @@ -0,0 +1,11 @@ +from odoo import models + +class IrQweb(models.AbstractModel): + _inherit = 'ir.qweb' + + def _register_hook(self): + super()._register_hook() + # if this module is installed, we are in a test environement. + # pregenerate assets at the end of the loading to speedup tests + if self.env.registry.updated_modules: + self._pregenerate_assets_bundles() diff --git a/odoo/modules/loading.py b/odoo/modules/loading.py index 2c4c9ae35d6..8ab9c44e3b6 100644 --- a/odoo/modules/loading.py +++ b/odoo/modules/loading.py @@ -579,7 +579,11 @@ def load_modules(registry, force_demo=False, status=None, update_module=False): else: _logger.error('At least one test failed when loading the modules.') - # STEP 8: call _register_hook on every model + + # STEP 8: save installed/updated modules for post-install tests and _register_hook + registry.updated_modules += processed_modules + + # STEP 9: call _register_hook on every model # This is done *exactly once* when the registry is being loaded. See the # management of those hooks in `Registry.setup_models`: all the calls to # setup_models() done here do not mess up with hooks, as registry.ready @@ -589,8 +593,6 @@ def load_modules(registry, force_demo=False, status=None, update_module=False): model._register_hook() env.flush_all() - # STEP 9: save installed/updated modules for post-install tests - registry.updated_modules += processed_modules def reset_modules_state(db_name): """ diff --git a/odoo/service/server.py b/odoo/service/server.py index ed5c1d96af8..605ba94a611 100644 --- a/odoo/service/server.py +++ b/odoo/service/server.py @@ -1307,7 +1307,12 @@ def preload_registries(dbnames): sorted(registry._init_modules)) _logger.info("Starting post tests") tests_before = registry._assertion_report.testsRun - result = loader.run_suite(loader.make_suite(module_names, 'post_install')) + post_install_suite = loader.make_suite(module_names, 'post_install') + if post_install_suite.countTestCases(): + with registry.cursor() as cr: + env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {}) + env['ir.qweb']._pregenerate_assets_bundles() + result = loader.run_suite(post_install_suite) registry._assertion_report.update(result) _logger.info("%d post-tests in %.2fs, %s queries", registry._assertion_report.testsRun - tests_before,