[IMP] base,web: Improve webclient startup

Today, some resources that varies very infrequentely are requested on
every page load: the menus, the translations and the qweb templates

After this commit, there will be 2 improvements that will make the
loading of the webclient faster:
1. cache almost static resources forever
when loading the /web/ page, the user session will
contain the cache key to be used to fetch these resources. As the content
of the resource itself it used to compute the hash, the request can be
put in cache by the browser forever (for a year).

The up side is that in general less requests are needed to load the web client
The down side is that the load time and number of queries of /web page
has increased to include what is needed to compute the hash of translations,
menus and qweb templates

2. trigger the loading of the menus before the download (and startup) of
the JS and CSS assets:
The menus contains the definition of the action, wich is the first thing
needed to load a page in odoo. Starting the loading of the menu much
earlier means that when they are not yet in the browser cache (see point 1),
the browser does not need to wait for the entire assets to be downloaded
and parsed before starting the 'load_menu' request, but instead start it
immediately and hold onto a promise to be used instead of the rpc 'load_menu'.

Because the request is done before anything else is loaded, including
JQuery, it is made in vanilla JS.

Co-authored-by: Julien Mougenot <jum@odoo.com>

addons/web/controllers/main.py

@@ -36,7 +36,7 @@ import odoo
 import odoo.modules.registry
 from odoo.api import call_kw, Environment
 from odoo.modules import get_module_path, get_resource_path
-from odoo.tools import image_process, topological_sort, html_escape, pycompat
+from odoo.tools import image_process, topological_sort, html_escape, pycompat, ustr
 from odoo.tools.mimetypes import guess_mimetype
 from odoo.tools.translate import _
 from odoo.tools.misc import str2bool, xlwt, file_open
@@ -63,6 +63,9 @@ env.filters["json"] = json.dumps
 # 1 week cache for asset bundles as advised by Google Page Speed
 BUNDLE_MAXAGE = 60 * 60 * 24 * 7
 
+# 1 year cache for content (menus, translations, static qweb)
+CONTENT_MAXAGE = 60 * 60 * 24 * 356
+
 DBNAME_PATTERN = '^[a-zA-Z0-9][a-zA-Z0-9_.-]+$'
 
 #----------------------------------------------------------
@@ -451,6 +454,23 @@ class Home(http.Controller):
         except AccessError:
             return werkzeug.utils.redirect('/web/login?error=access')
 
+    @http.route('/web/webclient/load_menus/<string:unique>', type='http', auth='user', methods=['GET'])
+    def web_load_menus(self, unique):
+        """
+        Loads the menus for the webclient
+        :param unique: this parameters is not used, but mandatory: it is used by the HTTP stack to make a unique request
+        :return: the menus (including the images in Base64)
+        """
+        menus = request.env["ir.ui.menu"].load_menus(request.debug)
+        body = json.dumps(menus, default=ustr)
+        response = request.make_response(body, [
+            # this method must specify a content-type application/json instead of using the default text/html set because
+            # the type of the route is set to HTTP, but the rpc is made with a get and expects JSON
+            ('Content-Type', 'application/json'),
+            ('Cache-Control', 'public, max-age=31536000'),
+        ])
+        return response
+
     @http.route('/web/dbredirect', type='http', auth="none")
     def web_db_redirect(self, redirect='/', **kw):
         ensure_db()
@@ -549,18 +569,14 @@ class WebClient(http.Controller):
             ('Cache-Control', 'max-age=36000'),
         ])
 
-    @http.route('/web/webclient/qweb', type='http', auth="none", cors="*")
+    @http.route('/web/webclient/qweb/<string:unique>', type='http', auth="none", cors="*")
-    def qweb(self, mods=None, db=None):
+    def qweb(self, unique, mods=None, db=None):
         files = [f[0] for f in manifest_glob('qweb', addons=mods, db=db)]
-        last_modified = get_last_modified(files)
+        content, _dummy = concat_xml(files)
-        if request.httprequest.if_modified_since and request.httprequest.if_modified_since >= last_modified:
+        return request.make_response(content, [
-            return werkzeug.wrappers.Response(status=304)
+                ('Content-Type', 'text/xml'),
-
+                ('Cache-Control','public, max-age=' + str(CONTENT_MAXAGE))
-        content, checksum = concat_xml(files)
+            ])
-
-        return make_conditional(
-            request.make_response(content, [('Content-Type', 'text/xml')]),
-            last_modified, checksum)
 
     @http.route('/web/webclient/bootstrap_translations', type='json', auth="none")
     def bootstrap_translations(self, mods):
@@ -585,41 +601,32 @@ class WebClient(http.Controller):
         return {"modules": translations_per_module,
                 "lang_parameters": None}
 
-    @http.route('/web/webclient/translations', type='json', auth="none")
+    @http.route('/web/webclient/translations/<string:unique>', type='http', auth="none")
-    def translations(self, mods=None, lang=None):
+    def translations(self, unique, mods=None, lang=None):
-        request.disable_db = False
+        """
-        if mods is None:
+        Load the translations for the specified language and modules
-            mods = [x['name'] for x in request.env['ir.module.module'].sudo().search_read(
-                [('state', '=', 'installed')], ['name'])]
-        if lang is None:
-            lang = request.context["lang"]
-        langs = request.env['res.lang'].sudo().search([("code", "=", lang)])
-        lang_params = None
-        if langs:
-            lang_params = langs.read([
-                "name", "direction", "date_format", "time_format",
-                "grouping", "decimal_point", "thousands_sep", "week_start"])[0]
-            lang_params['week_start'] = int(lang_params['week_start'])
 
-        # Regional languages (ll_CC) must inherit/override their parent lang (ll), but this is
+        :param unique: this parameters is not used, but mandatory: it is used by the HTTP stack to make a unique request
-        # done server-side when the language is loaded, so we only need to load the user's lang.
+        :param mods: the modules, a comma separated list
-        translations_per_module = {}
+        :param lang: the language of the user
-        messages = request.env['ir.translation'].sudo().search_read([
+        :return:
-            ('module', 'in', mods), ('lang', '=', lang),
+        """
-            ('comments', 'like', 'openerp-web'), ('value', '!=', False),
+        request.disable_db = False
-            ('value', '!=', '')],
+
-            ['module', 'src', 'value', 'lang'], order='module')
+        translations_per_module, lang_params = request.env["ir.translation"].get_translations_for_webclient(mods, lang)
-        for mod, msg_group in itertools.groupby(messages, key=operator.itemgetter('module')):
+
-            translations_per_module.setdefault(mod, {'messages': []})
+        body = json.dumps({
-            translations_per_module[mod]['messages'].extend({
-                'id': m['src'],
-                'string': m['value']}
-                for m in msg_group)
-        return {
             'lang_parameters': lang_params,
             'modules': translations_per_module,
             'multi_lang': len(request.env['res.lang'].sudo().get_installed()) > 1,
-        }
+        })
+        response = request.make_response(body, [
+            # this method must specify a content-type application/json instead of using the default text/html set because
+            # the type of the route is set to HTTP, but the rpc is made with a get and expects JSON
+            ('Content-Type', 'application/json'),
+            ('Cache-Control', 'public, max-age=' + str(CONTENT_MAXAGE)),
+        ])
+        return response
 
     @http.route('/web/webclient/version_info', type='json', auth="none")
     def version_info(self):

addons/web/models/ir_http.py

@@ -1,10 +1,13 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import hashlib
 import json
 
 from odoo import models
 from odoo.http import request
+from odoo.tools import ustr
+
+from odoo.addons.web.controllers.main import concat_xml, manifest_glob, module_boot
 
 import odoo
 
@@ -21,6 +24,19 @@ class Http(models.AbstractModel):
     def session_info(self):
         user = request.env.user
         version_info = odoo.service.common.exp_version()
+
+        user_context = request.session.get_context() if request.session.uid else {}
+
+        mods = ','.join(module_boot())
+        files = [f[0] for f in manifest_glob('qweb', addons=mods)]
+        _, qweb_checksum = concat_xml(files)
+
+        lang = user_context.get("lang")
+        translations_per_module, _ = request.env['ir.translation'].get_translations_for_webclient(mods, lang)
+
+        menu_json_utf8 = json.dumps(request.env['ir.ui.menu'].load_menus(request.debug), default=ustr, sort_keys=True).encode()
+        translations_json_utf8 = json.dumps(translations_per_module,  sort_keys=True).encode()
+
         return {
             "uid": request.session.uid,
             "is_system": user._is_system() if request.session.uid else False,
@@ -41,7 +57,11 @@ class Http(models.AbstractModel):
             "show_effect": True,
             "display_switch_company_menu": user.has_group('base.group_multi_company') and len(user.company_ids) > 1,
             "toggle_company": user.has_group('base.group_toggle_company'),
-
+            "cache_hashes": {
+                "load_menus": hashlib.sha1(menu_json_utf8).hexdigest(),
+                "qweb": qweb_checksum,
+                "translations": hashlib.sha1(translations_json_utf8).hexdigest(),
+            },
         }
 
     def get_currencies(self):

addons/web/static/src/js/chrome/abstract_web_client.js

@@ -101,7 +101,7 @@ var AbstractWebClient = Widget.extend(ServiceProviderMixin, KeyboardNavigationMi
         KeyboardNavigationMixin.init.call(this);
         this.origin = undefined;
         this._current_state = null;
-        this.menu_dm = new concurrency.DropMisordered();
+        this.menu_dp = new concurrency.DropPrevious();
         this.action_mutex = new concurrency.Mutex();
         this.set('title_part', {"zopenerp": "Odoo"});
     },

addons/web/static/src/js/chrome/web_client.js

@@ -55,12 +55,7 @@ return AbstractWebClient.extend({
         });
     },
     load_menus: function () {
-        return this._rpc({
+        return (odoo.loadMenusPromise || odoo.reloadMenus())
-                model: 'ir.ui.menu',
-                method: 'load_menus',
-                args: [config.debug],
-                context: session.user_context,
-            })
             .then(function (menuData) {
                 // Compute action_id if not defined on a top menu item
                 for (var i = 0; i < menuData.children.length; i++) {
@@ -75,6 +70,7 @@ return AbstractWebClient.extend({
                         }
                     }
                 }
+                odoo.loadMenusPromise = null;
                 return menuData;
             });
     },
@@ -82,18 +78,18 @@ return AbstractWebClient.extend({
         var self = this;
         this.set_title();
 
-        return this.instanciate_menu_widgets().then(function () {
+        return this.menu_dp.add(this.instanciate_menu_widgets()).then(function () {
             $(window).bind('hashchange', self.on_hashchange);
 
             // If the url's state is empty, we execute the user's home action if there is one (we
             // show the first app if not)
             var state = $.bbq.getState(true);
             if (_.keys(state).length === 1 && _.keys(state)[0] === "cids") {
-                return self._rpc({
+                return self.menu_dp.add(self._rpc({
                         model: 'res.users',
                         method: 'read',
                         args: [session.uid, ["action_id"]],
-                    })
+                    }))
                     .then(function (result) {
                         var data = result[0];
                         if (data.action_id) {
@@ -143,7 +139,7 @@ return AbstractWebClient.extend({
             if (!_.isEqual(self._current_state, stringstate)) {
                 var state = $.bbq.getState(true);
                 if (state.action || (state.model && (state.view_type || state.id))) {
-                    return self.action_manager.loadState(state, !!self._current_state).then(function () {
+                    return self.menu_dp.add(self.action_manager.loadState(state, !!self._current_state)).then(function () {
                         if (state.menu_id) {
                             if (state.menu_id !== self.menu.current_primary_menu) {
                                 core.bus.trigger('change_menu_section', state.menu_id);
@@ -158,7 +154,7 @@ return AbstractWebClient.extend({
                     });
                 } else if (state.menu_id) {
                     var action_id = self.menu.menu_id_to_action_id(state.menu_id);
-                    return self.do_action(action_id, {clear_breadcrumbs: true}).then(function () {
+                    return self.menu_dp.add(self.do_action(action_id, {clear_breadcrumbs: true})).then(function () {
                         core.bus.trigger('change_menu_section', state.menu_id);
                     });
                 } else {
@@ -179,7 +175,7 @@ return AbstractWebClient.extend({
     // --------------------------------------------------------------
     on_app_clicked: function (ev) {
         var self = this;
-        return this.menu_dm.add(data_manager.load_action(ev.data.action_id))
+        return this.menu_dp.add(data_manager.load_action(ev.data.action_id))
             .then(function (result) {
                 return self.action_mutex.exec(function () {
                     var completed = new Promise(function (resolve, reject) {
@@ -210,7 +206,7 @@ return AbstractWebClient.extend({
     },
     on_menu_clicked: function (ev) {
         var self = this;
-        return this.menu_dm.add(data_manager.load_action(ev.data.action_id))
+        return this.menu_dp.add(data_manager.load_action(ev.data.action_id))
             .then(function (result) {
                 self.$el.removeClass('o_mobile_menu_opened');
 

addons/web/static/src/js/core/session.js

@@ -241,8 +241,11 @@ var Session = core.Class.extend(mixins.EventDispatcherMixin, {
         });
     },
     load_qweb: function (mods) {
+        var self = this;
         var lock = this.qweb_mutex.exec(function () {
-            return $.get('/web/webclient/qweb?mods=' + mods).then(function (doc) {
+            var cacheId = self.cache_hashes && self.cache_hashes.qweb;
+            var route  = '/web/webclient/qweb/' + (cacheId ? cacheId : Date.now()) + '?mods=' + mods;
+            return $.get(route).then(function (doc) {
                 if (!doc) { return; }
                 qweb.add_template(doc);
             });
@@ -344,10 +347,18 @@ var Session = core.Class.extend(mixins.EventDispatcherMixin, {
     getTZOffset: function (date) {
         return -new Date(date).getTimezoneOffset();
     },
-
     //--------------------------------------------------------------------------
     // Public
     //--------------------------------------------------------------------------
+    /**
+     * Replaces the value of a key in cache_hashes (the hash of some resource computed on the back-end by a unique value
+     * @param {string} key the key in the cache_hashes to invalidate
+     */
+    invalidateCacheKey: function(key) {
+        if (this.cache_hashes && this.cache_hashes[key]) {
+            this.cache_hashes[key] = Date.now();
+        }
+    },
 
     /**
      * Reload the currencies (initially given in session_info). This is meant to

addons/web/static/src/js/core/translation.js

@@ -62,9 +62,12 @@ var TranslationDataBase = Class.extend(/** @lends instance.TranslationDataBase#
     */
     load_translations: function(session, modules, lang, url) {
         var self = this;
-        return session.rpc(url || '/web/webclient/translations', {
+        var cacheId = session.cache_hashes && session.cache_hashes.translations;
-            "mods": modules || null,
+        url = url || '/web/webclient/translations';
-            "lang": lang || null
+        url += '/' + (cacheId ? cacheId : Date.now());
+        return $.get(url, {
+            mods: modules ? modules.join(',') : null,
+            lang: lang || null,
         }).then(function (trans) {
             self.set_bundle(trans);
         });

addons/web/static/src/js/services/data_manager.js

@@ -4,6 +4,7 @@ odoo.define('web.DataManager', function (require) {
 var config = require('web.config');
 var core = require('web.core');
 var rpc = require('web.rpc');
+var session = require('web.session');
 var utils = require('web.utils');
 
 return core.Class.extend({
@@ -26,6 +27,7 @@ return core.Class.extend({
      * Suggestion: could be refined to invalidate some part of the cache
      */
     invalidate: function () {
+        session.invalidateCacheKey('load_menus');
         this._init_cache();
     },
 
@@ -45,7 +47,7 @@ return core.Class.extend({
                 route: "/web/action/load",
                 params: {
                     action_id: action_id,
-                    additional_context : additional_context,
+                    additional_context: additional_context,
                 },
             }).then(function (action) {
                 self._cache.actions[key] = action.no_cache ? null : self._cache.actions[key];

addons/web/views/webclient_templates.xml

@@ -747,8 +747,21 @@
             <t t-set="head_web">
                 <script type="text/javascript">
                     odoo.session_info = <t t-raw="session_info"/>;
+                    odoo.reloadMenus = function () {
+                        return new Promise(function(resolve, reject) {
+                            function onLoaded() {
+                                resolve(oReq.response);
+                            }
+                            var oReq = new XMLHttpRequest();
+                            oReq.responseType = "json";
+                            oReq.addEventListener("load", onLoaded);
+                            oReq.addEventListener("error", reject);
+                            oReq.open("GET", '/web/webclient/load_menus/' + odoo.session_info.cache_hashes.load_menus);
+                            oReq.send();
+                        });
+                    }
+                    odoo.loadMenusPromise = odoo.reloadMenus();
                 </script>
-
                 <t t-call-assets="web.assets_common" t-js="false"/>
                 <t t-call-assets="web.assets_backend" t-js="false"/>
                 <t t-call-assets="web.assets_common" t-css="false"/>

odoo/addons/base/models/ir_translation.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import itertools
 import logging
+import operator
 from collections import defaultdict
 from difflib import get_close_matches
 
@@ -851,3 +852,37 @@ class IrTranslation(models.Model):
                          ('name', 'like', model_name+','),
             ],
         }
+
+    @api.model
+    def get_translations_for_webclient(self, mods, lang):
+        if mods is None:
+            mods = [x['name'] for x in self.env['ir.module.module'].sudo().search_read(
+                [('state', '=', 'installed')], ['name'])]
+        else:
+            mods = mods.split(',')
+        if lang is None:
+            lang = self._context["lang"]
+        langs = self.env['res.lang'].sudo().search([("code", "=", lang)])
+        lang_params = None
+        if langs:
+            lang_params = langs.read([
+                "name", "direction", "date_format", "time_format",
+                "grouping", "decimal_point", "thousands_sep", "week_start"])[0]
+            lang_params['week_start'] = int(lang_params['week_start'])
+
+        # Regional languages (ll_CC) must inherit/override their parent lang (ll), but this is
+        # done server-side when the language is loaded, so we only need to load the user's lang.
+        translations_per_module = {}
+        messages = self.env['ir.translation'].sudo().search_read([
+            ('module', 'in', mods), ('lang', '=', lang),
+            ('comments', 'like', 'openerp-web'), ('value', '!=', False),
+            ('value', '!=', '')],
+            ['module', 'src', 'value', 'lang'], order='module')
+        for mod, msg_group in itertools.groupby(messages, key=operator.itemgetter('module')):
+            translations_per_module.setdefault(mod, {'messages': []})
+            translations_per_module[mod]['messages'].extend({
+                'id': m['src'],
+                'string': m['value']}
+                for m in msg_group)
+
+        return translations_per_module, lang_params