[FW][FIX] website: perf - reduce SQL queries on hot execution on website page

This PR is the continuity of #46925 It is a bunch of small commits, retro compatible, that should reduce the number of sql query for each page. -- Optimistic stats ``` | | Query | Time | | --------------------- | ----- | ------ | | website.page | | | | Before | 25 | 0.0218 | | Now | 12 | 0.0106 | | tracked website.page | | | | Before | 34 | 0.0276 | | Now | 20 | 0.0123 | | tracked homepage | | | | Before | 34 | 0.0223 | | Now | 19 | 0.0194 | ``` task-2211013 closes odoo/odoo#48048 Forward-port-of: odoo/odoo#47891 Forward-port-of: odoo/odoo#47257 Signed-off-by: Jérémy Kersten (jke) <jke@openerp.com> Signed-off-by: Romain Derie <rdeodoo@users.noreply.github.com>
2020-03-19 17:01:17 +01:00
parent 96486de1c1 ffd6ebff72
commit 04e07ba9aa
17 changed files with 179 additions and 73 deletions
@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-

-import hashlib
-import json
 import logging
 import os
 import re
@@ -152,7 +150,7 @@ def url_lang(path_or_uri, lang_code=None):
        location = werkzeug.urls.url_join(request.httprequest.path, location)
        lang_url_codes = [url_code for _, url_code, _ in Lang.get_available()]
        lang_code = pycompat.to_text(lang_code or request.context['lang'])
-        lang_url_code = Lang._lang_get(lang_code).url_code
+        lang_url_code = Lang._lang_code_to_urlcode(lang_code)
        lang_url_code = lang_url_code if lang_url_code in lang_url_codes else lang_code

        if (len(lang_url_codes) > 1 or force_lang) and is_multilang_url(location, lang_url_codes):
@@ -302,18 +300,12 @@ class IrHttp(models.AbstractModel):
        modules = IrHttpModel.get_translation_frontend_modules()
        user_context = request.session.get_context() if request.session.uid else {}
        lang = user_context.get('lang')
-        translations, lang_params = request.env['ir.translation'].get_translations_for_webclient(modules, lang)
-        translation_cache = {
-            'lang_parameters': lang_params,
-            'modules': translations,
-            'multi_lang': len(request.env['res.lang'].sudo().get_installed()) > 1,
-            'lang': lang,
-        }
+        translation_hash = request.env['ir.translation'].get_web_translations_hash(modules, lang)

        session_info.update({
            'translationURL': '/website/translations',
            'cache_hashes': {
-                'translations': hashlib.sha512(json.dumps(translation_cache, sort_keys=True).encode()).hexdigest()[:64],  # sha512/256
+                'translations': translation_hash,
            },
        })
        return session_info
@@ -321,17 +313,28 @@ class IrHttp(models.AbstractModel):
    @api.model
    def get_translation_frontend_modules(self):
        Modules = request.env['ir.module.module'].sudo()
-        domain = self._get_translation_frontend_modules_domain()
-        return Modules.search(
-            expression.AND([domain, [('state', '=', 'installed')]])
-        ).mapped('name')
+        extra_modules_domain = self._get_translation_frontend_modules_domain()
+        extra_modules_name = self._get_translation_frontend_modules_name()
+        if extra_modules_domain:
+            new = Modules.search(
+                expression.AND([extra_modules_domain, [('state', '=', 'installed')]])
+            ).mapped('name')
+            extra_modules_name += new
+        return extra_modules_name

    @classmethod
    def _get_translation_frontend_modules_domain(cls):
        """ Return a domain to list the domain adding web-translations and
            dynamic resources that may be used frontend views
        """
-        return [('name', '=', 'web')]
+        return []
+
+    @classmethod
+    def _get_translation_frontend_modules_name(cls):
+        """ Return a list of module name where web-translations and
+            dynamic resources may be used in frontend views
+        """
+        return ['web']

    bots = "bot|crawl|slurp|spider|curl|wget|facebookexternalhit".split("|")

@@ -2,13 +2,12 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.

 from odoo import models
-from odoo.osv import expression


 class IrHttp(models.AbstractModel):
    _inherit = 'ir.http'

    @classmethod
-    def _get_translation_frontend_modules_domain(cls):
-        domain = super(IrHttp, cls)._get_translation_frontend_modules_domain()
-        return expression.OR([domain, [('name', '=', 'payment')]])
+    def _get_translation_frontend_modules_name(cls):
+        mods = super(IrHttp, cls)._get_translation_frontend_modules_name()
+        return mods + ['payment']
@@ -2,13 +2,12 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.

 from odoo import models
-from odoo.osv import expression


 class IrHttp(models.AbstractModel):
    _inherit = 'ir.http'

    @classmethod
-    def _get_translation_frontend_modules_domain(cls):
-        domain = super(IrHttp, cls)._get_translation_frontend_modules_domain()
-        return expression.OR([domain, [('name', '=', 'portal')]])
+    def _get_translation_frontend_modules_name(cls):
+        mods = super(IrHttp, cls)._get_translation_frontend_modules_name()
+        return mods + ['portal']
@@ -2,7 +2,7 @@
 <odoo>
    <template id="frontend_layout" name="Main Frontend Layout" inherit_id="web.frontend_layout">
        <xpath expr="//div[@id='wrapwrap']" position="attributes">
-            <attribute name="t-attf-class" add="#{request.env['res.lang'].search([('code', '=', request.env.lang)]).direction == 'rtl' and 'o_rtl' or ''}" separator=" "/>
+            <attribute name="t-attf-class" add="#{request.env['res.lang']._lang_get(request.env.lang).direction == 'rtl' and 'o_rtl' or ''}" separator=" "/>
            <attribute name="t-attf-class" add="#{'o_portal' if is_portal else ''}" separator=" "/>
        </xpath>
        <xpath expr="//div[@id='wrapwrap']/header/img" position="replace">
@@ -50,19 +50,12 @@ class Http(models.AbstractModel):
            mods = module_boot()
            qweb_checksum = HomeStaticTemplateHelpers.get_qweb_templates_checksum(addons=mods, debug=request.session.debug)
            lang = user_context.get("lang")
-            translations_per_module, lang_params = request.env['ir.translation'].get_translations_for_webclient(mods, lang)
-            translation_cache = {
-                'lang': lang,
-                'lang_parameters': lang_params,
-                'modules': translations_per_module,
-                'multi_lang': len(request.env['res.lang'].sudo().get_installed()) > 1,
-            }
+            translation_hash = request.env['ir.translation'].get_web_translations_hash(mods, lang)
            menu_json_utf8 = json.dumps(request.env['ir.ui.menu'].load_menus(request.session.debug), default=ustr, sort_keys=True).encode()
-            translations_json_utf8 = json.dumps(translation_cache, sort_keys=True).encode()
            cache_hashes = {
                "load_menus": hashlib.sha512(menu_json_utf8).hexdigest()[:64], # sha512/256
                "qweb": qweb_checksum,
-                "translations": hashlib.sha512(translations_json_utf8).hexdigest()[:64], # sha512/256
+                "translations": translation_hash,
            }
            session_info.update({
                # current_company should be default_company
@@ -3,7 +3,6 @@

 from odoo import models
 from odoo.http import request
-from odoo.osv import expression


 class IrHttp(models.AbstractModel):
@@ -22,6 +21,6 @@ class IrHttp(models.AbstractModel):
        return super(IrHttp, cls)._dispatch()

    @classmethod
-    def _get_translation_frontend_modules_domain(cls):
-        domain = super(IrHttp, cls)._get_translation_frontend_modules_domain()
-        return expression.OR([domain, [('name', '=', 'web_editor')]])
+    def _get_translation_frontend_modules_name(cls):
+        mods = super(IrHttp, cls)._get_translation_frontend_modules_name()
+        return mods + ['web_editor']
@@ -278,7 +278,7 @@
            <field name="domain"></field>
            <field name="company_id" ref="base.main_company"/>
            <field name="user_id" ref="base.public_user"/>
-            <field name="homepage_id" ref="homepage_page"/>
+            <!-- Correct homepage will be set during bootstraping --> 
        </record>

        <!-- Open website on install -->
@@ -17,7 +17,7 @@ from odoo import api, models
 from odoo import registry, SUPERUSER_ID
 from odoo.http import request
 from odoo.tools.safe_eval import safe_eval
-from odoo.osv.expression import FALSE_DOMAIN, OR
+from odoo.osv.expression import FALSE_DOMAIN

 from odoo.addons.http_routing.models.ir_http import ModelConverter, _guess_mimetype
 from odoo.addons.portal.controllers.portal import _build_url_w_params
@@ -226,9 +226,10 @@ class Http(models.AbstractModel):
        return super(Http, cls)._get_default_lang()

    @classmethod
-    def _get_translation_frontend_modules_domain(cls):
-        domain = super(Http, cls)._get_translation_frontend_modules_domain()
-        return OR([domain, [('name', 'ilike', 'website')]])
+    def _get_translation_frontend_modules_name(cls):
+        mods = super(Http, cls)._get_translation_frontend_modules_name()
+        installed = request.registry._init_modules | set(odoo.conf.server_wide_modules)
+        return mods + [mod for mod in installed if mod.startswith('website')]

    @classmethod
    def _serve_page(cls):
@@ -236,22 +237,15 @@ class Http(models.AbstractModel):
        page_domain = [('url', '=', req_page)] + request.website.website_domain()

        published_domain = page_domain
-        # need to bypass website_published, to apply is_most_specific
-        # filter later if not publisher
-        pages = request.env['website.page'].sudo().search(published_domain, order='website_id')
-        pages = pages.filtered(pages._is_most_specific_page)
-
-        if not request.website.is_publisher():
-            pages = pages.filtered('is_visible')
-
-        mypage = pages[0] if pages else False
-        _, ext = os.path.splitext(req_page)
-        if mypage:
-            return request.render(mypage.get_view_identifier(), {
-                # 'path': req_page[1:],
+        # specific page first
+        page = request.env['website.page'].sudo().search(published_domain, order='website_id asc', limit=1)
+        if page and (request.website.is_publisher() or page.is_visible):
+            _, ext = os.path.splitext(req_page)
+            return request.render(page.get_view_identifier(), {
                'deletable': True,
-                'main_object': mypage,
+                'main_object': page,
            }, mimetype=_guess_mimetype(ext))
+        return False

    @classmethod
    def _serve_redirect(cls):
@@ -384,8 +384,9 @@ class View(models.Model):
                elif request.env.user.has_group('website.group_website_publisher'):
                    new_context = dict(self._context, inherit_branding_auto=True)
            if values and 'main_object' in values:
-                func = getattr(values['main_object'], 'get_backend_menu_id', False)
-                values['backend_menu_id'] = func and func() or self.env.ref('website.menu_website_configuration').id
+                if request.env.user.has_group('website.group_website_publisher'):
+                    func = getattr(values['main_object'], 'get_backend_menu_id', False)
+                    values['backend_menu_id'] = func and func() or self.env.ref('website.menu_website_configuration').id

                # Fallback incase main_object dont't inherit 'website.seo.metadata'
                if not hasattr(values['main_object'], 'get_website_meta'):
@@ -133,9 +133,23 @@ class Website(models.Model):
            self.default_lang_id = language_ids[0]

    def _compute_menu(self):
-        Menu = self.env['website.menu']
        for website in self:
-            website.menu_id = Menu.search([('parent_id', '=', False), ('website_id', '=', website.id)], order='id', limit=1).id
+            menus = self.env['website.menu'].browse(website._get_menu_ids())
+
+            # use field parent_id (1 query) to determine field child_id (2 queries by level)"
+            for menu in menus:
+                menu._cache['child_id'] = ()
+            for menu in menus:
+                # don't add child menu if parent is forbidden
+                if menu.parent_id and menu.parent_id in menus:
+                    menu.parent_id._cache['child_id'] += (menu.id,)
+
+            website.menu_id = menus and menus.filtered(lambda m: not m.parent_id)[0].id or False
+
+    # self.env.uid for ir.rule groups on menu
+    @tools.ormcache('self.env.uid', 'self.id')
+    def _get_menu_ids(self):
+        return self.env['website.menu'].search([('website_id', '=', self.id)]).ids

    @api.model
    def create(self, vals):
@@ -44,7 +44,7 @@ class Menu(models.Model):
    group_ids = fields.Many2many('res.groups', string='Visible Groups',
                                 help="User need to be at least in one of these groups to see the menu")
    is_mega_menu = fields.Boolean(compute=_compute_field_is_mega_menu, inverse=_set_field_is_mega_menu)
-    mega_menu_content = fields.Html(translate=html_translate, sanitize=False)
+    mega_menu_content = fields.Html(translate=html_translate, sanitize=False, prefetch=True)
    mega_menu_classes = fields.Char()

    def name_get(self):
@@ -68,6 +68,7 @@ class Menu(models.Model):
                  Be careful to return correct record for ir.model.data xml_id in case
                  of default main menus creation.
        '''
+        self.clear_caches()
        # Only used when creating website_data.xml default menu
        if vals.get('url') == '/default-main-menu':
            return super(Menu, self).create(vals)
@@ -91,7 +92,14 @@ class Menu(models.Model):
                res = super(Menu, self).create(vals)
        return res  # Only one record is returned but multiple could have been created

+    def write(self, values):
+        res = super().write(values)
+        if 'website_id' in values:
+            self.clear_caches()
+        return res
+
    def unlink(self):
+        self.clear_caches()
        default_menu = self.env.ref('website.main_menu', raise_if_not_found=False)
        menus_to_remove = self
        for menu in self.filtered(lambda m: default_menu and m.parent_id.id == default_menu.id):
@@ -172,6 +172,9 @@ class WebsiteVisitor(models.Model):
        access_token = request.httprequest.cookies.get('visitor_uuid')
        if access_token:
            visitor = Visitor.with_context(active_test=False).search([('access_token', '=', access_token)])
+            # Prefetch access_token and other fields. Since access_token has a restricted group and we access
+            # a non restricted field (partner_id) first it is not fetched and will require an additional query to be retrieved.
+            visitor.access_token

        if not self.env.user._is_public():
            partner_id = self.env.user.partner_id
@@ -8,6 +8,7 @@ from . import test_get_current_website
 from . import test_lang_url
 from . import test_menu
 from . import test_page
+from . import test_performance
 from . import test_qweb
 from . import test_res_users
 from . import test_theme
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.tests.common import HttpCase
+
+EXTRA_REQUEST = 5
+""" During tests, the query on 'base_registry_signaling, base_cache_signaling'
+    won't be executed on hot state, but 6 new queries related to the test cursor
+    will be added:
+        SAVEPOINT "test_cursor_4"
+        SAVEPOINT "test_cursor_4"
+        ROLLBACK TO SAVEPOINT "test_cursor_4"
+        SAVEPOINT "test_cursor_5"
+        [.. usual SQL Queries .. ]
+        SAVEPOINT "test_cursor_5"
+        ROLLBACK TO SAVEPOINT "test_cursor_5"
+"""
+
+
+class TestWebsitePerformance(HttpCase):
+
+    def setUp(self):
+        super().setUp()
+
+        self.page = self.env['website.page'].create({
+            'url': '/sql_page',
+            'name': 'Page Test SQL Perf',
+            'type': 'qweb',
+            'arch': '<t name="Contact Us" t-name="website.contactus"> \
+                       <t t-call="website.layout"> \
+                         <div id="wrap"><div class="oe_structure"/></div> \
+                       </t> \
+                     </t>',
+            'key': 'website.page_test_perf_sql',
+            'is_published': True,
+        })
+
+    def _get_url_hot_query(self, url):
+        # ensure worker is in hot state
+        self.url_open(url)
+        self.url_open(url)
+
+        sql_count = self.registry.test_cr.sql_log_count
+        self.url_open(url)
+        return self.registry.test_cr.sql_log_count - sql_count
+
+    def test_10_perf_sql_queries_page(self):
+        # standard website.page
+        expected_sql = 14 + EXTRA_REQUEST
+        self.assertEqual(self._get_url_hot_query(self.page.url), expected_sql)
+
+    def test_20_perf_sql_queries_homepage(self):
+        # homepage "/" has its own controller
+        # add 5 queries for test environment (RELEASE/SAVEPOINT/ROLLBACK test_cursor)
+        expected_sql = 21 + EXTRA_REQUEST
+        self.assertEqual(self._get_url_hot_query('/'), expected_sql)
+
+    def test_30_perf_sql_queries_page_no_layout(self):
+        # website.page with no call to layout templates
+        self.page.arch = '<div>I am a blank page</div>'
+        expected_sql = 8 + EXTRA_REQUEST
+        self.assertEqual(self._get_url_hot_query(self.page.url), expected_sql)
@@ -1,14 +1,13 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.

-from odoo import api, fields, models
-from odoo.osv import expression
+from odoo import models


 class IrHttp(models.AbstractModel):
    _inherit = 'ir.http'

    @classmethod
-    def _get_translation_frontend_modules_domain(cls):
-        domain = super(IrHttp, cls)._get_translation_frontend_modules_domain()
-        return expression.OR([domain, [('name', '=', 'im_livechat')]])
+    def _get_translation_frontend_modules_name(cls):
+        mods = super(IrHttp, cls)._get_translation_frontend_modules_name()
+        return mods + ['im_livechat']
@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
+import hashlib
 import itertools
+import json
 import logging
 import operator
 from collections import defaultdict
@@ -237,9 +239,11 @@ class IrTranslation(models.Model):
                        record = model.browse(trans.res_id)
                        record.modified([field.name])
        for trans in self:
-            if trans.type != 'model' or trans.name.split(',')[0] in self.CACHED_MODELS:
-                self.clear_caches()
-                break
+            if (trans.type != 'model' or
+               (trans.name.split(',')[0] in self.CACHED_MODELS) or
+               (trans.comments and 'openerp-web' in trans.comments)):  # clear get_web_trans_hash
+                        self.clear_caches()
+                        break

    @api.model
    def _set_ids(self, name, tt, lang, ids, value, src=None):
@@ -869,9 +873,16 @@ class IrTranslation(models.Model):
        langs = self.env['res.lang']._lang_get(lang)
        lang_params = None
        if langs:
-            lang_params = langs.read([
-                "name", "direction", "date_format", "time_format",
-                "grouping", "decimal_point", "thousands_sep", "week_start"])[0]
+            lang_params = {
+                "name": langs.name,
+                "direction": langs.direction,
+                "date_format": langs.date_format,
+                "time_format": langs.time_format,
+                "grouping": langs.grouping,
+                "decimal_point": langs.decimal_point,
+                "thousands_sep": langs.thousands_sep,
+                "week_start": langs.week_start,
+            }
            lang_params['week_start'] = int(lang_params['week_start'])
            lang_params['code'] = lang

@@ -891,3 +902,15 @@ class IrTranslation(models.Model):
                for m in msg_group)

        return translations_per_module, lang_params
+
+    @api.model
+    @tools.ormcache('frozenset(mods)', 'lang')
+    def get_web_translations_hash(self, mods, lang):
+        translations, lang_params = self.get_translations_for_webclient(mods, lang)
+        translation_cache = {
+            'lang_parameters': lang_params,
+            'modules': translations,
+            'lang': lang,
+            'multi_lang': len(self.env['res.lang'].sudo().get_installed()) > 1,
+        }
+        return hashlib.sha1(json.dumps(translation_cache, sort_keys=True).encode()).hexdigest()
@@ -213,6 +213,14 @@ class Lang(models.Model):
        langs = self.with_context(active_test=False).search([])
        return sorted([(lang.code, lang.url_code, lang.name) for lang in langs], key=itemgetter(2))

+    @api.model
+    @tools.ormcache('code')
+    def _lang_code_to_urlcode(self, code):
+        for c, urlc, name in self.get_available():
+            if c == code:
+                return urlc
+        return self._lang_get(code).url_code
+
    @api.model
    @tools.ormcache()
    def get_installed(self):