From 041fe5e21e625e6ea80d26ecd041491c89cbba12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Voet=20=28ryv=29?= Date: Thu, 10 Feb 2022 15:41:34 +0000 Subject: [PATCH] [IMP] base: add prefetching group feature The field prefetching mechanism was poorly customizable. Before this, we could only tell if a field was prefetched with other fields or not at all. We have no way to inform the framework, like: "When I need data of that field, prefetch these other fields, which are likely be used in the same transaction". From now on, the `prefetch` attribute is used as a grouping key for prefetching fields. When a field is fetched, all the fields with the same value for `prefetch` are taken for prefetching. For example, consider a small set of fields that are rarely used, except for one flow A using them. You want to prefetch those fields only in the flow A, and you want to fetch them in a single query. With the new feature, simply set `prefetch=A` for some string `A` on those fields, and they will be grouped for prefetching. Part-of: odoo/odoo#83818 --- addons/event/models/event_stage.py | 6 +-- addons/website/models/mixins.py | 6 +-- addons/website_blog/tests/test_performance.py | 10 ++-- .../test_new_api/models/test_new_api.py | 11 ++-- .../test_new_api/security/ir.model.access.csv | 2 +- .../test_new_api/tests/test_new_fields.py | 54 ++++++++++++++++++- odoo/fields.py | 2 +- odoo/models.py | 8 +-- 8 files changed, 78 insertions(+), 21 deletions(-) diff --git a/addons/event/models/event_stage.py b/addons/event/models/event_stage.py index d535ea75051..fff7823717e 100644 --- a/addons/event/models/event_stage.py +++ b/addons/event/models/event_stage.py @@ -17,11 +17,11 @@ class EventStage(models.Model): string='End Stage', default=False, help='Events will automatically be moved into this stage when they are finished. The event moved into this stage will automatically be set as green.') legend_blocked = fields.Char( - 'Red Kanban Label', default=lambda s: _('Blocked'), translate=True, required=True, + 'Red Kanban Label', default=lambda s: _('Blocked'), translate=True, prefetch='legend', required=True, help='Override the default value displayed for the blocked state for kanban selection.') legend_done = fields.Char( - 'Green Kanban Label', default=lambda s: _('Ready for Next Stage'), translate=True, required=True, + 'Green Kanban Label', default=lambda s: _('Ready for Next Stage'), translate=True, prefetch='legend', required=True, help='Override the default value displayed for the done state for kanban selection.') legend_normal = fields.Char( - 'Grey Kanban Label', default=lambda s: _('In Progress'), translate=True, required=True, + 'Grey Kanban Label', default=lambda s: _('In Progress'), translate=True, prefetch='legend', required=True, help='Override the default value displayed for the normal state for kanban selection.') diff --git a/addons/website/models/mixins.py b/addons/website/models/mixins.py index 1b161f7c833..2a1bf021222 100644 --- a/addons/website/models/mixins.py +++ b/addons/website/models/mixins.py @@ -21,9 +21,9 @@ class SeoMetadata(models.AbstractModel): _description = 'SEO metadata' is_seo_optimized = fields.Boolean("SEO optimized", compute='_compute_is_seo_optimized') - website_meta_title = fields.Char("Website meta title", translate=True) - website_meta_description = fields.Text("Website meta description", translate=True) - website_meta_keywords = fields.Char("Website meta keywords", translate=True) + website_meta_title = fields.Char("Website meta title", translate=True, prefetch="website_meta") + website_meta_description = fields.Text("Website meta description", translate=True, prefetch="website_meta") + website_meta_keywords = fields.Char("Website meta keywords", translate=True, prefetch="website_meta") website_meta_og_img = fields.Char("Website opengraph image") seo_name = fields.Char("Seo name", translate=True, prefetch=True) diff --git a/addons/website_blog/tests/test_performance.py b/addons/website_blog/tests/test_performance.py index 0def0db1d4b..7371636f2e4 100644 --- a/addons/website_blog/tests/test_performance.py +++ b/addons/website_blog/tests/test_performance.py @@ -13,7 +13,7 @@ class TestBlogPerformance(UtilPerf): self.env['website'].search([]).channel_id = False def test_10_perf_sql_blog_standard_data(self): - self.assertEqual(self._get_url_hot_query('/blog'), 35) + self.assertEqual(self._get_url_hot_query('/blog'), 28) def test_20_perf_sql_blog_bigger_data_scaling(self): BlogPost = self.env['blog.post'] @@ -25,8 +25,8 @@ class TestBlogPerformance(UtilPerf): for blog_post in blog_posts: blog_post.tag_ids += blog_tags blog_tags = blog_tags[:-1] - self.assertEqual(self._get_url_hot_query('/blog'), 42) - self.assertEqual(self._get_url_hot_query(blog_post[0].website_url), 39) + self.assertEqual(self._get_url_hot_query('/blog'), 28) + self.assertEqual(self._get_url_hot_query(blog_post[0].website_url), 33) def test_30_perf_sql_blog_bigger_data_scaling(self): BlogPost = self.env['blog.post'] @@ -38,5 +38,5 @@ class TestBlogPerformance(UtilPerf): for blog_post in blog_posts: blog_post.write({'tag_ids': [[6, 0, random.choices(blog_tags.ids, k=random.randint(0, len(blog_tags)))]]}) - self.assertLessEqual(self._get_url_hot_query('/blog'), 79) - self.assertLessEqual(self._get_url_hot_query(blog_post[0].website_url), 39) + self.assertLessEqual(self._get_url_hot_query('/blog'), 31) + self.assertLessEqual(self._get_url_hot_query(blog_post[0].website_url), 33) diff --git a/odoo/addons/test_new_api/models/test_new_api.py b/odoo/addons/test_new_api/models/test_new_api.py index b1e2c7b8f8a..6b9fe9d9faa 100644 --- a/odoo/addons/test_new_api/models/test_new_api.py +++ b/odoo/addons/test_new_api/models/test_new_api.py @@ -1566,12 +1566,17 @@ class PrecomputeMonetary(models.Model): self.currency_id = 1 # EUR -class PrefetchTranslateField(models.Model): - _name = 'test_new_api.prefetch.translate' - _description = 'A model with some translate fields to check prefetch' +class Prefetch(models.Model): + _name = 'test_new_api.prefetch' + _description = 'A model to check the prefetching of fields (translated and group)' name = fields.Char('Name', translate=True) description = fields.Char('Description', translate=True, prefetch=True) html_description = fields.Html('Styled description', translate=True, prefetch=True) rare_description = fields.Char('Rare Description', translate=True) rare_html_description = fields.Html('Rare Styled description', translate=True) + harry = fields.Integer('Harry Potter', prefetch='Harry Potter') + hermione = fields.Char('Hermione Granger', prefetch='Harry Potter') + ron = fields.Float('Ron Weasley', prefetch='Harry Potter') + hansel = fields.Integer('Hansel', prefetch="Hansel and Gretel") + gretel = fields.Char('Gretel', prefetch="Hansel and Gretel") diff --git a/odoo/addons/test_new_api/security/ir.model.access.csv b/odoo/addons/test_new_api/security/ir.model.access.csv index 5ba86c2ce93..a10e066c051 100644 --- a/odoo/addons/test_new_api/security/ir.model.access.csv +++ b/odoo/addons/test_new_api/security/ir.model.access.csv @@ -87,4 +87,4 @@ access_test_new_api_precompute_combo,access_test_new_api_precompute_combo,model_ access_test_new_api_precompute_editable,access_test_new_api_precompute_editable,model_test_new_api_precompute_editable,,1,0,0,0 access_test_new_api_precompute_required,access_test_new_api_precompute_required,model_test_new_api_precompute_required,,1,0,0,0 access_test_new_api_precompute_monetary,access_test_new_api_precompute_monetary,model_test_new_api_precompute_monetary,,1,0,0,0 -access_test_new_api_prefetch_translate,access_test_new_api_prefetch_translate,model_test_new_api_prefetch_translate,,1,0,0,0 +access_test_new_api_prefetch,access_test_new_api_prefetch,model_test_new_api_prefetch,,1,0,0,0 diff --git a/odoo/addons/test_new_api/tests/test_new_fields.py b/odoo/addons/test_new_api/tests/test_new_fields.py index b26f66d9a51..0ca14ca4418 100644 --- a/odoo/addons/test_new_api/tests/test_new_fields.py +++ b/odoo/addons/test_new_api/tests/test_new_fields.py @@ -2523,7 +2523,7 @@ class TestFields(TransactionCaseWithUserDemo): record_user.read(['tags']) def test_98_prefetch_translate(self): - Model = self.registry['test_new_api.prefetch.translate'] + Model = self.registry['test_new_api.prefetch'] # translated '_rec_name' field should be prefetched self.assertTrue(Model.name.prefetch) @@ -2536,6 +2536,58 @@ class TestFields(TransactionCaseWithUserDemo): self.assertFalse(Model.rare_description.prefetch) self.assertFalse(Model.rare_html_description.prefetch) + def test_99_prefetch_group(self): + records = self.env['test_new_api.prefetch'].create([{} for _ in range(10)]) + records.flush() + records.invalidate_cache() + + with self.assertQueries([""" + SELECT + "test_new_api_prefetch"."id" AS "id", + "test_new_api_prefetch"."name" AS "name", + "test_new_api_prefetch"."description" AS "description", + "test_new_api_prefetch"."html_description" AS "html_description", + "test_new_api_prefetch"."create_uid" AS "create_uid", + "test_new_api_prefetch"."create_date" AS "create_date", + "test_new_api_prefetch"."write_uid" AS "write_uid", + "test_new_api_prefetch"."write_date" AS "write_date" + FROM "test_new_api_prefetch" + WHERE "test_new_api_prefetch".id IN %s + """]): + records.mapped('name') # fetch all fields with prefetch=True + + with self.assertQueries([""" + SELECT + "test_new_api_prefetch"."id" AS "id", + "test_new_api_prefetch"."harry" AS "harry", + "test_new_api_prefetch"."hermione" AS "hermione", + "test_new_api_prefetch"."ron" AS "ron" + FROM "test_new_api_prefetch" + WHERE "test_new_api_prefetch".id IN %s + """]): + records.mapped('harry') # fetch all fields with prefetch='Harry Potter' + records.mapped('hermione') # fetched already + records.mapped('ron') # fetched already + + with self.assertQueries([""" + SELECT + "test_new_api_prefetch"."id" AS "id", + "test_new_api_prefetch"."hansel" AS "hansel", + "test_new_api_prefetch"."gretel" AS "gretel" + FROM "test_new_api_prefetch" + WHERE "test_new_api_prefetch".id IN %s + """]): + records.mapped('hansel') # fetch all fields with prefetch='Hansel and Gretel' + records.mapped('gretel') # fetched already + + records.invalidate_cache() + + with self.assertQueryCount(4): + records.mapped('name') # fetch all fields with prefetch=True + records.mapped('hansel') # fetch all fields with prefetch='Hansel and Gretel' + records.mapped('harry') # fetch all fields with prefetch='Harry Potter' + records.mapped('rare_description') # fetch that field only + class TestX2many(common.TransactionCase): def test_definition_many2many(self): diff --git a/odoo/fields.py b/odoo/fields.py index 9a0f3330566..b2c22b74101 100644 --- a/odoo/fields.py +++ b/odoo/fields.py @@ -288,7 +288,7 @@ class Field(MetaField('DummyField', (object,), {})): related_field = None # corresponding related field group_operator = None # operator for aggregating values group_expand = None # name of method to expand groups in read_group() - prefetch = True # whether the field is prefetched + prefetch = True # the prefetch group (False means no group) def __init__(self, string=Default, **kwargs): kwargs['string'] = string diff --git a/odoo/models.py b/odoo/models.py index 9f82cb7ce08..8eb1f5ad2c3 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -3227,7 +3227,7 @@ Fields: fields = self.check_field_access_rights('read', fields) # fetch stored fields from the database to the cache - stored_fields = set() + stored_fields = OrderedSet() for name in fields: field = self._fields.get(name) if not field: @@ -3238,7 +3238,7 @@ Fields: # optimization: prefetch direct field dependencies for dotname in self.pool.field_depends[field]: f = self._fields[dotname.split('.')[0]] - if f.prefetch and (not f.groups or self.user_has_groups(f.groups)): + if f.prefetch is True and (not f.groups or self.user_has_groups(f.groups)): stored_fields.add(f.name) self._read(stored_fields) @@ -3279,8 +3279,8 @@ Fields: fnames = [ name for name, f in self._fields.items() - # select fields that can be prefetched - if f.prefetch + # select fields with the same prefetch group + if f.prefetch == field.prefetch # discard fields with groups that the user may not access if not (f.groups and not self.user_has_groups(f.groups)) # discard fields that must be recomputed