[IMP] base: add prefetching group feature

The field prefetching mechanism was poorly customizable.  Before this,
we could only tell if a field was prefetched with other fields or not at
all.  We have no way to inform the framework, like: "When I need data of
that field, prefetch these other fields, which are likely be used in the
same transaction".

From now on, the `prefetch` attribute is used as a grouping key for
prefetching fields.  When a field is fetched, all the fields with the
same value for `prefetch` are taken for prefetching.

For example, consider a small set of fields that are rarely used, except
for one flow A using them.  You want to prefetch those fields only in
the flow A, and you want to fetch them in a single query.  With the new
feature, simply set `prefetch=A` for some string `A` on those fields,
and they will be grouped for prefetching.

Part-of: odoo/odoo#83818

addons/event/models/event_stage.py

@@ -17,11 +17,11 @@ class EventStage(models.Model):
         string='End Stage', default=False,
         help='Events will automatically be moved into this stage when they are finished. The event moved into this stage will automatically be set as green.')
     legend_blocked = fields.Char(
-        'Red Kanban Label', default=lambda s: _('Blocked'), translate=True, required=True,
+        'Red Kanban Label', default=lambda s: _('Blocked'), translate=True, prefetch='legend', required=True,
         help='Override the default value displayed for the blocked state for kanban selection.')
     legend_done = fields.Char(
-        'Green Kanban Label', default=lambda s: _('Ready for Next Stage'), translate=True, required=True,
+        'Green Kanban Label', default=lambda s: _('Ready for Next Stage'), translate=True, prefetch='legend', required=True,
         help='Override the default value displayed for the done state for kanban selection.')
     legend_normal = fields.Char(
-        'Grey Kanban Label', default=lambda s: _('In Progress'), translate=True, required=True,
+        'Grey Kanban Label', default=lambda s: _('In Progress'), translate=True, prefetch='legend', required=True,
         help='Override the default value displayed for the normal state for kanban selection.')

addons/website/models/mixins.py

@@ -21,9 +21,9 @@ class SeoMetadata(models.AbstractModel):
     _description = 'SEO metadata'
 
     is_seo_optimized = fields.Boolean("SEO optimized", compute='_compute_is_seo_optimized')
-    website_meta_title = fields.Char("Website meta title", translate=True)
-    website_meta_description = fields.Text("Website meta description", translate=True)
-    website_meta_keywords = fields.Char("Website meta keywords", translate=True)
+    website_meta_title = fields.Char("Website meta title", translate=True, prefetch="website_meta")
+    website_meta_description = fields.Text("Website meta description", translate=True, prefetch="website_meta")
+    website_meta_keywords = fields.Char("Website meta keywords", translate=True, prefetch="website_meta")
     website_meta_og_img = fields.Char("Website opengraph image")
     seo_name = fields.Char("Seo name", translate=True, prefetch=True)
 

addons/website_blog/tests/test_performance.py

@@ -13,7 +13,7 @@ class TestBlogPerformance(UtilPerf):
             self.env['website'].search([]).channel_id = False
 
     def test_10_perf_sql_blog_standard_data(self):
-        self.assertEqual(self._get_url_hot_query('/blog'), 35)
+        self.assertEqual(self._get_url_hot_query('/blog'), 28)
 
     def test_20_perf_sql_blog_bigger_data_scaling(self):
         BlogPost = self.env['blog.post']
@@ -25,8 +25,8 @@ class TestBlogPerformance(UtilPerf):
         for blog_post in blog_posts:
             blog_post.tag_ids += blog_tags
             blog_tags = blog_tags[:-1]
-        self.assertEqual(self._get_url_hot_query('/blog'), 42)
-        self.assertEqual(self._get_url_hot_query(blog_post[0].website_url), 39)
+        self.assertEqual(self._get_url_hot_query('/blog'), 28)
+        self.assertEqual(self._get_url_hot_query(blog_post[0].website_url), 33)
 
     def test_30_perf_sql_blog_bigger_data_scaling(self):
         BlogPost = self.env['blog.post']
@@ -38,5 +38,5 @@ class TestBlogPerformance(UtilPerf):
         for blog_post in blog_posts:
             blog_post.write({'tag_ids': [[6, 0, random.choices(blog_tags.ids, k=random.randint(0, len(blog_tags)))]]})
 
-        self.assertLessEqual(self._get_url_hot_query('/blog'), 79)
-        self.assertLessEqual(self._get_url_hot_query(blog_post[0].website_url), 39)
+        self.assertLessEqual(self._get_url_hot_query('/blog'), 31)
+        self.assertLessEqual(self._get_url_hot_query(blog_post[0].website_url), 33)

odoo/addons/test_new_api/models/test_new_api.py

@@ -1566,12 +1566,17 @@ class PrecomputeMonetary(models.Model):
         self.currency_id = 1  # EUR
 
 
-class PrefetchTranslateField(models.Model):
-    _name = 'test_new_api.prefetch.translate'
-    _description = 'A model with some translate fields to check prefetch'
+class Prefetch(models.Model):
+    _name = 'test_new_api.prefetch'
+    _description = 'A model to check the prefetching of fields (translated and group)'
 
     name = fields.Char('Name', translate=True)
     description = fields.Char('Description', translate=True, prefetch=True)
     html_description = fields.Html('Styled description', translate=True, prefetch=True)
     rare_description = fields.Char('Rare Description', translate=True)
     rare_html_description = fields.Html('Rare Styled description', translate=True)
+    harry = fields.Integer('Harry Potter', prefetch='Harry Potter')
+    hermione = fields.Char('Hermione Granger', prefetch='Harry Potter')
+    ron = fields.Float('Ron Weasley', prefetch='Harry Potter')
+    hansel = fields.Integer('Hansel', prefetch="Hansel and Gretel")
+    gretel = fields.Char('Gretel', prefetch="Hansel and Gretel")

odoo/addons/test_new_api/security/ir.model.access.csv

@@ -87,4 +87,4 @@ access_test_new_api_precompute_combo,access_test_new_api_precompute_combo,model_
 access_test_new_api_precompute_editable,access_test_new_api_precompute_editable,model_test_new_api_precompute_editable,,1,0,0,0
 access_test_new_api_precompute_required,access_test_new_api_precompute_required,model_test_new_api_precompute_required,,1,0,0,0
 access_test_new_api_precompute_monetary,access_test_new_api_precompute_monetary,model_test_new_api_precompute_monetary,,1,0,0,0
-access_test_new_api_prefetch_translate,access_test_new_api_prefetch_translate,model_test_new_api_prefetch_translate,,1,0,0,0
+access_test_new_api_prefetch,access_test_new_api_prefetch,model_test_new_api_prefetch,,1,0,0,0

odoo/addons/test_new_api/tests/test_new_fields.py

@@ -2523,7 +2523,7 @@ class TestFields(TransactionCaseWithUserDemo):
             record_user.read(['tags'])
 
     def test_98_prefetch_translate(self):
-        Model = self.registry['test_new_api.prefetch.translate']
+        Model = self.registry['test_new_api.prefetch']
 
         # translated '_rec_name' field should be prefetched
         self.assertTrue(Model.name.prefetch)
@@ -2536,6 +2536,58 @@ class TestFields(TransactionCaseWithUserDemo):
         self.assertFalse(Model.rare_description.prefetch)
         self.assertFalse(Model.rare_html_description.prefetch)
 
+    def test_99_prefetch_group(self):
+        records = self.env['test_new_api.prefetch'].create([{} for _ in range(10)])
+        records.flush()
+        records.invalidate_cache()
+
+        with self.assertQueries(["""
+            SELECT
+                "test_new_api_prefetch"."id" AS "id",
+                "test_new_api_prefetch"."name" AS "name",
+                "test_new_api_prefetch"."description" AS "description",
+                "test_new_api_prefetch"."html_description" AS "html_description",
+                "test_new_api_prefetch"."create_uid" AS "create_uid",
+                "test_new_api_prefetch"."create_date" AS "create_date",
+                "test_new_api_prefetch"."write_uid" AS "write_uid",
+                "test_new_api_prefetch"."write_date" AS "write_date"
+            FROM "test_new_api_prefetch"
+            WHERE "test_new_api_prefetch".id IN %s
+        """]):
+            records.mapped('name')  # fetch all fields with prefetch=True
+
+        with self.assertQueries(["""
+            SELECT
+                "test_new_api_prefetch"."id" AS "id",
+                "test_new_api_prefetch"."harry" AS "harry",
+                "test_new_api_prefetch"."hermione" AS "hermione",
+                "test_new_api_prefetch"."ron" AS "ron"
+            FROM "test_new_api_prefetch"
+            WHERE "test_new_api_prefetch".id IN %s
+        """]):
+            records.mapped('harry')  # fetch all fields with prefetch='Harry Potter'
+            records.mapped('hermione')  # fetched already
+            records.mapped('ron')  # fetched already
+
+        with self.assertQueries(["""
+            SELECT
+                "test_new_api_prefetch"."id" AS "id",
+                "test_new_api_prefetch"."hansel" AS "hansel",
+                "test_new_api_prefetch"."gretel" AS "gretel"
+            FROM "test_new_api_prefetch"
+            WHERE "test_new_api_prefetch".id IN %s
+        """]):
+            records.mapped('hansel')  # fetch all fields with prefetch='Hansel and Gretel'
+            records.mapped('gretel')  # fetched already
+
+        records.invalidate_cache()
+
+        with self.assertQueryCount(4):
+            records.mapped('name')  # fetch all fields with prefetch=True
+            records.mapped('hansel')  # fetch all fields with prefetch='Hansel and Gretel'
+            records.mapped('harry')  # fetch all fields with prefetch='Harry Potter'
+            records.mapped('rare_description')  # fetch that field only
+
 
 class TestX2many(common.TransactionCase):
     def test_definition_many2many(self):

odoo/fields.py

@@ -288,7 +288,7 @@ class Field(MetaField('DummyField', (object,), {})):
     related_field = None                # corresponding related field
     group_operator = None               # operator for aggregating values
     group_expand = None                 # name of method to expand groups in read_group()
-    prefetch = True                     # whether the field is prefetched
+    prefetch = True                     # the prefetch group (False means no group)
 
     def __init__(self, string=Default, **kwargs):
         kwargs['string'] = string

odoo/models.py

@@ -3227,7 +3227,7 @@ Fields:
         fields = self.check_field_access_rights('read', fields)
 
         # fetch stored fields from the database to the cache
-        stored_fields = set()
+        stored_fields = OrderedSet()
         for name in fields:
             field = self._fields.get(name)
             if not field:
@@ -3238,7 +3238,7 @@ Fields:
                 # optimization: prefetch direct field dependencies
                 for dotname in self.pool.field_depends[field]:
                     f = self._fields[dotname.split('.')[0]]
-                    if f.prefetch and (not f.groups or self.user_has_groups(f.groups)):
+                    if f.prefetch is True and (not f.groups or self.user_has_groups(f.groups)):
                         stored_fields.add(f.name)
         self._read(stored_fields)
 
@@ -3279,8 +3279,8 @@ Fields:
             fnames = [
                 name
                 for name, f in self._fields.items()
-                # select fields that can be prefetched
-                if f.prefetch
+                # select fields with the same prefetch group
+                if f.prefetch == field.prefetch
                 # discard fields with groups that the user may not access
                 if not (f.groups and not self.user_has_groups(f.groups))
                 # discard fields that must be recomputed