[IMP] core: improve mapped and filtered performance

Previously, mapped was following a very naive approach, which was simply
calling the field name passed as input for every record in a recordset,
sequentially.

The problem with this approach is that we will potentially recompute the
same fields multiple times for differents records, when this could be
done once per field for ALL records, and store this value in cache for
further access.

Another potential problem is that we don't take advantage of the ORM's
prefetching to fetch all the records that are not in cache at once,
instead of doing the same query for every record in the recordset.

Yet another problem is the conversion of each cache value to a record
format and then combining all of the individual records into a single
recordset, which, depending on the size of the recordset, can take an
unbelievable amount of CPU time.

With this new implementation of `mapped()` we take care of all of these
problems:

This is done by first delegating `mapped()` from the model to the field,
this mapped takes a recordset as input and it will try to batch compute
and prefetch as much as possible for the entire recordset, but it will
not keep these values for the actual output, it just stores everything
in cache and then at the end, retrieves everything from the cache to
guarantee the same order.

After the mapped, the conversion from cache format to record format is
delegated to the new `convert_to_record_multi` which will fetch all the
ids and then perform a single browse to encapsulate all of the records
into a single recordset with the least amount of overhead possible.

Part of Task 2170344

closes odoo/odoo#42611

Signed-off-by: Raphael Collet (rco) <rco@openerp.com>

odoo/api.py

@@ -827,6 +827,21 @@ class Cache(object):
             except KeyError:
                 pass
 
+    def get_until_miss(self, records, field):
+        """ Return the cached values of ``field`` for ``records`` until a value is not found. """
+        field_cache = self._data[field]
+        key = records.env.cache_key(field) if field.depends_context else None
+        vals = []
+        for record_id in records._ids:
+            try:
+                if key is not None:
+                    vals.append(field_cache[record_id][key])
+                else:
+                    vals.append(field_cache[record_id])
+            except KeyError:
+                break
+        return vals
+
     def get_records_different_from(self, records, field, value):
         """ Return the subset of ``records`` that has not ``value`` for ``field``. """
         field_cache = self._data[field]

odoo/fields.py

@@ -757,6 +757,15 @@ class Field(MetaField('DummyField', (object,), {})):
         """
         return False if value is None else value
 
+    def convert_to_record_multi(self, values, records):
+        """ Convert a list of values from the cache format to the record format.
+        Some field classes may override this method to add optimizations for
+        batch processing.
+        """
+        # spare the method lookup overhead
+        convert = self.convert_to_record
+        return [convert(value, records) for value in values]
+
     def convert_to_read(self, value, record, use_name_get=True):
         """ Convert ``value`` from the record format to the format returned by
         method :meth:`BaseModel.read`.
@@ -1008,6 +1017,38 @@ class Field(MetaField('DummyField', (object,), {})):
 
         return self.convert_to_record(value, record)
 
+    def mapped(self, records):
+        """ Return the values of ``self`` for ``records``, either as a list
+        (scalar fields), or as a recordset (relational fields).
+
+        This method is meant to be used internally and has very little benefit
+        over a simple call to `~odoo.models.BaseModel.mapped()` on a recordset.
+        """
+        if self.name == 'id':
+            # not stored in cache
+            return list(records._ids)
+
+        if self.compute:
+            # Force the computation of the subset of 'records' to compute. This
+            # is necessary because the values in cache are not valid for the
+            # records to compute. Note that this explicitly prevents an infinite
+            # loop upon recompute, which invokes mapped() on the records, which
+            # fetches record values, which calls flush, which calls recompute.
+            to_compute_ids = records.env.all.tocompute.get(self, ())
+            ids = [id_ for id_ in records._ids if id_ in to_compute_ids]
+            for record in records.browse(ids):
+                self.__get__(record, type(records))
+
+        # retrieve values in cache, and fetch missing ones
+        vals = records.env.cache.get_until_miss(records, self)
+        while len(vals) < len(records):
+            # trigger prefetching on remaining records, and continue retrieval
+            remaining = records[len(vals):]
+            self.__get__(first(remaining), type(remaining))
+            vals += records.env.cache.get_until_miss(remaining, self)
+
+        return self.convert_to_record_multi(vals, records)
+
     def __set__(self, records, value):
         """ set the value of field ``self`` on ``records`` """
         protected_ids = []
@@ -2239,10 +2280,8 @@ class _Relational(Field):
         # base case: do the regular access
         if records is None or len(records._ids) <= 1:
             return super().__get__(records, owner)
-        # multirecord case: return the union of the values of 'self' on records
-        get = super().__get__
-        comodel = records.env[self.comodel_name]
-        return comodel.union(*[get(record, owner) for record in records])
+        # multirecord case: use mapped
+        return self.mapped(records)
 
     def _setup_regular_base(self, model):
         super(_Relational, self)._setup_regular_base(model)
@@ -2425,6 +2464,12 @@ class Many2one(_Relational):
         prefetch_ids = IterableGenerator(prefetch_many2one_ids, record, self)
         return record.pool[self.comodel_name]._browse(record.env, ids, prefetch_ids)
 
+    def convert_to_record_multi(self, values, records):
+        # return the ids as a recordset without duplicates
+        prefetch_ids = IterableGenerator(prefetch_many2one_ids, records, self)
+        ids = tuple(unique(id_ for id_ in values if id_ is not None))
+        return records.pool[self.comodel_name]._browse(records.env, ids, prefetch_ids)
+
     def convert_to_read(self, value, record, use_name_get=True):
         if use_name_get and value:
             # evaluate name_get() as superuser, because the visibility of a
@@ -2694,6 +2739,19 @@ class _RelationalMulti(_Relational):
             corecords = corecords.filtered(Comodel._active_name).with_prefetch(prefetch_ids)
         return corecords
 
+    def convert_to_record_multi(self, values, records):
+        # return the list of ids as a recordset without duplicates
+        prefetch_ids = IterableGenerator(prefetch_x2many_ids, records, self)
+        Comodel = records.pool[self.comodel_name]
+        ids = tuple(unique(id_ for ids in values for id_ in ids))
+        corecords = Comodel._browse(records.env, ids, prefetch_ids)
+        if (
+            Comodel._active_name
+            and self.context.get('active_test', records.env.context.get('active_test', True))
+        ):
+            corecords = corecords.filtered(Comodel._active_name).with_prefetch(prefetch_ids)
+        return corecords
+
     def convert_to_read(self, value, record, use_name_get=True):
         return value.ids
 

odoo/models.py

@@ -5242,7 +5242,7 @@ Record ids: %(records)s
         if isinstance(func, str):
             recs = self
             for name in func.split('.'):
-                recs = recs._mapped_func(operator.itemgetter(name))
+                recs = recs._fields[name].mapped(recs)
             return recs
         else:
             return self._mapped_func(func)
@@ -5279,6 +5279,8 @@ Record ids: %(records)s
         if isinstance(func, str):
             name = func
             func = lambda rec: any(rec.mapped(name))
+            # populate cache
+            self.mapped(name)
         return self.browse([rec.id for rec in self if func(rec)])
 
     def filtered_domain(self, domain):