From bc0dd33e5bffe87a9789644f61c88040ac2c6a06 Mon Sep 17 00:00:00 2001
From: Raphael Collet <rco@odoo.com>
Date: Wed, 16 Sep 2020 10:58:52 +0000
Subject: [PATCH] [FIX] core: compute on batches of maximum PREFETCH_MAX
 records
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Calling a compute method on 1M records inevitably brings a worst case in
cache prefetching: the code that determines which records to fetch has
time complexity O(N²).  We avoid this situation by calling the compute
method on maximum 1K records.  Each computed batch has a "prefetch set"
of maximum 1K records too, which avoids the worst-case scenario above.

closes odoo/odoo#58664

X-original-commit: 0b1aa660b1d94fe46dea09670b875f2d81ca36a3
Signed-off-by: Raphael Collet (rco) <rco@openerp.com>
---
 odoo/fields.py |  8 +++-----
 odoo/models.py | 30 ++++++++++++++++++------------
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/odoo/fields.py b/odoo/fields.py
index effdec15c78..f840585414f 100644
--- a/odoo/fields.py
+++ b/odoo/fields.py
@@ -932,10 +932,8 @@ class Field(MetaField('DummyField', (object,), {})):
             if self.recursive:
                 recs = record
             else:
-                recs = env.records_to_compute(self)
-                # compute the field on real records only (if 'record' is real)
-                # or new records only (if 'record' is new)
-                recs = recs.filtered(lambda rec: bool(rec.id) == bool(record.id))
+                ids = expand_ids(record.id, env.all.tocompute[self])
+                recs = record.browse(itertools.islice(ids, PREFETCH_MAX))
             try:
                 self.compute_value(recs)
             except (AccessError, MissingError):
@@ -3755,4 +3753,4 @@ def apply_required(model, field_name):
 
 # imported here to avoid dependency cycle issues
 from .exceptions import AccessError, MissingError, UserError
-from .models import check_pg_name, BaseModel, NewId, IdType
+from .models import check_pg_name, BaseModel, NewId, IdType, expand_ids, PREFETCH_MAX
diff --git a/odoo/models.py b/odoo/models.py
index c21221012a3..936b99633fc 100644
--- a/odoo/models.py
+++ b/odoo/models.py
@@ -55,7 +55,7 @@ from .exceptions import AccessError, MissingError, ValidationError, UserError
 from .osv.query import Query
 from .tools import frozendict, lazy_classproperty, ormcache, \
                    Collector, LastOrderedSet, OrderedSet, IterableGenerator, \
-                   groupby, unique
+                   groupby
 from .tools.config import config
 from .tools.func import frame_codeinfo
 from .tools.misc import CountingStream, clean_context, DEFAULT_SERVER_DATETIME_FORMAT, DEFAULT_SERVER_DATE_FORMAT, get_lang
@@ -219,6 +219,19 @@ def origin_ids(ids):
     return ((id_ or id_.origin) for id_ in ids if (id_ or getattr(id_, "origin", None)))
 
 
+def expand_ids(id0, ids):
+    """ Return an iterator of unique ids from the concatenation of ``[id0]`` and
+        ``ids``, and of the same kind (all real or all new).
+    """
+    yield id0
+    seen = {id0}
+    kind = bool(id0)
+    for id_ in ids:
+        if id_ not in seen and bool(id_) == kind:
+            yield id_
+            seen.add(id_)
+
+
 IdType = (int, str, NewId)
 
 
@@ -5621,27 +5634,20 @@ Fields:
         """ Return the cache of ``self``, mapping field names to values. """
         return RecordCache(self)
 
-    @api.model
     def _in_cache_without(self, field, limit=PREFETCH_MAX):
         """ Return records to prefetch that have no value in cache for ``field``
             (:class:`Field` instance), including ``self``.
             Return at most ``limit`` records.
         """
-        # This method returns records that are either all real, or all new.
+        ids = expand_ids(self.id, self._prefetch_ids)
+        ids = self.env.cache.get_missing_ids(self.browse(ids), field)
+        if limit:
+            ids = itertools.islice(ids, limit)
         # Those records are aimed at being either fetched, or computed.  But the
         # method '_fetch_field' is not correct with new records: it considers
         # them as forbidden records, and clears their cache!  On the other hand,
         # compute methods are not invoked with a mix of real and new records for
         # the sake of code simplicity.
-        kind = bool(self.id)
-        recs = self.browse(unique(self._prefetch_ids))
-        ids = [self.id]
-        for record_id in self.env.cache.get_missing_ids(recs - self, field):
-            if bool(record_id) != kind:
-                continue
-            ids.append(record_id)
-            if limit and limit <= len(ids):
-                break
         return self.browse(ids)
 
     @api.model