From aa4fad64ce2d0e48675d2d526c78121219e1711d Mon Sep 17 00:00:00 2001 From: Alvaro Fuentes Date: Thu, 15 Jul 2021 11:55:08 +0000 Subject: [PATCH] [FIX] web: fix web_read_group total groups count Fetching all the groups from the DB causes MemoryError on some DBs Example Accounting > Accounting > Journals > Miscellaneous menu: ``` Traceback (most recent call last): File "/tmp/tmpbnah9jtp/migrations/base/tests/test_mock_crawl.py", line 176, in crawl_menu self.mock_action(action_vals) File "/tmp/tmpbnah9jtp/migrations/base/tests/test_mock_crawl.py", line 267, in mock_action mock_method(model, view, fields_list, domain, group_by) File "/tmp/tmpbnah9jtp/migrations/base/tests/test_mock_crawl.py", line 380, in mock_view_tree self.mock_web_read_group(model, view, domain, group_by, fields_list, limit_group=5) File "/tmp/tmpbnah9jtp/migrations/base/tests/test_mock_crawl.py", line 425, in mock_web_read_group data = model.web_read_group(domain, fields_list, group_by, limit=limit)["groups"] File "/home/odoo/src/odoo/14.0/addons/web/models/models.py", line 96, in web_read_group all_groups = self.read_group(domain, ['display_name'], groupby, lazy=True) File "/home/odoo/src/odoo/14.0/odoo/models.py", line 2248, in read_group result = self._read_group_raw(domain, fields, groupby, offset=offset, limit=limit, orderby=orderby, lazy=lazy) File "/home/odoo/src/odoo/14.0/odoo/models.py", line 2387, in _read_group_raw result = [self._read_group_format_result(d, annotated_groupbys, groupby, domain) for d in data] File "/home/odoo/src/odoo/14.0/odoo/models.py", line 2387, in result = [self._read_group_format_result(d, annotated_groupbys, groupby, domain) for d in data] MemoryError ``` This issue was observed during the upgrade requests upg-18830, target 14.0; and upg-61934 (legacy), target 13.0 The issue can be reproduced on a clean DB with just account_accountant installed and ~2 millions account moves on a Misc journal. closes odoo/odoo#73855 X-original-commit: a9b3d9cf9bfd2bf4c9b0904059f1606ad6e04ee4 Signed-off-by: Christophe Simonis --- addons/web/models/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/addons/web/models/models.py b/addons/web/models/models.py index 093b824d614..14c697a4900 100644 --- a/addons/web/models/models.py +++ b/addons/web/models/models.py @@ -102,8 +102,15 @@ class Base(models.AbstractModel): if not groups: length = 0 elif limit and len(groups) == limit: - all_groups = self.read_group(domain, ['display_name'], groupby, lazy=True) - length = len(all_groups) + # We need to fetch all groups to know the total number + # this cannot be done all at once to avoid MemoryError + length = limit + chunk_size = 100000 + while True: + more = len(self.read_group(domain, ['display_name'], groupby, offset=length, limit=chunk_size, lazy=True)) + length += more + if more < chunk_size: + break else: length = len(groups) + offset return {