diff --git a/odoo/addons/base/models/ir_actions_report.py b/odoo/addons/base/models/ir_actions_report.py index 1945afd2559..eff55808e4a 100644 --- a/odoo/addons/base/models/ir_actions_report.py +++ b/odoo/addons/base/models/ir_actions_report.py @@ -6,7 +6,7 @@ from odoo import api, fields, models, tools, SUPERUSER_ID, _ from odoo.exceptions import UserError, AccessError from odoo.tools.safe_eval import safe_eval, time from odoo.tools.misc import find_in_path, ustr -from odoo.tools import check_barcode_encoding, config, is_html_empty, parse_version +from odoo.tools import check_barcode_encoding, config, is_html_empty, parse_version, split_every from odoo.http import request from odoo.osv.expression import NEGATIVE_TERM_OPERATORS, FALSE_DOMAIN @@ -26,6 +26,8 @@ from PyPDF2 import PdfFileWriter, PdfFileReader from collections import OrderedDict from collections.abc import Iterable from PIL import Image, ImageFile +from itertools import islice + # Allow truncated images ImageFile.LOAD_TRUNCATED_IMAGES = True @@ -50,6 +52,25 @@ except Exception: def _get_wkhtmltopdf_bin(): return find_in_path('wkhtmltopdf') +def _split_table(tree, max_rows): + """ + Walks through the etree and splits tables with more than max_rows rows into + multiple tables with max_rows rows. + + This function is needed because wkhtmltopdf has a exponential processing + time growth when processing tables with many rows. This function is a + workaround for this problem. + + :param tree: The etree to process + :param max_rows: The maximum number of rows per table + """ + for table in list(tree.iter('table')): + prev = table + for rows in islice(split_every(max_rows, table), 1, None): + sibling = etree.Element('table', attrib=table.attrib) + sibling.extend(rows) + prev.addnext(sibling) + prev = sibling # Check the presence of Wkhtmltopdf and return its version at Odoo start-up wkhtmltopdf_state = 'install' @@ -440,7 +461,19 @@ class IrActionsReport(models.Model): prefix = '%s%d.' % ('report.body.tmp.', i) body_file_fd, body_file_path = tempfile.mkstemp(suffix='.html', prefix=prefix) with closing(os.fdopen(body_file_fd, 'wb')) as body_file: - body_file.write(body.encode()) + # HACK: wkhtmltopdf doesn't like big table at all and the + # processing time become exponential with the number + # of rows (like 1H for 250k rows). + # + # So we split the table into multiple tables containing + # 500 rows each. This reduce the processing time to 1min + # for 250k rows. The number 500 was taken from opw-1689673 + if len(body) < 4 * 1024 * 1024: # 4Mib + body_file.write(body.encode()) + else: + tree = lxml.html.fromstring(body) + _split_table(tree, 500) + body_file.write(lxml.html.tostring(tree)) paths.append(body_file_path) temporary_files.append(body_file_path) diff --git a/odoo/addons/base/tests/__init__.py b/odoo/addons/base/tests/__init__.py index 73f5b6e2526..3af1f825589 100644 --- a/odoo/addons/base/tests/__init__.py +++ b/odoo/addons/base/tests/__init__.py @@ -40,6 +40,7 @@ from . import test_qweb from . import test_res_config from . import test_res_lang from . import test_search +from . import test_split_table from . import test_translate # from . import test_uninstall # loop from . import test_user_has_group diff --git a/odoo/addons/base/tests/split_table/copy_attributes.split1.xml b/odoo/addons/base/tests/split_table/copy_attributes.split1.xml new file mode 100644 index 00000000000..fd554eeb6b1 --- /dev/null +++ b/odoo/addons/base/tests/split_table/copy_attributes.split1.xml @@ -0,0 +1,18 @@ + + + + + + + +
111
+ + + + + + + +
+ 222
+ diff --git a/odoo/addons/base/tests/split_table/copy_attributes.xml b/odoo/addons/base/tests/split_table/copy_attributes.xml new file mode 100644 index 00000000000..85c82fe219f --- /dev/null +++ b/odoo/addons/base/tests/split_table/copy_attributes.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + +
111
+ 222
+ diff --git a/odoo/addons/base/tests/split_table/first_nested.split2.xml b/odoo/addons/base/tests/split_table/first_nested.split2.xml new file mode 100644 index 00000000000..5cc387e9608 --- /dev/null +++ b/odoo/addons/base/tests/split_table/first_nested.split2.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + +
+ + + + + + + + + + + +
444
555
+ + + + + + +
666
+ 		77
111
+ + + + + + + + + + + +
222
333
+ diff --git a/odoo/addons/base/tests/split_table/first_nested.xml b/odoo/addons/base/tests/split_table/first_nested.xml new file mode 100644 index 00000000000..9a5d1be2fa6 --- /dev/null +++ b/odoo/addons/base/tests/split_table/first_nested.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + +
444
555
666
+ 		77
111
222
333
+ diff --git a/odoo/addons/base/tests/split_table/nested.split2.xml b/odoo/addons/base/tests/split_table/nested.split2.xml new file mode 100644 index 00000000000..2cd017702c9 --- /dev/null +++ b/odoo/addons/base/tests/split_table/nested.split2.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + +
111
222
+ + + + + + + + + + + +
333
+ + + + + + + + + + + +
444
555
+ + + + + + +
666
+ 		77
+ diff --git a/odoo/addons/base/tests/split_table/nested.xml b/odoo/addons/base/tests/split_table/nested.xml new file mode 100644 index 00000000000..d317dba66d9 --- /dev/null +++ b/odoo/addons/base/tests/split_table/nested.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + +
111
222
333
+ + + + + + + + + + + + + + + + +
444
555
666
+ 		77
+ diff --git a/odoo/addons/base/tests/split_table/simple.split1.xml b/odoo/addons/base/tests/split_table/simple.split1.xml new file mode 100644 index 00000000000..cc13f8178f0 --- /dev/null +++ b/odoo/addons/base/tests/split_table/simple.split1.xml @@ -0,0 +1,23 @@ + + + + + + + +
111
+ + + + + + +
222
+ + + + + + +
333
+ diff --git a/odoo/addons/base/tests/split_table/simple.split2.xml b/odoo/addons/base/tests/split_table/simple.split2.xml new file mode 100644 index 00000000000..c022649c988 --- /dev/null +++ b/odoo/addons/base/tests/split_table/simple.split2.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + +
111
222
+ + + + + + +
333
+ diff --git a/odoo/addons/base/tests/split_table/simple.xml b/odoo/addons/base/tests/split_table/simple.xml new file mode 100644 index 00000000000..68380f33269 --- /dev/null +++ b/odoo/addons/base/tests/split_table/simple.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + +
111
222
333
+ diff --git a/odoo/addons/base/tests/test_split_table.py b/odoo/addons/base/tests/test_split_table.py new file mode 100644 index 00000000000..64a448ad879 --- /dev/null +++ b/odoo/addons/base/tests/test_split_table.py @@ -0,0 +1,30 @@ +from odoo.tests.common import TransactionCase +from odoo.tools import file_open +from odoo.addons.base.models.ir_actions_report import _split_table +from lxml import etree + +def cleanup_string(s): + return ''.join(s.split()) + +class TestSplitTable(TransactionCase): + def test_split_table(self): + # NOTE: All the tests's xml are in split_table/ relative to this file + CASES = ( + ("Table's len is equal to max_rows and should not be split", "simple", "simple", 3), + ("Table's len is greater to max_rows and should not be split", "simple", "simple", 4), + ("max_rows is 1 and every table should be split", "simple", "simple.split1", 1), + ("max_row is 2 and the table should be split", "simple", "simple.split2", 2), + ("Nested tables should be split", "nested", "nested.split2", 2), + ("Nested tables at the start should be split", "first_nested", "first_nested.split2", 2), + ("Attributes should be copied", "copy_attributes", "copy_attributes.split1", 1), + ) + + for description, actual, expected, max_rows in CASES: + with self.subTest(description), \ + file_open(f"base/tests/split_table/{actual}.xml") as actual, \ + file_open(f"base/tests/split_table/{expected}.xml") as expected: + + tree = etree.fromstring(actual.read()) + _split_table(tree, max_rows) + processed = etree.tostring(tree, encoding='unicode') + self.assertEqual(cleanup_string(processed), cleanup_string(expected.read()))