From 78a8e0a98e5376dff9c21c90b8ce8a5b2b13f7b7 Mon Sep 17 00:00:00 2001 From: oco-odoo Date: Mon, 31 Jul 2023 11:35:13 +0000 Subject: [PATCH] [FIX] l10n_fr_fec: alleviate memory error when exporting too many move lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now split the export of the FEC files per batch of move lines. This was necessary, as the fetchall() ran on the cursor to get the results could return a huge amount of data on big databases, saturating the RAM. This fix is not exhaustive, and memory consumption could be lowered by removing the need to convert the whole file to base 64 and write it in attachment. Those changes are not doable in stable ; further improvements will come to master. OPW 3442579 closes odoo/odoo#130884 X-original-commit: 86a221fa3773b5b23ae8f1ad653c1aaac607482e Signed-off-by: de Wouters de Bouchout Jean-Benoît (jbw) Signed-off-by: Olivier Colson (oco) --- addons/l10n_fr_fec/wizard/account_fr_fec.py | 39 +++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/addons/l10n_fr_fec/wizard/account_fr_fec.py b/addons/l10n_fr_fec/wizard/account_fr_fec.py index 1e3b0943006..1da8fd008fd 100644 --- a/addons/l10n_fr_fec/wizard/account_fr_fec.py +++ b/addons/l10n_fr_fec/wizard/account_fr_fec.py @@ -302,6 +302,9 @@ class AccountFrFec(models.TransientModel): aj_name = f"COALESCE(aj.name->>'{lang}', aj.name->>'en_US')" else: aj_name = "aj.name" + + query_limit = int(self.env['ir.config_parameter'].sudo().get_param('l10n_fr_fec.batch_size', 500000)) # To prevent memory errors when fetching the results + sql_query = f''' SELECT REGEXP_REPLACE(replace(aj.code, '|', '/'), '[\\t\\r\\n]', ' ', 'g') AS JournalCode, @@ -358,20 +361,44 @@ class AccountFrFec(models.TransientModel): am.date, am.name, aml.id + LIMIT %(limit)s + OFFSET %(offset)s ''' - self._cr.execute(sql_query, {**params, 'date_to': self.date_to}) - for row in self._cr.fetchall(): - rows_to_write.append(list(row)) + with io.BytesIO() as fecfile: + csv_writer = pycompat.csv_writer(fecfile, delimiter='|', lineterminator='') + + # Write header and initial balances + for initial_row in rows_to_write: + initial_row = list(initial_row) + # We don't skip \n at then end of the file if there are only initial balances, for simplicity. An empty period export shouldn't happen IRL. + initial_row[-1] += u'\r\n' + csv_writer.writerow(initial_row) + + # Write current period's data + query_offset = 0 + has_more_results = True + while has_more_results: + self._cr.execute(sql_query, {**params, 'date_to': self.date_to, 'limit': query_limit + 1, 'offset': query_offset}) + query_offset += query_limit + has_more_results = self._cr.rowcount > query_limit # we load one more result than the limit to check if there is more + query_results = self._cr.fetchall() + for i, row in enumerate(query_results[:query_limit]): + if i < len(query_results) - 1: + # The file is not allowed to end with an empty line, so we can't use lineterminator on the writer + row = list(row) + row[-1] += u'\r\n' + csv_writer.writerow(row) + + base64_result = base64.encodebytes(fecfile.getvalue()) - fecvalue = self._csv_write_rows(rows_to_write) end_date = fields.Date.to_string(self.date_to).replace('-', '') suffix = '' if self.export_type == "nonofficial": suffix = '-NONOFFICIAL' self.write({ - 'fec_data': base64.encodebytes(fecvalue), + 'fec_data': base64_result, # Filename = FECYYYYMMDD where YYYMMDD is the closing date 'filename': '%sFEC%s%s.csv' % (company_legal_data, end_date, suffix), }) @@ -387,7 +414,7 @@ class AccountFrFec(models.TransientModel): 'target': 'new', } - def _csv_write_rows(self, rows, lineterminator=u'\r\n'): + def _csv_write_rows(self, rows, lineterminator=u'\r\n'): #DEPRECATED; will disappear in master """ Write FEC rows into a file It seems that Bercy's bureaucracy is not too happy about the