diff --git a/addons/l10n_fr_fec/wizard/account_fr_fec.py b/addons/l10n_fr_fec/wizard/account_fr_fec.py index 1e3b0943006..1da8fd008fd 100644 --- a/addons/l10n_fr_fec/wizard/account_fr_fec.py +++ b/addons/l10n_fr_fec/wizard/account_fr_fec.py @@ -302,6 +302,9 @@ class AccountFrFec(models.TransientModel): aj_name = f"COALESCE(aj.name->>'{lang}', aj.name->>'en_US')" else: aj_name = "aj.name" + + query_limit = int(self.env['ir.config_parameter'].sudo().get_param('l10n_fr_fec.batch_size', 500000)) # To prevent memory errors when fetching the results + sql_query = f''' SELECT REGEXP_REPLACE(replace(aj.code, '|', '/'), '[\\t\\r\\n]', ' ', 'g') AS JournalCode, @@ -358,20 +361,44 @@ class AccountFrFec(models.TransientModel): am.date, am.name, aml.id + LIMIT %(limit)s + OFFSET %(offset)s ''' - self._cr.execute(sql_query, {**params, 'date_to': self.date_to}) - for row in self._cr.fetchall(): - rows_to_write.append(list(row)) + with io.BytesIO() as fecfile: + csv_writer = pycompat.csv_writer(fecfile, delimiter='|', lineterminator='') + + # Write header and initial balances + for initial_row in rows_to_write: + initial_row = list(initial_row) + # We don't skip \n at then end of the file if there are only initial balances, for simplicity. An empty period export shouldn't happen IRL. + initial_row[-1] += u'\r\n' + csv_writer.writerow(initial_row) + + # Write current period's data + query_offset = 0 + has_more_results = True + while has_more_results: + self._cr.execute(sql_query, {**params, 'date_to': self.date_to, 'limit': query_limit + 1, 'offset': query_offset}) + query_offset += query_limit + has_more_results = self._cr.rowcount > query_limit # we load one more result than the limit to check if there is more + query_results = self._cr.fetchall() + for i, row in enumerate(query_results[:query_limit]): + if i < len(query_results) - 1: + # The file is not allowed to end with an empty line, so we can't use lineterminator on the writer + row = list(row) + row[-1] += u'\r\n' + csv_writer.writerow(row) + + base64_result = base64.encodebytes(fecfile.getvalue()) - fecvalue = self._csv_write_rows(rows_to_write) end_date = fields.Date.to_string(self.date_to).replace('-', '') suffix = '' if self.export_type == "nonofficial": suffix = '-NONOFFICIAL' self.write({ - 'fec_data': base64.encodebytes(fecvalue), + 'fec_data': base64_result, # Filename = FECYYYYMMDD where YYYMMDD is the closing date 'filename': '%sFEC%s%s.csv' % (company_legal_data, end_date, suffix), }) @@ -387,7 +414,7 @@ class AccountFrFec(models.TransientModel): 'target': 'new', } - def _csv_write_rows(self, rows, lineterminator=u'\r\n'): + def _csv_write_rows(self, rows, lineterminator=u'\r\n'): #DEPRECATED; will disappear in master """ Write FEC rows into a file It seems that Bercy's bureaucracy is not too happy about the