pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FIX] l10n_fr_fec: alleviate memory error when exporting too many move lines

Browse Source 
We now split the export of the FEC files per batch of move lines. This was necessary, as the fetchall() ran on the cursor to get the results could return a huge amount of data on big databases, saturating the RAM.

This fix is not exhaustive, and memory consumption could be lowered by removing the need to convert the whole file to base 64 and write it in attachment. Those changes are not doable in stable ; further improvements will come to master.

OPW 3442579

closes odoo/odoo#130884

X-original-commit: 86a221fa3773b5b23ae8f1ad653c1aaac607482e
Signed-off-by: de Wouters de Bouchout Jean-Benoît (jbw) <jbw@odoo.com>
Signed-off-by: Olivier Colson (oco) <oco@odoo.com>
This commit is contained in:
oco-odoo
2023-08-04 16:35:05 +02:00
parent 7f4da03e36
commit 78a8e0a98e
1 changed files with 33 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/l10n_fr_fec/wizard/account_fr_fec.py
+33 -6
View File
@@ -302,6 +302,9 @@ class AccountFrFec(models.TransientModel):
aj_name = f"COALESCE(aj.name->>'{lang}', aj.name->>'en_US')"
else:
aj_name = "aj.name"
query_limit = int(self.env['ir.config_parameter'].sudo().get_param('l10n_fr_fec.batch_size', 500000)) # To prevent memory errors when fetching the results
sql_query = f'''
SELECT
REGEXP_REPLACE(replace(aj.code, '|', '/'), '[\\t\\r\\n]', ' ', 'g') AS JournalCode,
@@ -358,20 +361,44 @@ class AccountFrFec(models.TransientModel):
am.date,
am.name,
aml.id
LIMIT %(limit)s
OFFSET %(offset)s
'''
self._cr.execute(sql_query, {**params, 'date_to': self.date_to})
for row in self._cr.fetchall():
rows_to_write.append(list(row))
with io.BytesIO() as fecfile:
csv_writer = pycompat.csv_writer(fecfile, delimiter='|', lineterminator='')
# Write header and initial balances
for initial_row in rows_to_write:
initial_row = list(initial_row)
# We don't skip \n at then end of the file if there are only initial balances, for simplicity. An empty period export shouldn't happen IRL.
initial_row[-1] += u'\r\n'
csv_writer.writerow(initial_row)
# Write current period's data
query_offset = 0
has_more_results = True
while has_more_results:
self._cr.execute(sql_query, {**params, 'date_to': self.date_to, 'limit': query_limit + 1, 'offset': query_offset})
query_offset += query_limit
has_more_results = self._cr.rowcount > query_limit # we load one more result than the limit to check if there is more
query_results = self._cr.fetchall()
for i, row in enumerate(query_results[:query_limit]):
if i < len(query_results) - 1:
# The file is not allowed to end with an empty line, so we can't use lineterminator on the writer
row = list(row)
row[-1] += u'\r\n'
csv_writer.writerow(row)
base64_result = base64.encodebytes(fecfile.getvalue())
fecvalue = self._csv_write_rows(rows_to_write)
end_date = fields.Date.to_string(self.date_to).replace('-', '')
suffix = ''
if self.export_type == "nonofficial":
suffix = '-NONOFFICIAL'
self.write({
'fec_data': base64.encodebytes(fecvalue),
'fec_data': base64_result,
# Filename = <siren>FECYYYYMMDD where YYYMMDD is the closing date
'filename': '%sFEC%s%s.csv' % (company_legal_data, end_date, suffix),
})
@@ -387,7 +414,7 @@ class AccountFrFec(models.TransientModel):
'target': 'new',
}
def _csv_write_rows(self, rows, lineterminator=u'\r\n'):
def _csv_write_rows(self, rows, lineterminator=u'\r\n'): #DEPRECATED; will disappear in master
"""
Write FEC rows into a file
It seems that Bercy's bureaucracy is not too happy about the