From 7496a710b983eeb18a1f3ac8f817ae08fcdad0c3 Mon Sep 17 00:00:00 2001 From: Stanislas Sobieski Date: Thu, 27 Apr 2023 11:32:06 +0000 Subject: [PATCH] [FIX] cloc: avoid memory issue on big file Before this commit: Files that should be ignored in the manifest but aren't (js library for example) it can happen that files have huge lines, the regex to substract the comments will overuse memory. For example, a file of 13M with a line of more that 8M characters, the memory consumptions peak at 1.7G The results might be different, but it's an acceptable compromise closes odoo/odoo#120376 X-original-commit: 63b13af2e49a83168d4304a06b6489c4b86eabf6 Signed-off-by: Thibault Francois --- odoo/tools/cloc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/odoo/tools/cloc.py b/odoo/tools/cloc.py index 83f6cf73927..23c39dff980 100644 --- a/odoo/tools/cloc.py +++ b/odoo/tools/cloc.py @@ -22,6 +22,7 @@ DEFAULT_EXCLUDE = [ STANDARD_MODULES = ['web', 'web_enterprise', 'theme_common', 'base'] MAX_FILE_SIZE = 25 * 2**20 # 25 MB +MAX_LINE_SIZE = 100000 VALID_EXTENSION = ['.py', '.js', '.xml', '.css', '.scss'] class Cloc(object): @@ -62,6 +63,10 @@ class Cloc(object): # Based on https://stackoverflow.com/questions/241327 s = s.strip() + "\n" total = s.count("\n") + # To avoid to use too much memory we don't try to count file + # with very large line, usually minified file + if max(len(l) for l in s.split('\n')) > MAX_LINE_SIZE: + return -1, "Max line size exceeded" def replacer(match): s = match.group(0)