From 431b4e69e33d39aea741471382a44b20e1ef9718 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Thu, 31 Aug 2023 10:03:37 +0200 Subject: [PATCH] [FIX] base: correctly parse utf8 html module descriptions Apparently `lxml.html.document_fromstring` (and possibly other `lxml.html` loaders) parses byte-strings as latin1 regardless of their actual encoding, maybe because python2, maybe because there's a super legacy html4 parser underlying it. Either way that means ever since loading `static/description/index.html` files was added 10 years ago (4bf6a7ea4c1703a02aa547db477ad7162ac1378c) `_get_desc` has been loading these files in latin1 rather than the utf8 most people would expect. Add an explicit decoding phase to try and load html description files in UTF8. Fall back to latin1 in case there are description files which are genuinely in latin1, or even just some random-ass broken stuff which very much isn't utf8 (the extended-ascii encodings -- of which latin1 is one -- will happily accept and mangle any input as every byte value is valid, utf8 is a lot more structured). Closes #127846 closes odoo/odoo#133859 X-original-commit: 4dbc3b00e587f3d64cfd964a685f2bddd1b499ad Signed-off-by: Xavier Morel (xmo) --- odoo/addons/base/models/ir_module.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/odoo/addons/base/models/ir_module.py b/odoo/addons/base/models/ir_module.py index 6d078212142..039429209a1 100644 --- a/odoo/addons/base/models/ir_module.py +++ b/odoo/addons/base/models/ir_module.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 +import warnings from collections import defaultdict, OrderedDict from decorator import decorator from operator import attrgetter @@ -169,7 +170,15 @@ class Module(models.Model): if module_path and path: with tools.file_open(path, 'rb') as desc_file: doc = desc_file.read() - html = lxml.html.document_fromstring(doc) + try: + contents = doc.decode() + except UnicodeDecodeError: + warnings.warn( + f"Non-UTF8 module descriptions are deprecated since Odoo 17 ({module.name}'s description is not)", + category=DeprecationWarning, + ) + contents = doc + html = lxml.html.document_fromstring(contents) for element, attribute, link, pos in html.iterlinks(): if element.get('src') and not '//' in element.get('src') and not 'static/' in element.get('src'): element.set('src', "/%s/static/description/%s" % (module.name, element.get('src')))