From e12b6013d5e52bfbad93dc7fad40f17658490475 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 6 May 2019 09:16:25 +0000 Subject: [PATCH] [FIX] base_import: tests failing with older chardet Some distributions still bundle chardet 2.3, which have some guessing divergences / incompatibilities with python (resolved in 3.x): 1. UTF-{16,32} with BOM is guessed as LE/BE, which when used to decode the string doesn't strip out the BOM. Handle this by checking if the BOM is present and converting the encoding name to the non-marked version in that case. 2. The ISO-8859-1 test string is guessed as ISO-8859-2 (TBF the decoding does make some sense). Allow multiple targets/guesses to "fix" that. closes odoo/odoo#33179 Signed-off-by: Xavier Morel (xmo) --- addons/base_import/models/base_import.py | 14 ++++++++++++++ addons/base_import/tests/test_csv_magic.py | 15 ++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index 4e4945fd37c..dd29bf396d1 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -2,6 +2,7 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 +import codecs import collections import unicodedata @@ -32,6 +33,12 @@ DEFAULT_IMAGE_REGEX = r"(?:http|https)://.*(?:png|jpe?g|tiff?|gif|bmp)" DEFAULT_IMAGE_CHUNK_SIZE = 32768 IMAGE_FIELDS = ["icon", "image", "logo", "picture"] _logger = logging.getLogger(__name__) +BOM_MAP = { + 'utf-16le': codecs.BOM_UTF16_LE, + 'utf-16be': codecs.BOM_UTF16_BE, + 'utf-32le': codecs.BOM_UTF32_LE, + 'utf-32be': codecs.BOM_UTF32_BE, +} try: import xlrd @@ -321,6 +328,13 @@ class Import(models.TransientModel): encoding = options.get('encoding') if not encoding: encoding = options['encoding'] = chardet.detect(csv_data)['encoding'].lower() + # some versions of chardet (e.g. 2.3.0 but not 3.x) will return + # utf-(16|32)(le|be), which for python means "ignore / don't strip + # BOM". We don't want that, so rectify the encoding to non-marked + # IFF the guessed encoding is LE/BE and csv_data starts with a BOM + bom = BOM_MAP.get(encoding) + if bom and csv_data.startswith(bom): + encoding = options['encoding'] = encoding[:-2] if encoding != 'utf-8': csv_data = csv_data.decode(encoding).encode('utf-8') diff --git a/addons/base_import/tests/test_csv_magic.py b/addons/base_import/tests/test_csv_magic.py index 0a35a4731c1..b7975337361 100644 --- a/addons/base_import/tests/test_csv_magic.py +++ b/addons/base_import/tests/test_csv_magic.py @@ -26,18 +26,27 @@ class TestEncoding(ImportCase): options.setdefault('separator', '\t') test_text = "text\tnumber\tdate\tdatetime\n%s\t1.23.45,67\t\t\n" % text for encoding in ['utf-8', 'utf-16', 'utf-32', *encodings]: + if isinstance(encoding, tuple): + encoding, es = encoding + else: + es = [encoding] preview = self._make_import( test_text.encode(encoding)).parse_preview(dict(options)) self.assertIsNone(preview.get('error')) guessed = preview['options']['encoding'] - self.assertIsNotNone(guessed, encoding) - self.assertEqual(codecs.lookup(guessed).name, codecs.lookup(encoding).name) + self.assertIsNotNone(guessed) + self.assertIn( + codecs.lookup(guessed).name, [ + codecs.lookup(e).name + for e in es + ] + ) def test_autodetect_encoding(self): """ Check that import preview can detect & return encoding """ - self._check_text("Iñtërnâtiônàlizætiøn", ['iso-8859-1']) + self._check_text("Iñtërnâtiônàlizætiøn", [('iso-8859-1', ['iso-8859-1', 'iso-8859-2'])]) self._check_text("やぶら小路の藪柑子。海砂利水魚の、食う寝る処に住む処、パイポパイポ パイポのシューリンガン。", ['eucjp', 'shift_jis', 'iso2022_jp'])