From 6de2091d4d3a142b9c09846ed207387f58f7f027 Mon Sep 17 00:00:00 2001 From: Nicolas Lempereur Date: Fri, 22 Apr 2016 13:13:01 +0200 Subject: [PATCH] [IMP] base_import: only feed utf-8 to csv reader As noted in the csv module documentation*, up to python 2.7 it is best to used UTF-8 as input (and worst to use UTF-16 csv). closes #11792 opw-674756 * https://docs.python.org/2/library/csv.html --- addons/base_import/models.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/addons/base_import/models.py b/addons/base_import/models.py index 91d54b64588..2703dd218aa 100644 --- a/addons/base_import/models.py +++ b/addons/base_import/models.py @@ -250,15 +250,21 @@ class ir_import(orm.TransientModel): :throws csv.Error: if an error is detected during CSV parsing :throws UnicodeDecodeError: if ``options.encoding`` is incorrect """ - csv_iterator = csv.reader( - StringIO(record.file), - quotechar=str(options['quoting']), - delimiter=str(options['separator'])) + csv_data = record.file # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet encoding = options.get('encoding', 'utf-8') + if encoding != 'utf-8': + # csv module expect utf-8, see http://docs.python.org/2/library/csv.html + csv_data = csv_data.decode(encoding).encode('utf-8') + + csv_iterator = csv.reader( + StringIO(csv_data), + quotechar=str(options['quoting']), + delimiter=str(options['separator'])) + return ( - [item.decode(encoding) for item in row] + [item.decode('utf-8') for item in row] for row in csv_iterator if any(x for x in row if x.strip()) )