diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index f2bab8b52b3..7201b194193 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import csv import datetime import io import itertools @@ -262,14 +261,13 @@ class Import(models.TransientModel): # csv module expect utf-8, see http://docs.python.org/2/library/csv.html csv_data = csv_data.decode(encoding).encode('utf-8') - csv_iterator = csv.reader( + csv_iterator = pycompat.csv_reader( io.BytesIO(csv_data), quotechar=str(options['quoting']), delimiter=str(options['separator'])) return ( - [item.decode('utf-8') for item in row] - for row in csv_iterator + row for row in csv_iterator if any(x for x in row if x.strip()) ) diff --git a/addons/base_import/tests/test_base_import.py b/addons/base_import/tests/test_base_import.py index e3357a5c3a1..6d15119ca02 100644 --- a/addons/base_import/tests/test_base_import.py +++ b/addons/base_import/tests/test_base_import.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 -import csv import io import unittest from odoo.tests.common import TransactionCase, can_import from odoo.modules.module import get_module_resource -from odoo.tools import mute_logger +from odoo.tools import mute_logger, pycompat ID_FIELD = { 'id': 'id', @@ -501,11 +500,11 @@ class test_convert_import_data(TransactionCase): Ensure importing keep newlines """ output = io.BytesIO() - writer = csv.writer(output, quoting=csv.QUOTE_ALL) + writer = pycompat.csv_writer(output, quoting=1) - data_row = ["\tfoo\n\tbar", " \"hello\" \n\n 'world' "] + data_row = [u"\tfoo\n\tbar", u" \"hello\" \n\n 'world' "] - writer.writerow(["name", "Some Value"]) + writer.writerow([u"name", u"Some Value"]) writer.writerow(data_row) import_wizard = self.env['base_import.import'].create({ @@ -527,16 +526,15 @@ class test_failures(TransactionCase): Ensure big fields (e.g. b64-encoded image data) can be imported and we're not hitting limits of the default CSV parser config """ - import csv, io from PIL import Image im = Image.new('RGB', (1920, 1080)) fout = io.BytesIO() - writer = csv.writer(fout, dialect=None) + writer = pycompat.csv_writer(fout, dialect=None) writer.writerows([ - ['name', 'db_datas'], - ['foo', base64.b64encode(im.tobytes())] + [u'name', u'db_datas'], + [u'foo', base64.b64encode(im.tobytes()).decode('ascii')] ]) import_wizard = self.env['base_import.import'].create({ diff --git a/addons/l10n_fr_fec/wizard/account_fr_fec.py b/addons/l10n_fr_fec/wizard/account_fr_fec.py index f82df08dc99..b8c4b571124 100644 --- a/addons/l10n_fr_fec/wizard/account_fr_fec.py +++ b/addons/l10n_fr_fec/wizard/account_fr_fec.py @@ -4,11 +4,11 @@ # Copyright (C) 2013-2015 Akretion (http://www.akretion.com) import base64 -import csv import io from odoo import api, fields, models, _ from odoo.exceptions import Warning +from odoo.tools import pycompat class AccountFrFec(models.TransientModel): @@ -86,24 +86,24 @@ class AccountFrFec(models.TransientModel): # So it will be easier for the accountant to check the file before # sending it to the fiscal administration header = [ - 'JournalCode', # 0 - 'JournalLib', # 1 - 'EcritureNum', # 2 - 'EcritureDate', # 3 - 'CompteNum', # 4 - 'CompteLib', # 5 - 'CompAuxNum', # 6 We use partner.id - 'CompAuxLib', # 7 - 'PieceRef', # 8 - 'PieceDate', # 9 - 'EcritureLib', # 10 - 'Debit', # 11 - 'Credit', # 12 - 'EcritureLet', # 13 - 'DateLet', # 14 - 'ValidDate', # 15 - 'Montantdevise', # 16 - 'Idevise', # 17 + u'JournalCode', # 0 + u'JournalLib', # 1 + u'EcritureNum', # 2 + u'EcritureDate', # 3 + u'CompteNum', # 4 + u'CompteLib', # 5 + u'CompAuxNum', # 6 We use partner.id + u'CompAuxLib', # 7 + u'PieceRef', # 8 + u'PieceDate', # 9 + u'EcritureLib', # 10 + u'Debit', # 11 + u'Credit', # 12 + u'EcritureLet', # 13 + u'DateLet', # 14 + u'ValidDate', # 15 + u'Montantdevise', # 16 + u'Idevise', # 17 ] company = self.env.user.company_id @@ -115,7 +115,7 @@ class AccountFrFec(models.TransientModel): _("FEC is for French companies only !")) fecfile = io.BytesIO() - w = csv.writer(fecfile, delimiter='|') + w = pycompat.csv_writer(fecfile, delimiter='|') w.writerow(header) # INITIAL BALANCE @@ -190,7 +190,7 @@ class AccountFrFec(models.TransientModel): else: listrow[11] = '0,00' listrow[12] = str(-listrow_amount).replace('.', ',') - w.writerow([s.encode("utf-8") for s in listrow]) + w.writerow(listrow) #if the unaffected earnings account wasn't in the selection yet: add it manually if (not unaffected_earnings_line and unaffected_earnings_results @@ -201,7 +201,7 @@ class AccountFrFec(models.TransientModel): if unaffected_earnings_account: unaffected_earnings_results[4] = unaffected_earnings_account.code unaffected_earnings_results[5] = unaffected_earnings_account.name - w.writerow([s.encode("utf-8") for s in unaffected_earnings_results]) + w.writerow(unaffected_earnings_results) # LINES sql_query = ''' @@ -266,8 +266,7 @@ class AccountFrFec(models.TransientModel): sql_query, (self.date_from, self.date_to, company.id)) for row in self._cr.fetchall(): - listrow = list(row) - w.writerow([s.encode("utf-8") for s in listrow]) + w.writerow(list(row)) siren = company.vat[4:13] end_date = self.date_to.replace('-', '') diff --git a/addons/l10n_lu/scripts/tax2csv.py b/addons/l10n_lu/scripts/tax2csv.py index 013df32e3d3..f4080284064 100644 --- a/addons/l10n_lu/scripts/tax2csv.py +++ b/addons/l10n_lu/scripts/tax2csv.py @@ -1,17 +1,8 @@ from collections import OrderedDict -import csv import xlrd from odoo.tools import pycompat -def _e(s): - if s is None: - return '' - if pycompat.PY2 and type(s) == pycompat.text_type: - return s.encode('utf8') - return str(s) - - def _is_true(s): return s not in ('F', 'False', 0, '', None, False) @@ -55,9 +46,9 @@ class LuxTaxGenerator: yield OrderedDict(pycompat.izip(keys, row)) def tax_codes_to_csv(self): - writer = csv.writer(open('account.tax.code.template-%s.csv' % + writer = pycompat.csv_writer(open('account.tax.code.template-%s.csv' % self.suffix, 'wb')) - tax_codes_iterator = self.iter_tax_codes + tax_codes_iterator = self.iter_tax_codes() keys = next(tax_codes_iterator) writer.writerow(keys) @@ -68,7 +59,7 @@ class LuxTaxGenerator: if tax_code in tax_codes: raise RuntimeError('duplicate tax code %s' % tax_code) tax_codes[tax_code] = row['id'] - writer.writerow(pycompat.imap(_e, pycompat.values(row))) + writer.writerow([pycompat.to_text(v) for v in pycompat.values(row)]) # read taxes and add leaf tax codes new_tax_codes = {} # id: parent_code @@ -144,17 +135,19 @@ class LuxTaxGenerator: for tax_code_id in sorted(new_tax_codes): name, parent_code = new_tax_codes[tax_code_id] - writer.writerow((tax_code_id, - 'lu_tct_m' + parent_code, - tax_code_id.replace('lu_tax_code_template_', ''), - '1', - '', - _e(name), - '')) + writer.writerow([ + tax_code_id, + u'lu_tct_m' + parent_code, + tax_code_id.replace('lu_tax_code_template_', u''), + u'1', + u'', + pycompat.to_text(name), + u'' + ]) def taxes_to_csv(self): - writer = csv.writer(open('account.tax.template-%s.csv' % - self.suffix, 'wb')) + writer = pycompat.csv_writer(open('account.tax.template-%s.csv' % + self.suffix, 'wb')) taxes_iterator = self.iter_taxes() keys = next(taxes_iterator) writer.writerow(keys[3:] + ['sequence']) @@ -167,17 +160,20 @@ class LuxTaxGenerator: cur_seq = seq + 1000 else: cur_seq = seq - writer.writerow(list(pycompat.imap(_e, list(pycompat.values(row))[3:])) + [cur_seq]) + writer.writerow([ + pycompat.to_text(v) + for v in list(pycompat.values(row))[3:] + ] + [cur_seq]) def fiscal_pos_map_to_csv(self): - writer = csv.writer(open('account.fiscal.' - 'position.tax.template-%s.csv' % - self.suffix, 'wb')) + writer = pycompat.csv_writer(open('account.fiscal.' + 'position.tax.template-%s.csv' % + self.suffix, 'wb')) fiscal_pos_map_iterator = self.iter_fiscal_pos_map() keys = next(fiscal_pos_map_iterator) writer.writerow(keys) for row in fiscal_pos_map_iterator: - writer.writerow(pycompat.imap(_e, pycompat.values(row))) + writer.writerow([pycompat.to_text(s) for s in pycompat.values(row)]) if __name__ == '__main__': diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 9ac7b7a2fd8..ebe9e8aa16d 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -3,7 +3,6 @@ import babel.messages.pofile import base64 -import csv import datetime import functools import glob @@ -1389,31 +1388,21 @@ class CSVExport(ExportFormat, http.Controller): def from_data(self, fields, rows): fp = io.BytesIO() - writer = csv.writer(fp, quoting=csv.QUOTE_ALL) + writer = pycompat.csv_writer(fp, quoting=1) - writer.writerow([name.encode('utf-8') for name in fields]) + writer.writerow(fields) for data in rows: row = [] for d in data: - if type(d) != str and isinstance(d, pycompat.string_types): - try: - d = d.encode('utf-8') - except UnicodeError: - pass - if d is False: d = None - # Spreadsheet apps tend to detect formulas on leading =, + and - - if type(d) is str and d.startswith(('=', '-', '+')): + if isinstance(d, pycompat.string_types) and d.startswith(('=', '-', '+')): d = "'" + d - row.append(d) + row.append(pycompat.to_text(d)) writer.writerow(row) - fp.seek(0) - data = fp.read() - fp.close() - return data + return fp.getvalue() class ExcelExport(ExportFormat, http.Controller): # Excel needs raw data to correctly handle numbers and date values diff --git a/doc/python3.rst b/doc/python3.rst index a7d143b85b7..7ca86708ced 100644 --- a/doc/python3.rst +++ b/doc/python3.rst @@ -452,6 +452,26 @@ conversions: text is *encoded* to bytes and bytes are *decoded* to text. migrated to using ``base64.b64encode`` and ``base64.b64decode`` respectively. +csv +--- + +``csv`` is a fairly vicious one: not only is it not a very good format, the +Python 2 and Python 3 versions of the library are text-model incompatible in +significant ways: + +* Python 2's CSV only works on *ascii-compatible byte streams* (it has no + encoding support at all) and extracts bytestring values +* Python 3's CSV only works on *text streams* and extract text values +* And ``io`` doesn't provide "native string" streaming facilities. + +However with respect to Odoo it turns out most or all uses of ``csv`` fit +inside a model of *byte stream to and from text values*. + +The latter is thus a model implemented by cross-version wrappers +:func:`odoo.tools.pycompat.csv_reader` and +:func:`odoo.tools.pycompat.csv_writer`: they take a *UTF-8 byte stream* and +read or write *text* values. + .. _hash randomisation: http://bugs.python.org/issue13703 .. _requests: http://docs.python-requests.org/ diff --git a/odoo/addons/test_pylint/tests/test_pylint.py b/odoo/addons/test_pylint/tests/test_pylint.py index cc73ca5a709..92764eaeb84 100644 --- a/odoo/addons/test_pylint/tests/test_pylint.py +++ b/odoo/addons/test_pylint/tests/test_pylint.py @@ -79,6 +79,7 @@ class TestPyLint(TransactionCase): BAD_MODULES = [ 'commands', 'cPickle', + 'csv', 'cStringIO', 'md5', 'urllib', diff --git a/odoo/cli/server.py b/odoo/cli/server.py index 180efcfb70f..d740dc7282a 100644 --- a/odoo/cli/server.py +++ b/odoo/cli/server.py @@ -12,7 +12,7 @@ GNU Public Licence. """ import atexit -import csv +import csv # pylint: disable=deprecated-module import logging import os import signal diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py index 7da9f4e4412..f979df7a28c 100644 --- a/odoo/tools/convert.py +++ b/odoo/tools/convert.py @@ -2,7 +2,6 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 import io -import csv import logging import os.path import re @@ -844,28 +843,21 @@ def convert_csv_import(cr, module, fname, csvcontent, idref=None, mode='init', quote: " delimiter: , encoding: utf-8''' - if not idref: - idref={} - model = ('.'.join(fname.split('.')[:-1]).split('-'))[0] - #remove folder path from model - head, model = os.path.split(model) + filename, _ext = os.path.splitext(os.path.basename(fname)) + model = filename.split('-')[0] - input = io.BytesIO(csvcontent) #FIXME - reader = csv.reader(input, quotechar='"', delimiter=',') + reader = pycompat.csv_reader(io.BytesIO(csvcontent), quotechar='"', delimiter=',') fields = next(reader) if not (mode == 'init' or 'id' in fields): _logger.error("Import specification does not contain 'id' and we are in init mode, Cannot continue.") return - datas = [] - for line in reader: - if not (line and any(line)): - continue - try: - datas.append([ustr(v) for v in line]) - except Exception: - _logger.error("Cannot import the line: %s", line) + # filter out empty lines (any([]) == False) and lines containing only empty cells + datas = [ + line for line in reader + if any(line) + ] context = { 'mode': mode, diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py index 05a5a91f356..274ae4a6d24 100644 --- a/odoo/tools/misc.py +++ b/odoo/tools/misc.py @@ -30,7 +30,6 @@ from lxml import etree from .which import which import traceback -import csv from operator import itemgetter try: @@ -403,14 +402,15 @@ def scan_languages(): csvpath = odoo.modules.module.get_resource_path('base', 'res', 'res.lang.csv') try: # read (code, name) from languages in base/res/res.lang.csv - result = [] - with open(csvpath) as csvfile: - reader = csv.reader(csvfile, delimiter=',', quotechar='"') + with open(csvpath, 'rb') as csvfile: + reader = pycompat.csv_reader(csvfile, delimiter=',', quotechar='"') fields = next(reader) code_index = fields.index("code") name_index = fields.index("name") - for row in reader: - result.append((ustr(row[code_index]), ustr(row[name_index]))) + result = [ + (row[code_index], row[name_index]) + for row in reader + ] except Exception: _logger.error("Could not read %s", csvpath) result = [] diff --git a/odoo/tools/pycompat.py b/odoo/tools/pycompat.py index e4111668d24..89bf96c3dd6 100644 --- a/odoo/tools/pycompat.py +++ b/odoo/tools/pycompat.py @@ -1,10 +1,16 @@ # -*- coding: utf-8 -*- # to remove if we decide to add a dependency on six or future # very strongly inspired by https://github.com/pallets/werkzeug/blob/master/werkzeug/_compat.py +#pylint: disable=deprecated-module +import csv +import collections +import io import sys + PY2 = sys.version_info[0] == 2 +_Writer = collections.namedtuple('_Writer', 'writerow writerows') if PY2: # pylint: disable=long-builtin,unichr-builtin,unicode-builtin unichr = unichr @@ -39,6 +45,19 @@ if PY2: return cls exec ('def reraise(tp, value, tb=None):\n raise tp, value, tb') + + def csv_reader(stream, **params): + for row in csv.reader(stream, **params): + yield [c.decode('utf-8') for c in row] + def csv_writer(stream, **params): + w = csv.writer(stream, **params) + return _Writer( + writerow=lambda r: w.writerow([c.encode('utf-8') for c in r]), + writerows=lambda rs: w.writerows( + [c.encode('utf-8') for c in r] + for r in rs + ) + ) else: # pylint: disable=bad-functions unichr = chr @@ -74,6 +93,15 @@ else: raise value.with_traceback(tb) raise value + def csv_reader(stream, **params): + assert not isinstance(stream, io.TextIOBase),\ + "For cross-compatibility purposes, csv_reader takes a bytes stream" + return csv.reader(io.TextIOWrapper(stream, encoding='utf-8'), **params) + def csv_writer(stream, **params): + assert not isinstance(stream, io.TextIOBase), \ + "For cross-compatibility purposes, csv_writer takes a bytes stream" + return csv.writer(io.TextIOWrapper(stream, encoding='utf-8', line_buffering=True), **params) + def to_text(source): """ Generates a text value (an instance of text_type) from an arbitrary source. diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py index dbf995a5161..58bbf6bcec5 100644 --- a/odoo/tools/translate.py +++ b/odoo/tools/translate.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. import codecs -import csv import fnmatch import inspect import io @@ -121,6 +120,8 @@ _LOCALE2WIN32 = { ENGLISH_SMALL_WORDS = set("as at by do go if in me no of ok on or to up us we".split()) +# these direct uses of CSV are ok. +import csv # pylint: disable=deprecated-module class UNIX_LINE_TERMINATOR(csv.excel): lineterminator = '\n' @@ -657,7 +658,7 @@ def trans_export(lang, modules, buffer, format, cr): def _process(format, modules, rows, buffer, lang): if format == 'csv': - writer = csv.writer(buffer, 'UNIX') + writer = pycompat.csv_writer(buffer, dialect='UNIX') # write header first writer.writerow(("module","type","name","res_id","src","value","comments")) for module, type, name, res_id, src, trad, comments in rows: @@ -1035,11 +1036,9 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, # now, the serious things: we read the language file fileobj.seek(0) if fileformat == 'csv': - reader = csv.reader(fileobj, quotechar='"', delimiter=',') + reader = pycompat.csv_reader(fileobj, quotechar='"', delimiter=',') # read the first line of the file (it contains columns titles) - for row in reader: - fields = row - break + fields = next(reader) elif fileformat == 'po': reader = PoFile(fileobj)