pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FIX] P3: CSV reading & writing

Browse Source
This commit is contained in:
Xavier Morel
2017-08-20 23:25:54 +02:00
committed by Olivier Dony
parent 0797cfd355
commit be7c5aefdf
12 changed files with 128 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/base_import/models/base_import.py
+2 -4
View File
@@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import csv
import datetime
import io
import itertools
@@ -262,14 +261,13 @@ class Import(models.TransientModel):
# csv module expect utf-8, see http://docs.python.org/2/library/csv.html
csv_data = csv_data.decode(encoding).encode('utf-8')
csv_iterator = csv.reader(
csv_iterator = pycompat.csv_reader(
io.BytesIO(csv_data),
quotechar=str(options['quoting']),
delimiter=str(options['separator']))
return (
[item.decode('utf-8') for item in row]
for row in csv_iterator
row for row in csv_iterator
if any(x for x in row if x.strip())
)
addons/base_import/tests/test_base_import.py
+7 -9
View File
@@ -1,13 +1,12 @@
# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import base64
import csv
import io
import unittest
from odoo.tests.common import TransactionCase, can_import
from odoo.modules.module import get_module_resource
from odoo.tools import mute_logger
from odoo.tools import mute_logger, pycompat
ID_FIELD = {
'id': 'id',
@@ -501,11 +500,11 @@ class test_convert_import_data(TransactionCase):
Ensure importing keep newlines
"""
output = io.BytesIO()
writer = csv.writer(output, quoting=csv.QUOTE_ALL)
writer = pycompat.csv_writer(output, quoting=1)
data_row = ["\tfoo\n\tbar", " \"hello\" \n\n 'world' "]
data_row = [u"\tfoo\n\tbar", u" \"hello\" \n\n 'world' "]
writer.writerow(["name", "Some Value"])
writer.writerow([u"name", u"Some Value"])
writer.writerow(data_row)
import_wizard = self.env['base_import.import'].create({
@@ -527,16 +526,15 @@ class test_failures(TransactionCase):
Ensure big fields (e.g. b64-encoded image data) can be imported and
we're not hitting limits of the default CSV parser config
"""
import csv, io
from PIL import Image
im = Image.new('RGB', (1920, 1080))
fout = io.BytesIO()
writer = csv.writer(fout, dialect=None)
writer = pycompat.csv_writer(fout, dialect=None)
writer.writerows([
['name', 'db_datas'],
['foo', base64.b64encode(im.tobytes())]
[u'name', u'db_datas'],
[u'foo', base64.b64encode(im.tobytes()).decode('ascii')]
])
import_wizard = self.env['base_import.import'].create({
addons/l10n_fr_fec/wizard/account_fr_fec.py
+23 -24
View File
@@ -4,11 +4,11 @@
# Copyright (C) 2013-2015 Akretion (http://www.akretion.com)
import base64
import csv
import io
from odoo import api, fields, models, _
from odoo.exceptions import Warning
from odoo.tools import pycompat
class AccountFrFec(models.TransientModel):
@@ -86,24 +86,24 @@ class AccountFrFec(models.TransientModel):
# So it will be easier for the accountant to check the file before
# sending it to the fiscal administration
header = [
'JournalCode', # 0
'JournalLib', # 1
'EcritureNum', # 2
'EcritureDate', # 3
'CompteNum', # 4
'CompteLib', # 5
'CompAuxNum', # 6 We use partner.id
'CompAuxLib', # 7
'PieceRef', # 8
'PieceDate', # 9
'EcritureLib', # 10
'Debit', # 11
'Credit', # 12
'EcritureLet', # 13
'DateLet', # 14
'ValidDate', # 15
'Montantdevise', # 16
'Idevise', # 17
u'JournalCode', # 0
u'JournalLib', # 1
u'EcritureNum', # 2
u'EcritureDate', # 3
u'CompteNum', # 4
u'CompteLib', # 5
u'CompAuxNum', # 6 We use partner.id
u'CompAuxLib', # 7
u'PieceRef', # 8
u'PieceDate', # 9
u'EcritureLib', # 10
u'Debit', # 11
u'Credit', # 12
u'EcritureLet', # 13
u'DateLet', # 14
u'ValidDate', # 15
u'Montantdevise', # 16
u'Idevise', # 17
]
company = self.env.user.company_id
@@ -115,7 +115,7 @@ class AccountFrFec(models.TransientModel):
_("FEC is for French companies only !"))
fecfile = io.BytesIO()
w = csv.writer(fecfile, delimiter='|')
w = pycompat.csv_writer(fecfile, delimiter='|')
w.writerow(header)
# INITIAL BALANCE
@@ -190,7 +190,7 @@ class AccountFrFec(models.TransientModel):
else:
listrow[11] = '0,00'
listrow[12] = str(-listrow_amount).replace('.', ',')
w.writerow([s.encode("utf-8") for s in listrow])
w.writerow(listrow)
#if the unaffected earnings account wasn't in the selection yet: add it manually
if (not unaffected_earnings_line
and unaffected_earnings_results
@@ -201,7 +201,7 @@ class AccountFrFec(models.TransientModel):
if unaffected_earnings_account:
unaffected_earnings_results[4] = unaffected_earnings_account.code
unaffected_earnings_results[5] = unaffected_earnings_account.name
w.writerow([s.encode("utf-8") for s in unaffected_earnings_results])
w.writerow(unaffected_earnings_results)
# LINES
sql_query = '''
@@ -266,8 +266,7 @@ class AccountFrFec(models.TransientModel):
sql_query, (self.date_from, self.date_to, company.id))
for row in self._cr.fetchall():
listrow = list(row)
w.writerow([s.encode("utf-8") for s in listrow])
w.writerow(list(row))
siren = company.vat[4:13]
end_date = self.date_to.replace('-', '')
addons/l10n_lu/scripts/tax2csv.py
+22 -26
View File
@@ -1,17 +1,8 @@
from collections import OrderedDict
import csv
import xlrd
from odoo.tools import pycompat
def _e(s):
if s is None:
return ''
if pycompat.PY2 and type(s) == pycompat.text_type:
return s.encode('utf8')
return str(s)
def _is_true(s):
return s not in ('F', 'False', 0, '', None, False)
@@ -55,9 +46,9 @@ class LuxTaxGenerator:
yield OrderedDict(pycompat.izip(keys, row))
def tax_codes_to_csv(self):
writer = csv.writer(open('account.tax.code.template-%s.csv' %
writer = pycompat.csv_writer(open('account.tax.code.template-%s.csv' %
self.suffix, 'wb'))
tax_codes_iterator = self.iter_tax_codes
tax_codes_iterator = self.iter_tax_codes()
keys = next(tax_codes_iterator)
writer.writerow(keys)
@@ -68,7 +59,7 @@ class LuxTaxGenerator:
if tax_code in tax_codes:
raise RuntimeError('duplicate tax code %s' % tax_code)
tax_codes[tax_code] = row['id']
writer.writerow(pycompat.imap(_e, pycompat.values(row)))
writer.writerow([pycompat.to_text(v) for v in pycompat.values(row)])
# read taxes and add leaf tax codes
new_tax_codes = {} # id: parent_code
@@ -144,17 +135,19 @@ class LuxTaxGenerator:
for tax_code_id in sorted(new_tax_codes):
name, parent_code = new_tax_codes[tax_code_id]
writer.writerow((tax_code_id,
'lu_tct_m' + parent_code,
tax_code_id.replace('lu_tax_code_template_', ''),
'1',
'',
_e(name),
''))
writer.writerow([
tax_code_id,
u'lu_tct_m' + parent_code,
tax_code_id.replace('lu_tax_code_template_', u''),
u'1',
u'',
pycompat.to_text(name),
u''
])
def taxes_to_csv(self):
writer = csv.writer(open('account.tax.template-%s.csv' %
self.suffix, 'wb'))
writer = pycompat.csv_writer(open('account.tax.template-%s.csv' %
self.suffix, 'wb'))
taxes_iterator = self.iter_taxes()
keys = next(taxes_iterator)
writer.writerow(keys[3:] + ['sequence'])
@@ -167,17 +160,20 @@ class LuxTaxGenerator:
cur_seq = seq + 1000
else:
cur_seq = seq
writer.writerow(list(pycompat.imap(_e, list(pycompat.values(row))[3:])) + [cur_seq])
writer.writerow([
pycompat.to_text(v)
for v in list(pycompat.values(row))[3:]
] + [cur_seq])
def fiscal_pos_map_to_csv(self):
writer = csv.writer(open('account.fiscal.'
'position.tax.template-%s.csv' %
self.suffix, 'wb'))
writer = pycompat.csv_writer(open('account.fiscal.'
'position.tax.template-%s.csv' %
self.suffix, 'wb'))
fiscal_pos_map_iterator = self.iter_fiscal_pos_map()
keys = next(fiscal_pos_map_iterator)
writer.writerow(keys)
for row in fiscal_pos_map_iterator:
writer.writerow(pycompat.imap(_e, pycompat.values(row)))
writer.writerow([pycompat.to_text(s) for s in pycompat.values(row)])
if __name__ == '__main__':
addons/web/controllers/main.py
+5 -16
View File
@@ -3,7 +3,6 @@
import babel.messages.pofile
import base64
import csv
import datetime
import functools
import glob
@@ -1389,31 +1388,21 @@ class CSVExport(ExportFormat, http.Controller):
def from_data(self, fields, rows):
fp = io.BytesIO()
writer = csv.writer(fp, quoting=csv.QUOTE_ALL)
writer = pycompat.csv_writer(fp, quoting=1)
writer.writerow([name.encode('utf-8') for name in fields])
writer.writerow(fields)
for data in rows:
row = []
for d in data:
if type(d) != str and isinstance(d, pycompat.string_types):
try:
d = d.encode('utf-8')
except UnicodeError:
pass
if d is False: d = None
# Spreadsheet apps tend to detect formulas on leading =, + and -
if type(d) is str and d.startswith(('=', '-', '+')):
if isinstance(d, pycompat.string_types) and d.startswith(('=', '-', '+')):
d = "'" + d
row.append(d)
row.append(pycompat.to_text(d))
writer.writerow(row)
fp.seek(0)
data = fp.read()
fp.close()
return data
return fp.getvalue()
class ExcelExport(ExportFormat, http.Controller):
# Excel needs raw data to correctly handle numbers and date values
doc/python3.rst
+20
View File
@@ -452,6 +452,26 @@ conversions: text is *encoded* to bytes and bytes are *decoded* to text.
migrated to using ``base64.b64encode`` and ``base64.b64decode``
respectively.
csv
---
``csv`` is a fairly vicious one: not only is it not a very good format, the
Python 2 and Python 3 versions of the library are text-model incompatible in
significant ways:
* Python 2's CSV only works on *ascii-compatible byte streams* (it has no
encoding support at all) and extracts bytestring values
* Python 3's CSV only works on *text streams* and extract text values
* And ``io`` doesn't provide "native string" streaming facilities.
However with respect to Odoo it turns out most or all uses of ``csv`` fit
inside a model of *byte stream to and from text values*.
The latter is thus a model implemented by cross-version wrappers
:func:`odoo.tools.pycompat.csv_reader` and
:func:`odoo.tools.pycompat.csv_writer`: they take a *UTF-8 byte stream* and
read or write *text* values.
.. _hash randomisation: http://bugs.python.org/issue13703
.. _requests: http://docs.python-requests.org/
odoo/addons/test_pylint/tests/test_pylint.py
+1
View File
@@ -79,6 +79,7 @@ class TestPyLint(TransactionCase):
BAD_MODULES = [
'commands',
'cPickle',
'csv',
'cStringIO',
'md5',
'urllib',
odoo/cli/server.py
+1 -1
View File
@@ -12,7 +12,7 @@ GNU Public Licence.
"""
import atexit
import csv
import csv # pylint: disable=deprecated-module
import logging
import os
import signal
odoo/tools/convert.py
+8 -16
View File
@@ -2,7 +2,6 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import base64
import io
import csv
import logging
import os.path
import re
@@ -844,28 +843,21 @@ def convert_csv_import(cr, module, fname, csvcontent, idref=None, mode='init',
quote: "
delimiter: ,
encoding: utf-8'''
if not idref:
idref={}
model = ('.'.join(fname.split('.')[:-1]).split('-'))[0]
#remove folder path from model
head, model = os.path.split(model)
filename, _ext = os.path.splitext(os.path.basename(fname))
model = filename.split('-')[0]
input = io.BytesIO(csvcontent) #FIXME
reader = csv.reader(input, quotechar='"', delimiter=',')
reader = pycompat.csv_reader(io.BytesIO(csvcontent), quotechar='"', delimiter=',')
fields = next(reader)
if not (mode == 'init' or 'id' in fields):
_logger.error("Import specification does not contain 'id' and we are in init mode, Cannot continue.")
return
datas = []
for line in reader:
if not (line and any(line)):
continue
try:
datas.append([ustr(v) for v in line])
except Exception:
_logger.error("Cannot import the line: %s", line)
# filter out empty lines (any([]) == False) and lines containing only empty cells
datas = [
line for line in reader
if any(line)
]
context = {
'mode': mode,
odoo/tools/misc.py
+6 -6
View File
@@ -30,7 +30,6 @@ from lxml import etree
from .which import which
import traceback
import csv
from operator import itemgetter
try:
@@ -403,14 +402,15 @@ def scan_languages():
csvpath = odoo.modules.module.get_resource_path('base', 'res', 'res.lang.csv')
try:
# read (code, name) from languages in base/res/res.lang.csv
result = []
with open(csvpath) as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='"')
with open(csvpath, 'rb') as csvfile:
reader = pycompat.csv_reader(csvfile, delimiter=',', quotechar='"')
fields = next(reader)
code_index = fields.index("code")
name_index = fields.index("name")
for row in reader:
result.append((ustr(row[code_index]), ustr(row[name_index])))
result = [
(row[code_index], row[name_index])
for row in reader
]
except Exception:
_logger.error("Could not read %s", csvpath)
result = []
odoo/tools/pycompat.py
+28
View File
@@ -1,10 +1,16 @@
# -*- coding: utf-8 -*-
# to remove if we decide to add a dependency on six or future
# very strongly inspired by https://github.com/pallets/werkzeug/blob/master/werkzeug/_compat.py
#pylint: disable=deprecated-module
import csv
import collections
import io
import sys
PY2 = sys.version_info[0] == 2
_Writer = collections.namedtuple('_Writer', 'writerow writerows')
if PY2:
# pylint: disable=long-builtin,unichr-builtin,unicode-builtin
unichr = unichr
@@ -39,6 +45,19 @@ if PY2:
return cls
exec ('def reraise(tp, value, tb=None):\n raise tp, value, tb')
def csv_reader(stream, **params):
for row in csv.reader(stream, **params):
yield [c.decode('utf-8') for c in row]
def csv_writer(stream, **params):
w = csv.writer(stream, **params)
return _Writer(
writerow=lambda r: w.writerow([c.encode('utf-8') for c in r]),
writerows=lambda rs: w.writerows(
[c.encode('utf-8') for c in r]
for r in rs
)
)
else:
# pylint: disable=bad-functions
unichr = chr
@@ -74,6 +93,15 @@ else:
raise value.with_traceback(tb)
raise value
def csv_reader(stream, **params):
assert not isinstance(stream, io.TextIOBase),\
"For cross-compatibility purposes, csv_reader takes a bytes stream"
return csv.reader(io.TextIOWrapper(stream, encoding='utf-8'), **params)
def csv_writer(stream, **params):
assert not isinstance(stream, io.TextIOBase), \
"For cross-compatibility purposes, csv_writer takes a bytes stream"
return csv.writer(io.TextIOWrapper(stream, encoding='utf-8', line_buffering=True), **params)
def to_text(source):
""" Generates a text value (an instance of text_type) from an arbitrary
source.
odoo/tools/translate.py
+5 -6
View File
@@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import codecs
import csv
import fnmatch
import inspect
import io
@@ -121,6 +120,8 @@ _LOCALE2WIN32 = {
ENGLISH_SMALL_WORDS = set("as at by do go if in me no of ok on or to up us we".split())
# these direct uses of CSV are ok.
import csv # pylint: disable=deprecated-module
class UNIX_LINE_TERMINATOR(csv.excel):
lineterminator = '\n'
@@ -657,7 +658,7 @@ def trans_export(lang, modules, buffer, format, cr):
def _process(format, modules, rows, buffer, lang):
if format == 'csv':
writer = csv.writer(buffer, 'UNIX')
writer = pycompat.csv_writer(buffer, dialect='UNIX')
# write header first
writer.writerow(("module","type","name","res_id","src","value","comments"))
for module, type, name, res_id, src, trad, comments in rows:
@@ -1035,11 +1036,9 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
# now, the serious things: we read the language file
fileobj.seek(0)
if fileformat == 'csv':
reader = csv.reader(fileobj, quotechar='"', delimiter=',')
reader = pycompat.csv_reader(fileobj, quotechar='"', delimiter=',')
# read the first line of the file (it contains columns titles)
for row in reader:
fields = row
break
fields = next(reader)
elif fileformat == 'po':
reader = PoFile(fileobj)