diff --git a/addons/base_import/models.py b/addons/base_import/models.py index 784ae7f49f1..7463a02028e 100644 --- a/addons/base_import/models.py +++ b/addons/base_import/models.py @@ -1,7 +1,10 @@ import csv +import datetime +import io import itertools import logging import operator +import os try: from cStringIO import StringIO @@ -12,10 +15,38 @@ import psycopg2 from openerp.osv import orm, fields from openerp.tools.translate import _ +from openerp.tools import DEFAULT_SERVER_DATE_FORMAT, \ + DEFAULT_SERVER_DATETIME_FORMAT FIELDS_RECURSION_LIMIT = 2 ERROR_PREVIEW_BYTES = 200 _logger = logging.getLogger(__name__) + +try: + import xlrd + try: + from xlrd import xlsx + except ImportError: + xlsx = None +except ImportError: + xlrd = xlsx = None + +try: + import odf_ods_reader +except ImportError: + odf_ods_reader = None + +FILE_TYPE_DICT = { + 'text/csv': ('csv', True, None), + 'application/vnd.ms-excel': ('xls', xlrd, 'xlrd'), + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ('xlsx', xlsx, 'xlrd >= 0.8'), + 'application/vnd.oasis.opendocument.spreadsheet': ('ods', odf_ods_reader, 'odfpy') +} +EXTENSIONS = { + '.' + ext: handler + for mime, (ext, handler, req) in FILE_TYPE_DICT.iteritems() +} + class ir_import(orm.TransientModel): _name = 'base_import.import' # allow imports to survive for 12h in case user is slow @@ -26,7 +57,7 @@ class ir_import(orm.TransientModel): 'file': fields.binary( 'File', help="File to check and/or import, raw binary (not base64)"), 'file_name': fields.char('File Name'), - 'file_type': fields.char('File Type'), + 'file_type': fields.char(string='File Type'), } def get_fields(self, cr, uid, model, context=None, @@ -126,6 +157,69 @@ class ir_import(orm.TransientModel): # TODO: cache on model? return fields + def _read_file(self, file_type, record, options): + (file_extension, handler, req) = FILE_TYPE_DICT.get(file_type, (None, None, None)) + if handler: + return getattr(self, '_read_' + file_extension)(record, options) + # fallback on file extensions as mime types can be unreliable (e.g. + # software setting incorrect mime types, or non-installed software + # leading to browser not sending mime types) + if record.file_name: + p, ext = os.path.splitext(record.file_name) + if ext and EXTENSIONS.get(ext): + return getattr(self, '_read_' + ext[1:])(record, options) + + if req: + raise ImportError(_("Unable to load \"{extension}\" file: requires Python module \"{modname}\"").format(extension=file_extension, modname=req)) + raise ValueError(_("Unsupported file format \"{}\", import only supports CSV, ODS, XLS and XLSX").format(file_type)) + + def _read_xls(self, record, options): + book = xlrd.open_workbook(file_contents=record.file) + sheet = book.sheet_by_index(0) + # emulate Sheet.get_rows for pre-0.9.4 + for row in itertools.imap(sheet.row, range(sheet.nrows)): + values = [] + for cell in row: + if cell.ctype is xlrd.XL_CELL_NUMBER: + is_float = cell.value % 1 != 0.0 + values.append( + unicode(cell.value) + if is_float + else unicode(int(cell.value)) + ) + elif cell.ctype is xlrd.XL_CELL_DATE: + is_datetime = cell.value % 1 != 0.0 + # emulate xldate_as_datetime for pre-0.9.3 + dt = datetime.datetime(*xlrd.xldate.xldate_as_tuple( + cell.value, book.datemode)) + values.append( + dt.strftime(DEFAULT_SERVER_DATETIME_FORMAT) + if is_datetime + else dt.strftime(DEFAULT_SERVER_DATE_FORMAT) + ) + elif cell.ctype is xlrd.XL_CELL_BOOLEAN: + values.append(u'True' if cell.value else u'False') + elif cell.ctype is xlrd.XL_CELL_ERROR: + raise ValueError( + _("Error cell found while reading XLS/XLSX file: %s") % + xlrd.error_text_from_code.get( + cell.value, "unknown error code %s" % cell.value) + ) + else: + values.append(cell.value) + if any(x for x in values if x.strip()): + yield values + _read_xlsx = _read_xls + + def _read_ods(self, record, options): + doc = odf_ods_reader.ODSReader(file=io.BytesIO(record.file)) + + return ( + row + for row in doc.getFirstSheet() + if any(x for x in row if x.strip()) + ) + def _read_csv(self, record, options): """ Returns a CSV-parsed iterator of all empty lines in the file @@ -137,15 +231,13 @@ class ir_import(orm.TransientModel): quotechar=str(options['quoting']), delimiter=str(options['separator'])) - def nonempty(row): - return any(x for x in row if x.strip()) - - csv_nonempty = itertools.ifilter(nonempty, csv_iterator) # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet encoding = options.get('encoding', 'utf-8') - return itertools.imap( - lambda row: [item.decode(encoding) for item in row], - csv_nonempty) + return ( + [item.decode(encoding) for item in row] + for row in csv_iterator + if any(x for x in row if x.strip()) + ) def _match_header(self, header, fields, options): """ Attempts to match a given header to a field of the @@ -211,10 +303,10 @@ class ir_import(orm.TransientModel): return None, None headers = next(rows) - return headers, dict( - (index, [field['name'] for field in self._match_header(header, fields, options)] or None) + return headers, { + index: [field['name'] for field in self._match_header(header, fields, options)] or None for index, header in enumerate(headers) - ) + } def parse_preview(self, cr, uid, id, options, count=10, context=None): """ Generates a preview of the uploaded files, and performs @@ -236,7 +328,7 @@ class ir_import(orm.TransientModel): fields = self.get_fields(cr, uid, record.res_model, context=context) try: - rows = self._read_csv(record, options) + rows = self._read_file(record.file_type, record, options) headers, matches = self._match_headers(rows, fields, options) # Match should have consumed the first row (iif headers), get @@ -253,15 +345,17 @@ class ir_import(orm.TransientModel): # Due to lazy generators, UnicodeDecodeError (for # instance) may only be raised when serializing the # preview to a list in the return. - _logger.debug("Error during CSV parsing preview", exc_info=True) + _logger.debug("Error during parsing preview", exc_info=True) + preview = None + if record.file_type == 'text/csv': + preview = record.file[:ERROR_PREVIEW_BYTES].decode('iso-8859-1') return { 'error': str(e), # iso-8859-1 ensures decoding will always succeed, # even if it yields non-printable characters. This is # in case of UnicodeDecodeError (or csv.Error # compounded with UnicodeDecodeError) - 'preview': record.file[:ERROR_PREVIEW_BYTES] - .decode( 'iso-8859-1'), + 'preview': preview, } def _convert_import_data(self, record, fields, options, context=None): @@ -287,7 +381,7 @@ class ir_import(orm.TransientModel): # Get only list of actually imported fields import_fields = filter(None, fields) - rows_to_import = self._read_csv(record, options) + rows_to_import = self._read_file(record.file_type, record, options) if options.get('headers'): rows_to_import = itertools.islice( rows_to_import, 1, None) diff --git a/addons/base_import/odf_ods_reader.py b/addons/base_import/odf_ods_reader.py new file mode 100644 index 00000000000..0f0142f3f04 --- /dev/null +++ b/addons/base_import/odf_ods_reader.py @@ -0,0 +1,98 @@ +# Copyright 2011 Marco Conti + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# sourced from https://github.com/marcoconti83/read-ods-with-odfpy +# further altered locally + +from odf import opendocument +from odf.table import Table, TableRow, TableCell +from odf.text import P + + +class ODSReader(object): + + # loads the file + def __init__(self, file=None, content=None, clonespannedcolumns=None): + if not content: + self.clonespannedcolumns = clonespannedcolumns + self.doc = opendocument.load(file) + else: + self.clonespannedcolumns = clonespannedcolumns + self.doc = content + self.SHEETS = {} + for sheet in self.doc.spreadsheet.getElementsByType(Table): + self.readSheet(sheet) + + # reads a sheet in the sheet dictionary, storing each sheet as an + # array (rows) of arrays (columns) + def readSheet(self, sheet): + name = sheet.getAttribute("name") + rows = sheet.getElementsByType(TableRow) + arrRows = [] + + # for each row + for row in rows: + arrCells = [] + cells = row.getElementsByType(TableCell) + + # for each cell + for count, cell in enumerate(cells, start=1): + # repeated value? + repeat = 0 + if count != len(cells): + repeat = cell.getAttribute("numbercolumnsrepeated") + if not repeat: + repeat = 1 + spanned = int(cell.getAttribute('numbercolumnsspanned') or 0) + # clone spanned cells + if self.clonespannedcolumns is not None and spanned > 1: + repeat = spanned + + ps = cell.getElementsByType(P) + textContent = u"" + + # for each text/text:span node + for p in ps: + for n in p.childNodes: + if n.nodeType == 1 and n.tagName == "text:span": + for c in n.childNodes: + if c.nodeType == 3: + textContent = u'{}{}'.format(textContent, n.data) + + if n.nodeType == 3: + textContent = u'{}{}'.format(textContent, n.data) + + if textContent: + if not textContent.startswith("#"): # ignore comments cells + for rr in range(int(repeat)): # repeated? + arrCells.append(textContent) + else: + for rr in range(int(repeat)): + arrCells.append("") + + # if row contained something + if arrCells: + arrRows.append(arrCells) + + #else: + # print ("Empty or commented row (", row_comment, ")") + + self.SHEETS[name] = arrRows + + # returns a sheet as an array (rows) of arrays (columns) + def getSheet(self, name): + return self.SHEETS[name] + + def getFirstSheet(self): + return next(iter(self.SHEETS.itervalues())) diff --git a/addons/base_import/static/src/js/import.js b/addons/base_import/static/src/js/import.js index 4854df76410..5c57ff708f5 100644 --- a/addons/base_import/static/src/js/import.js +++ b/addons/base_import/static/src/js/import.js @@ -144,7 +144,7 @@ var DataImport = Widget.extend(ControlPanelMixin, { this.id = null; this.Import = new Model('base_import.import'); this.session = session; - action.display_name = _t('Import a CSV File'); // Displayed in the breadcrumbs + action.display_name = _t('Import a File'); // Displayed in the breadcrumbs }, start: function () { var self = this; @@ -237,6 +237,7 @@ var DataImport = Widget.extend(ControlPanelMixin, { if (!this.$('input.oe_import_file').val()) { return; } this.$el.removeClass('oe_import_preview oe_import_error'); + this.$el.find('.oe_import_toggle').toggle((this.$('input.oe_import_file')[0].files[0].type == "text/csv")); jsonp(this.$el, { url: '/base_import/set_file' }, this.proxy('settings_changed')); diff --git a/addons/base_import/static/src/xml/import.xml b/addons/base_import/static/src/xml/import.xml index c3eb2dc2b0d..ae5c78c497d 100644 --- a/addons/base_import/static/src/xml/import.xml +++ b/addons/base_import/static/src/xml/import.xml @@ -5,15 +5,12 @@ -

Select the .CSV - file to import. If you need a sample importable file, you +

Select the file to import. If you need a sample importable file, you can use the export tool to generate one.

- + + name="file" class="oe_import_file" accept=".csv, .xls, .xlsx, .ods"/>
- File Format Options… + CSV Format Options…

@@ -363,11 +360,11 @@

-

Import preview failed due to: . The issue is - usually an incorrect file encoding.

-

Here is the start of the file we could not import:

+

Import preview failed due to: .

+

For CSV files, the issue could be an incorrect encoding.

+

Here is the start of the file we could not import:

-
+
  • can be imported, returns ``True`` if it can be, + ``False`` otherwise. + + To use with ``unittest.skipUnless`` for tests conditional on *optional* + dependencies, which may or may be present but must still be tested if + possible. + """ + try: + importlib.import_module(module) + except ImportError: + return False + else: + return True