diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index 0eb7e7b7580..4f2c262fab6 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import chardet import datetime import io import itertools @@ -45,6 +46,37 @@ EXTENSIONS = { for mime, (ext, handler, req) in FILE_TYPE_DICT.items() } +class Base(models.AbstractModel): + _inherit = 'base' + + @api.model + def get_import_templates(self): + """ + Get the import templates label and path. + + :return: a list(dict) containing label and template path + like ``[{'label': 'foo', 'template': 'path'}]`` + """ + return [] + +class ImportMapping(models.Model): + """ mapping of previous column:field selections + + This is useful when repeatedly importing from a third-party + system: column names generated by the external system may + not match Odoo's field names or labels. This model is used + to save the mapping between column names and fields so that + next time a user imports from the same third-party systems + we can automatically match the columns to the correct field + without them having to re-enter the mapping every single + time. + """ + _name = 'base_import.mapping' + + res_model = fields.Char(index=True) + column_name = fields.Char() + field_name = fields.Char() + class Import(models.TransientModel): @@ -251,15 +283,10 @@ class Import(models.TransientModel): def _read_csv(self, options): """ Returns a CSV-parsed iterator of all non-empty lines in the file :throws csv.Error: if an error is detected during CSV parsing - :throws UnicodeDecodeError: if ``options.encoding`` is incorrect """ csv_data = self.file - - # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet - encoding = options.get('encoding', 'utf-8') - if encoding != 'utf-8': - # csv module expect utf-8, see http://docs.python.org/2/library/csv.html - csv_data = csv_data.decode(encoding).encode('utf-8') + encoding = chardet.detect(csv_data)['encoding'] + csv_data = csv_data.decode(encoding).encode('utf-8') csv_iterator = pycompat.csv_reader( io.BytesIO(csv_data), @@ -376,7 +403,7 @@ class Import(models.TransientModel): if results: return results - return ['id', 'text', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html'] + return ['id', 'text', 'boolean', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html'] @api.model def _find_type_from_preview(self, options, preview): @@ -400,6 +427,7 @@ class Import(models.TransientModel): :rtype: list(Field) """ string_match = None + IrTranslation = self.env['ir.translation'] for field in fields: # FIXME: should match all translations & original # TODO: use string distance (levenshtein? hamming?) @@ -409,6 +437,9 @@ class Import(models.TransientModel): # matching string are not reliable way because # strings have no unique constraint string_match = field + translated_header = IrTranslation._get_source('ir.model.fields,field_description', 'model', self.env.lang, header).lower() + if translated_header == field['string'].lower(): + string_match = field if string_match: # this behavior is only applied if there is no matching field['name'] return [string_match] @@ -440,23 +471,31 @@ class Import(models.TransientModel): Will consume the first line of the ``rows`` iterator. - Returns a pair of (None, None) if headers were not requested - or the list of headers and a dict mapping cell indices - to key paths in the ``fields`` tree + Returns the list of headers and a dict mapping cell indices + to key paths in the ``fields`` tree. If headers were not + requested, both collections are empty. :param Iterator rows: :param dict fields: :param dict options: - :rtype: (None, None) | (list(str), dict(int: list(str))) + :rtype: (list(str), dict(int: list(str))) """ if not options.get('headers'): return [], {} headers = next(rows) - return headers, { - index: [field['name'] for field in self._match_header(header, fields, options)] or None - for index, header in enumerate(headers) - } + matches = {} + mapping_records = self.env['base_import.mapping'].search_read([('res_model', '=', self.res_model)], ['column_name', 'field_name']) + mapping_fields = {rec['column_name']: rec['field_name'] for rec in mapping_records} + for index, header in enumerate(headers): + match_field = [] + mapping_field_name = mapping_fields.get(header.lower()) + if mapping_field_name: + match_field = mapping_field_name.split('/') + if not match_field: + match_field = [field['name'] for field in self._match_header(header, fields, options)] + matches[index] = match_field or None + return headers, matches @api.multi def parse_preview(self, options, count=10): @@ -469,7 +508,7 @@ class Import(models.TransientModel): :param int count: number of preview lines to generate :param options: format-specific options. - CSV: {encoding, quoting, separator, headers} + CSV: {quoting, separator, headers} :type options: {str, str, str, bool} :returns: {fields, matches, headers, preview} | {error, preview} :rtype: {dict(str: dict(...)), dict(int, list(str)), list(str), list(list(str))} | {str, str} @@ -482,14 +521,23 @@ class Import(models.TransientModel): # Match should have consumed the first row (iif headers), get # the ``count`` next rows for preview preview = list(itertools.islice(rows, count)) - assert preview, "CSV file seems to have no content" + assert preview, "file seems to have no content" header_types = self._find_type_from_preview(options, preview) - if options.get('keep_matches', False) and len(options.get('fields', [])): + if options.get('keep_matches') and len(options.get('fields', [])): matches = {} for index, match in enumerate(options.get('fields')): if match: matches[index] = match.split('/') + if options.get('keep_matches'): + advanced_mode = options.get('advanced') + else: + # Check is label contain relational field + has_relational_header = any(len(models.fix_import_export_id_paths(col)) > 1 for col in headers) + # Check is matches fields have relational field + has_relational_match = any(len(match) > 1 for field, match in matches.items() if match) + advanced_mode = has_relational_header or has_relational_match + return { 'fields': fields, 'matches': matches or False, @@ -497,6 +545,7 @@ class Import(models.TransientModel): 'headers_type': header_types or False, 'preview': preview, 'options': options, + 'advanced_mode': advanced_mode, 'debug': self.user_has_groups('base.group_no_one'), } except Exception as error: @@ -505,7 +554,7 @@ class Import(models.TransientModel): # preview to a list in the return. _logger.debug("Error during parsing preview", exc_info=True) preview = None - if self.file_type == 'text/csv': + if self.file_type == 'text/csv' and self.file: preview = self.file[:ERROR_PREVIEW_BYTES].decode('iso-8859-1') return { 'error': str(error), @@ -627,7 +676,13 @@ class Import(models.TransientModel): try: line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format) except ValueError as e: - raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e)) + try: + # Allow to import date in datetime fields + if field['type'] == 'datetime': + user_format = pycompat.to_native(options.get('date_format')) + line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format) + except ValueError as e: + raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e)) except Exception as e: raise ValueError(_("Error Parsing Date [%s:L%d]: %s") % (name, num + 1, e)) # Check if the field is in import_field and is a relational (followed by /) @@ -643,12 +698,14 @@ class Import(models.TransientModel): return data @api.multi - def do(self, fields, options, dryrun=False): + def do(self, fields, columns, options, dryrun=False): """ Actual execution of the import :param fields: import mapping: maps each column to a field, ``False`` for the columns to ignore :type fields: list(str|bool) + :param columns: columns label + :type columns: list(str|bool) :param dict options: :param bool dryrun: performs all import operations (and validations) but rollbacks writes, allows @@ -661,7 +718,7 @@ class Import(models.TransientModel): error message associated with the error (a string) and ``record`` the data which failed to import (or ``false`` if that data isn't available or provided) - :rtype: list({type, message, record}) + :rtype: dict(ids: list(int), messages: list({type, message, record})) """ self.ensure_one() self._cr.execute('SAVEPOINT import') @@ -671,11 +728,13 @@ class Import(models.TransientModel): # Parse date and float field data = self._parse_import_data(data, import_fields, options) except ValueError as error: - return [{ - 'type': 'error', - 'message': pycompat.text_type(error), - 'record': False, - }] + return { + 'messages': [{ + 'type': 'error', + 'message': pycompat.text_type(error), + 'record': False, + }] + } _logger.info('importing %d rows...', len(data)) @@ -698,4 +757,20 @@ class Import(models.TransientModel): except psycopg2.InternalError: pass - return import_result['messages'] + # Insert/Update mapping columns when import complete successfully + if import_result['ids'] and options.get('headers'): + BaseImportMapping = self.env['base_import.mapping'] + for index, column_name in enumerate(columns): + if column_name: + # Update to latest selected field + exist_records = BaseImportMapping.search([('res_model', '=', self.res_model), ('column_name', '=', column_name)]) + if exist_records: + exist_records.write({'field_name': fields[index]}) + else: + BaseImportMapping.create({ + 'res_model': self.res_model, + 'column_name': column_name, + 'field_name': fields[index] + }) + + return import_result diff --git a/addons/base_import/security/ir.model.access.csv b/addons/base_import/security/ir.model.access.csv index 7ed063df1c2..6844f024d3e 100644 --- a/addons/base_import/security/ir.model.access.csv +++ b/addons/base_import/security/ir.model.access.csv @@ -12,4 +12,5 @@ access_base_import_tests_models_m2o_required_related,base.import.tests.models.m2 access_base_import_tests_models_o2m,base.import.tests.models.o2m,model_base_import_tests_models_o2m,base.group_user,1,1,1,1 access_base_import_tests_models_o2m_child,base.import.tests.models.o2m.child,model_base_import_tests_models_o2m_child,base.group_user,1,1,1,1 access_base_import_tests_models_preview,base.import.tests.models.preview,model_base_import_tests_models_preview,base.group_user,1,1,1,1 +access_base_import_mapping,base.import.mapping,model_base_import_mapping,base.group_user,1,1,1,1 diff --git a/addons/base_import/static/csv/res.partner.csv b/addons/base_import/static/csv/res.partner.csv deleted file mode 100644 index 5414512f4df..00000000000 --- a/addons/base_import/static/csv/res.partner.csv +++ /dev/null @@ -1,7 +0,0 @@ -ID,Name,Company Type,Related Company,Customer,Supplier,Address Type,Street,Street2,City,State,ZIP,Country,Website,Phone,Mobile,Email -res_partner_1,Grocery Outlet,Company,,TRUE,FALSE,,51 Federal Street,#401,San Francisco,California,94107,United States,groceryoutlet.com,123456789,, -res_partner_2,Bob,Individual,Grocery Outlet,,,Contact,,,,,,,,,4157654321, -res_partner_3,Bill,Individual,Grocery Outlet,,,Other address,645 21st Street,,San Francisco,California,94110,United States,,,4517864512, -res_partner_4,Warehouse,Individual,Grocery Outlet,,,Shipping address,645 21st Street,,San Francisco,California,94110,United States,,415659874,,bill@odoo.com -res_partner_5,Accounting dep.,Individual,Grocery Outlet,,,Invoice address,51 Federal Street,#404,San Francisco,California,94107,United States,,123456780,, -res_partner_6,Agrolait,Company,,TRUE,FALSE,,3442 2nd Avenue,,New York,New York,11201,United States,agrolait.com,654978123,, diff --git a/addons/base_import/static/src/js/import_action.js b/addons/base_import/static/src/js/import_action.js index d995e9cf055..68dc6c663fa 100644 --- a/addons/base_import/static/src/js/import_action.js +++ b/addons/base_import/static/src/js/import_action.js @@ -47,7 +47,6 @@ function jsonp(form, attributes, callback) { var DataImport = AbstractAction.extend(ControlPanelMixin, { template: 'ImportView', opts: [ - {name: 'encoding', label: _lt("Encoding:"), value: 'utf-8'}, {name: 'separator', label: _lt("Separator:"), value: ','}, {name: 'quoting', label: _lt("Text Delimiter:"), value: '"'} ], @@ -60,7 +59,6 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { events: { // 'change .oe_import_grid input': 'import_dryrun', 'change .oe_import_file': 'loaded_file', - 'click .oe_import_file_reload': 'loaded_file', 'change input.oe_import_has_header, .js_import_options input': 'settings_changed', 'change input.oe_import_advanced_mode': function (e) { this.do_not_change_match = true; @@ -111,9 +109,21 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { action.display_name = _t('Import a File'); // Displayed in the breadcrumbs this.do_not_change_match = false; }, + /** + * @override + */ + willStart: function () { + var self = this; + return this._rpc({ + model: this.res_model, + method: 'get_import_templates', + context: this.parent_context, + }).then(function (result) { + self.importTemplates = result; + }); + }, start: function () { var self = this; - this.setup_encoding_picker(); this.setup_separator_picker(); this.setup_float_format_picker(); @@ -124,7 +134,6 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { self.$('input[name=import_id]').val(id); self.renderButtons(); - self.renderImportLink(); var status = { cp_content: {$buttons: self.$buttons}, }; @@ -145,36 +154,15 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { this.$buttons = $(QWeb.render("ImportView.buttons", this)); this.$buttons.filter('.o_import_validate').on('click', this.validate.bind(this)); this.$buttons.filter('.o_import_import').on('click', this.import.bind(this)); + this.$buttons.filter('.o_import_file_reload').on('click', this.loaded_file.bind(this)); + this.$buttons.filter('.oe_import_file').on('click', function () { + self.$('.oe_import_file').click(); + }); this.$buttons.filter('.o_import_cancel').on('click', function(e) { e.preventDefault(); self.exit(); }); }, - renderImportLink: function() { - if (this.res_model == 'res.partner') { - this.$(".import-link").prop({"text": _t(" Import Template for Customers"), "href": "/base_import/static/csv/res.partner.csv"}); - this.$(".template-import").removeClass("hidden"); - } - }, - setup_encoding_picker: function () { - this.$('input.oe_import_encoding').select2({ - width: '160px', - query: function (q) { - var make = function (term) { return {id: term, text: term}; }; - var suggestions = _.map( - ('utf-8 utf-16 windows-1252 latin1 latin2 big5 ' + - 'gb18030 shift_jis windows-1251 koir8_r').split(/\s+/), - make); - if (q.term) { - suggestions.unshift(make(q.term)); - } - q.callback({results: suggestions}); - }, - initSelection: function (e, c) { - return c({id: 'utf-8', text: 'utf-8'}); - } - }).select2('val', 'utf-8'); - }, setup_separator_picker: function () { this.$('input.oe_import_separator').select2({ width: '160px', @@ -196,26 +184,33 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { }); }, setup_float_format_picker: function () { - var sug_query = function (q) { - var suggestions = [ - {id: ',', text: _t("Comma")}, - {id: '.', text: _t("Dot")}, - ]; - if (q.term) { - suggestions.unshift({id: q.term, text: q.term}); - } - q.callback({results: suggestions}); - }; + var sub_query = function (q, options) { + var suggestions = [ + {id: ',', text: _t("Comma")}, + {id: '.', text: _t("Dot")}, + ]; + if (options && options.no_separator) { + suggestions.push({id: '', text: _t("No Separator")}); + } + if (q.term) { + suggestions.unshift({id: q.term, text: q.term}); + } + q.callback({results: suggestions}); + }; this.$('input.oe_import_float_thousand_separator').select2({ width: '160px', - query: sug_query, + query: function (q) { + sub_query(q, {no_separator: true}) + }, initSelection: function (e, c) { return c({id: ',', text: _t("Comma")}); }, }); this.$('input.oe_import_float_decimal_separator').select2({ width: '160px', - query: sug_query, + query: function (q) { + sub_query(q); + }, initSelection: function (e, c) { return c({id: ',', text: _t("Dot")}); }, @@ -254,9 +249,7 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { //- File & settings change section onfile_loaded: function () { var file = this.$('.oe_import_file')[0].files[0]; - this.$('.oe_import_file_show').val(file !== undefined && file.name || ''); - this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload')) - .prop('disabled', true); + this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').addClass('hidden'); if (!this.$('input.oe_import_file').val()) { return this['settings_changed'](); } this.$('.oe_import_date_format').val(''); this.$('.oe_import_datetime_format').val(''); @@ -268,15 +261,14 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { if ((file.type && _.last(file.type.split('/')) === "csv") || ( _.last(file.name.split('.')) === "csv")) { import_toggle = true; } - this.$el.find('.oe_import_toggle').toggle(import_toggle); + this.$el.find('.oe_import_box').toggle(import_toggle); jsonp(this.$el, { url: '/base_import/set_file' }, this.proxy('settings_changed')); }, onpreviewing: function () { var self = this; - this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload')) - .prop('disabled', true); + this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').addClass('hidden'); this.$el.addClass('oe_import_with_file'); // TODO: test that write // succeeded? this.$el.removeClass('oe_import_preview_error oe_import_error'); @@ -295,26 +287,32 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { }, onpreview_error: function (event, from, to, result) { this.$('.oe_import_options').show(); - this.$('.oe_import_file_reload').prop('disabled', false); + this.$buttons.filter('.o_import_file_reload').removeClass('hidden'); this.$el.addClass('oe_import_preview_error oe_import_error'); + this.$el.find('.oe_import_box, .oe_import_with_file').removeClass('hidden'); + this.$el.find('.o_view_nocontent').addClass('hidden'); this.$('.oe_import_error_report').html( QWeb.render('ImportView.preview.error', result)); }, onpreview_success: function (event, from, to, result) { var self = this; - this.$buttons.filter('.o_import_import').removeClass('btn-primary'); - this.$buttons.filter('.o_import_validate').addClass('btn-primary'); - this.$buttons.filter('.o_import_button').add(this.$('.oe_import_file_reload')) - .prop('disabled', false); + this.$buttons.filter('.oe_import_file') + .text(_t('Load New File')) + .removeClass('btn-primary').addClass('btn-default') + .blur(); + this.$buttons.filter('.o_import_import, .o_import_validate, .o_import_file_reload').removeClass('hidden'); + this.$el.find('.oe_import_box, .oe_import_with_file').removeClass('hidden'); + this.$el.find('.o_view_nocontent').addClass('hidden'); this.$el.addClass('oe_import_preview'); + this.$('input.oe_import_advanced_mode').prop('checked', result.advanced_mode); this.$('.oe_import_grid').html(QWeb.render('ImportView.preview', result)); if (result.headers.length === 1) { this.$('.oe_import_options').show(); - this.onresults(null, null, null, [{ + this.onresults(null, null, null, {'messages': [{ type: 'warning', message: _t("A single column was found in the file, this often means the file separator is incorrect") - }]); + }]}); } this.$('.oe_import_date_format').val(time.strftime_to_moment_format(result.options.date_format)); @@ -461,6 +459,9 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { var fields = this.$('.oe_import_fields input.oe_import_match_field').map(function (index, el) { return $(el).select2('val') || false; }).get(); + var columns = this.$('.oe_import_grid-header .oe_import_grid-cell .o_import_header_name').map(function () { + return $(this).text().trim().toLowerCase() || false; + }).get(); var tracking_disable = 'tracking_disable' in kwargs ? kwargs.tracking_disable : !this.$('#oe_import_tracking').prop('checked') var defer_parent_store = 'defer_parent_store' in kwargs ? kwargs.defer_parent_store : !!this.$('#oe_import_deferparentstore').prop('checked') delete kwargs.tracking_disable; @@ -472,18 +473,18 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { return this._rpc({ model: 'base_import.import', method: 'do', - args: [this.id, fields, this.import_options()], + args: [this.id, fields, columns, this.import_options()], kwargs : kwargs, }).fail(function (error, event) { // In case of unexpected exception, convert // "JSON-RPC error" to an import failure, and // prevent default handling (warning dialog) if (event) { event.preventDefault(); } - return $.when([{ + return $.when({'messages': [{ type: 'error', record: false, message: error.data.arguments && error.data.arguments[1] || error.message, - }]); + }]}); }) ; }, onvalidate: function () { @@ -492,27 +493,26 @@ var DataImport = AbstractAction.extend(ControlPanelMixin, { }, onimport: function () { var self = this; - return this.call_import({ dryrun: false }).done(function (message) { + return this.call_import({ dryrun: false }).done(function (results) { + var message = results.messages; if (!_.any(message, function (message) { return message.type === 'error'; })) { - self['import_succeeded'](); + self['import_succeeded'](results); return; } - self['import_failed'](message); + self['import_failed'](results); }); }, - onimported: function () { + onimported: function (event, from, to, results) { + this.do_notify(_t("Import completed"), _.str.sprintf(_t("%d records were successfully imported"), results.ids.length)); this.exit(); }, exit: function () { this.trigger_up('history_back'); }, - onresults: function (event, from, to, message) { + onresults: function (event, from, to, results) { + var message = results.messages; var no_messages = _.isEmpty(message); - this.$buttons.filter('.o_import_import').toggleClass('btn-primary', no_messages); - this.$buttons.filter('.o_import_import').toggleClass('btn-default', !no_messages); - this.$buttons.filter('.o_import_validate').toggleClass('btn-primary', !no_messages); - this.$buttons.filter('.o_import_validate').toggleClass('btn-default', no_messages); if (no_messages) { message.push({ type: 'info', @@ -591,7 +591,7 @@ StateMachine.create({ { name: 'import', from: ['preview_success', 'results'], to: 'importing' }, { name: 'import_succeeded', from: 'importing', to: 'imported'}, { name: 'import_failed', from: 'importing', to: 'results' } - ] + ], }); return { diff --git a/addons/base_import/static/src/scss/base_import.scss b/addons/base_import/static/src/scss/base_import.scss index 9a4ce56f5c2..1ba7b05043f 100644 --- a/addons/base_import/static/src/scss/base_import.scss +++ b/addons/base_import/static/src/scss/base_import.scss @@ -1,6 +1,5 @@ .oe_import { @include o-webclient-padding($top: 8px); - background-color: white; overflow: auto; position: absolute; // Needed for chrome top: 0; @@ -27,7 +26,7 @@ } .oe_import_box { - padding: 16px; + padding: 8px; background: #F0EEEE; border-radius: $border-radius-base; border: solid 1px #dddddd; @@ -39,9 +38,6 @@ } } - .oe_import_toggle { - margin-top: 8px; - } a.oe_import_toggle { display: block; &:before { @@ -94,6 +90,12 @@ margin: 2px 0; list-style: none; border-radius: $border-radius-base; + &.bg-error { + @extend .bg-danger; + } + &.text-error { + @extend .text-danger; + } } } diff --git a/addons/base_import/static/src/xml/base_import.xml b/addons/base_import/static/src/xml/base_import.xml index fcd5b239cb6..df9fbc3e5d1 100644 --- a/addons/base_import/static/src/xml/base_import.xml +++ b/addons/base_import/static/src/xml/base_import.xml @@ -6,34 +6,12 @@ -
-
-

Select a CSV or Excel file to import. Help

-
-
-
- - - - - - - -
- -
- -
+