diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 7803f658f39..0e9f4406191 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -26,7 +26,7 @@ import werkzeug.exceptions import werkzeug.utils import werkzeug.wrappers import werkzeug.wsgi -from collections import OrderedDict, defaultdict +from collections import OrderedDict, defaultdict, Counter from werkzeug.urls import url_decode, iri_to_uri from lxml import etree import unicodedata @@ -607,6 +607,162 @@ class HomeStaticTemplateHelpers(object): return cls(addons, db, debug=debug)._get_qweb_templates()[0] +class GroupsTreeNode: + """ + This class builds an ordered tree of groups from the result of a `read_group(lazy=False)`. + The `read_group` returns a list of dictionnaries and each dictionnary is used to + build a leaf. The entire tree is built by inserting all leaves. + """ + + def __init__(self, model, fields, groupby, root=None): + self._model = model + self._fields = fields + self._groupby = groupby + + self.count = 0 # Total number of records in the subtree + self.aggregated_values = Counter() # Fields aggregated values {field_name: aggregated value} + self.children = OrderedDict() + self.data = [] # Only leaf nodes have data + if root: + self.insert_leaf(root) + + def child(self, key): + """ + Return the child identified by `key`. + If it doesn't exists inserts a default node and returns it. + :param key: child key identifier (groupby value as returned by read_group, + usually (id, display_name)) + :return: the child node + """ + if key not in self.children: + self.children[key] = GroupsTreeNode(self._model, self._fields, self._groupby) + return self.children[key] + + def insert_leaf(self, group): + """ + Build a leaf from `group` and insert it in the tree. + :param group: dict as returned by `read_group(lazy=False)` + """ + # Pop every known key in the group dict (__domain, __count and grouped values) + # Remaining keys are aggregated fields. + leaf_path = [group.pop(groupby_field) for groupby_field in self._groupby] + domain = group.pop('__domain') + count = group.pop('__count') + aggregated_values = group + + records = self._model.search(domain, offset=0, limit=False, order=False) + + # Follow the path from the top level group to the deepest + # group which actually contains the records' data. + node = self # root + node.count += count + for node_key in leaf_path: + # Go down to the next node or create one if it does not exist yet. + node = node.child(node_key) + # Update count value and aggregated value. + node.count += count + node.aggregated_values += Counter(aggregated_values) + + node.data = records.export_data(self._fields).get('datas',[]) + + +class ExportXlsxWriter: + + def __init__(self, field_names, row_count=0): + self.field_names = field_names + self.output = io.BytesIO() + self.workbook = xlsxwriter.Workbook(self.output, {'in_memory': True}) + self.base_style = self.workbook.add_format({'text_wrap': True}) + self.header_style = self.workbook.add_format({'bold': True}) + self.header_bold_style = self.workbook.add_format({'text_wrap': True, 'bold': True, 'bg_color': '#e9ecef'}) + self.date_style = self.workbook.add_format({'text_wrap': True, 'num_format': 'yyyy-mm-dd'}) + self.datetime_style = self.workbook.add_format({'text_wrap': True, 'num_format': 'yyyy-mm-dd hh:mm:ss'}) + self.worksheet = self.workbook.add_worksheet() + self.value = False + + if row_count > self.worksheet.xls_rowmax: + raise UserError(_('There are too many rows (%s rows, limit: %s) to export as Excel 2007-2013 (.xlsx) format. Consider splitting the export.') % (row_count, self.worksheet.xls_rowmax)) + + def __enter__(self): + self.write_header() + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + + def write_header(self): + # Write main header + for i, fieldname in enumerate(self.field_names): + self.write(0, i, fieldname, self.header_style) + self.worksheet.set_column(0, i, 30) # around 220 pixels + + def close(self): + self.workbook.close() + with self.output: + self.value = self.output.getvalue() + + def write(self, row, column, cell_value, style=None): + self.worksheet.write(row, column, cell_value, style) + + def write_cell(self, row, column, cell_value): + cell_style = self.base_style + + if isinstance(cell_value, bytes): + try: + # because xlsx uses raw export, we can get a bytes object + # here. xlsxwriter does not support bytes values in Python 3 -> + # assume this is base64 and decode to a string, if this + # fails note that you can't export + cell_value = pycompat.to_text(cell_value) + except UnicodeDecodeError: + raise UserError(_("Binary fields can not be exported to Excel unless their content is base64-encoded. That does not seem to be the case for %s.") % self.field_names[column]) + + if isinstance(cell_value, str): + if len(cell_value) > self.worksheet.xls_strmax: + cell_value = _("The content of this cell is too long for an XLSX file (more than %s characters). Please use the CSV format for this export.") % self.worksheet.xls_strmax + else: + cell_value = cell_value.replace("\r", " ") + elif isinstance(cell_value, datetime.datetime): + cell_style = self.datetime_style + elif isinstance(cell_value, datetime.date): + cell_style = self.date_style + self.write(row, column, cell_value, cell_style) + +class GroupExportXlsxWriter(ExportXlsxWriter): + + def __init__(self, fields, row_count=0): + super().__init__([f['label'].strip() for f in fields], row_count) + self.fields = fields + + def write_group(self, row, column, group_name, group, group_depth=0): + group_name = group_name if isinstance(group_name, str) else group_name and group_name[1] or _("Undefined") + row, column = self._write_group_header(row, column, group_name, group, group_depth) + + # Recursively write sub-groups + for child_group_name, child_group in group.children.items(): + row, column = self.write_group(row, column, child_group_name, child_group, group_depth + 1) + + for record in group.data: + row, column = self._write_row(row, column, record) + return row, column + + def _write_row(self, row, column, data): + for value in data: + self.write_cell(row, column, value) + column += 1 + return row + 1, 0 + + def _write_group_header(self, row, column, label, group, group_depth=0): + aggregates = group.aggregated_values + + label = '%s%s (%s)' % (' ' * group_depth, label, group.count) + self.write(row, column, label, self.header_bold_style) + for field in self.fields[1:]: # No aggregates allowed in the first column because of the group title + column += 1 + self.write(row, column, aggregates.get(field['name'], ''), self.header_bold_style) + return row + 1, 0 + + #---------------------------------------------------------- # Odoo Web web Controllers #---------------------------------------------------------- @@ -1602,26 +1758,43 @@ class ExportFormat(object): """ raise NotImplementedError() + def from_group_data(self, fields, groups): + raise NotImplementedError() + def base(self, data, token): params = json.loads(data) model, fields, ids, domain, import_compat = \ operator.itemgetter('model', 'fields', 'ids', 'domain', 'import_compat')(params) - Model = request.env[model].with_context(import_compat=import_compat, **params.get('context', {})) - records = Model.browse(ids) or Model.search(domain, offset=0, limit=False, order=False) - - if not Model._is_an_ordinary_table(): - fields = [field for field in fields if field['name'] != 'id'] - field_names = [f['name'] for f in fields] - import_data = records.export_data(field_names).get('datas',[]) - if import_compat: columns_headers = field_names else: columns_headers = [val['label'].strip() for val in fields] - return request.make_response(self.from_data(columns_headers, import_data), + Model = request.env[model].with_context(**params.get('context', {})) + groupby = params.get('groupby') + if not import_compat and groupby: + domain = [('id', 'in', ids)] if ids else domain + groups_data = Model.read_group(domain, field_names, groupby, lazy=False) + + # read_group(lazy=False) returns a dict only for final groups (with actual data), + # not for intermediary groups. The full group tree must be re-constructed. + tree = GroupsTreeNode(Model, field_names, groupby) + for leaf in groups_data: + tree.insert_leaf(leaf) + + response_data = self.from_group_data(fields, tree) + else: + Model = Model.with_context(import_compat=import_compat) + records = Model.browse(ids) if ids else Model.search(domain, offset=0, limit=False, order=False) + + if not Model._is_an_ordinary_table(): + fields = [field for field in fields if field['name'] != 'id'] + + export_data = records.export_data(field_names).get('datas',[]) + response_data = self.from_data(columns_headers, export_data) + return request.make_response(response_data, headers=[('Content-Disposition', content_disposition(self.filename(model))), ('Content-Type', self.content_type)], @@ -1641,6 +1814,9 @@ class CSVExport(ExportFormat, http.Controller): def filename(self, base): return base + '.csv' + def from_group_data(self, fields, groups): + raise UserError(_("Exporting grouped data to csv is not supported.")) + def from_data(self, fields, rows): fp = io.BytesIO() writer = pycompat.csv_writer(fp, quoting=1) @@ -1673,49 +1849,21 @@ class ExcelExport(ExportFormat, http.Controller): def filename(self, base): return base + '.xlsx' + def from_group_data(self, fields, groups): + with GroupExportXlsxWriter(fields, groups.count) as xlsx_writer: + x, y = 1, 0 + for group_name, group in groups.children.items(): + x, y = xlsx_writer.write_group(x, y, group_name, group) + + return xlsx_writer.value + def from_data(self, fields, rows): - output = io.BytesIO() - workbook = xlsxwriter.Workbook(output, {'in_memory': True}) - worksheet = workbook.add_worksheet() - if len(rows) > worksheet.xls_rowmax: - raise UserError(_('There are too many rows (%s rows, limit: %s) to export as Excel 2007-2013 (.xlsx) format. Consider splitting the export.') % (len(rows), worksheet.xls_rowmax)) + with ExportXlsxWriter(fields, len(rows)) as xlsx_writer: + for row_index, row in enumerate(rows): + for cell_index, cell_value in enumerate(row): + xlsx_writer.write_cell(row_index + 1, cell_index, cell_value) - for i, fieldname in enumerate(fields): - worksheet.write(0, i, fieldname) - worksheet.set_column(0, len(fields)-1, 30) # around 220 pixels - - base_style = workbook.add_format({'text_wrap': True}) - date_style = workbook.add_format({'text_wrap': True, 'num_format': 'yyyy-mm-dd'}) - datetime_style = workbook.add_format({'text_wrap': True, 'num_format': 'yyyy-mm-dd hh:mm:ss'}) - - for row_index, row in enumerate(rows): - for cell_index, cell_value in enumerate(row): - cell_style = base_style - - if isinstance(cell_value, bytes): - try: - # because xlsx uses raw export, we can get a bytes object - # here. xlsxwriter does not support bytes values in Python 3 -> - # assume this is base64 and decode to a string, if this - # fails note that you can't export - cell_value = pycompat.to_text(cell_value) - except UnicodeDecodeError: - raise UserError(_("Binary fields can not be exported to Excel unless their content is base64-encoded. That does not seem to be the case for %s.") % fields[cell_index]) - - if isinstance(cell_value, str): - if len(cell_value) > worksheet.xls_strmax: - cell_value = _("The content of this cell is too long for an XLSX file (more than %s characters). Please use the CSV format for this export.") % worksheet.xls_strmax - else: - cell_value = cell_value.replace("\r", " ") - elif isinstance(cell_value, datetime.datetime): - cell_style = datetime_style - elif isinstance(cell_value, datetime.date): - cell_style = date_style - worksheet.write(row_index + 1, cell_index, cell_value, cell_style) - - workbook.close() - with output: - return output.getvalue() + return xlsx_writer.value class Apps(http.Controller): @http.route('/apps/', auth='user') diff --git a/addons/web/static/src/js/views/list/list_controller.js b/addons/web/static/src/js/views/list/list_controller.js index dd2adea5a84..b90d903739b 100644 --- a/addons/web/static/src/js/views/list/list_controller.js +++ b/addons/web/static/src/js/views/list/list_controller.js @@ -22,6 +22,9 @@ var ListController = BasicController.extend({ * the list view. It can be overridden to add buttons in specific child views. */ buttons_template: 'ListView.buttons', + events: _.extend({}, BasicController.prototype.events, { + 'click .o_list_export_xlsx': '_onDirectExportData', + }), custom_events: _.extend({}, BasicController.prototype.custom_events, { activate_next_widget: '_onActivateNextWidget', add_record: '_onAddRecord', @@ -69,14 +72,11 @@ var ListController = BasicController.extend({ * @returns {Promise} a promise that resolve to the active domain */ getActiveDomain: function () { - // TODO: this method should be synchronous... var self = this; if (this.$('thead .o_list_record_selector input').prop('checked')) { var searchQuery = this._controlPanel ? this._controlPanel.getSearchQuery() : {}; var record = self.model.get(self.handle, {raw: true}); - return Promise.all(record.getDomain().concat(searchQuery.domain || [])); - } else { - return Promise.resolve(); + return record.getDomain().concat(searchQuery.domain || []); } }, /* @@ -340,6 +340,17 @@ var ListController = BasicController.extend({ self._updateButtons('readonly'); }); }, + /** + * @returns {DataExport} the export dialog widget + * @private + */ + _getExportDialogWidget() { + let state = this.model.get(this.handle); + let defaultExportFields = this.renderer.columns.map(field => field.attrs.name); + let groupedBy = this.renderer.state.groupedBy; + return new DataExport(this, state, defaultExportFields, groupedBy, + this.getActiveDomain(), this.getSelectedIds()); + }, /** * @override * @private @@ -499,6 +510,8 @@ var ListController = BasicController.extend({ * @returns {Promise} */ _update: function () { + let visilibityFunction = this.renderer.state.count ? 'show' : 'hide' + this.$('.o_list_export_xlsx')[visilibityFunction]() return this._super.apply(this, arguments) .then(this._toggleSidebar.bind(this)) .then(this._toggleCreateButton.bind(this)) @@ -640,11 +653,15 @@ var ListController = BasicController.extend({ * @private */ _onExportData: function () { - var record = this.model.get(this.handle); - var defaultExportFields = _.map(this.renderer.columns, function (field) { - return field.attrs.name; - }); - new DataExport(this, record, defaultExportFields).open(); + this._getExportDialogWidget().open(); + }, + /** + * Export Records in a xls file + * + * @private + */ + _onDirectExportData() { + this._getExportDialogWidget().export(); }, /** * Opens the related form view. diff --git a/addons/web/static/src/js/views/list/list_view.js b/addons/web/static/src/js/views/list/list_view.js index baba4d4a35f..f3f125ba8e3 100644 --- a/addons/web/static/src/js/views/list/list_view.js +++ b/addons/web/static/src/js/views/list/list_view.js @@ -52,6 +52,7 @@ var ListView = BasicView.extend({ } }); + this.controllerParams.activeActions.export_xlsx = this.arch.attrs.export_xlsx ? !!JSON.parse(this.arch.attrs.export_xlsx): true; this.controllerParams.editable = (!this.arch.attrs.edit || !!JSON.parse(this.arch.attrs.edit)) ? this.arch.attrs.editable : false; this.controllerParams.hasSidebar = params.hasSidebar; diff --git a/addons/web/static/src/js/widgets/data_export.js b/addons/web/static/src/js/widgets/data_export.js index 070b1927c5f..97ec75e6b47 100644 --- a/addons/web/static/src/js/widgets/data_export.js +++ b/addons/web/static/src/js/widgets/data_export.js @@ -34,11 +34,11 @@ var DataExport = Dialog.extend({ * @param {Object} record * @param {string[]} defaultExportFields */ - init: function (parent, record, defaultExportFields) { + init: function (parent, record, defaultExportFields, groupedBy, activeDomain, idsToExport) { var options = { title: _t("Export Data"), buttons: [ - {text: _t("Export"), click: this._exportData, classes: 'btn-primary'}, + {text: _t("Export"), click: this._onExportData, classes: 'btn-primary'}, {text: _t("Close"), close: true}, ], }; @@ -46,9 +46,13 @@ var DataExport = Dialog.extend({ this.records = {}; this.record = record; this.defaultExportFields = defaultExportFields; + this.groupby = groupedBy; this.exports = new data.DataSetSearch(this, 'ir.exports', this.record.getContext()); this.rowIndex = 0; this.rowIndexLevel = 0; + this.isCompatibleMode = false; + this.domain = activeDomain || this.record.domain; + this.idsToExport = activeDomain ? false: idsToExport; }, /** * @override @@ -73,16 +77,6 @@ var DataExport = Dialog.extend({ }); })); - proms.push(this.getParent().getActiveDomain().then(function (domain) { - if (domain === undefined) { - self.idsToExport = self.getParent().getSelectedIds(); - self.domain = self.record.domain; - } else { - self.idsToExport = false; - self.domain = domain; - } - })); - proms.push(this._showExportsList()); // Bind sortable events after Dialog is open @@ -117,6 +111,17 @@ var DataExport = Dialog.extend({ } }, + /** + * Export all data with default values (fields, domain) + */ + export() { + let exportedFields = this.defaultExportFields.map(field => ({ + name: field, + label: this.record.fields[field].string, + })); + this._exportData(exportedFields, 'xlsx', false); + }, + //-------------------------------------------------------------------------- // Private //-------------------------------------------------------------------------- @@ -146,13 +151,7 @@ var DataExport = Dialog.extend({ * * @private */ - _exportData: function () { - var exportedFields = this.$('.o_export_field').map(function () { - return { - name: $(this).data('field_id'), - label: this.textContent - }; - }).get(); + _exportData(exportedFields, exportFormat, idsToExport) { if (_.isEmpty(exportedFields)) { Dialog.alert(this, _t("Please select fields to export...")); @@ -162,8 +161,6 @@ var DataExport = Dialog.extend({ exportedFields.unshift({ name: 'id', label: _t('External ID') }); } - var exportFormat = this.$exportFormatInputs.filter(':checked').val(); - framework.blockUI(); this.getSession().get_file({ url: '/web/export/' + exportFormat, @@ -171,8 +168,9 @@ var DataExport = Dialog.extend({ data: JSON.stringify({ model: this.record.model, fields: exportedFields, - ids: this.idsToExport, + ids: idsToExport, domain: this.domain, + groupby: this.groupby, context: pyUtils.eval('contexts', [this.record.getContext()]), import_compat: this.isCompatibleMode, }) @@ -545,6 +543,20 @@ var DataExport = Dialog.extend({ } } }, + /** + * Submit the user data and export the file + * + * @private + */ + _onExportData() { + let exportedFields = this.$('.o_export_field').map((i, field) => ({ + name: $(field).data('field_id'), + label: field.textContent, + } + )).get(); + let exportFormat = this.$exportFormatInputs.filter(':checked').val(); + this._exportData(exportedFields, exportFormat, this.idsToExport); + }, /** * Add a field to export field list on double click * diff --git a/addons/web/static/src/xml/base.xml b/addons/web/static/src/xml/base.xml index 20c76190c7f..54ffb93a059 100644 --- a/addons/web/static/src/xml/base.xml +++ b/addons/web/static/src/xml/base.xml @@ -275,6 +275,9 @@ Create + +