[MERGE] forward port branch saas-17 up to 877e709871

addons/account/models/account.py

@@ -5,7 +5,7 @@ import math
 
 from odoo.osv import expression
 from odoo.tools.float_utils import float_round as round
-from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT, pycompat
+from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
 from odoo.exceptions import UserError, ValidationError
 from odoo import api, fields, models, _
 

addons/account/models/account_bank_statement.py

@@ -305,9 +305,9 @@ class AccountBankStatement(models.Model):
         st_lines_left = self.env['account.bank.statement.line'].browse([line.get('id') for line in self.env.cr.dictfetchall()])
 
         #try to assign partner to bank_statement_line
-        stl_to_assign_partner = [stl.id for stl in st_lines_left if not stl.partner_id]
-        refs = list(set([st.name for st in st_lines_left if not stl.partner_id]))
-        if st_lines_left and stl_to_assign_partner and refs\
+        stl_to_assign = st_lines_left.filtered(lambda stl: not stl.partner_id)
+        refs = {stl_to_assign.mapped('name')}
+        if stl_to_assign and refs\
            and st_lines_left[0].journal_id.default_credit_account_id\
            and st_lines_left[0].journal_id.default_debit_account_id:
 
@@ -327,7 +327,7 @@ class AccountBankStatement(models.Model):
             params = (self.env.user.company_id.id, (st_lines_left[0].journal_id.default_credit_account_id.id, st_lines_left[0].journal_id.default_debit_account_id.id), tuple(refs))
             if statements:
                 sql_query += 'AND stl.id IN %s'
-                params += (tuple(stl_to_assign_partner),)
+                params += (tuple(stl_to_assign.ids),)
             self.env.cr.execute(sql_query, params)
             results = self.env.cr.dictfetchall()
             st_line = self.env['account.bank.statement.line']
@@ -499,7 +499,6 @@ class AccountBankStatementLine(models.Model):
                 'st_line': st_line.get_statement_line_for_reconciliation_widget(),
                 'reconciliation_proposition': rp
             })
-
         return ret
 
     def get_statement_line_for_reconciliation_widget(self):

addons/account/models/account_config_settings.py

@@ -82,10 +82,8 @@ class AccountConfigSettings(models.TransientModel):
             self.env.ref('base.group_user').write({'implied_ids': [(4, self.env.ref('product.group_sale_pricelist').id)]})
         """ Set the product taxes if they have changed """
         ir_values_obj = self.env['ir.values']
-        if self.default_sale_tax_id:
-            ir_values_obj.sudo().set_default('product.template', "taxes_id", [self.default_sale_tax_id.id], for_all_users=True, company_id=self.company_id.id)
-        if self.default_purchase_tax_id:
-            ir_values_obj.sudo().set_default('product.template', "supplier_taxes_id", [self.default_purchase_tax_id.id], for_all_users=True, company_id=self.company_id.id)
+        ir_values_obj.sudo().set_default('product.template', "taxes_id", [self.default_sale_tax_id.id] if self.default_sale_tax_id else False, for_all_users=True, company_id=self.company_id.id)
+        ir_values_obj.sudo().set_default('product.template', "supplier_taxes_id", [self.default_purchase_tax_id.id] if self.default_purchase_tax_id else False, for_all_users=True, company_id=self.company_id.id)
         """ install a chart of accounts for the given company (if required) """
         if self.chart_template_id and self.chart_template_id != self.company_id.chart_template_id:
             wizard = self.env['wizard.multi.charts.accounts'].create({

addons/account/models/account_invoice.py

@@ -341,9 +341,9 @@ class AccountInvoice(models.Model):
         related='partner_id.commercial_partner_id', store=True, readonly=True,
         help="The commercial entity that will be used on Journal Entries for this invoice")
 
-    outstanding_credits_debits_widget = fields.Text(compute='_get_outstanding_info_JSON')
-    payments_widget = fields.Text(compute='_get_payment_info_JSON')
-    has_outstanding = fields.Boolean(compute='_get_outstanding_info_JSON')
+    outstanding_credits_debits_widget = fields.Text(compute='_get_outstanding_info_JSON', groups="account.group_account_invoice")
+    payments_widget = fields.Text(compute='_get_payment_info_JSON', groups="account.group_account_invoice")
+    has_outstanding = fields.Boolean(compute='_get_outstanding_info_JSON', groups="account.group_account_invoice")
 
     #fields use to set the sequence, on the first invoice of the journal
     sequence_number_next = fields.Char(string='Next Number', compute="_get_sequence_prefix", inverse="_set_sequence_next")
@@ -411,7 +411,7 @@ class AccountInvoice(models.Model):
             '_onchange_partner_id': ['account_id', 'payment_term_id', 'fiscal_position_id', 'partner_bank_id'],
             '_onchange_journal_id': ['currency_id'],
         }
-        for onchange_method, changed_fields in pycompat.items(onchanges):
+        for onchange_method, changed_fields in onchanges.items():
             if any(f not in vals for f in changed_fields):
                 invoice = self.new(vals)
                 getattr(invoice, onchange_method)()
@@ -532,7 +532,7 @@ class AccountInvoice(models.Model):
             tax_grouped = invoice.get_taxes_values()
 
             # Create new tax lines
-            for tax in pycompat.values(tax_grouped):
+            for tax in tax_grouped.values():
                 account_invoice_tax.create(tax)
 
         # dummy write on self to trigger recomputations
@@ -551,7 +551,7 @@ class AccountInvoice(models.Model):
     def _onchange_invoice_line_ids(self):
         taxes_grouped = self.get_taxes_values()
         tax_lines = self.tax_line_ids.filtered('manual')
-        for tax in pycompat.values(taxes_grouped):
+        for tax in taxes_grouped.values():
             tax_lines += tax_lines.new(tax)
         self.tax_line_ids = tax_lines
         return
@@ -799,7 +799,7 @@ class AccountInvoice(models.Model):
         self.ensure_one()
         credit_aml = self.env['account.move.line'].browse(credit_aml_id)
         if not credit_aml.currency_id and self.currency_id != self.company_id.currency_id:
-            credit_aml.with_context(allow_amount_currency=True).write({
+            credit_aml.with_context(allow_amount_currency=True, check_move_validity=False).write({
                 'amount_currency': self.company_id.currency_id.with_context(date=credit_aml.date).compute(credit_aml.balance, self.currency_id),
                 'currency_id': self.currency_id.id})
         if credit_aml.payment_id:
@@ -944,7 +944,7 @@ class AccountInvoice(models.Model):
                 else:
                     line2[tmp] = l
             line = []
-            for key, val in pycompat.items(line2):
+            for key, val in line2.items():
                 line.append((0, 0, val))
         return line
 
@@ -1137,7 +1137,7 @@ class AccountInvoice(models.Model):
         result = []
         for line in lines:
             values = {}
-            for name, field in pycompat.items(line._fields):
+            for name, field in line._fields.items():
                 if name in MAGIC_COLUMNS:
                     continue
                 elif field.type == 'many2one':
@@ -1307,7 +1307,7 @@ class AccountInvoice(models.Model):
         for line in self.tax_line_ids:
             res.setdefault(line.tax_id.tax_group_id, 0.0)
             res[line.tax_id.tax_group_id] += line.amount
-        res = sorted(pycompat.items(res), key=lambda l: l[0].sequence)
+        res = sorted(res.items(), key=lambda l: l[0].sequence)
         res = [(l[0].name, l[1]) for l in res]
         return res
 
@@ -1408,7 +1408,7 @@ class AccountInvoiceLine(models.Model):
                         node.set('domain', "[('purchase_ok', '=', True)]")
                 else:
                     node.set('domain', "[('sale_ok', '=', True)]")
-            res['arch'] = etree.tostring(doc)
+            res['arch'] = etree.tostring(doc, encoding='unicode')
         return res
 
     @api.v8

addons/account/models/account_move.py

@@ -987,26 +987,10 @@ class AccountMoveLine(models.Model):
             vals['amount_currency'] = sign * abs(sum([r.amount_residual_currency for r in self]))
 
         # Writeoff line in the account of self
-        first_line_dict = vals.copy()
-        first_line_dict['account_id'] = self[0].account_id.id
-        if 'analytic_account_id' in first_line_dict:
-            del first_line_dict['analytic_account_id']
-        if 'tax_ids' in first_line_dict:
-            tax_ids = []
-            #vals['tax_ids'] is a list of commands [[4, tax_id, None], ...]
-            for tax_id in vals['tax_ids']:
-                tax_ids.append(tax_id[1])
-            amount = first_line_dict['credit'] - first_line_dict['debit']
-            amount_tax = self.env['account.tax'].browse(tax_ids).compute_all(amount)['total_included']
-            first_line_dict['credit'] = amount_tax > 0 and amount_tax or 0.0
-            first_line_dict['debit'] = amount_tax < 0 and abs(amount_tax) or 0.0
-            del first_line_dict['tax_ids']
+        first_line_dict = self._prepare_writeoff_first_line_values(vals)
 
         # Writeoff line in specified writeoff account
-        second_line_dict = vals.copy()
-        second_line_dict['debit'], second_line_dict['credit'] = second_line_dict['credit'], second_line_dict['debit']
-        if 'amount_currency' in vals:
-            second_line_dict['amount_currency'] = -second_line_dict['amount_currency']
+        second_line_dict = self._prepare_writeoff_second_line_values(vals)
 
         # Create the move
         writeoff_move = self.env['account.move'].with_context(apply_taxes=True).create({
@@ -1020,6 +1004,32 @@ class AccountMoveLine(models.Model):
         # Return the writeoff move.line which is to be reconciled
         return writeoff_move.line_ids.filtered(lambda r: r.account_id == self[0].account_id)
 
+    @api.multi
+    def _prepare_writeoff_first_line_values(self, values):
+        line_values = values.copy()
+        line_values['account_id'] = self[0].account_id.id
+        if 'analytic_account_id' in line_values:
+            del line_values['analytic_account_id']
+        if 'tax_ids' in line_values:
+            tax_ids = []
+            # vals['tax_ids'] is a list of commands [[4, tax_id, None], ...]
+            for tax_id in values['tax_ids']:
+                tax_ids.append(tax_id[1])
+            amount = line_values['credit'] - line_values['debit']
+            amount_tax = self.env['account.tax'].browse(tax_ids).compute_all(amount)['total_included']
+            line_values['credit'] = amount_tax > 0 and amount_tax or 0.0
+            line_values['debit'] = amount_tax < 0 and abs(amount_tax) or 0.0
+            del line_values['tax_ids']
+        return line_values
+
+    @api.multi
+    def _prepare_writeoff_second_line_values(self, values):
+        line_values = values.copy()
+        line_values['debit'], line_values['credit'] = line_values['credit'], line_values['debit']
+        if 'amount_currency' in values:
+            line_values['amount_currency'] = -line_values['amount_currency']
+        return line_values
+
     def force_full_reconcile(self):
         """ After running the manual reconciliation wizard and making full reconciliation, we need to run this method to create
             potentially exchange rate entries that will balance the remaining amount_residual_currency (possibly several aml in
@@ -1031,7 +1041,7 @@ class AccountMoveLine(models.Model):
         partial_rec_set = self.env['account.partial.reconcile']
         maxdate = '0000-00-00'
 
-        #gather the max date for the move creation, and all aml that are unbalanced
+        # gather the max date for the move creation, and all aml that are unbalanced
         for aml in self:
             maxdate = max(aml.date, maxdate)
             if aml.amount_residual_currency:
@@ -1771,6 +1781,6 @@ class AccountFullReconcile(models.Model):
         # The move date should be the maximum date between payment and invoice
         # (in case of payment in advance). However, we should make sure the
         # move date is not recorded after the end of year closing.
-        if move_date > company.fiscalyear_lock_date:
+        if move_date > (company.fiscalyear_lock_date or '0000-00-00'):
             res['date'] = move_date
         return res

addons/account/models/chart_template.py

@@ -7,8 +7,6 @@ from odoo.exceptions import UserError
 
 import logging
 
-from odoo.tools import pycompat
-
 _logger = logging.getLogger(__name__)
 
 def migrate_set_tags_and_taxes_updatable(cr, registry, module):
@@ -327,7 +325,7 @@ class AccountChartTemplate(models.Model):
 
         # writing account values after creation of accounts
         company.transfer_account_id = account_template_ref[transfer_account_id.id]
-        for key, value in pycompat.items(generated_tax_res['account_dict']):
+        for key, value in generated_tax_res['account_dict'].items():
             if value['refund_account_id'] or value['account_id'] or value['cash_basis_account']:
                 AccountTaxObj.browse(key).write({
                     'refund_account_id': account_ref.get(value['refund_account_id'], False),

addons/account/models/partner.py

@@ -5,7 +5,7 @@ from operator import itemgetter
 import time
 
 from odoo import api, fields, models, _
-from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT, pycompat
+from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
 from odoo.exceptions import ValidationError
 from odoo.addons.base.res.res_partner import WARNING_MESSAGE, WARNING_HELP
 
@@ -73,7 +73,7 @@ class AccountFiscalPosition(models.Model):
         ref_dict = {}
         for line in self.account_ids:
             ref_dict[line.account_src_id] = line.account_dest_id
-        for key, acc in list(pycompat.items(accounts)):
+        for key, acc in accounts.items():
             if acc in ref_dict:
                 accounts[key] = ref_dict[acc]
         return accounts
@@ -293,7 +293,7 @@ class ResPartner(models.Model):
                 """ % where_clause
         self.env.cr.execute(query, where_clause_params)
         price_totals = self.env.cr.dictfetchall()
-        for partner, child_ids in pycompat.items(all_partners_and_children):
+        for partner, child_ids in all_partners_and_children.items():
             partner.total_invoiced = sum(price['total'] for price in price_totals if price['partner_id'] in child_ids)
 
     @api.multi

addons/account/report/account_report_financial.py

@@ -3,7 +3,6 @@
 import time
 from odoo import api, models, _
 from odoo.exceptions import UserError
-from odoo.tools import pycompat
 
 
 class ReportFinancial(models.AbstractModel):
@@ -28,7 +27,7 @@ class ReportFinancial(models.AbstractModel):
             if where_clause.strip():
                 wheres.append(where_clause.strip())
             filters = " AND ".join(wheres)
-            request = "SELECT account_id as id, " + ', '.join(pycompat.values(mapping)) + \
+            request = "SELECT account_id as id, " + ', '.join(mapping.values()) + \
                        " FROM " + tables + \
                        " WHERE account_id IN %s " \
                             + filters + \
@@ -55,26 +54,26 @@ class ReportFinancial(models.AbstractModel):
             if report.type == 'accounts':
                 # it's the sum of the linked accounts
                 res[report.id]['account'] = self._compute_account_balance(report.account_ids)
-                for value in pycompat.values(res[report.id]['account']):
+                for value in res[report.id]['account'].values():
                     for field in fields:
                         res[report.id][field] += value.get(field)
             elif report.type == 'account_type':
                 # it's the sum the leaf accounts with such an account type
                 accounts = self.env['account.account'].search([('user_type_id', 'in', report.account_type_ids.ids)])
                 res[report.id]['account'] = self._compute_account_balance(accounts)
-                for value in pycompat.values(res[report.id]['account']):
+                for value in res[report.id]['account'].values():
                     for field in fields:
                         res[report.id][field] += value.get(field)
             elif report.type == 'account_report' and report.account_report_id:
                 # it's the amount of the linked report
                 res2 = self._compute_report_balance(report.account_report_id)
-                for key, value in pycompat.items(res2):
+                for key, value in res2.items():
                     for field in fields:
                         res[report.id][field] += value[field]
             elif report.type == 'sum':
                 # it's the sum of the children of this account.report
                 res2 = self._compute_report_balance(report.children_ids)
-                for key, value in pycompat.items(res2):
+                for key, value in res2.items():
                     for field in fields:
                         res[report.id][field] += value[field]
         return res
@@ -86,11 +85,11 @@ class ReportFinancial(models.AbstractModel):
         res = self.with_context(data.get('used_context'))._compute_report_balance(child_reports)
         if data['enable_filter']:
             comparison_res = self.with_context(data.get('comparison_context'))._compute_report_balance(child_reports)
-            for report_id, value in pycompat.items(comparison_res):
+            for report_id, value in comparison_res.items():
                 res[report_id]['comp_bal'] = value['balance']
                 report_acc = res[report_id].get('account')
                 if report_acc:
-                    for account_id, val in pycompat.items(comparison_res[report_id].get('account')):
+                    for account_id, val in comparison_res[report_id].get('account').items():
                         report_acc[account_id]['comp_bal'] = val['balance']
 
         for report in child_reports:
@@ -115,7 +114,7 @@ class ReportFinancial(models.AbstractModel):
 
             if res[report.id].get('account'):
                 sub_lines = []
-                for account_id, value in pycompat.items(res[report.id]['account']):
+                for account_id, value in res[report.id]['account'].items():
                     #if there are accounts to display, we add them to the lines with a level equals to their level in
                     #the COA + 1 (to avoid having them with a too low level that would conflicts with the level of data
                     #financial reports for Assets, liabilities...)

addons/account/static/src/js/account_payment_field.js

@@ -46,7 +46,7 @@ var ShowPaymentLineWidget = AbstractField.extend({
             k.index = v;
             k.amount = field_utils.format.float(k.amount, {digits: k.digits});
             if (k.date){
-                k.date = field_utils.format.date(field_utils.parse.date(k.date, {isUTC: true}));
+                k.date = field_utils.format.date(field_utils.parse.date(k.date, {}, {isUTC: true}));
             }
         });
         this.$el.html(QWeb.render('ShowPaymentInfo', {

addons/account/static/src/js/reconciliation/reconciliation_action.js

@@ -30,6 +30,7 @@ var StatementAction = Widget.extend(ControlPanelMixin, {
         validate_all_balanced: '_onValidate',
         change_name: '_onChangeName',
         close_statement: '_onCloseStatement',
+        load_more: '_onLoadMore',
     },
     config: {
         // used to instanciate the model
@@ -71,6 +72,7 @@ var StatementAction = Widget.extend(ControlPanelMixin, {
                     'bank_statement_id': self.model.bank_statement_id,
                     'valuenow': self.model.valuenow,
                     'valuemax': self.model.valuemax,
+                    'defaultDisplayQty': self.model.defaultDisplayQty,
                     'title': self.title,
                 });
             });
@@ -89,12 +91,7 @@ var StatementAction = Widget.extend(ControlPanelMixin, {
         this.update_control_panel({breadcrumbs: breadcrumbs, search_view_hidden: true}, {clear: true});
 
         this.renderer.prependTo(self.$('.o_form_sheet'));
-        _.each(this.model.lines, function (line, handle) {
-            var widget = new self.config.LineRenderer(self, self.model, line);
-            widget.handle = handle;
-            self.widgets.push(widget);
-            widget.appendTo(self.$('.o_reconciliation_lines'));
-        });
+        this._renderLines();
         this._openFirstLine();
     },
 
@@ -134,6 +131,15 @@ var StatementAction = Widget.extend(ControlPanelMixin, {
         return _.find(this.widgets, function (widget) {return widget.handle===handle;});
     },
 
+    /**
+     *
+     */
+    _loadMore: function(qty) {
+        var self = this;
+        return this.model.loadMore(qty).then(function () {
+            self._renderLines();
+        });
+    },
     /**
      * sitch to 'match' the first available line
      *
@@ -153,6 +159,24 @@ var StatementAction = Widget.extend(ControlPanelMixin, {
         }
         return handle;
     },
+    /**
+     * render line widget and append to view
+     *
+     * @private
+     */
+    _renderLines: function () {
+        var self = this;
+        var linesToDisplay = this.model.getStatementLines();
+        _.each(linesToDisplay, function (line, handle) {
+            var widget = new self.config.LineRenderer(self, self.model, line);
+            widget.handle = handle;
+            self.widgets.push(widget);
+            widget.appendTo(self.$('.o_reconciliation_lines'));
+        });
+        if (this.model.hasMoreLines() === false) {
+            this.renderer.hideLoadMoreButton();
+        }
+    },
 
     //--------------------------------------------------------------------------
     // Handlers
@@ -223,6 +247,14 @@ var StatementAction = Widget.extend(ControlPanelMixin, {
             });
         });
     },
+    /**
+     * Load more statement and render them
+     *
+     * @param {OdooEvent} event
+     */
+    _onLoadMore: function (event) {
+        return this._loadMore(this.model.defaultDisplayQty);
+    },
     /**
      * call 'validate' or 'autoReconciliation' model method then destroy the
      * validated lines and update the action renderer with the new status bar 
@@ -246,7 +278,15 @@ var StatementAction = Widget.extend(ControlPanelMixin, {
             });
             _.each(result.handles, function (handle) {
                 self._getWidget(handle).destroy();
+                var index = _.findIndex(self.widgets, function (widget) {return widget.handle===handle;});
+                self.widgets.splice(index, 1);
             });
+            // Get number of widget and if less than constant and if there are more to laod, load until constant
+            if (self.widgets.length < self.model.defaultDisplayQty 
+                && self.model.valuemax - self.model.valuenow >= self.model.defaultDisplayQty) {
+                var toLoad = self.model.defaultDisplayQty - self.widgets.length;
+                self._loadMore(toLoad);
+            }
             self._openFirstLine();
         });
     },

addons/account/static/src/js/reconciliation/reconciliation_model.js

@@ -100,6 +100,8 @@ var StatementModel = BasicModel.extend({
         this.lines = {};
         this.valuenow = 0;
         this.valuemax = 0;
+        this.alreadyDisplayed = [];
+        this.defaultDisplayQty = 10;
     },
 
     //--------------------------------------------------------------------------
@@ -230,10 +232,10 @@ var StatementModel = BasicModel.extend({
         var line = this.getLine(handle);
         line.st_line.partner_id = partner && partner.id;
         line.st_line.partner_name = partner && partner.display_name || '';
-        return this._changePartner(handle, partner.id)
+        return $.when(partner && this._changePartner(handle, partner.id))
                 .then(function() {
                     line.reconciliation_proposition = [];
-                    return self._performMoveLine(handle)
+                    return self.changeMode(handle, 'match');
                 })
                 .then(function () {
                     if (line.mode === 'create') {
@@ -253,7 +255,7 @@ var StatementModel = BasicModel.extend({
                 args: [self.bank_statement_id.id],
             })
             .then(function () {
-                return self.bank_statement_id.id
+                return self.bank_statement_id.id;
             });
     },
     /**
@@ -283,6 +285,34 @@ var StatementModel = BasicModel.extend({
     getContext: function () {
         return this.context;
     },
+    /**
+     * Return the lines that needs to be displayed by the widget
+     *
+     * @returns {Object} lines that are loaded and not yet displayed
+     */
+    getStatementLines: function () {
+        var self = this;
+        var linesToDisplay = _.pick(this.lines, function(value, key, object) { 
+            if (value.visible === true && self.alreadyDisplayed.indexOf(key) === -1) {
+                self.alreadyDisplayed.push(key);
+                return object;
+            }
+        });
+        return linesToDisplay;
+    },
+    /**
+     * Return a boolean telling if load button needs to be displayed or not
+     *
+     * @returns {boolean} true if load more button needs to be displayed
+     */
+    hasMoreLines: function () {
+        var self = this;
+        var notDisplayed = _.filter(this.lines, function(line) { return !line.visible; });
+        if (notDisplayed.length > 0) {
+            return true;
+        }
+        return false;
+    },
     /**
      * get the line data for this handle
      *
@@ -355,14 +385,42 @@ var StatementModel = BasicModel.extend({
                 line.reconcileModels = self.reconcileModels;
             });
             var ids = _.pluck(self.lines, 'id');
-            return self._rpc({
-                    model: 'account.bank.statement.line',
-                    method: 'get_data_for_reconciliation_widget',
-                    args: [ids],
-                })
-                .then(self._formatLine.bind(self));
+            ids = ids.splice(0, self.defaultDisplayQty);
+            self.pagerIndex = ids.length;
+            return self.loadData(ids, []);
         });
     },
+    /**
+     * Load more bank statement line
+     *
+     * @param {integer} quantity to load
+     * @returns {Deferred}
+     */
+    loadMore: function(qty) {
+        if (qty === undefined) {
+            qty = this.defaultDisplayQty;
+        }
+        var ids = _.pluck(this.lines, 'id');
+        ids = ids.splice(this.pagerIndex, qty);
+        this.pagerIndex += qty;
+        return this.loadData(ids, this._getExcludedIds());
+    },
+    /**
+     * RPC method to load informations on lines
+     * 
+     * @param {Array} ids of bank statement line passed to rpc call
+     * @param {Array} list of move_line ids that needs to be excluded from search
+     * @returns {Deferred}
+     */
+    loadData: function(ids, excluded_ids) {
+        var self = this;
+        return self._rpc({
+            model: 'account.bank.statement.line',
+            method: 'get_data_for_reconciliation_widget',
+            args: [ids, excluded_ids],
+        })
+        .then(self._formatLine.bind(self));
+    },
     /**
      * Add lines into the propositions from the reconcile model
      * Can add 2 lines, and each with its taxes. The second line become editable
@@ -770,6 +828,7 @@ var StatementModel = BasicModel.extend({
             var line = _.find(self.lines, function (l) {
                 return l.id === data.st_line.id;
             });
+            line.visible = true;
             _.extend(line, data);
             self._formatLineProposition(line, line.reconciliation_proposition);
             if (!line.reconciliation_proposition.length) {
@@ -850,6 +909,26 @@ var StatementModel = BasicModel.extend({
         prop.amount = prop.base_amount;
         return prop;
     },
+    /**
+     * Return list of account_move_line that has been selected and needs to be removed
+     * from other calls.
+     *
+     * @private
+     * @returns {Array} list of excluded ids
+     */
+    _getExcludedIds: function () {
+        var excludedIds = [];
+        _.each(this.lines, function(line) {
+            if (line.reconciliation_proposition) {
+                _.each(line.reconciliation_proposition, function(prop) {
+                    if (parseInt(prop['id'])) {
+                        excludedIds.push(prop['id']);
+                    }
+                })
+            }
+        });
+        return excludedIds;
+    },
     /**
      * Defined whether the line is to be displayed or not. Here, we only display
      * the line if it comes from the server or if an account is defined when it
@@ -1168,7 +1247,8 @@ var ManualModel = StatementModel.extend({
             'filter': "",
             'reconcileModels': [],
             'account_id': this._formatNameGet([data.account_id, data.account_name]),
-            'st_line': data
+            'st_line': data,
+            'visible': true
         });
         this._formatLineProposition(line, line.reconciliation_proposition);
         if (!line.reconciliation_proposition.length) {

addons/account/static/src/js/reconciliation/reconciliation_renderer.js

@@ -24,6 +24,7 @@ var StatementRenderer = Widget.extend(FieldManagerMixin, {
         "click *[rel='do_action']": "_onDoAction",
         'click button.button_back_to_statement': '_onGoToBankStatement',
         'click button.button_close_statement': '_onCloseBankStatement',
+        'click button.js_load_more': '_onLoadMore',
     },
     /**
      * @override
@@ -87,6 +88,12 @@ var StatementRenderer = Widget.extend(FieldManagerMixin, {
     //--------------------------------------------------------------------------
     // Public
     //--------------------------------------------------------------------------
+    /*
+     * hide the button to load more statement line
+     */
+    hideLoadMoreButton: function () {
+        this.$('.js_load_more').hide();
+    },
     /**
      * update the statement rendering
      *
@@ -214,6 +221,14 @@ var StatementRenderer = Widget.extend(FieldManagerMixin, {
             view_mode: 'form',
         });
     },
+    /**
+     * Load more statement lines for reconciliation
+     * @private
+     * @param {MouseEvent} event
+     */
+    _onLoadMore: function (e) {
+        this.trigger_up('load_more');
+    },
     /**
      * @private
      */

addons/account/static/src/less/account_reconciliation.less

@@ -166,7 +166,6 @@
                 vertical-align: top;
                 padding-right: 10px;
                 font-weight: bold;
-                white-space: nowrap;
             }
         }
 
@@ -209,7 +208,7 @@
             .o-transition(max-height, 250ms);
         }
         &[data-mode="match"] > .match {
-            max-height: 130px;
+            max-height: none;
             .o-transition(max-height, 400ms);
         }
         &[data-mode="create"] > .create {

addons/account/static/src/xml/account_reconciliation.xml

@@ -25,6 +25,9 @@
             <div class="notification_area"/>
         </div>
         <div class="o_reconciliation_lines"/>
+        <div t-if="widget._initialState.valuemax > widget._initialState.defaultDisplayQty">
+            <button class="btn btn-default js_load_more">Load more</button>
+        </div>
         <div class="pull-right text-muted">Tip: Hit CTRL-Enter to reconcile all the balanced items in the sheet.</div>
     </div>
     <div t-else="" class="o_view_nocontent">
@@ -203,7 +206,16 @@
         <td class="cell_action"><span class="fa fa-add-remove"/></td>
         <td class="cell_account_code"><t t-esc="line.account_code"/></td>
         <td class="cell_due_date"><t t-esc="line.date_maturity === false ? line.date : line.date_maturity"/></td>
-        <td class="cell_label"><t t-if="line.partner_id &amp;&amp; line.partner_id !== state.st_line.partner_id"><t t-if="line.partner_name.length"><t t-esc="line.partner_name"/>: </t><t t-esc="line.label || line.name"/><t t-if="line.ref &amp;&amp; line.ref.length"> : </t><t t-esc="line.ref"/></t><t t-else=""><t t-esc="line.ref &amp;&amp; line.ref.length ? line.ref : line.label || line.name"/></t></td>
+        <td class="cell_label">
+            <t t-if="line.partner_id &amp;&amp; line.partner_id !== state.st_line.partner_id">
+                <t t-if="line.partner_name.length">
+                    <t t-esc="line.partner_name"/>: 
+                </t>   
+            </t>
+            <t t-esc="line.label || line.name"/>
+            <t t-if="line.ref &amp;&amp; line.ref.length"> : </t>
+            <t t-esc="line.ref"/>
+        </td>
         <td class="cell_left"><t t-if="line.amount &lt; 0"><span t-if="line.amount_currency_str" t-attf-class="o_multi_currency o_multi_currency_color_#{line.currency_id%8} line_info_button fa fa-money" t-att-data-content="line.amount_currency_str"/><t t-raw="line.amount_str"/></t></td>
         <td class="cell_right"><t t-if="line.amount &gt; 0"><span t-if="line.amount_currency_str" t-attf-class="o_multi_currency o_multi_currency_color_#{line.currency_id%8} line_info_button fa fa-money" t-att-data-content="line.amount_currency_str"/><t t-raw="line.amount_str"/></t></td>
         <td class="cell_info_popover"></td>

addons/account/static/tests/reconciliation_tests.js

@@ -1,7 +1,9 @@
 odoo.define('account.reconciliation_tests.data', function () {
 "use strict";
 
-var data = {
+var Datas = {};
+
+var db = {
     'res.partner': {
         fields: {
             id: {string: "ID", type: 'integer'},
@@ -58,7 +60,7 @@ var data = {
             {id: 7, display_name: "Tax 10.00% include", amount: 10, price_include: true, account_id: 288},
         ],
         json_friendly_compute_all: function (args) {
-            var tax = _.find(data['account.tax'].records, {'id': args[0][0]});
+            var tax = _.find(db['account.tax'].records, {'id': args[0][0]});
             var amount = args[1];
             var tax_base = tax.price_include ? amount*100/(100+tax.amount) : amount;
             return $.when({
@@ -103,7 +105,7 @@ var data = {
     'account.bank.statement': {
         fields: {},
         reconciliation_widget_preprocess: function () {
-            return $.when(data_preprocess);
+            return $.when(Datas.used.data_preprocess);
         },
     },
     'account.bank.statement.line': {
@@ -122,19 +124,19 @@ var data = {
             var partner_id = args.splice(1, 1)[0];
             var excluded_ids = args.splice(1, 1)[0];
             var key = JSON.stringify(args);
-            if (!mv_lines[key]) {
+            if (!Datas.used.mv_lines[key]) {
                 throw new Error("Unknown parameters for get_move_lines_for_reconciliation_widget: '"+ key + "'");
             }
-            return $.when(_.filter(mv_lines[key], function (line) {
+            return $.when(_.filter(Datas.used.mv_lines[key], function (line) {
                 return excluded_ids.indexOf(line.id) === -1 && (!partner_id || partner_id === line.partner_id);
             }));
         },
         get_data_for_reconciliation_widget: function (args) {
             var ids = args[0];
-            return $.when(_.filter(data_widget, function (w) {return _.contains(ids, w.st_line.id);}));
+            return $.when(_.filter(Datas.used.data_widget, function (w) {return _.contains(ids, w.st_line.id);}));
         },
         reconciliation_widget_auto_reconcile: function () {
-            return $.when(auto_reconciliation);
+            return $.when(Datas.used.auto_reconciliation);
         },
         process_reconciliations: function (args) {
             var datas = args[1];
@@ -142,8 +144,8 @@ var data = {
             ids = ids.concat(_.flatten(_.pluck(datas, 'payment_aml_ids')));
             ids = _.compact(ids);
 
-            for (var key in move_lines_for_manual_reconciliation) {
-                move_lines_for_manual_reconciliation[key] = _.filter(move_lines_for_manual_reconciliation[key], function (mv_line) {
+            for (var key in Datas.used.move_lines_for_manual_reconciliation) {
+                Datas.used.move_lines_for_manual_reconciliation[key] = _.filter(Datas.used.move_lines_for_manual_reconciliation[key], function (mv_line) {
                     return ids.indexOf(mv_line.id) === -1;
                 });
             }
@@ -154,18 +156,18 @@ var data = {
         fields: {},
         get_data_for_manual_reconciliation_widget: function (args) {
             var key = JSON.stringify(args);
-            if (!data_for_manual_reconciliation_widget[key]) {
+            if (!Datas.used.data_for_manual_reconciliation_widget[key]) {
                 throw new Error("Unknown parameters for get_data_for_manual_reconciliation_widget: '"+ key + "'");
             }
-            return $.when(data_for_manual_reconciliation_widget[key]);
+            return $.when(Datas.used.data_for_manual_reconciliation_widget[key]);
         },
         get_move_lines_for_manual_reconciliation: function (args) {
             var excluded_ids = args.splice(2, 1)[0];
             var key = JSON.stringify(args);
-            if (!move_lines_for_manual_reconciliation[key]) {
+            if (!Datas.used.move_lines_for_manual_reconciliation[key]) {
                 throw new Error("Unknown parameters for get_move_lines_for_manual_reconciliation: '"+ key + "'");
             }
-            return $.when(_.filter(move_lines_for_manual_reconciliation[key], function (line) {
+            return $.when(_.filter(Datas.used.move_lines_for_manual_reconciliation[key], function (line) {
                 return excluded_ids.indexOf(line.id) === -1;
             }));
         },
@@ -174,8 +176,8 @@ var data = {
             var datas = args[0];
             for (var i in datas) {
                 var data = datas[i];
-                for (var key in move_lines_for_manual_reconciliation) {
-                    move_lines_for_manual_reconciliation[key] = _.filter(move_lines_for_manual_reconciliation[key], function (mv_line) {
+                for (var key in Datas.used.move_lines_for_manual_reconciliation) {
+                    Datas.used.move_lines_for_manual_reconciliation[key] = _.filter(Datas.used.move_lines_for_manual_reconciliation[key], function (mv_line) {
                         return data.mv_line_ids.indexOf(mv_line.id) === -1;
                     });
                 }
@@ -367,7 +369,12 @@ var mv_lines = {
     '[7,"",0,6]': [
         {'account_type': "receivable", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-02-07", 'date': "2017-01-08", 'total_amount_str': "$ 650.00", 'partner_id': 12, 'account_name': "101200 Account Receivable", 'name': "INV/2017/0012", 'partner_name': "Camptocamp", 'total_amount_currency_str': "", 'id': 133, 'credit': 0.0, 'journal_id': [1, "Customer Invoices"], 'amount_str': "$ 650.00", 'debit': 650.0, 'account_id': [287, "101200 Account Receivable"], 'account_code': "101200", 'ref': "", 'already_paid': false},
         {'account_type': "receivable", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-02-28", 'date': "2017-01-01", 'total_amount_str': "$ 4,610.00", 'partner_id': 12, 'account_name': "101200 Account Receivable", 'name': "INV/2017/0001", 'partner_name': "Camptocamp", 'total_amount_currency_str': "", 'id': 106, 'credit': 0.0, 'journal_id': [1, "Customer Invoices"], 'amount_str': "$ 4,610.00", 'debit': 4610.0, 'account_id': [287, "101200 Account Receivable"], 'account_code': "101200", 'ref': "", 'already_paid': false},
-        {'account_type': "payable", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-02-28", 'date': "2017-01-01", 'total_amount_str': "$ 10,000.00", 'partner_id': 12, 'account_name': "Account Payable", 'name': "BILL/2017/0001", 'partner_name': "Camptocamp", 'total_amount_currency_str': "", 'id': 114, 'credit': 10000.0, 'journal_id': [2, "Vendor Bills"], 'amount_str': "$ 10,000.00", 'debit': 0.0, 'account_id': [284, "101110 Stock Valuation Account"], 'account_code': "111100", 'ref': "", 'already_paid': false}
+        {'account_type': "payable", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-02-28", 'date': "2017-01-01", 'total_amount_str': "$ 10,000.00", 'partner_id': 12, 'account_name': "Account Payable", 'name': "BILL/2017/0001", 'partner_name': "Camptocamp", 'total_amount_currency_str': "", 'id': 114, 'credit': 10000.0, 'journal_id': [2, "Vendor Bills"], 'amount_str': "$ 10,000.00", 'debit': 0.0, 'account_id': [284, "101110 Stock Valuation Account"], 'account_code': "111100", 'ref': "", 'already_paid': false},
+        {'account_type': "liquidity", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-01-23", 'date': "2017-01-23", 'total_amount_str': "$ 376.00", 'partner_id': 7, 'account_name': "Bank", 'name': "BNK1/2017/0002: SUPP.OUT/2017/0002", 'partner_name': "ASUSTeK", 'total_amount_currency_str': "", 'id': 392, 'credit': 376.0, 'journal_id': "Bank", 'amount_str': "$ 376.00", 'debit': 0.0, 'account_code': "101401", 'ref': "BILL/2017/0003", 'already_paid': true},
+        {'account_type': "liquidity", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-01-23", 'date': "2017-01-23", 'total_amount_str': "$ 100.00", 'partner_id': 8, 'account_name': "Bank", 'name': "BNK1/2017/0003: CUST.IN/2017/0001", 'partner_name': "Agrolait", 'total_amount_currency_str': "", 'id': 394, 'credit': 0.0, 'journal_id': "Bank", 'amount_str': "$ 100.00", 'debit': 100.0, 'account_code': "101401", 'ref': "", 'already_paid': true},
+        {'account_type': "liquidity", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-01-23", 'date': "2017-01-23", 'total_amount_str': "$ 525.50", 'partner_id': 8, 'account_name': "Bank", 'name': "BNK1/2017/0004: CUST.IN/2017/0002", 'partner_name': "Agrolait", 'total_amount_currency_str': "", 'id': 396, 'credit': 0.0, 'journal_id': "Bank", 'amount_str': "$ 525.50", 'debit': 525.5, 'account_code': "101401", 'ref': "INV/2017/0003", 'already_paid': true},
+        {'account_type': "receivable", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-02-07", 'date': "2017-01-08", 'total_amount_str': "$ 650.00", 'partner_id': 8, 'account_name': "101200 Account Receivable", 'name': "INV/2017/0002", 'partner_name': "Agrolait", 'total_amount_currency_str': "", 'id': 109, 'credit': 0.0, 'journal_id': [1, "Customer Invoices"], 'amount_str': "$ 650.00", 'debit': 650.0, 'account_code': "101200", 'ref': "", 'already_paid': false},
+        {'account_type': "receivable", 'amount_currency_str': "", 'currency_id': false, 'date_maturity': "2017-02-22", 'date': "2017-01-23", 'total_amount_str': "$ 525.00", 'partner_id': 8, 'account_name': "101200 Account Receivable", 'name': "INV/2017/0004", 'partner_name': "Agrolait", 'total_amount_currency_str': "", 'id': 399, 'credit': 0.0, 'journal_id': [1, "Customer Invoices"], 'amount_str': "$ 525.00", 'debit': 525.0, 'account_code': "101200", 'ref': "", 'already_paid': false},
     ],
     '[8,"",0,6]': [],
 };
@@ -468,23 +475,22 @@ var options = {
     }
 };
 
-return {
-    params: {
-        data: data,
-        data_preprocess: data_preprocess,
-        data_widget: data_widget,
-        mv_lines: mv_lines,
-        auto_reconciliation: auto_reconciliation,
-        data_for_manual_reconciliation_widget: data_for_manual_reconciliation_widget,
-        move_lines_for_manual_reconciliation: move_lines_for_manual_reconciliation,
-        session: session,
-        options: options,
-    },
-    // this is the main function for this module. Its job is to export (and clone) all data for a test.
-    getParams: function () {
-        return $.extend(true, {}, this.params);
-    }
+Datas.params = {
+    data: db,
+    data_preprocess: data_preprocess,
+    data_widget: data_widget,
+    mv_lines: mv_lines,
+    auto_reconciliation: auto_reconciliation,
+    data_for_manual_reconciliation_widget: data_for_manual_reconciliation_widget,
+    move_lines_for_manual_reconciliation: move_lines_for_manual_reconciliation,
+    session: session,
+    options: options,
 };
+    // this is the main function for this module. Its job is to export (and clone) all data for a test.
+Datas.getParams = function () {
+    return (this.used = $.extend(true, {}, this.params));
+};
+return Datas;
 });
 
 odoo.define('account.reconciliation_tests', function (require) {
@@ -534,7 +540,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation fields', function (assert) {
         assert.expect(6);
 
@@ -561,7 +566,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation basic data', function (assert) {
         assert.expect(14);
 
@@ -630,7 +634,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation partial', function (assert) {
         assert.expect(10);
 
@@ -701,7 +704,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation currencies', function (assert) {
         assert.expect(2);
 
@@ -736,7 +738,7 @@ QUnit.module('account', {
     });
 
     QUnit.test('Reconciliation change partner', function (assert) {
-        assert.expect(4);
+        assert.expect(10);
 
         var clientAction = new ReconciliationClientAction.StatementAction(null, this.params.options);
 
@@ -754,7 +756,21 @@ QUnit.module('account', {
         $('.ui-autocomplete .ui-menu-item a:contains(Camptocamp)').trigger('mouseenter').trigger('click');
         clientAction._onAction({target: widget, name: 'change_partner', data: {data: {display_name: 'Camptocamp', id: 12}}, stopped: false});
         assert.strictEqual(widget.$('.o_input_dropdown input').val(), "Camptocamp", "the partner many2one should display Camptocamp");
-        assert.strictEqual(widget.$('.match table tr').length, 3, "camptocamp should have 3 propositions for reconciliation");
+        assert.strictEqual(widget.$('.match table tr.mv_line').length, 3, "camptocamp should have 3 propositions for reconciliation");
+
+        widget = clientAction.widgets[2];
+        widget.$('.accounting_view thead td:first').trigger('click');
+        widget.$('.accounting_view .mv_line .cell_label').trigger('click');
+        assert.strictEqual(widget.$('.o_input_dropdown input').val(), "Camptocamp", "the partner many2one should display agrolait");
+        assert.strictEqual(widget.$('.match table tr').length, 3, "Camptocamp should have 3 propositions for reconciliation");
+        assert.strictEqual(widget.$('.match .match_controls .fa:not(.disabled)').length, 0, "should not display the right arrow");
+
+        // Simulate remove partner
+        widget.$('.o_input_dropdown input').trigger('click');
+        widget.$('.o_input_dropdown input').val('').trigger('keyup').trigger('blur');
+        assert.strictEqual(widget.$('.o_input_dropdown input').val(), "", "the partner many2one should be empty");
+        assert.strictEqual(widget.$('.match table tr.mv_line').length, 5, "should have 5 propositions for reconciliation if partner is false");
+        assert.strictEqual(widget.$('.match .match_controls .fa:not(.disabled)').length, 1, "should not display the right arrow");
 
         clientAction.destroy();
     });
@@ -839,7 +855,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation create line with taxes', function (assert) {
         assert.expect(13);
 
@@ -883,7 +898,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation create line from reconciliation model', function (assert) {
         assert.expect(6);
 
@@ -900,7 +914,7 @@ QUnit.module('account', {
         widget.$('.create .quick_add button:contains(ATOS)').trigger('click');
 
         assert.strictEqual(widget.$('.accounting_view tbody .cell_label, .accounting_view tbody .cell_right').text().replace(/[\n\r\s$,]+/g, ' '),
-            "ATOS Banque1145.62Tax 20.00%229.12ATOS Frais26.71Tax 10.00% include2.67", "should display 4 lines");
+            " ATOS Banque 1145.62 Tax 20.00% 229.12 ATOS Frais 26.71 Tax 10.00% include 2.67", "should display 4 lines");
         assert.strictEqual(widget.$('.accounting_view tfoot .cell_label, .accounting_view tfoot .cell_left').text().replace(/[\n\r\s$,]+/g, ' '),
             "Create Write-off229.12", "should display the 'Create Write-off' line with value in left column");
 
@@ -926,7 +940,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation auto reconciliation', function (assert) {
         assert.expect(6);
 
@@ -958,7 +971,6 @@ QUnit.module('account', {
         clientAction.destroy();
     });
 
-
     QUnit.test('Reconciliation manual', function (assert) {
         assert.expect(8);
 
@@ -992,5 +1004,229 @@ QUnit.module('account', {
 
         clientAction.destroy();
     });
+
+
+    QUnit.test('Reconciliation pager', function (assert) {
+        assert.expect(6);
+
+        this.params.data['account.bank.statement.line'].records.push(
+            {id: 9, display_name: "add 9"},
+            {id: 10, display_name: "add 10"},
+            {id: 11, display_name: "add 11"},
+            {id: 12, display_name: "add 12"},
+            {id: 13, display_name: "add 13"},
+            {id: 14, display_name: "add 14"},
+            {id: 15, display_name: "add 15"},
+            {id: 16, display_name: "add 16"}
+        );
+
+        var data_preprocess = this.params.data_preprocess;
+        data_preprocess.st_lines_ids.push(9, 10, 11, 12, 13, 14, 15, 16);
+
+        this.params.data_widget.push(
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 9",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 9,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            },
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 10",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 10,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            },
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 11",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 11,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            },
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 12",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 12,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            },
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 13",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 13,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            },
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 14",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 14,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            },
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 15",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 15,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            },
+            {
+                'st_line': {
+                    'currency_id': 3,
+                    'communication_partner_name': false,
+                    'open_balance_account_id': 287,
+                    'name': "add 16",
+                    'partner_name': "Camptocamp",
+                    'partner_id': 12,
+                    'has_no_partner': false,
+                    'journal_id': 84,
+                    'account_name': "Bank",
+                    'note': "",
+                    'amount': 650.0,
+                    'amount_str': "$ 650.00",
+                    'amount_currency_str': "",
+                    'date': "2017-01-01",
+                    'account_code': "101401",
+                    'ref': "",
+                    'id': 16,
+                    'statement_id': 2
+                },
+                'reconciliation_proposition': []
+            }
+        );
+
+        var clientAction = new ReconciliationClientAction.StatementAction(null, this.params.options);
+        testUtils.addMockEnvironment(clientAction, {
+            'data': this.params.data,
+        });
+
+        clientAction.appendTo($('#qunit-fixture'));
+
+        assert.strictEqual(clientAction.$('.o_reconciliation_line').length, 10, "should display 10 lines");
+        assert.strictEqual(clientAction.$('.js_load_more:visible').length, 1, "should display the load more button");
+        clientAction.$('.o_reconcile:visible').click();
+        assert.strictEqual(clientAction.$('.o_reconcile:visible').length, 0, "should remove the reconciled line");
+        assert.strictEqual(clientAction.$('.o_reconciliation_line').length, 10, "should load one line to complete the 10");
+        clientAction.$('.js_load_more').click();
+        assert.strictEqual(clientAction.$('.o_reconciliation_line').length, 11, "should load the last record");
+        assert.strictEqual(clientAction.$('.js_load_more:visible').length, 0, "should hide the load more button");
+
+        clientAction.destroy();
+    });
 });
 });

addons/account/tests/test_reconciliation.py

@@ -2,8 +2,6 @@ from odoo.addons.account.tests.account_test_classes import AccountingTestCase
 import time
 import unittest
 
-from odoo.tools import pycompat
-
 
 class TestReconciliation(AccountingTestCase):
 
@@ -339,8 +337,8 @@ class TestReconciliation(AccountingTestCase):
         self.assertTrue(exchange_loss_line, 'There should be one move line of 0.01 EUR in credit')
         # The journal items of the reconciliation should have their debit and credit total equal
         # Besides, the total debit and total credit should be 60.61 EUR (2.00 USD)
-        self.assertEquals(sum(res['debit'] for res in pycompat.values(result)), 60.61)
-        self.assertEquals(sum(res['credit'] for res in pycompat.items(result)), 60.61)
+        self.assertEquals(sum(res['debit'] for res in result.values()), 60.61)
+        self.assertEquals(sum(res['credit'] for res in result.items()), 60.61)
         counterpart_exchange_loss_line = None
         for line in exchange_loss_line.move_id.line_id:
             if line.account_id.id == self.account_fx_expense_id:

addons/account/views/account_invoice_view.xml

@@ -223,7 +223,7 @@
                         <button name="action_invoice_draft" states="cancel" string="Set to Draft" type="object" groups="account.group_account_invoice"/>
                     <field name="state" widget="statusbar" statusbar_visible="draft,open,paid" />
                 </header>
-                <div class="alert alert-info" role="alert" style="margin-bottom:0px;" attrs="{'invisible': [('has_outstanding','=',False)]}">
+                <div groups="account.group_account_invoice" class="alert alert-info" role="alert" style="margin-bottom:0px;" attrs="{'invisible': [('has_outstanding','=',False)]}">
                     You have <bold><a class="alert-link" href="#outstanding" role="button">outstanding debits</a></bold> for this supplier. You can allocate them to mark this bill as paid.
                 </div>
                 <field name="has_outstanding" invisible="1"/>
@@ -363,7 +363,7 @@
                     <button name="action_invoice_draft" states="cancel" string="Reset to Draft" type="object" groups="account.group_account_invoice"/>
                     <field name="state" widget="statusbar" nolabel="1" statusbar_visible="draft,open,paid"/>
                 </header>
-                <div class="alert alert-info" role="alert" style="margin-bottom:0px;" attrs="{'invisible': [('has_outstanding','=',False)]}">
+                <div groups="account.group_account_invoice" class="alert alert-info" role="alert" style="margin-bottom:0px;" attrs="{'invisible': [('has_outstanding','=',False)]}">
                     You have <bold><a class="alert-link" href="#outstanding" role="button">outstanding payments</a></bold> for this customer. You can allocate them to mark this invoice as paid.
                 </div>
                 <field name="has_outstanding" invisible="1"/>

addons/account/wizard/account_invoice_refund.py

@@ -74,7 +74,7 @@ class AccountInvoiceRefund(models.TransientModel):
                     for tmpline in refund.move_id.line_ids:
                         if tmpline.account_id.id == inv.account_id.id:
                             to_reconcile_lines += tmpline
-                            to_reconcile_lines.filtered(lambda l: l.reconciled == False).reconcile()
+                    to_reconcile_lines.filtered(lambda l: l.reconciled == False).reconcile()
                     if mode == 'modify':
                         invoice = inv.read(inv_obj._get_refund_modify_read_fields())
                         invoice = invoice[0]

addons/account_asset/models/account_asset.py

@@ -7,7 +7,7 @@ from dateutil.relativedelta import relativedelta
 
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError, ValidationError
-from odoo.tools import DEFAULT_SERVER_DATE_FORMAT as DF, pycompat
+from odoo.tools import DEFAULT_SERVER_DATE_FORMAT as DF
 from odoo.tools import float_compare, float_is_zero
 
 
@@ -374,7 +374,7 @@ class AccountAssetAsset(models.Model):
         vals = self.onchange_category_id_values(self.category_id.id)
         # We cannot use 'write' on an object that doesn't exist yet
         if vals:
-            for k, v in pycompat.items(vals['value']):
+            for k, v in vals['value'].items():
                 setattr(self, k, v)
 
     def onchange_category_id_values(self, category_id):
@@ -583,7 +583,7 @@ class AccountAssetDepreciationLine(models.Model):
             message = ''
             if message_description:
                 message = '<span>%s</span>' % message_description
-            for name, values in pycompat.items(tracked_values):
+            for name, values in tracked_values.items():
                 message += '<div> &nbsp; &nbsp; &bull; <b>%s</b>: ' % name
                 message += '%s</div>' % values
             return message

addons/account_asset/wizard/asset_modify.py

@@ -39,7 +39,7 @@ class AssetModify(models.TransientModel):
                 node = doc.xpath("//field[@name='method_number']")[0]
                 node.set('invisible', '1')
                 setup_modifiers(node, result['fields']['method_number'])
-            result['arch'] = etree.tostring(doc)
+            result['arch'] = etree.tostring(doc, encoding='unicode')
         return result
 
     @api.model

addons/account_voucher/models/account_voucher.py

@@ -5,7 +5,6 @@
 from odoo import fields, models, api, _
 from odoo.addons import decimal_precision as dp
 from odoo.exceptions import UserError
-from odoo.tools import pycompat
 
 
 class AccountVoucher(models.Model):
@@ -358,7 +357,7 @@ class AccountVoucherLine(models.Model):
             self.company_id.id,
             self.voucher_id.currency_id.id,
             self.voucher_id.voucher_type)
-        for fname, fvalue in pycompat.items(onchange_res['value']):
+        for fname, fvalue in onchange_res['value'].items():
             setattr(self, fname, fvalue)
 
     def _get_account(self, product, fpos, type):
@@ -394,7 +393,7 @@ class AccountVoucherLine(models.Model):
             if product.description_purchase:
                 values['name'] += '\n' + product.description_purchase
         else:
-            values['price_unit'] = product.lst_price
+            values['price_unit'] = price_unit or product.lst_price
             taxes = product.taxes_id or account.tax_ids
             if product.description_sale:
                 values['name'] += '\n' + product.description_sale
@@ -404,7 +403,7 @@ class AccountVoucherLine(models.Model):
         if company and currency:
             if company.currency_id != currency:
                 if type == 'purchase':
-                    values['price_unit'] = product.standard_price
+                    values['price_unit'] = price_unit or product.standard_price
                 values['price_unit'] = values['price_unit'] * currency.rate
 
         return {'value': values, 'domain': {}}

addons/anonymization/models/anonymization.py

@@ -6,6 +6,7 @@ from operator import itemgetter
 
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
+from odoo.tools import pycompat
 
 FIELD_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('not_existing', 'Not Existing'), ('new', 'New')]
 ANONYMIZATION_HISTORY_STATE = [('started', 'Started'), ('done', 'Done'), ('in_exception', 'Exception occured')]
@@ -13,7 +14,7 @@ ANONYMIZATION_DIRECTION = [('clear -> anonymized', 'clear -> anonymized'), ('ano
 
 
 def group(lst, cols):
-    if isinstance(cols, basestring):
+    if isinstance(cols, pycompat.string_types):
         cols = [cols]
     return dict((k, [v for v in itr]) for k, itr in groupby(sorted(lst, key=itemgetter(*cols)), itemgetter(*cols)))
 

addons/anonymization/wizard/anonymize_wizard.py

@@ -120,7 +120,7 @@ class IrModelFieldsAnonymizeWizard(models.TransientModel):
             else:
                 raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized,"
                                   " while some fields are not anonymized. You should try to solve this problem before trying to do anything else."))
-            res['arch'] = etree.tostring(eview)
+            res['arch'] = etree.tostring(eview, encoding='unicode')
         return res
 
     @api.multi

addons/auth_signup/controllers/main.py

@@ -8,7 +8,6 @@ from odoo.addons.auth_signup.models.res_users import SignupError
 from odoo.addons.web.controllers.main import ensure_db, Home
 from odoo.exceptions import UserError
 from odoo.http import request
-from odoo.tools import pycompat
 
 _logger = logging.getLogger(__name__)
 
@@ -106,7 +105,7 @@ class AuthSignupHome(Home):
             try:
                 # retrieve the user info (name, login or email) corresponding to a signup token
                 token_infos = request.env['res.partner'].sudo().signup_retrieve_info(qcontext.get('token'))
-                for k, v in pycompat.items(token_infos):
+                for k, v in token_infos.items():
                     qcontext.setdefault(k, v)
             except:
                 qcontext['error'] = _("Invalid signup token")

addons/auth_signup/models/res_partner.py

@@ -8,8 +8,6 @@ from collections import defaultdict
 from datetime import datetime, timedelta
 
 from odoo import api, exceptions, fields, models, _
-from odoo.tools import pycompat
-
 
 class SignupError(Exception):
     pass

addons/base_address_city/models/res_partner.py

@@ -35,5 +35,5 @@ class Partner(models.Model):
             city_id_node = etree.fromstring(replacement_xml)
             city_node.getparent().replace(city_node, city_id_node)
 
-        arch = etree.tostring(doc)
+        arch = etree.tostring(doc, encoding='unicode')
         return arch

addons/base_address_extended/models/base_address_extended.py

@@ -5,7 +5,6 @@ import re
 
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
-from odoo.tools import pycompat
 
 STREET_FIELDS = ('street_name', 'street_number', 'street_number2')
 
@@ -135,7 +134,7 @@ class Partner(models.Model):
                 vals[field_name] = street_raw
             # assign the values to the fields
             # /!\ Note that a write(vals) would cause a recursion since it would bypass the cache
-            for k, v in pycompat.items(vals):
+            for k, v in vals.items():
                 partner[k] = v
 
 

addons/base_automation/models/base_automation.py

@@ -12,7 +12,6 @@ from dateutil.relativedelta import relativedelta
 
 from odoo import api, fields, models, SUPERUSER_ID
 from odoo.modules.registry import Registry
-from odoo.tools import pycompat
 from odoo.tools.safe_eval import safe_eval
 
 _logger = logging.getLogger(__name__)
@@ -264,7 +263,7 @@ class BaseAutomation(models.Model):
                 if res:
                     if 'value' in res:
                         res['value'].pop('id', None)
-                        self.update({key: val for key, val in pycompat.items(res['value']) if key in self._fields})
+                        self.update({key: val for key, val in res['value'].items() if key in self._fields})
                     if 'domain' in res:
                         result.setdefault('domain', {}).update(res['domain'])
                     if 'warning' in res:

addons/base_gengo/wizard/base_gengo_translations.py

@@ -8,7 +8,6 @@ import uuid
 
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import UserError, ValidationError
-from odoo.tools import pycompat
 
 _logger = logging.getLogger(__name__)
 
@@ -194,7 +193,7 @@ class BaseGengoTranslations(models.TransientModel):
         term_ids.write(vals)
         jobs = response.get('jobs', [])
         if jobs:
-            for t_id, job in pycompat.items(jobs):
+            for t_id, job in jobs.items():
                 self._update_terms_job(job)
 
         return

addons/base_import/models/base_import.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import csv
 import datetime
 import io
 import itertools
@@ -17,12 +16,6 @@ from odoo.tools.mimetypes import guess_mimetype
 from odoo.tools.misc import ustr
 from odoo.tools import DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT, pycompat
 
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
-
 FIELDS_RECURSION_LIMIT = 2
 ERROR_PREVIEW_BYTES = 200
 _logger = logging.getLogger(__name__)
@@ -49,7 +42,7 @@ FILE_TYPE_DICT = {
 }
 EXTENSIONS = {
     '.' + ext: handler
-    for mime, (ext, handler, req) in pycompat.items(FILE_TYPE_DICT)
+    for mime, (ext, handler, req) in FILE_TYPE_DICT.items()
 }
 
 
@@ -121,7 +114,7 @@ class Import(models.TransientModel):
         }]
         model_fields = Model.fields_get()
         blacklist = models.MAGIC_COLUMNS + [Model.CONCURRENCY_CHECK_FIELD]
-        for name, field in pycompat.items(model_fields):
+        for name, field in model_fields.items():
             if name in blacklist:
                 continue
             # an empty string means the field is deprecated, @deprecated must
@@ -134,7 +127,7 @@ class Import(models.TransientModel):
                     continue
                 # states = {state: [(attr, value), (attr2, value2)], state2:...}
                 if not any(attr == 'readonly' and value is False
-                           for attr, value in itertools.chain.from_iterable(pycompat.values(states))):
+                           for attr, value in itertools.chain.from_iterable(states.values())):
                     continue
             field_value = {
                 'id': name,
@@ -214,9 +207,9 @@ class Import(models.TransientModel):
                 if cell.ctype is xlrd.XL_CELL_NUMBER:
                     is_float = cell.value % 1 != 0.0
                     values.append(
-                        unicode(cell.value)
+                        pycompat.text_type(cell.value)
                         if is_float
-                        else unicode(int(cell.value))
+                        else pycompat.text_type(int(cell.value))
                     )
                 elif cell.ctype is xlrd.XL_CELL_DATE:
                     is_datetime = cell.value % 1 != 0.0
@@ -268,14 +261,13 @@ class Import(models.TransientModel):
             # csv module expect utf-8, see http://docs.python.org/2/library/csv.html
             csv_data = csv_data.decode(encoding).encode('utf-8')
 
-        csv_iterator = csv.reader(
-            StringIO(csv_data),
+        csv_iterator = pycompat.csv_reader(
+            io.BytesIO(csv_data),
             quotechar=str(options['quoting']),
             delimiter=str(options['separator']))
 
         return (
-            [item.decode('utf-8') for item in row]
-            for row in csv_iterator
+            row for row in csv_iterator
             if any(x for x in row if x.strip())
         )
 
@@ -603,7 +595,7 @@ class Import(models.TransientModel):
     def _parse_import_data(self, data, import_fields, options):
         # Get fields of type date/datetime
         all_fields = self.env[self.res_model].fields_get()
-        for name, field in pycompat.items(all_fields):
+        for name, field in all_fields.items():
             if field['type'] in ('date', 'datetime') and name in import_fields:
                 # Parse date
                 index = import_fields.index(name)
@@ -611,11 +603,13 @@ class Import(models.TransientModel):
                 server_format = DEFAULT_SERVER_DATE_FORMAT if field['type'] == 'date' else DEFAULT_SERVER_DATETIME_FORMAT
 
                 if options.get('%s_format' % field['type'], server_format) != server_format:
-                    user_format = ustr(options.get('%s_format' % field['type'])).encode('utf-8')
+                    # datetime.str[fp]time takes *native strings* in both
+                    # versions, for both data and pattern
+                    user_format = pycompat.to_native(options.get('%s_format' % field['type']))
                     for num, line in enumerate(data):
                         if line[index]:
                             try:
-                                line[index] = dt.strftime(dt.strptime(ustr(line[index].strip()).encode('utf-8'), user_format), server_format)
+                                line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index].strip()), user_format), server_format)
                             except ValueError as e:
                                 raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e))
                             except Exception as e:
@@ -659,12 +653,18 @@ class Import(models.TransientModel):
         except ValueError as error:
             return [{
                 'type': 'error',
-                'message': unicode(error),
+                'message': pycompat.text_type(error),
                 'record': False,
             }]
 
         _logger.info('importing %d rows...', len(data))
-        import_result = self.env[self.res_model].with_context(import_file=True).load(import_fields, data)
+
+        model = self.env[self.res_model].with_context(import_file=True)
+        defer_parent_store = self.env.context.get('defer_parent_store_computation', True)
+        if defer_parent_store and model._parent_store:
+            model = model.with_context(defer_parent_store_computation=True)
+        
+        import_result = model.load(import_fields, data)
         _logger.info('done')
 
         # If transaction aborted, RELEASE SAVEPOINT is going to raise

addons/base_import/models/odf_ods_reader.py

@@ -19,8 +19,6 @@ from odf import opendocument
 from odf.table import Table, TableRow, TableCell
 from odf.text import P
 
-from odoo.tools import pycompat
-
 
 class ODSReader(object):
 
@@ -97,4 +95,4 @@ class ODSReader(object):
         return self.SHEETS[name]
 
     def getFirstSheet(self):
-        return next(iter(pycompat.values(self.SHEETS)))
+        return next(iter(self.SHEETS.values()))

addons/base_import/tests/test_base_import.py

@@ -1,13 +1,12 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
-import csv
+import base64
 import io
 import unittest
 
 from odoo.tests.common import TransactionCase, can_import
 from odoo.modules.module import get_module_resource
-from odoo.tools import mute_logger
+from odoo.tools import mute_logger, pycompat
 
 ID_FIELD = {
     'id': 'id',
@@ -231,10 +230,10 @@ class TestPreview(TransactionCase):
     def test_csv_success(self):
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'base_import.tests.models.preview',
-            'file': 'name,Some Value,Counter\n'
-                    'foo,1,2\n'
-                    'bar,3,4\n'
-                    'qux,5,6\n',
+            'file': b'name,Some Value,Counter\n'
+                    b'foo,1,2\n'
+                    b'bar,3,4\n'
+                    b'qux,5,6\n',
             'file_type': 'text/csv'
         })
 
@@ -359,10 +358,10 @@ class test_convert_import_data(TransactionCase):
     def test_all(self):
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'base_import.tests.models.preview',
-            'file': 'name,Some Value,Counter\n'
-                    'foo,1,2\n'
-                    'bar,3,4\n'
-                    'qux,5,6\n',
+            'file': b'name,Some Value,Counter\n'
+                    b'foo,1,2\n'
+                    b'bar,3,4\n'
+                    b'qux,5,6\n',
             'file_type': 'text/csv'
 
         })
@@ -381,8 +380,8 @@ class test_convert_import_data(TransactionCase):
     def test_date_fields(self):
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'res.partner',
-            'file': 'name,date,create_date\n'
-                    '"foo","2013年07月18日","2016-10-12 06:06"\n',
+            'file': u'name,date,create_date\n'
+                    u'"foo","2013年07月18日","2016-10-12 06:06"\n'.encode('utf-8'),
             'file_type': 'text/csv'
 
         })
@@ -409,10 +408,10 @@ class test_convert_import_data(TransactionCase):
         """
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'base_import.tests.models.preview',
-            'file': 'name,Some Value,Counter\n'
-                    'foo,1,2\n'
-                    'bar,3,4\n'
-                    'qux,5,6\n',
+            'file': b'name,Some Value,Counter\n'
+                    b'foo,1,2\n'
+                    b'bar,3,4\n'
+                    b'qux,5,6\n',
             'file_type': 'text/csv'
         })
         data, fields = import_wizard._convert_import_data(
@@ -433,10 +432,10 @@ class test_convert_import_data(TransactionCase):
         """
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'base_import.tests.models.preview',
-            'file': 'name,Some Value,Counter\n'
-                    'foo,1,2\n'
-                    ',3,\n'
-                    ',5,6\n',
+            'file': b'name,Some Value,Counter\n'
+                    b'foo,1,2\n'
+                    b',3,\n'
+                    b',5,6\n',
             'file_type': 'text/csv'
         })
         data, fields = import_wizard._convert_import_data(
@@ -453,12 +452,12 @@ class test_convert_import_data(TransactionCase):
     def test_empty_rows(self):
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'base_import.tests.models.preview',
-            'file': 'name,Some Value\n'
-                    'foo,1\n'
-                    '\n'
-                    'bar,2\n'
-                    '     \n'
-                    '\t \n',
+            'file': b'name,Some Value\n'
+                    b'foo,1\n'
+                    b'\n'
+                    b'bar,2\n'
+                    b'     \n'
+                    b'\t \n',
             'file_type': 'text/csv'
         })
         data, fields = import_wizard._convert_import_data(
@@ -475,8 +474,8 @@ class test_convert_import_data(TransactionCase):
     def test_nofield(self):
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'base_import.tests.models.preview',
-            'file': 'name,Some Value,Counter\n'
-                    'foo,1,2\n',
+            'file': b'name,Some Value,Counter\n'
+                    b'foo,1,2\n',
             'file_type': 'text/csv'
 
         })
@@ -485,8 +484,8 @@ class test_convert_import_data(TransactionCase):
     def test_falsefields(self):
         import_wizard = self.env['base_import.import'].create({
             'res_model': 'base_import.tests.models.preview',
-            'file': 'name,Some Value,Counter\n'
-                    'foo,1,2\n',
+            'file': b'name,Some Value,Counter\n'
+                    b'foo,1,2\n',
             'file_type': 'text/csv'
         })
 
@@ -501,11 +500,11 @@ class test_convert_import_data(TransactionCase):
         Ensure importing keep newlines
         """
         output = io.BytesIO()
-        writer = csv.writer(output, quoting=csv.QUOTE_ALL)
+        writer = pycompat.csv_writer(output, quoting=1)
 
-        data_row = ["\tfoo\n\tbar", " \"hello\" \n\n 'world' "]
+        data_row = [u"\tfoo\n\tbar", u" \"hello\" \n\n 'world' "]
 
-        writer.writerow(["name", "Some Value"])
+        writer.writerow([u"name", u"Some Value"])
         writer.writerow(data_row)
 
         import_wizard = self.env['base_import.import'].create({
@@ -527,16 +526,15 @@ class test_failures(TransactionCase):
         Ensure big fields (e.g. b64-encoded image data) can be imported and
         we're not hitting limits of the default CSV parser config
         """
-        import csv, cStringIO
         from PIL import Image
 
         im = Image.new('RGB', (1920, 1080))
-        fout = cStringIO.StringIO()
+        fout = io.BytesIO()
 
-        writer = csv.writer(fout, dialect=None)
+        writer = pycompat.csv_writer(fout, dialect=None)
         writer.writerows([
-            ['name', 'db_datas'],
-            ['foo', im.tobytes().encode('base64')]
+            [u'name', u'db_datas'],
+            [u'foo', base64.b64encode(im.tobytes()).decode('ascii')]
         ])
 
         import_wizard = self.env['base_import.import'].create({

addons/base_import_module/models/ir_module.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import base64
 import logging
 import os
 import sys
@@ -8,7 +9,7 @@ from os.path import join as opj
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
 from odoo.modules import load_information_from_description_file
-from odoo.tools import convert_file, exception_to_unicode, pycompat
+from odoo.tools import convert_file, exception_to_unicode
 from odoo.tools.osutil import tempdir
 
 _logger = logging.getLogger(__name__)
@@ -63,8 +64,8 @@ class IrModule(models.Model):
             for root, dirs, files in os.walk(path_static):
                 for static_file in files:
                     full_path = opj(root, static_file)
-                    with open(full_path, 'r') as fp:
-                        data = fp.read().encode('base64')
+                    with open(full_path, 'rb') as fp:
+                        data = base64.b64encode(fp.read())
                     url_path = '/%s%s' % (module, full_path.split(path)[1].replace(os.path.sep, '/'))
                     url_path = url_path.decode(sys.getfilesystemencoding())
                     filename = os.path.split(url_path)[1]
@@ -118,6 +119,6 @@ class IrModule(models.Model):
                 finally:
                     addons.module.ad_paths.remove(module_dir)
         r = ["Successfully imported module '%s'" % mod for mod in success]
-        for mod, error in pycompat.items(errors):
+        for mod, error in errors.items():
             r.append("Error while importing module '%s': %r" % (mod, error))
         return '\n'.join(r), module_names

addons/board/controllers/main.py

@@ -29,7 +29,7 @@ class Board(Controller):
                         'domain': str(domain)
                     })
                     column.insert(0, new_action)
-                    arch = ElementTree.tostring(xml, 'utf-8')
+                    arch = ElementTree.tostring(xml, encoding='unicode')
                     request.env['ir.ui.view.custom'].create({
                         'user_id': request.session.uid,
                         'ref_id': view_id,

addons/board/models/board.py

@@ -2,6 +2,7 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from odoo import api, models
+from odoo.tools import pycompat
 
 
 class Board(models.AbstractModel):
@@ -41,13 +42,8 @@ class Board(models.AbstractModel):
                 if child.tag == 'action' and child.get('invisible'):
                     node.remove(child)
                 else:
-                    child = remove_unauthorized_children(child)
+                    remove_unauthorized_children(child)
             return node
 
-        def encode(s):
-            if isinstance(s, unicode):
-                return s.encode('utf8')
-            return s
-
-        archnode = etree.fromstring(encode(arch))
-        return etree.tostring(remove_unauthorized_children(archnode), pretty_print=True)
+        archnode = etree.fromstring(arch)
+        return etree.tostring(remove_unauthorized_children(archnode), pretty_print=True, encoding='unicode')

addons/bus/controllers/main.py

@@ -4,6 +4,8 @@ from odoo import exceptions
 from odoo.http import Controller, request, route
 from odoo.addons.bus.models.bus import dispatch
 
+from odoo.tools import pycompat
+
 
 class BusController(Controller):
     """ Examples:
@@ -14,7 +16,7 @@ class BusController(Controller):
 
     @route('/longpolling/send', type="json", auth="public")
     def send(self, channel, message):
-        if not isinstance(channel, basestring):
+        if not isinstance(channel, pycompat.string_types):
             raise Exception("bus.Bus only string channels are allowed.")
         return request.env['bus.bus'].sendone(channel, message)
 
@@ -33,7 +35,7 @@ class BusController(Controller):
             options = {}
         if not dispatch:
             raise Exception("bus.Bus unavailable")
-        if [c for c in channels if not isinstance(c, basestring)]:
+        if [c for c in channels if not isinstance(c, pycompat.string_types)]:
             raise Exception("bus.Bus only string channels are allowed.")
         if request.registry.in_test_mode():
             raise exceptions.UserError("bus.Bus not available in test mode")

addons/calendar/models/calendar.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
+import base64
 
 import babel.dates
 import collections
@@ -33,7 +34,7 @@ def calendar_id2real_id(calendar_id=None, with_date=False):
         :param with_date: if a value is passed to this param it will return dates based on value of withdate + calendar_id
         :return: real event id
     """
-    if calendar_id and isinstance(calendar_id, (basestring)):
+    if calendar_id and isinstance(calendar_id, pycompat.string_types):
         res = [bit for bit in calendar_id.split('-') if bit]
         if len(res) == 2:
             real_id = res[0]
@@ -47,7 +48,7 @@ def calendar_id2real_id(calendar_id=None, with_date=False):
 
 
 def get_real_ids(ids):
-    if isinstance(ids, (basestring, pycompat.integer_types)):
+    if isinstance(ids, (pycompat.string_types, pycompat.integer_types)):
         return calendar_id2real_id(ids)
 
     if isinstance(ids, (list, tuple)):
@@ -173,6 +174,7 @@ class Attendee(models.Model):
         mails_to_send = self.env['mail.mail']
         for attendee in self:
             if attendee.email or attendee.partner_id.email:
+                # FIXME: is ics_file text or bytes?
                 ics_file = ics_files.get(attendee.event_id.id)
                 mail_id = invitation_template.send_mail(attendee.id)
 
@@ -181,7 +183,7 @@ class Attendee(models.Model):
                     vals['attachment_ids'] = [(0, 0, {'name': 'invitation.ics',
                                                       'mimetype': 'text/calendar',
                                                       'datas_fname': 'invitation.ics',
-                                                      'datas': str(ics_file).encode('base64')})]
+                                                      'datas': base64.b64encode(ics_file)})]
                 vals['model'] = None  # We don't want to have the mail in the tchatter while in queue!
                 vals['res_id'] = False
                 current_mail = self.env['mail.mail'].browse(mail_id)
@@ -468,7 +470,7 @@ class Alarm(models.Model):
     name = fields.Char('Name', required=True)
     type = fields.Selection([('notification', 'Notification'), ('email', 'Email')], 'Type', required=True, default='email')
     duration = fields.Integer('Remind Before', required=True, default=1)
-    interval = fields.Selection(list(pycompat.items(_interval_selection)), 'Unit', required=True, default='hours')
+    interval = fields.Selection(list(_interval_selection.items()), 'Unit', required=True, default='hours')
     duration_minutes = fields.Integer('Duration in minutes', compute='_compute_duration_minutes', store=True, help="Duration in minutes")
 
     @api.onchange('duration', 'interval')
@@ -630,8 +632,8 @@ class Meeting(models.Model):
             }
 
         # formats will be used for str{f,p}time() which do not support unicode in Python 2, coerce to str
-        format_date = lang_params.get("date_format", '%B-%d-%Y').encode('utf-8')
-        format_time = lang_params.get("time_format", '%I-%M %p').encode('utf-8')
+        format_date = pycompat.to_native(lang_params.get("date_format", '%B-%d-%Y'))
+        format_time = pycompat.to_native(lang_params.get("time_format", '%I-%M %p'))
         return (format_date, format_time)
 
     @api.model
@@ -917,11 +919,11 @@ class Meeting(models.Model):
                     elif interval == 'minutes':
                         delta = timedelta(minutes=duration)
                     trigger.value = delta
-                    valarm.add('DESCRIPTION').value = alarm.name or 'Odoo'
+                    valarm.add('DESCRIPTION').value = alarm.name or u'Odoo'
             for attendee in meeting.attendee_ids:
                 attendee_add = event.add('attendee')
-                attendee_add.value = 'MAILTO:' + (attendee.email or '')
-            result[meeting.id] = cal.serialize()
+                attendee_add.value = u'MAILTO:' + (attendee.email or u'')
+            result[meeting.id] = cal.serialize().encode('utf-8')
 
         return result
 
@@ -1058,7 +1060,7 @@ class Meeting(models.Model):
                 pile.reverse()
                 new_pile = []
                 for item in pile:
-                    if not isinstance(item, basestring):
+                    if not isinstance(item, pycompat.string_types):
                         res = item
                     elif str(item) == str('&'):
                         first = new_pile.pop()
@@ -1207,7 +1209,7 @@ class Meeting(models.Model):
 
         if interval == 'day':
             # Day number (1-31)
-            result = unicode(date.day)
+            result = pycompat.text_type(date.day)
 
         elif interval == 'month':
             # Localized month name and year
@@ -1219,6 +1221,7 @@ class Meeting(models.Model):
 
         elif interval == 'time':
             # Localized time
+            # FIXME: formats are specifically encoded to bytes, maybe use babel?
             dummy, format_time = self._get_date_formats()
             result = tools.ustr(date.strftime(format_time + " %Z"))
 
@@ -1289,7 +1292,7 @@ class Meeting(models.Model):
     @api.multi
     def _get_message_unread(self):
         id_map = {x: calendar_id2real_id(x) for x in self.ids}
-        real = self.browse(set(pycompat.values(id_map)))
+        real = self.browse(set(id_map.values()))
         super(Meeting, real)._get_message_unread()
         for event in self:
             if event.id == id_map[event.id]:
@@ -1301,7 +1304,7 @@ class Meeting(models.Model):
     @api.multi
     def _get_message_needaction(self):
         id_map = {x: calendar_id2real_id(x) for x in self.ids}
-        real = self.browse(set(pycompat.values(id_map)))
+        real = self.browse(set(id_map.values()))
         super(Meeting, real)._get_message_needaction()
         for event in self:
             if event.id == id_map[event.id]:
@@ -1314,7 +1317,7 @@ class Meeting(models.Model):
     @api.returns('self', lambda value: value.id)
     def message_post(self, **kwargs):
         thread_id = self.id
-        if isinstance(self.id, basestring):
+        if isinstance(self.id, pycompat.string_types):
             thread_id = get_real_ids(self.id)
         if self.env.context.get('default_date'):
             context = dict(self.env.context)
@@ -1350,7 +1353,7 @@ class Meeting(models.Model):
         for arg in args:
             if arg[0] == 'id':
                 for n, calendar_id in enumerate(arg[2]):
-                    if isinstance(calendar_id, basestring):
+                    if isinstance(calendar_id, pycompat.string_types):
                         arg[2][n] = calendar_id.split('-')[0]
         return super(Meeting, self)._name_search(name=name, args=args, operator=operator, limit=limit, name_get_uid=name_get_uid)
 
@@ -1388,7 +1391,7 @@ class Meeting(models.Model):
             # set end_date for calendar searching
             if any(field in values for field in ['recurrency', 'end_type', 'count', 'rrule_type', 'start', 'stop']):
                 for real_meeting in real_meetings:
-                    if real_meeting.recurrency and real_meeting.end_type in ('count', unicode('count')):
+                    if real_meeting.recurrency and real_meeting.end_type == u'count':
                         final_date = real_meeting._get_recurrency_end_date()
                         super(Meeting, real_meeting).write({'final_date': final_date})
 
@@ -1472,7 +1475,7 @@ class Meeting(models.Model):
         for calendar_id, real_id in select:
             res = real_data[real_id].copy()
             ls = calendar_id2real_id(calendar_id, with_date=res and res.get('duration', 0) > 0 and res.get('duration') or 1)
-            if not isinstance(ls, (basestring, pycompat.integer_types)) and len(ls) >= 2:
+            if not isinstance(ls, (pycompat.string_types, pycompat.integer_types)) and len(ls) >= 2:
                 res['start'] = ls[1]
                 res['stop'] = ls[2]
 

addons/calendar/models/ir_attachment.py

@@ -5,6 +5,8 @@ from odoo import api, models
 
 from odoo.addons.calendar.models.calendar import get_real_ids
 
+from odoo.tools import pycompat
+
 
 class Attachment(models.Model):
 
@@ -16,13 +18,13 @@ class Attachment(models.Model):
         args = list(args)
         if any([leaf for leaf in args if leaf[0] == "res_model" and leaf[2] == 'calendar.event']):
             for index in range(len(args)):
-                if args[index][0] == "res_id" and isinstance(args[index][2], basestring):
+                if args[index][0] == "res_id" and isinstance(args[index][2], pycompat.string_types):
                     args[index] = (args[index][0], args[index][1], get_real_ids(args[index][2]))
         return super(Attachment, self).search(args, offset=offset, limit=limit, order=order, count=count)
 
     @api.multi
     def write(self, vals):
         """ When posting an attachment (new or not), convert the virtual ids in real ids. """
-        if isinstance(vals.get('res_id'), basestring):
+        if isinstance(vals.get('res_id'), pycompat.string_types):
             vals['res_id'] = get_real_ids(vals.get('res_id'))
         return super(Attachment, self).write(vals)

addons/calendar/models/mail_message.py

@@ -5,6 +5,8 @@ from odoo import api, models
 
 from odoo.addons.calendar.models.calendar import get_real_ids
 
+from odoo.tools import pycompat
+
 
 class Message(models.Model):
 
@@ -16,7 +18,7 @@ class Message(models.Model):
         args = list(args)
         for index in range(len(args)):
             if args[index][0] == "res_id":
-                if isinstance(args[index][2], basestring):
+                if isinstance(args[index][2], pycompat.string_types):
                     args[index] = (args[index][0], args[index][1], get_real_ids(args[index][2]))
                 elif isinstance(args[index][2], list):
                     args[index] = (args[index][0], args[index][1], [get_real_ids(x) for x in args[index][2]])

addons/crm/wizard/base_partner_merge.py

@@ -2,56 +2,18 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from ast import literal_eval
-from email.utils import parseaddr
 import functools
-try:
-    from html.entities import entitydefs
-except ImportError:
-    from htmlentitydefs import entitydefs # pylint: disable=deprecated-module
 import itertools
 import logging
-import operator
 import psycopg2
-import re
-
-# Validation Library https://pypi.python.org/pypi/validate_email/1.1
-from .validate_email import validate_email
 
 from odoo import api, fields, models
 from odoo import SUPERUSER_ID, _
 from odoo.exceptions import ValidationError, UserError
-from odoo.tools import mute_logger, pycompat
+from odoo.tools import mute_logger
 
 _logger = logging.getLogger('base.partner.merge')
 
-
-# http://www.php2python.com/wiki/function.html-entity-decode/
-def html_entity_decode_char(m, defs=entitydefs):
-    try:
-        return defs[m.group(1)]
-    except KeyError:
-        return m.group(0)
-
-
-def html_entity_decode(string):
-    pattern = re.compile("&(\w+?);")
-    return pattern.sub(html_entity_decode_char, string)
-
-
-def sanitize_email(email):
-    assert isinstance(email, basestring) and email
-
-    result = re.subn(r';|/|:', ',', html_entity_decode(email or ''))[0].split(',')
-
-    emails = [parseaddr(email)[1]
-              for item in result
-              for email in item.split()]
-
-    return [email.lower()
-            for email in emails
-            if validate_email(email)]
-
-
 class MergePartnerLine(models.TransientModel):
 
     _name = 'base.partner.merge.line'
@@ -271,7 +233,7 @@ class MergePartnerAutomatic(models.TransientModel):
                 return item
         # get all fields that are not computed or x2many
         values = dict()
-        for column, field in pycompat.items(model_fields):
+        for column, field in model_fields.items():
             if field.type not in ('many2many', 'one2many') and field.compute is None:
                 for item in itertools.chain(src_partners, [dst_partner]):
                     if item[column]:
@@ -409,7 +371,7 @@ class MergePartnerAutomatic(models.TransientModel):
         """
         return any(
             self.env[model].search_count([(field, 'in', aggr_ids)])
-            for model, field in pycompat.items(models)
+            for model, field in models.items()
         )
 
     @api.model

addons/crm/wizard/validate_email.py

@@ -1,123 +0,0 @@
-# RFC 2822 - style email validation for Python
-# (c) 2012 Syrus Akbary <me@syrusakbary.com>
-# Extended from (c) 2011 Noel Bush <noel@aitools.org>
-# for support of mx and user check
-# This code is made available to you under the GNU LGPL v3.
-#
-# This module provides a single method, valid_email_address(),
-# which returns True or False to indicate whether a given address
-# is valid according to the 'addr-spec' part of the specification
-# given in RFC 2822.  Ideally, we would like to find this
-# in some other library, already thoroughly tested and well-
-# maintained.  The standard Python library email.utils
-# contains a parse_addr() function, but it is not sufficient
-# to detect many malformed addresses.
-#
-# This implementation aims to be faithful to the RFC, with the
-# exception of a circular definition (see comments below), and
-# with the omission of the pattern components marked as "obsolete".
-
-import re
-import smtplib
-import socket
-
-try:
-    import DNS
-    ServerError = DNS.ServerError
-except:
-    DNS = None
-    class ServerError(Exception): pass
-# All we are really doing is comparing the input string to one
-# gigantic regular expression.  But building that regexp, and
-# ensuring its correctness, is made much easier by assembling it
-# from the "tokens" defined by the RFC.  Each of these tokens is
-# tested in the accompanying unit test file.
-#
-# The section of RFC 2822 from which each pattern component is
-# derived is given in an accompanying comment.
-#
-# (To make things simple, every string below is given as 'raw',
-# even when it's not strictly necessary.  This way we don't forget
-# when it is necessary.)
-#
-WSP = r'[ \t]'                                       # see 2.2.2. Structured Header Field Bodies
-CRLF = r'(?:\r\n)'                                   # see 2.2.3. Long Header Fields
-NO_WS_CTL = r'\x01-\x08\x0b\x0c\x0f-\x1f\x7f'        # see 3.2.1. Primitive Tokens
-QUOTED_PAIR = r'(?:\\.)'                             # see 3.2.2. Quoted characters
-FWS = r'(?:(?:' + WSP + r'*' + CRLF + r')?' + \
-            WSP + r'+)'                                    # see 3.2.3. Folding white space and comments
-CTEXT = r'[' + NO_WS_CTL + \
-                r'\x21-\x27\x2a-\x5b\x5d-\x7e]'              # see 3.2.3
-CCONTENT = r'(?:' + CTEXT + r'|' + \
-                     QUOTED_PAIR + r')'                        # see 3.2.3 (NB: The RFC includes COMMENT here
-                                                                                                         # as well, but that would be circular.)
-COMMENT = r'\((?:' + FWS + r'?' + CCONTENT + \
-                    r')*' + FWS + r'?\)'                       # see 3.2.3
-CFWS = r'(?:' + FWS + r'?' + COMMENT + ')*(?:' + \
-             FWS + '?' + COMMENT + '|' + FWS + ')'         # see 3.2.3
-ATEXT = r'[\w!#$%&\'\*\+\-/=\?\^`\{\|\}~]'           # see 3.2.4. Atom
-ATOM = CFWS + r'?' + ATEXT + r'+' + CFWS + r'?'      # see 3.2.4
-DOT_ATOM_TEXT = ATEXT + r'+(?:\.' + ATEXT + r'+)*'   # see 3.2.4
-DOT_ATOM = CFWS + r'?' + DOT_ATOM_TEXT + CFWS + r'?' # see 3.2.4
-QTEXT = r'[' + NO_WS_CTL + \
-                r'\x21\x23-\x5b\x5d-\x7e]'                   # see 3.2.5. Quoted strings
-QCONTENT = r'(?:' + QTEXT + r'|' + \
-                     QUOTED_PAIR + r')'                        # see 3.2.5
-QUOTED_STRING = CFWS + r'?' + r'"(?:' + FWS + \
-                                r'?' + QCONTENT + r')*' + FWS + \
-                                r'?' + r'"' + CFWS + r'?'
-LOCAL_PART = r'(?:' + DOT_ATOM + r'|' + \
-                         QUOTED_STRING + r')'                    # see 3.4.1. Addr-spec specification
-DTEXT = r'[' + NO_WS_CTL + r'\x21-\x5a\x5e-\x7e]'    # see 3.4.1
-DCONTENT = r'(?:' + DTEXT + r'|' + \
-                     QUOTED_PAIR + r')'                        # see 3.4.1
-DOMAIN_LITERAL = CFWS + r'?' + r'\[' + \
-                                 r'(?:' + FWS + r'?' + DCONTENT + \
-                                 r')*' + FWS + r'?\]' + CFWS + r'?'  # see 3.4.1
-DOMAIN = r'(?:' + DOT_ATOM + r'|' + \
-                 DOMAIN_LITERAL + r')'                       # see 3.4.1
-ADDR_SPEC = LOCAL_PART + r'@' + DOMAIN               # see 3.4.1
-
-# A valid address will match exactly the 3.4.1 addr-spec.
-VALID_ADDRESS_REGEXP = '^' + ADDR_SPEC + '$'
-
-def validate_email(email, check_mx=False,verify=False):
-
-    """Indicate whether the given string is a valid email address
-    according to the 'addr-spec' portion of RFC 2822 (see section
-    3.4.1).  Parts of the spec that are marked obsolete are *not*
-    included in this test, and certain arcane constructions that
-    depend on circular definitions in the spec may not pass, but in
-    general this should correctly identify any email address likely
-    to be in use as of 2011."""
-    try:
-        assert re.match(VALID_ADDRESS_REGEXP, email) is not None
-        check_mx |= verify
-        if check_mx:
-            if not DNS: raise Exception('For check the mx records or check if the email exists you must have installed pyDNS python package')
-            DNS.DiscoverNameServers()
-            hostname = email[email.find('@')+1:]
-            mx_hosts = DNS.mxlookup(hostname)
-            for mx in mx_hosts:
-                try:
-                    smtp = smtplib.SMTP()
-                    smtp.connect(mx[1])
-                    if not verify: return True
-                    status, _ = smtp.helo()
-                    if status != 250: continue
-                    smtp.mail('')
-                    status, _ = smtp.rcpt(email)
-                    if status != 250: return False
-                    break
-                except smtplib.SMTPServerDisconnected: #Server not permits verify user
-                    break
-                except smtplib.SMTPConnectError:
-                    continue
-    except (AssertionError, ServerError): 
-        return False
-    return True
-
-# import sys
-
-# sys.modules[__name__],sys.modules['validate_email_module'] = validate_email,sys.modules[__name__]
-# from validate_email_module import *

addons/document/models/ir_attachment.py

@@ -1,30 +1,16 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
+import io
 import logging
 import PyPDF2
 import xml.dom.minidom
 import zipfile
 
-from StringIO import StringIO
-
 from odoo import api, models
 
 _logger = logging.getLogger(__name__)
 FTYPES = ['docx', 'pptx', 'xlsx', 'opendoc', 'pdf']
 
-# Keep function in case it is necessary to do toUnicode(buf.encode('ascii', 'replace'))
-def toUnicode(s):
-    try:
-        return s.decode('utf-8')
-    except UnicodeError:
-        try:
-            return s.decode('latin')
-        except UnicodeError:
-            try:
-                return s.encode('ascii')
-            except UnicodeError:
-                return s
-
 def textToString(element):
     buff = u""
     for node in element.childNodes:
@@ -41,7 +27,7 @@ class IrAttachment(models.Model):
     def _index_docx(self, bin_data):
         '''Index Microsoft .docx documents'''
         buf = u""
-        f = StringIO(bin_data)
+        f = io.BytesIO(bin_data)
         if zipfile.is_zipfile(f):
             try:
                 zf = zipfile.ZipFile(f)
@@ -57,7 +43,7 @@ class IrAttachment(models.Model):
         '''Index Microsoft .pptx documents'''
 
         buf = u""
-        f = StringIO(bin_data)
+        f = io.BytesIO(bin_data)
         if zipfile.is_zipfile(f):
             try:
                 zf = zipfile.ZipFile(f)
@@ -75,7 +61,7 @@ class IrAttachment(models.Model):
         '''Index Microsoft .xlsx documents'''
 
         buf = u""
-        f = StringIO(bin_data)
+        f = io.BytesIO(bin_data)
         if zipfile.is_zipfile(f):
             try:
                 zf = zipfile.ZipFile(f)
@@ -91,7 +77,7 @@ class IrAttachment(models.Model):
         '''Index OpenDocument documents (.odt, .ods...)'''
 
         buf = u""
-        f = StringIO(bin_data)
+        f = io.BytesIO(bin_data)
         if zipfile.is_zipfile(f):
             try:
                 zf = zipfile.ZipFile(f)
@@ -107,8 +93,8 @@ class IrAttachment(models.Model):
         '''Index PDF documents'''
 
         buf = u""
-        if bin_data.startswith('%PDF-'):
-            f = StringIO(bin_data)
+        if bin_data.startswith(b'%PDF-'):
+            f = io.BytesIO(bin_data)
             try:
                 pdf = PyPDF2.PdfFileReader(f, overwriteWarnings=False)
                 for page in pdf.pages:

addons/event/models/event.py

@@ -5,7 +5,6 @@ import pytz
 from odoo import _, api, fields, models
 from odoo.addons.mail.models.mail_template import format_tz
 from odoo.exceptions import AccessError, UserError, ValidationError
-from odoo.tools import pycompat
 from odoo.tools.translate import html_translate
 
 from dateutil.relativedelta import relativedelta
@@ -389,7 +388,7 @@ class EventRegistration(models.Model):
             'partner_id': partner_id.id,
             'event_id': event_id and event_id.id or False,
         }
-        data.update({key: value for key, value in pycompat.items(registration) if key in self._fields})
+        data.update({key: value for key, value in registration.items() if key in self._fields})
         return data
 
     @api.one

addons/fleet/models/fleet_vehicle_cost.py

@@ -6,8 +6,6 @@ from odoo.exceptions import UserError
 
 from dateutil.relativedelta import relativedelta
 
-from odoo.tools import pycompat
-
 
 class FleetVehicleCost(models.Model):
     _name = 'fleet.vehicle.cost'
@@ -272,7 +270,7 @@ class FleetVehicleLogContract(models.Model):
                 res[contract.vehicle_id.id] = 1
 
         Vehicle = self.env['fleet.vehicle']
-        for vehicle, value in pycompat.items(res):
+        for vehicle, value in res.items():
             Vehicle.browse(vehicle).message_post(body=_('%s contract(s) will expire soon and should be renewed and/or closed!') % value)
         nearly_expired_contracts.write({'state': 'diesoon'})
 

addons/gamification/models/challenge.py

@@ -503,8 +503,10 @@ class Challenge(models.Model):
 
             line_data['own_goal_id'] = False,
             line_data['goals'] = []
-
-            goals = Goals.search(domain, order="completeness desc, current desc")
+            if line.condition=='higher':
+                goals = Goals.search(domain, order="completeness desc, current desc")
+            else:
+                goals = Goals.search(domain, order="completeness desc, current asc")
             for ranking, goal in enumerate(goals):
                 if user and goal.user_id == user:
                     line_data['own_goal_id'] = goal.id

addons/gamification/models/goal.py

@@ -262,7 +262,7 @@ class Goal(models.Model):
         for goal in self:
             goals_by_definition.setdefault(goal.definition_id, []).append(goal)
 
-        for definition, goals in pycompat.items(goals_by_definition):
+        for definition, goals in goals_by_definition.items():
             goals_to_write = {}
             if definition.computation_mode == 'manually':
                 for goal in goals:
@@ -307,9 +307,9 @@ class Goal(models.Model):
                         subqueries.setdefault((start_date, end_date), {}).update({goal.id:safe_eval(definition.batch_user_expression, {'user': goal.user_id})})
 
                     # the global query should be split by time periods (especially for recurrent goals)
-                    for (start_date, end_date), query_goals in pycompat.items(subqueries):
+                    for (start_date, end_date), query_goals in subqueries.items():
                         subquery_domain = list(general_domain)
-                        subquery_domain.append((field_name, 'in', list(set(pycompat.values(query_goals)))))
+                        subquery_domain.append((field_name, 'in', list(set(query_goals.values()))))
                         if start_date:
                             subquery_domain.append((field_date_name, '>=', start_date))
                         if end_date:
@@ -353,7 +353,7 @@ class Goal(models.Model):
 
                         goals_to_write.update(goal._get_write_values(new_value))
 
-            for goal, values in pycompat.items(goals_to_write):
+            for goal, values in goals_to_write.items():
                 if not values:
                     continue
                 goal.write(values)

addons/google_account/models/google_service.py

@@ -9,6 +9,7 @@ import requests
 from werkzeug import urls
 
 from odoo import api, fields, models, registry, _
+from odoo.exceptions import UserError
 from odoo.http import request
 
 
@@ -125,6 +126,9 @@ class GoogleService(models.TransientModel):
         client_id = get_param('google_%s_client_id' % (service,), default=False)
         client_secret = get_param('google_%s_client_secret' % (service,), default=False)
 
+        if not client_id or not client_secret:
+            raise UserError(_("The account for the Google service '%s' is not configured") % service)
+
         headers = {"content-type": "application/x-www-form-urlencoded"}
         data = {
             'refresh_token': refresh_token,

addons/google_calendar/models/google_calendar.py

@@ -12,7 +12,7 @@ import pytz
 from werkzeug import urls
 
 from odoo import api, fields, models, tools, _
-from odoo.tools import exception_to_unicode, pycompat
+from odoo.tools import exception_to_unicode
 
 _logger = logging.getLogger(__name__)
 
@@ -25,15 +25,13 @@ class Meta(type):
     """ This Meta class allow to define class as a structure, and so instancied variable
         in __init__ to avoid to have side effect alike 'static' variable """
     def __new__(typ, name, parents, attrs):
-        methods = {k: v for k, v in pycompat.items(attrs)
-                       if callable(v)}
-        attrs = {k: v for k, v in pycompat.items(attrs)
-                     if not callable(v)}
+        methods = {k: v for k, v in attrs.items() if callable(v)}
+        attrs = {k: v for k, v in attrs.items() if not callable(v)}
 
         def init(self, **kw):
-            for key, val in pycompat.items(attrs):
+            for key, val in attrs.items():
                 setattr(self, key, val)
-            for key, val in pycompat.items(kw):
+            for key, val in kw.items():
                 assert key in attrs
                 setattr(self, key, val)
 
@@ -161,7 +159,7 @@ class SyncOperation(object):
     def __init__(self, src, info, **kw):
         self.src = src
         self.info = info
-        for key, val in pycompat.items(kw):
+        for key, val in kw.items():
             setattr(self, key, val)
 
     def __str__(self):
@@ -689,7 +687,7 @@ class GoogleCalendar(models.AbstractModel):
 
             my_google_attendees = CalendarAttendee.with_context(context_novirtual).search([
                 ('partner_id', '=', my_partner_id),
-                ('google_internal_event_id', 'in', pycompat.keys(all_event_from_google))
+                ('google_internal_event_id', 'in', all_event_from_google.keys())
             ])
             my_google_att_ids = my_google_attendees.ids
 
@@ -753,7 +751,7 @@ class GoogleCalendar(models.AbstractModel):
             ev_to_sync.OE.status = event.active
             ev_to_sync.OE.synchro = att.oe_synchro_date
 
-        for event in pycompat.values(all_event_from_google):
+        for event in all_event_from_google.values():
             event_id = event.get('id')
             base_event_id = event_id.rsplit('_', 1)[0]
 
@@ -788,7 +786,7 @@ class GoogleCalendar(models.AbstractModel):
         #      DO ACTION     #
         ######################
         for base_event in event_to_synchronize:
-            event_to_synchronize[base_event] = sorted(pycompat.items(event_to_synchronize[base_event]), key=operator.itemgetter(0))
+            event_to_synchronize[base_event] = sorted(event_to_synchronize[base_event].items(), key=operator.itemgetter(0))
             for current_event in event_to_synchronize[base_event]:
                 self.env.cr.commit()
                 event = current_event[1]  # event is an Sync Event !

addons/hr/models/hr.py

@@ -1,13 +1,12 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import base64
 import logging
 
 from odoo import api, fields, models
 from odoo import tools, _
 from odoo.exceptions import ValidationError
 from odoo.modules.module import get_module_resource
-from odoo.tools import pycompat
 
 _logger = logging.getLogger(__name__)
 
@@ -103,7 +102,7 @@ class Employee(models.Model):
     @api.model
     def _default_image(self):
         image_path = get_module_resource('hr', 'static/src/img', 'default_image.png')
-        return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64'))
+        return tools.image_resize_image_big(base64.b64encode(open(image_path, 'rb').read()))
 
     # resource and user
     name = fields.Char(related='resource_id.name', store=True, oldname='name_related')
@@ -254,7 +253,7 @@ class Employee(models.Model):
         if auto_follow_fields is None:
             auto_follow_fields = ['user_id']
         user_field_lst = []
-        for name, field in pycompat.items(self._fields):
+        for name, field in self._fields.items():
             if name in auto_follow_fields and name in updated_fields and field.comodel_name == 'res.users':
                 user_field_lst.append(name)
         return user_field_lst

addons/hr_expense/models/hr_expense.py

@@ -5,7 +5,7 @@ import re
 
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
-from odoo.tools import email_split, float_is_zero, pycompat
+from odoo.tools import email_split, float_is_zero
 
 from odoo.addons import decimal_precision as dp
 
@@ -243,7 +243,7 @@ class HrExpense(models.Model):
             expense.sheet_id.write({'account_move_id': move.id})
             if expense.payment_mode == 'company_account':
                 expense.sheet_id.paid_expense_sheets()
-        for move in pycompat.values(move_group_by_sheet):
+        for move in move_group_by_sheet.values():
             move.post()
         return True
 

addons/hr_payroll/models/hr_payslip.py

@@ -11,8 +11,6 @@ import babel
 from odoo import api, fields, models, tools, _
 from odoo.addons import decimal_precision as dp
 from odoo.exceptions import UserError, ValidationError
-from odoo.tools import pycompat
-
 
 class HrPayslip(models.Model):
     _name = 'hr.payslip'
@@ -205,7 +203,7 @@ class HrPayslip(models.Model):
             }
 
             res.append(attendances)
-            res.extend(pycompat.values(leaves))
+            res.extend(leaves.values())
         return res
 
     @api.model
@@ -369,7 +367,7 @@ class HrPayslip(models.Model):
                     #blacklist this rule and its children
                     blacklist += [id for id, seq in rule._recursive_search_of_rules()]
 
-        return [value for code, value in pycompat.items(result_dict)]
+        return list(result_dict.values())
 
     # YTI TODO To rename. This method is not really an onchange, as it is not in any view
     # employee_id and contract_id could be browse records

addons/hr_payroll/report/report_payslip_details.py

@@ -2,8 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from odoo import api, models
-from odoo.tools import pycompat
-
 
 class PayslipDetailsReport(models.AbstractModel):
     _name = 'report.hr_payroll.report_payslipdetails'
@@ -38,9 +36,9 @@ class PayslipDetailsReport(models.AbstractModel):
                 result.setdefault(x[2], {})
                 result[x[2]].setdefault(x[1], [])
                 result[x[2]][x[1]].append(x[0])
-            for payslip_id, lines_dict in pycompat.items(result):
+            for payslip_id, lines_dict in result.items():
                 res.setdefault(payslip_id, [])
-                for rule_categ_id, line_ids in pycompat.items(lines_dict):
+                for rule_categ_id, line_ids in lines_dict.items():
                     rule_categories = RuleCateg.browse(rule_categ_id)
                     lines = PayslipLine.browse(line_ids)
                     level = 0
@@ -70,9 +68,9 @@ class PayslipDetailsReport(models.AbstractModel):
             result.setdefault(line.slip_id.id, {})
             result[line.slip_id.id].setdefault(line.register_id, line)
             result[line.slip_id.id][line.register_id] |= line
-        for payslip_id, lines_dict in pycompat.items(result):
+        for payslip_id, lines_dict in result.items():
             res.setdefault(payslip_id, [])
-            for register, lines in pycompat.items(lines_dict):
+            for register, lines in lines_dict.items():
                 res[payslip_id].append({
                     'register_name': register.name,
                     'total': sum(lines.mapped('total')),

addons/hr_recruitment/models/hr_recruitment.py

@@ -4,7 +4,6 @@
 from datetime import datetime
 
 from odoo import api, fields, models, tools, SUPERUSER_ID
-from odoo.tools import pycompat
 from odoo.tools.translate import _
 from odoo.exceptions import UserError
 
@@ -251,7 +250,7 @@ class Applicant(models.Model):
             self = self.with_context(default_department_id=vals.get('department_id'))
         if vals.get('job_id') or self._context.get('default_job_id'):
             job_id = vals.get('job_id') or self._context.get('default_job_id')
-            for key, value in pycompat.items(self._onchange_job_id_internal(job_id)['value']):
+            for key, value in self._onchange_job_id_internal(job_id)['value'].items():
                 if key not in vals:
                     vals[key] = value
         if vals.get('user_id'):

addons/hw_escpos/escpos/escpos.py

@@ -12,6 +12,8 @@ from hashlib import md5
 from PIL import Image
 from xml.etree import ElementTree as ET
 
+from odoo.tools import pycompat
+
 try:
     import jcconv
 except ImportError:
@@ -27,7 +29,7 @@ from .exceptions import *
 
 def utfstr(stuff):
     """ converts stuff to string and does without failing if stuff is a utf8 string """
-    if isinstance(stuff,basestring):
+    if isinstance(stuff,pycompat.string_types):
         return stuff
     else:
         return str(stuff)
@@ -431,7 +433,7 @@ class Escpos:
 
         print('print_b64_img')
 
-        id = md5(img).digest()
+        id = md5(img.encode('utf-8')).digest()
 
         if id not in self.img_cache:
             print('not in cache')

addons/hw_proxy/controllers/main.py

@@ -5,6 +5,7 @@ from __future__ import print_function
 import logging
 import subprocess
 import time
+import subprocess
 from threading import Lock
 
 
@@ -23,7 +24,7 @@ BANNED_DEVICES = {
 }
 
 
-# drivers modules must add to drivers an object with a get_status() method 
+# drivers modules must add to drivers an object with a get_status() method
 # so that 'status' can return the status of all active drivers
 drivers = {}
 
@@ -108,7 +109,7 @@ class Proxy(http.Controller):
             if not (device_id in BANNED_DEVICES):
                 resp += "<div class='device' data-device='"+device+"'>"+device_name+"</div>\n"
                 count += 1
-        
+
         if count == 0:
             resp += "<div class='device'>No USB Device Found</div>"
 
@@ -126,7 +127,7 @@ class Proxy(http.Controller):
             """ % subprocess.check_output('lsusb -v', shell=True)
 
         return request.make_response(resp,{
-            'Cache-Control': 'no-cache', 
+            'Cache-Control': 'no-cache',
             'Content-Type': 'text/html; charset=utf-8',
             'Access-Control-Allow-Origin':  '*',
             'Access-Control-Allow-Methods': 'GET',
@@ -167,7 +168,7 @@ class Proxy(http.Controller):
     @http.route('/hw_proxy/payment_request', type='json', auth='none', cors='*')
     def payment_request(self, price):
         """
-        The PoS will activate the method payment 
+        The PoS will activate the method payment
         """
         print("payment_request: price:%s" % price)
         return 'ok'
@@ -175,7 +176,7 @@ class Proxy(http.Controller):
     @http.route('/hw_proxy/payment_status', type='json', auth='none', cors='*')
     def payment_status(self):
         print("payment_status")
-        return { 'status':'waiting' } 
+        return { 'status':'waiting' }
 
     @http.route('/hw_proxy/payment_cancel', type='json', auth='none', cors='*')
     def payment_cancel(self):

addons/im_livechat/models/im_livechat_channel.py

@@ -1,13 +1,11 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import base64
 import random
 import re
 from datetime import datetime, timedelta
 
 from odoo import api, fields, models, modules, tools
-from odoo.tools import pycompat
-
 
 class ImLivechatChannel(models.Model):
     """ Livechat Channel
@@ -21,7 +19,7 @@ class ImLivechatChannel(models.Model):
 
     def _default_image(self):
         image_path = modules.get_module_resource('im_livechat', 'static/src/img', 'default.png')
-        return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64'))
+        return tools.image_resize_image_big(base64.b64encode(open(image_path, 'rb').read()))
 
     def _default_user_ids(self):
         return [(6, 0, [self._uid])]
@@ -96,7 +94,7 @@ class ImLivechatChannel(models.Model):
         for record in self:
             dt = fields.Datetime.to_string(datetime.utcnow() - timedelta(days=7))
             repartition = record.channel_ids.rating_get_grades([('create_date', '>=', dt)])
-            total = sum(pycompat.values(repartition))
+            total = sum(repartition.values())
             if total > 0:
                 happy = repartition['great']
                 record.rating_percentage_satisfaction = ((happy*100) / total) if happy > 0 else 0

addons/im_livechat/models/mail_channel.py

@@ -2,8 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from odoo import api, fields, models, _
-from odoo.tools import pycompat
-
 
 class ChannelPartner(models.Model):
     _inherit = 'mail.channel.partner'
@@ -78,7 +76,7 @@ class MailChannel(models.Model):
                 last_msg = self.env['mail.message'].search([("channel_ids", "in", [channel.id])], limit=1)
                 if last_msg:
                     channel_infos_dict[channel.id]['last_message_date'] = last_msg.date
-        return list(pycompat.values(channel_infos_dict))
+        return list(channel_infos_dict.values())
 
     @api.model
     def channel_fetch_slot(self):

addons/l10n_be_intrastat/wizard/xml_decl.py

@@ -6,6 +6,7 @@ from xml.etree import ElementTree as ET
 from collections import namedtuple
 
 from odoo import api, exceptions, fields, models, _
+from odoo.tools.pycompat import text_type
 
 INTRASTAT_XMLNS = 'http://www.onegate.eu/2010-01-01'
 
@@ -264,18 +265,18 @@ class XmlDeclaration(models.TransientModel):
                 continue
             numlgn += 1
             item = ET.SubElement(datas, 'Item')
-            self._set_Dim(item, 'EXSEQCODE', unicode(numlgn))
-            self._set_Dim(item, 'EXTRF', unicode(linekey.EXTRF))
-            self._set_Dim(item, 'EXCNT', unicode(linekey.EXCNT))
-            self._set_Dim(item, 'EXTTA', unicode(linekey.EXTTA))
-            self._set_Dim(item, 'EXREG', unicode(linekey.EXREG))
-            self._set_Dim(item, 'EXTGO', unicode(linekey.EXGO))
+            self._set_Dim(item, 'EXSEQCODE', text_type(numlgn))
+            self._set_Dim(item, 'EXTRF', text_type(linekey.EXTRF))
+            self._set_Dim(item, 'EXCNT', text_type(linekey.EXCNT))
+            self._set_Dim(item, 'EXTTA', text_type(linekey.EXTTA))
+            self._set_Dim(item, 'EXREG', text_type(linekey.EXREG))
+            self._set_Dim(item, 'EXTGO', text_type(linekey.EXGO))
             if extendedmode:
-                self._set_Dim(item, 'EXTPC', unicode(linekey.EXTPC))
-                self._set_Dim(item, 'EXDELTRM', unicode(linekey.EXDELTRM))
-            self._set_Dim(item, 'EXTXVAL', unicode(round(amounts[0], 0)).replace(".", ","))
-            self._set_Dim(item, 'EXWEIGHT', unicode(round(amounts[1], 0)).replace(".", ","))
-            self._set_Dim(item, 'EXUNITS', unicode(round(amounts[2], 0)).replace(".", ","))
+                self._set_Dim(item, 'EXTPC', text_type(linekey.EXTPC))
+                self._set_Dim(item, 'EXDELTRM', text_type(linekey.EXDELTRM))
+            self._set_Dim(item, 'EXTXVAL', text_type(round(amounts[0], 0)).replace(".", ","))
+            self._set_Dim(item, 'EXWEIGHT', text_type(round(amounts[1], 0)).replace(".", ","))
+            self._set_Dim(item, 'EXUNITS', text_type(round(amounts[2], 0)).replace(".", ","))
 
         if numlgn == 0:
             #no datas

addons/l10n_fr_fec/wizard/account_fr_fec.py

@@ -4,11 +4,11 @@
 # Copyright (C) 2013-2015 Akretion (http://www.akretion.com)
 
 import base64
-import csv
-import StringIO
+import io
 
 from odoo import api, fields, models, _
 from odoo.exceptions import Warning
+from odoo.tools import pycompat
 
 
 class AccountFrFec(models.TransientModel):
@@ -86,24 +86,24 @@ class AccountFrFec(models.TransientModel):
         # So it will be easier for the accountant to check the file before
         # sending it to the fiscal administration
         header = [
-            'JournalCode',    # 0
-            'JournalLib',     # 1
-            'EcritureNum',    # 2
-            'EcritureDate',   # 3
-            'CompteNum',      # 4
-            'CompteLib',      # 5
-            'CompAuxNum',     # 6  We use partner.id
-            'CompAuxLib',     # 7
-            'PieceRef',       # 8
-            'PieceDate',      # 9
-            'EcritureLib',    # 10
-            'Debit',          # 11
-            'Credit',         # 12
-            'EcritureLet',    # 13
-            'DateLet',        # 14
-            'ValidDate',      # 15
-            'Montantdevise',  # 16
-            'Idevise',        # 17
+            u'JournalCode',    # 0
+            u'JournalLib',     # 1
+            u'EcritureNum',    # 2
+            u'EcritureDate',   # 3
+            u'CompteNum',      # 4
+            u'CompteLib',      # 5
+            u'CompAuxNum',     # 6  We use partner.id
+            u'CompAuxLib',     # 7
+            u'PieceRef',       # 8
+            u'PieceDate',      # 9
+            u'EcritureLib',    # 10
+            u'Debit',          # 11
+            u'Credit',         # 12
+            u'EcritureLet',    # 13
+            u'DateLet',        # 14
+            u'ValidDate',      # 15
+            u'Montantdevise',  # 16
+            u'Idevise',        # 17
             ]
 
         company = self.env.user.company_id
@@ -114,8 +114,8 @@ class AccountFrFec(models.TransientModel):
             raise Warning(
                 _("FEC is for French companies only !"))
 
-        fecfile = StringIO.StringIO()
-        w = csv.writer(fecfile, delimiter='|')
+        fecfile = io.BytesIO()
+        w = pycompat.csv_writer(fecfile, delimiter='|')
         w.writerow(header)
 
         # INITIAL BALANCE
@@ -190,7 +190,7 @@ class AccountFrFec(models.TransientModel):
                     else:
                         listrow[11] = '0,00'
                         listrow[12] = str(-listrow_amount).replace('.', ',')
-            w.writerow([s.encode("utf-8") for s in listrow])
+            w.writerow(listrow)
         #if the unaffected earnings account wasn't in the selection yet: add it manually
         if (not unaffected_earnings_line
             and unaffected_earnings_results
@@ -201,7 +201,7 @@ class AccountFrFec(models.TransientModel):
             if unaffected_earnings_account:
                 unaffected_earnings_results[4] = unaffected_earnings_account.code
                 unaffected_earnings_results[5] = unaffected_earnings_account.name
-            w.writerow([s.encode("utf-8") for s in unaffected_earnings_results])
+            w.writerow(unaffected_earnings_results)
 
         # LINES
         sql_query = '''
@@ -266,8 +266,7 @@ class AccountFrFec(models.TransientModel):
             sql_query, (self.date_from, self.date_to, company.id))
 
         for row in self._cr.fetchall():
-            listrow = list(row)
-            w.writerow([s.encode("utf-8") for s in listrow])
+            w.writerow(list(row))
 
         siren = company.vat[4:13]
         end_date = self.date_to.replace('-', '')

addons/l10n_in_hr_payroll/report/report_hr_yearly_salary_detail.py

@@ -5,8 +5,6 @@ from datetime import date
 
 from odoo import api, models, _
 from odoo.exceptions import UserError
-from odoo.tools import pycompat
-
 
 class EmployeesYearlySalaryReport(models.AbstractModel):
     _name = 'report.l10n_in_hr_payroll.report_hryearlysalary'
@@ -99,7 +97,7 @@ class EmployeesYearlySalaryReport(models.AbstractModel):
 
     def salary_list(self, salaries):
         cat_salary_all = []
-        for category_name, amount in pycompat.items(salaries):
+        for category_name, amount in salaries.items():
             cat_salary = []
             total = 0.0
             cat_salary.append(category_name)

addons/l10n_lu/scripts/tax2csv.py

@@ -1,18 +1,8 @@
 from collections import OrderedDict
-import csv
 
 import xlrd
 from odoo.tools import pycompat
 
-def _e(s):
-    if type(s) is unicode:
-        return s.encode('utf8')
-    elif s is None:
-        return ''
-    else:
-        return str(s)
-
-
 def _is_true(s):
     return s not in ('F', 'False', 0, '', None, False)
 
@@ -56,9 +46,9 @@ class LuxTaxGenerator:
             yield OrderedDict(pycompat.izip(keys, row))
 
     def tax_codes_to_csv(self):
-        writer = csv.writer(open('account.tax.code.template-%s.csv' %
+        writer = pycompat.csv_writer(open('account.tax.code.template-%s.csv' %
                                  self.suffix, 'wb'))
-        tax_codes_iterator = self.iter_tax_codes
+        tax_codes_iterator = self.iter_tax_codes()
         keys = next(tax_codes_iterator)
         writer.writerow(keys)
 
@@ -69,7 +59,7 @@ class LuxTaxGenerator:
             if tax_code in tax_codes:
                 raise RuntimeError('duplicate tax code %s' % tax_code)
             tax_codes[tax_code] = row['id']
-            writer.writerow(pycompat.imap(_e, pycompat.values(row)))
+            writer.writerow([pycompat.to_text(v) for v in row.values()])
 
         # read taxes and add leaf tax codes
         new_tax_codes = {}  # id: parent_code
@@ -145,17 +135,19 @@ class LuxTaxGenerator:
 
         for tax_code_id in sorted(new_tax_codes):
             name, parent_code = new_tax_codes[tax_code_id]
-            writer.writerow((tax_code_id,
-                             'lu_tct_m' + parent_code,
-                             tax_code_id.replace('lu_tax_code_template_', ''),
-                             '1',
-                             '',
-                             _e(name),
-                             ''))
+            writer.writerow([
+                tax_code_id,
+                u'lu_tct_m' + parent_code,
+                tax_code_id.replace('lu_tax_code_template_', u''),
+                u'1',
+                u'',
+                pycompat.to_text(name),
+                u''
+            ])
 
     def taxes_to_csv(self):
-        writer = csv.writer(open('account.tax.template-%s.csv' %
-                                 self.suffix, 'wb'))
+        writer = pycompat.csv_writer(open('account.tax.template-%s.csv' %
+                                     self.suffix, 'wb'))
         taxes_iterator = self.iter_taxes()
         keys = next(taxes_iterator)
         writer.writerow(keys[3:] + ['sequence'])
@@ -168,17 +160,20 @@ class LuxTaxGenerator:
                 cur_seq = seq + 1000
             else:
                 cur_seq = seq
-            writer.writerow(list(pycompat.imap(_e, list(pycompat.values(row))[3:])) + [cur_seq])
+            writer.writerow([
+                pycompat.to_text(v)
+                for v in list(row.values())[3:]
+            ] + [cur_seq])
 
     def fiscal_pos_map_to_csv(self):
-        writer = csv.writer(open('account.fiscal.'
-                                 'position.tax.template-%s.csv' %
-                                 self.suffix, 'wb'))
+        writer = pycompat.csv_writer(open('account.fiscal.'
+                                     'position.tax.template-%s.csv' %
+                                     self.suffix, 'wb'))
         fiscal_pos_map_iterator = self.iter_fiscal_pos_map()
         keys = next(fiscal_pos_map_iterator)
         writer.writerow(keys)
         for row in fiscal_pos_map_iterator:
-            writer.writerow(pycompat.imap(_e, pycompat.values(row)))
+            writer.writerow([pycompat.to_text(s) for s in row.values()])
 
 
 if __name__ == '__main__':

addons/link_tracker/models/link_tracker.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import base64
 import datetime
 import random
 import re
@@ -12,9 +12,9 @@ from werkzeug import urls, utils
 
 
 from odoo import models, fields, api, _
-from odoo.tools import ustr, pycompat
+from odoo.tools import ustr
 
-URL_REGEX = r'(\bhref=[\'"](?!mailto:)([^\'"]+)[\'"])'
+URL_REGEX = r'(\bhref=[\'"](?!mailto:|tel:|sms:)([^\'"]+)[\'"])'
 
 def VALIDATE_URL(url):
     if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'):
@@ -90,7 +90,7 @@ class link_tracker(models.Model):
     @api.one
     @api.depends('favicon')
     def _compute_icon_src(self):
-        self.icon_src = 'data:image/png;base64,' + self.favicon
+        self.icon_src = b'data:image/png;base64,' + self.favicon
 
     @api.one
     @api.depends('url')
@@ -123,7 +123,7 @@ class link_tracker(models.Model):
     def _compute_favicon(self):
         try:
             icon = requests.get('http://www.google.com/s2/favicons', params={'domain': self.url}, timeout=5).content
-            icon_base64 = icon.encode('base64').replace("\n", "")
+            icon_base64 = base64.b64encode(icon).replace(b"\n", b"").decode('ascii')
         except:
             icon_base64 = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsSAAALEgHS3X78AAACiElEQVQ4EaVTzU8TURCf2tJuS7tQtlRb6UKBIkQwkRRSEzkQgyEc6lkOKgcOph78Y+CgjXjDs2i44FXY9AMTlQRUELZapVlouy3d7kKtb0Zr0MSLTvL2zb75eL838xtTvV6H/xELBptMJojeXLCXyobnyog4YhzXYvmCFi6qVSfaeRdXdrfaU1areV5KykmX06rcvzumjY/1ggkR3Jh+bNf1mr8v1D5bLuvR3qDgFbvbBJYIrE1mCIoCrKxsHuzK+Rzvsi29+6DEbTZz9unijEYI8ObBgXOzlcrx9OAlXyDYKUCzwwrDQx1wVDGg089Dt+gR3mxmhcUnaWeoxwMbm/vzDFzmDEKMMNhquRqduT1KwXiGt0vre6iSeAUHNDE0d26NBtAXY9BACQyjFusKuL2Ry+IPb/Y9ZglwuVscdHaknUChqLF/O4jn3V5dP4mhgRJgwSYm+gV0Oi3XrvYB30yvhGa7BS70eGFHPoTJyQHhMK+F0ZesRVVznvXw5Ixv7/C10moEo6OZXbWvlFAF9FVZDOqEABUMRIkMd8GnLwVWg9/RkJF9sA4oDfYQAuzzjqzwvnaRUFxn/X2ZlmGLXAE7AL52B4xHgqAUqrC1nSNuoJkQtLkdqReszz/9aRvq90NOKdOS1nch8TpL555WDp49f3uAMXhACRjD5j4ykuCtf5PP7Fm1b0DIsl/VHGezzP1KwOiZQobFF9YyjSRYQETRENSlVzI8iK9mWlzckpSSCQHVALmN9Az1euDho9Xo8vKGd2rqooA8yBcrwHgCqYR0kMkWci08t/R+W4ljDCanWTg9TJGwGNaNk3vYZ7VUdeKsYJGFNkfSzjXNrSX20s4/h6kB81/271ghG17l+rPTAAAAAElFTkSuQmCC'
 
@@ -165,7 +165,7 @@ class link_tracker(models.Model):
             create_vals['url'] = VALIDATE_URL(vals['url'])
 
         search_domain = []
-        for fname, value in pycompat.items(create_vals):
+        for fname, value in create_vals.items():
             search_domain.append((fname, '=', value))
 
         result = self.search(search_domain, limit=1)

addons/lunch/models/lunch.py

@@ -10,8 +10,6 @@ from odoo import api, fields, models, _
 from odoo.exceptions import AccessError, ValidationError
 from odoo.addons import decimal_precision as dp
 
-from odoo.tools import pycompat
-
 
 class LunchOrder(models.Model):
     """
@@ -26,10 +24,10 @@ class LunchOrder(models.Model):
         prev_order = self.env['lunch.order.line'].search([('user_id', '=', self.env.uid), ('product_id.active', '!=', False)], limit=20, order='id desc')
         # If we return return prev_order.ids, we will have duplicates (identical orders).
         # Therefore, this following part removes duplicates based on product_id and note.
-        return list(pycompat.values({
+        return list({
             (order.product_id, order.note): order.id
             for order in prev_order
-        }))
+        }.values())
 
     user_id = fields.Many2one('res.users', 'User', readonly=True,
                               states={'new': [('readonly', False)]},
@@ -88,10 +86,10 @@ class LunchOrder(models.Model):
         prev_order = self.env['lunch.order.line'].search([('user_id', '=', self.env.uid), ('product_id.active', '!=', False)], limit=20, order='date desc, id desc')
         # If we use prev_order.ids, we will have duplicates (identical orders).
         # Therefore, this following part removes duplicates based on product_id and note.
-        self.previous_order_ids = list(pycompat.values({
+        self.previous_order_ids = list({
             (order.product_id, order.note): order.id
             for order in prev_order
-        }))
+        }.values())
 
         if self.previous_order_ids:
             lunch_data = {}

addons/mail/controllers/main.py

@@ -13,7 +13,7 @@ from odoo import api, http, registry, SUPERUSER_ID, _
 from odoo.addons.web.controllers.main import binary_content
 from odoo.exceptions import AccessError
 from odoo.http import request
-from odoo.tools import consteq
+from odoo.tools import consteq, pycompat
 
 _logger = logging.getLogger(__name__)
 
@@ -101,7 +101,7 @@ class MailController(http.Controller):
         """ End-point to receive mail from an external SMTP server. """
         dbs = req.jsonrequest.get('databases')
         for db in dbs:
-            message = dbs[db].decode('base64')
+            message = base64.b64decode(dbs[db])
             try:
                 db_registry = registry(db)
                 with db_registry.cursor() as cr:
@@ -119,7 +119,7 @@ class MailController(http.Controller):
         follower_id = None
         follower_recs = request.env['mail.followers'].sudo().browse(follower_ids)
         res_ids = follower_recs.mapped('res_id')
-        request.env[res_model].browse(res_ids).check_access_rule("write")
+        request.env[res_model].browse(res_ids).check_access_rule("read")
         for follower in follower_recs:
             is_uid = partner_id == follower.partner_id
             follower_id = follower.id if is_uid else follower_id
@@ -182,7 +182,7 @@ class MailController(http.Controller):
             else:
                 # either a wrong message_id, either someone trying ids -> just go to messaging
                 return self._redirect_to_messaging()
-        elif res_id and isinstance(res_id, basestring):
+        elif res_id and isinstance(res_id, pycompat.string_types):
             res_id = int(res_id)
 
         return self._redirect_to_record(model, res_id, access_token)

addons/mail/models/ir_model.py

@@ -3,6 +3,7 @@
 
 from odoo import _, api, fields, models
 from odoo.exceptions import UserError
+from odoo.tools import pycompat
 
 
 class IrModel(models.Model):
@@ -41,7 +42,7 @@ class IrModel(models.Model):
         model_class = super(IrModel, self)._instanciate(model_data)
         if model_data.get('is_mail_thread') and model_class._name != 'mail.thread':
             parents = model_class._inherit or []
-            parents = [parents] if isinstance(parents, basestring) else parents
+            parents = [parents] if isinstance(parents, pycompat.string_types) else parents
             model_class._inherit = parents + ['mail.thread']
         return model_class
 

addons/mail/models/mail_channel.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import base64
 from email.utils import formataddr
 
 import re
@@ -9,7 +9,7 @@ import uuid
 from odoo import _, api, fields, models, modules, tools
 from odoo.exceptions import UserError
 from odoo.osv import expression
-from odoo.tools import ormcache, pycompat
+from odoo.tools import ormcache
 from odoo.tools.safe_eval import safe_eval
 
 
@@ -41,7 +41,7 @@ class Channel(models.Model):
 
     def _get_default_image(self):
         image_path = modules.get_module_resource('mail', 'static/src/img', 'groupdefault.png')
-        return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64'))
+        return tools.image_resize_image_big(base64.b64encode(open(image_path, 'rb').read()))
 
     @api.model
     def default_get(self, fields):
@@ -619,7 +619,7 @@ class Channel(models.Model):
             channel = channels_preview[message['id']]
             del(channel['message_id'])
             channel['last_message'] = message
-        return list(pycompat.values(channels_preview))
+        return list(channels_preview.values())
 
     #------------------------------------------------------
     # Commands

addons/mail/models/mail_followers.py

@@ -2,7 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from odoo import api, fields, models
-from odoo.tools import pycompat
 
 
 class Followers(models.Model):
@@ -24,7 +23,7 @@ class Followers(models.Model):
         index=True, required=True, ondelete='cascade',
         help='Model of the followed resource')
     res_model = fields.Char(
-        'Related Document Model Name', index=True, readonly=True, related='res_model_id.model', store=True)
+        'Related Document Model Name', index=True, readonly=True, related='res_model_id.model', compute_sudo=True, store=True)
     res_id = fields.Integer(
         'Related Document ID', index=True, help='Id of the followed resource')
     partner_id = fields.Many2one(
@@ -42,7 +41,7 @@ class Followers(models.Model):
                       using the subtypes given in the parameters
         """
         res_model_id = self.env['ir.model']._get(res_model).id
-        force_mode = force or (all(data for data in pycompat.values(partner_data)) and all(data for data in pycompat.values(channel_data)))
+        force_mode = force or (all(partner_data.values()) and all(channel_data.values()))
         generic = []
         specific = {}
         existing = {}  # {res_id: follower_ids}
@@ -71,13 +70,13 @@ class Followers(models.Model):
 
         if force_mode:
             employee_pids = self.env['res.users'].sudo().search([('partner_id', 'in', list(partner_data)), ('share', '=', False)]).mapped('partner_id').ids
-            for pid, data in pycompat.items(partner_data):
+            for pid, data in partner_data.items():
                 if not data:
                     if pid not in employee_pids:
                         partner_data[pid] = external_default_subtypes.ids
                     else:
                         partner_data[pid] = default_subtypes.ids
-            for cid, data in pycompat.items(channel_data):
+            for cid, data in channel_data.items():
                 if not data:
                     channel_data[cid] = default_subtypes.ids
 

addons/mail/models/mail_mail.py

@@ -13,7 +13,6 @@ from email.utils import formataddr
 from odoo import _, api, fields, models
 from odoo import tools
 from odoo.addons.base.ir.ir_mail_server import MailDeliveryException
-from odoo.tools import pycompat
 from odoo.tools.safe_eval import safe_eval
 
 _logger = logging.getLogger(__name__)
@@ -169,7 +168,7 @@ class MailMail(models.Model):
           - else fallback on mail.email_to splitting """
         self.ensure_one()
         if partner:
-            email_to = [formataddr((partner.name, partner.email))]
+            email_to = [formataddr((partner.name or 'False', partner.email or 'False'))]
         else:
             email_to = tools.email_split_and_format(self.email_to)
         return email_to
@@ -205,7 +204,7 @@ class MailMail(models.Model):
             groups[mail.mail_server_id.id].append(mail.id)
         sys_params = self.env['ir.config_parameter'].sudo()
         batch_size = int(sys_params.get_param('mail.session.batch.size', 1000))
-        for server_id, record_ids in pycompat.items(groups):
+        for server_id, record_ids in groups.items():
             for mail_batch in tools.split_every(batch_size, record_ids):
                 yield server_id, mail_batch
 

addons/mail/models/mail_message.py

@@ -9,7 +9,6 @@ from email.utils import formataddr
 from odoo import _, api, fields, models, modules, SUPERUSER_ID, tools
 from odoo.exceptions import UserError, AccessError
 from odoo.osv import expression
-from odoo.tools import pycompat
 
 _logger = logging.getLogger(__name__)
 _image_dataurl = re.compile(r'(data:image/[a-z]+?);base64,([a-z0-9+/]{3,}=*)([\'"])', re.I)
@@ -282,7 +281,7 @@ class Message(models.Model):
         partners = self.env['res.partner'].sudo()
         attachments = self.env['ir.attachment']
         trackings = self.env['mail.tracking.value']
-        for key, message in pycompat.items(message_tree):
+        for key, message in message_tree.items():
             if message.author_id:
                 partners |= message.author_id
             if message.subtype_id and message.partner_ids:  # take notified people of message with a subtype
@@ -446,7 +445,7 @@ class Message(models.Model):
     def _find_allowed_doc_ids(self, model_ids):
         IrModelAccess = self.env['ir.model.access']
         allowed_ids = set()
-        for doc_model, doc_dict in pycompat.items(model_ids):
+        for doc_model, doc_dict in model_ids.items():
             if not IrModelAccess.check(doc_model, 'read', False):
                 continue
             allowed_ids |= self._find_allowed_model_wise(doc_model, doc_dict)
@@ -607,17 +606,17 @@ class Message(models.Model):
         # Author condition (READ, WRITE, CREATE (private))
         author_ids = []
         if operation == 'read' or operation == 'write':
-            author_ids = [mid for mid, message in pycompat.items(message_values)
+            author_ids = [mid for mid, message in message_values.items()
                           if message.get('author_id') and message.get('author_id') == self.env.user.partner_id.id]
         elif operation == 'create':
-            author_ids = [mid for mid, message in pycompat.items(message_values)
+            author_ids = [mid for mid, message in message_values.items()
                           if not message.get('model') and not message.get('res_id')]
 
         # Parent condition, for create (check for received notifications for the created message parent)
         notified_ids = []
         if operation == 'create':
             # TDE: probably clean me
-            parent_ids = [message.get('parent_id') for mid, message in pycompat.items(message_values)
+            parent_ids = [message.get('parent_id') for message in message_values.values()
                           if message.get('parent_id')]
             self._cr.execute("""SELECT DISTINCT m.id, partner_rel.res_partner_id, channel_partner.partner_id FROM "%s" m
                 LEFT JOIN "mail_message_res_partner_rel" partner_rel
@@ -630,37 +629,37 @@ class Message(models.Model):
                 ON channel_partner.channel_id = channel.id AND channel_partner.partner_id = (%%s)
                 WHERE m.id = ANY (%%s)""" % self._table, (self.env.user.partner_id.id, self.env.user.partner_id.id, parent_ids,))
             not_parent_ids = [mid[0] for mid in self._cr.fetchall() if any([mid[1], mid[2]])]
-            notified_ids += [mid for mid, message in pycompat.items(message_values)
+            notified_ids += [mid for mid, message in message_values.items()
                              if message.get('parent_id') in not_parent_ids]
 
         # Recipients condition, for read and write (partner_ids) and create (message_follower_ids)
         other_ids = set(self.ids).difference(set(author_ids), set(notified_ids))
         model_record_ids = _generate_model_record_ids(message_values, other_ids)
         if operation in ['read', 'write']:
-            notified_ids = [mid for mid, message in pycompat.items(message_values) if message.get('notified')]
+            notified_ids = [mid for mid, message in message_values.items() if message.get('notified')]
         elif operation == 'create':
-            for doc_model, doc_ids in pycompat.items(model_record_ids):
+            for doc_model, doc_ids in model_record_ids.items():
                 followers = self.env['mail.followers'].sudo().search([
                     ('res_model', '=', doc_model),
                     ('res_id', 'in', list(doc_ids)),
                     ('partner_id', '=', self.env.user.partner_id.id),
                     ])
                 fol_mids = [follower.res_id for follower in followers]
-                notified_ids += [mid for mid, message in pycompat.items(message_values)
+                notified_ids += [mid for mid, message in message_values.items()
                                  if message.get('model') == doc_model and message.get('res_id') in fol_mids]
 
         # CRUD: Access rights related to the document
         other_ids = other_ids.difference(set(notified_ids))
         model_record_ids = _generate_model_record_ids(message_values, other_ids)
         document_related_ids = []
-        for model, doc_ids in pycompat.items(model_record_ids):
+        for model, doc_ids in model_record_ids.items():
             DocumentModel = self.env[model]
             mids = DocumentModel.browse(doc_ids).exists()
             if hasattr(DocumentModel, 'check_mail_message_access'):
                 DocumentModel.check_mail_message_access(mids.ids, operation)  # ?? mids ?
             else:
                 self.env['mail.thread'].check_mail_message_access(mids.ids, operation, model_name=model)
-            document_related_ids += [mid for mid, message in pycompat.items(message_values)
+            document_related_ids += [mid for mid, message in message_values.items()
                                      if message.get('model') == model and message.get('res_id') in mids.ids]
 
         # Calculate remaining ids: if not void, raise an error
@@ -745,7 +744,7 @@ class Message(models.Model):
                     values['attachment_ids'].append((4, attachment.id))
                     data_to_url[key] = '/web/image/%s' % attachment.id
                 return '%s%s alt="%s"' % (data_to_url[key], match.group(3), name)
-            values['body'] = _image_dataurl.sub(base64_to_boundary, values['body'])
+            values['body'] = _image_dataurl.sub(base64_to_boundary, tools.ustr(values['body']))
 
         message = super(Message, self).create(values)
         message._invalidate_documents()

addons/mail/models/mail_template.py

@@ -45,7 +45,7 @@ def format_tz(env, dt, tz=False, format=False):
         return format_datetime(ts, format or 'medium', locale=env.context.get("lang") or 'en_US')
 
     if format:
-        return ts.strftime(format)
+        return pycompat.text_type(ts.strftime(format))
     else:
         lang = env.context.get("lang")
         langs = env['res.lang']
@@ -54,9 +54,9 @@ def format_tz(env, dt, tz=False, format=False):
         format_date = langs.date_format or '%B-%d-%Y'
         format_time = langs.time_format or '%I-%M %p'
 
-        fdate = ts.strftime(format_date).decode('utf-8')
-        ftime = ts.strftime(format_time).decode('utf-8')
-        return "%s %s%s" % (fdate, ftime, (' (%s)' % tz) if tz else '')
+        fdate = pycompat.text_type(ts.strftime(format_date))
+        ftime = pycompat.text_type(ts.strftime(format_time))
+        return u"%s %s%s" % (fdate, ftime, (u' (%s)' % tz) if tz else u'')
 
 def format_amount(env, amount, currency):
     fmt = "%.{0}f".format(currency.decimal_places)
@@ -306,8 +306,8 @@ class MailTemplate(models.Model):
         # form a tree
         root = lxml.html.fromstring(html)
         if not len(root) and root.text is None and root.tail is None:
-            html = '<div>%s</div>' % html
-            root = lxml.html.fromstring(html)
+            html = u'<div>%s</div>' % html
+            root = lxml.html.fromstring(html, encoding='unicode')
 
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
         base = urls.url_parse(base_url)
@@ -324,12 +324,12 @@ class MailTemplate(models.Model):
         for node in root.iter():
             if node.tag == 'a' and node.get('href'):
                 node.set('href', _process_link(node.get('href')))
-            elif node.tag == 'img' and not node.get('src', 'data').startswith('data'):
+            elif node.tag == 'img' and not node.get('src', 'data').startswith(u'data'):
                 node.set('src', _process_link(node.get('src')))
 
-        html = lxml.html.tostring(root, pretty_print=False, method='html')
+        html = lxml.html.tostring(root, pretty_print=False, method='html', encoding='unicode')
         # this is ugly, but lxml/etree tostring want to put everything in a 'div' that breaks the editor -> remove that
-        if html.startswith('<div>') and html.endswith('</div>'):
+        if html.startswith(u'<div>') and html.endswith(u'</div>'):
             html = html[5:-6]
         return html
 
@@ -379,7 +379,7 @@ class MailTemplate(models.Model):
             'user': self.env.user,
             'ctx': self._context,  # context kw would clash with mako internals
         }
-        for res_id, record in pycompat.items(res_to_rec):
+        for res_id, record in res_to_rec.items():
             variables['object'] = record
             try:
                 render_result = template.render(variables)
@@ -391,7 +391,7 @@ class MailTemplate(models.Model):
             results[res_id] = render_result
 
         if post_process:
-            for res_id, result in pycompat.items(results):
+            for res_id, result in results.items():
                 results[res_id] = self.render_post_process(result)
 
         return multi_mode and results or results[res_ids[0]]
@@ -412,7 +412,7 @@ class MailTemplate(models.Model):
         self.ensure_one()
 
         langs = self.render_template(self.lang, self.model, res_ids)
-        for res_id, lang in pycompat.items(langs):
+        for res_id, lang in langs.items():
             if lang:
                 template = self.with_context(lang=lang)
             else:
@@ -431,11 +431,11 @@ class MailTemplate(models.Model):
 
         if self.use_default_to or self._context.get('tpl_force_default_to'):
             default_recipients = self.env['mail.thread'].message_get_default_recipients(res_model=self.model, res_ids=res_ids)
-            for res_id, recipients in pycompat.items(default_recipients):
+            for res_id, recipients in default_recipients.items():
                 results[res_id].pop('partner_to', None)
                 results[res_id].update(recipients)
 
-        for res_id, values in pycompat.items(results):
+        for res_id, values in results.items():
             partner_ids = values.get('partner_ids', list())
             if self._context.get('tpl_partners_only'):
                 mails = tools.email_split(values.pop('email_to', '')) + tools.email_split(values.pop('email_cc', ''))
@@ -474,11 +474,11 @@ class MailTemplate(models.Model):
 
         # templates: res_id -> template; template -> res_ids
         templates_to_res_ids = {}
-        for res_id, template in pycompat.items(res_ids_to_templates):
+        for res_id, template in res_ids_to_templates.items():
             templates_to_res_ids.setdefault(template, []).append(res_id)
 
         results = dict()
-        for template, template_res_ids in pycompat.items(templates_to_res_ids):
+        for template, template_res_ids in templates_to_res_ids.items():
             Template = self.env['mail.template']
             # generate fields value for all res_ids linked to the current template
             if template.lang:
@@ -488,7 +488,7 @@ class MailTemplate(models.Model):
                 generated_field_values = Template.render_template(
                     getattr(template, field), template.model, template_res_ids,
                     post_process=(field == 'body_html'))
-                for res_id, field_value in pycompat.items(generated_field_values):
+                for res_id, field_value in generated_field_values.items():
                     results.setdefault(res_id, dict())[field] = field_value
             # compute recipients
             if any(field in fields for field in ['email_to', 'partner_to', 'email_cc']):

addons/mail/models/mail_thread.py

@@ -238,7 +238,7 @@ class MailThread(models.AbstractModel):
 
         # auto_subscribe: take values and defaults into account
         create_values = dict(values)
-        for key, val in pycompat.items(self._context):
+        for key, val in self._context.items():
             if key.startswith('default_') and key[8:] not in create_values:
                 create_values[key[8:]] = val
         thread.message_auto_subscribe(list(create_values), values=create_values)
@@ -373,7 +373,7 @@ class MailThread(models.AbstractModel):
                 options['display_log_button'] = is_employee
                 # save options on the node
                 node.set('options', repr(options))
-            res['arch'] = etree.tostring(doc)
+            res['arch'] = etree.tostring(doc, encoding='unicode')
         return res
 
     # ------------------------------------------------------
@@ -388,7 +388,7 @@ class MailThread(models.AbstractModel):
                 always tracked fields and modified on_change fields
         """
         tracked_fields = []
-        for name, field in pycompat.items(self._fields):
+        for name, field in self._fields.items():
             if getattr(field, 'track_visibility', False):
                 tracked_fields.append(name)
 
@@ -416,13 +416,13 @@ class MailThread(models.AbstractModel):
 
     @api.multi
     def _message_track_post_template(self, tracking):
-        if not any(change for rec_id, (change, tracking_value_ids) in pycompat.items(tracking)):
+        if not any(change for rec_id, (change, tracking_value_ids) in tracking.items()):
             return True
         templates = self._track_template(tracking)
-        for field_name, (template, post_kwargs) in pycompat.items(templates):
+        for field_name, (template, post_kwargs) in templates.items():
             if not template:
                 continue
-            if isinstance(template, basestring):
+            if isinstance(template, pycompat.string_types):
                 self.message_post_with_view(template, **post_kwargs)
             else:
                 self.message_post_with_template(template.id, **post_kwargs)
@@ -450,7 +450,7 @@ class MailThread(models.AbstractModel):
         display_values_ids = []
 
         # generate tracked_values data structure: {'col_name': {col_info, new_value, old_value}}
-        for col_name, col_info in pycompat.items(tracked_fields):
+        for col_name, col_info in tracked_fields.items():
             track_visibility = getattr(self._fields[col_name], 'track_visibility', 'onchange')
             initial_value = initial[col_name]
             new_value = getattr(self, col_name)
@@ -569,7 +569,7 @@ class MailThread(models.AbstractModel):
     def _generate_notification_token(self, base_link, params):
         secret = self.env['ir.config_parameter'].sudo().get_param('database.secret')
         token = '%s?%s' % (base_link, ' '.join('%s=%s' % (key, params[key]) for key in sorted(params)))
-        hm = hmac.new(str(secret), token, hashlib.sha1).hexdigest()
+        hm = hmac.new(secret.encode('utf-8'), token.encode('utf-8'), hashlib.sha1).hexdigest()
         return hm
 
     @api.multi
@@ -816,7 +816,7 @@ class MailThread(models.AbstractModel):
     def message_capable_models(self):
         """ Used by the plugin addon, based for plugin_outlook and others. """
         ret_dict = {}
-        for model_name, model in pycompat.items(self.env):
+        for model_name, model in self.env.items():
             if hasattr(model, "message_process") and hasattr(model, "message_post"):
                 ret_dict[model_name] = model._description
         return ret_dict
@@ -1266,12 +1266,16 @@ class MailThread(models.AbstractModel):
         # we don't know its encoding until we parse its headers and hence can't
         # convert it to utf-8 for transport between the mailgate script and here.
         if isinstance(message, xmlrpclib.Binary):
-            message = str(message.data)
-        # Warning: message_from_string doesn't always work correctly on unicode,
-        # we must use utf-8 strings here :-(
-        if isinstance(message, unicode):
+            message = bytes(message.data)
+        # message_from_string parses from a *native string*, except apparently
+        # sometimes message is ISO-8859-1 binary data or some shit and the
+        # straightforward version (pycompat.to_native) won't work right ->
+        # always encode message to bytes then use the relevant method
+        # depending on ~python version
+        if isinstance(message, pycompat.text_type):
             message = message.encode('utf-8')
-        msg_txt = email.message_from_string(message)
+        extract = getattr(email, 'message_from_bytes', email.message_from_string)
+        msg_txt = extract(message)
 
         # parse the message, verify we are not in a loop by checking message_id is not duplicated
         msg = self.message_parse(msg_txt, save_original=save_original)
@@ -1492,9 +1496,9 @@ class MailThread(models.AbstractModel):
             'message_type': 'email',
         }
         if not isinstance(message, Message):
-            if isinstance(message, unicode):
-                # Warning: message_from_string doesn't always work correctly on unicode,
-                # we must use utf-8 strings here :-(
+            # message_from_string works on a native str, so on py2 we need to
+            # encode incoming unicode strings
+            if pycompat.PY2 and not isinstance(message, str):
                 message = message.encode('utf-8')
             message = email.message_from_string(message)
 
@@ -1723,11 +1727,11 @@ class MailThread(models.AbstractModel):
                 cid = info and info.get('cid')
             else:
                 continue
-            if isinstance(content, unicode):
+            if isinstance(content, pycompat.text_type):
                 content = content.encode('utf-8')
             data_attach = {
                 'name': name,
-                'datas': base64.b64encode(str(content)),
+                'datas': base64.b64encode(content),
                 'type': 'binary',
                 'datas_fname': name,
                 'description': name,
@@ -1920,7 +1924,7 @@ class MailThread(models.AbstractModel):
             values['slug'] = slug
         except ImportError:
             values['slug'] = lambda self: self.id
-        if isinstance(views_or_xmlid, basestring):
+        if isinstance(views_or_xmlid, pycompat.string_types):
             views = self.env.ref(views_or_xmlid, raise_if_not_found=False)
         else:
             views = views_or_xmlid
@@ -2052,7 +2056,7 @@ class MailThread(models.AbstractModel):
         if auto_follow_fields is None:
             auto_follow_fields = ['user_id']
         user_field_lst = []
-        for name, field in pycompat.items(self._fields):
+        for name, field in self._fields.items():
             if name in auto_follow_fields and name in updated_fields and getattr(field, 'track_visibility', False) and field.comodel_name == 'res.users':
                 user_field_lst.append(name)
         return user_field_lst
@@ -2148,10 +2152,10 @@ class MailThread(models.AbstractModel):
         for partner in to_add_users.mapped('partner_id'):
             new_partners.setdefault(partner.id, None)
 
-        for pid, subtypes in pycompat.items(new_partners):
+        for pid, subtypes in new_partners.items():
             subtypes = list(subtypes) if subtypes is not None else None
             self.message_subscribe(partner_ids=[pid], subtype_ids=subtypes, force=(subtypes != None))
-        for cid, subtypes in pycompat.items(new_channels):
+        for cid, subtypes in new_channels.items():
             subtypes = list(subtypes) if subtypes is not None else None
             self.message_subscribe(channel_ids=[cid], subtype_ids=subtypes, force=(subtypes != None))
 

addons/mail/models/res_partner.py

@@ -4,7 +4,6 @@
 import logging
 import threading
 
-from odoo.tools import pycompat
 from odoo.tools.misc import split_every
 
 from odoo import _, api, fields, models, registry, SUPERUSER_ID
@@ -175,7 +174,7 @@ class Partner(models.Model):
 
         emails = self.env['mail.mail']
         recipients_nbr, recipients_max = 0, 50
-        for email_type, recipient_template_values in pycompat.items(recipients):
+        for email_type, recipient_template_values in recipients.items():
             if recipient_template_values['followers']:
                 # generate notification email content
                 template_fol_values = dict(base_template_ctx, **recipient_template_values)  # fixme: set button_unfollow to none

addons/mail/static/src/xml/client_action.xml

@@ -84,7 +84,7 @@
     <t t-name="mail.chat.SidebarItems">
         <div t-if="!disable_add_channel" class="o_mail_add_channel" t-attf-data-type="#{channel_type}">
             <span t-if="display_hash">#</span>
-            <input type="text" t-attf-placeholder="#{input_placeholder}"/>
+            <input type="text" class="o_input" t-attf-placeholder="#{input_placeholder}"/>
         </div>
         <t t-foreach="channels" t-as="channel">
             <t t-set="counter" t-value="channel.is_chat ? channel.unread_counter : channel.needaction_counter"/>
@@ -134,6 +134,6 @@
 
     <!-- Invite dialog's content -->
     <div t-name="mail.PartnerInviteDialog">
-        <input type="text" class="o_mail_chat_partner_invite_input" id="mail_search_partners"/>
+        <input type="text" class="o_input o_mail_chat_partner_invite_input" id="mail_search_partners"/>
     </div>
 </templates>

addons/mail/static/src/xml/composer.xml

@@ -5,7 +5,7 @@
          <div class="o_composer_attachments_list"/>
          <t t-if="widget.extended">
              <div class="o_composer_subject">
-                 <input tabindex="1" type="text" placeholder="Subject"></input>
+                 <input tabindex="1" type="text" class="o_input" placeholder="Subject"></input>
                  <div class="btn-group o_composer_buttons">
                     <button tabindex="3" class="btn btn-sm btn-primary o_composer_button_send" type="button"><t t-esc="widget.options.send_text"/></button>
                     <button tabindex="4" class="btn btn-sm btn-icon fa fa-smile-o o_composer_button_emoji" type="button" data-toggle="popover"/>
@@ -15,7 +15,7 @@
          </t>
          <div t-attf-class="o_composer #{widget.extended ? 'o_extended_composer' : ''}">
              <div class="o_composer_input">
-                 <textarea class="o_composer_text_field" tabindex="2" placeholder="Write something..."/>
+                 <textarea class="o_input o_composer_text_field" tabindex="2" placeholder="Write something..."/>
              </div>
              <t t-if="!widget.extended">
                  <div class="btn-group o_composer_buttons">

addons/mail/tests/test_mail_features.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
+import base64
 
 from odoo.addons.mail.tests.common import TestMail
 from odoo.addons.mail.tests.test_mail_gateway import MAIL_TEMPLATE_PLAINTEXT
@@ -89,8 +90,8 @@ class TestMessagePost(TestMail):
         _body, _body_alt = '<p>Test Body</p>', 'Test Body'
         _subject = 'Test Subject'
         _attachments = [
-            ('List1', 'My first attachment'),
-            ('List2', 'My second attachment')
+            ('List1', b'My first attachment'),
+            ('List2', b'My second attachment')
         ]
         _attach_1 = self.env['ir.attachment'].sudo(self.user_employee).create({
             'name': 'Attach1', 'datas_fname': 'Attach1',
@@ -129,8 +130,8 @@ class TestMessagePost(TestMail):
                          'message_post: all atttachments should be linked to the mail.test model')
         self.assertEqual(set(msg.attachment_ids.mapped('res_id')), set([self.test_pigs.id]),
                          'message_post: all atttachments should be linked to the pigs group')
-        self.assertEqual(set([x.decode('base64') for x in msg.attachment_ids.mapped('datas')]),
-                         set(['migration test', _attachments[0][1], _attachments[1][1]]))
+        self.assertEqual(set([base64.b64decode(x) for x in msg.attachment_ids.mapped('datas')]),
+                         set([b'migration test', _attachments[0][1], _attachments[1][1]]))
         self.assertTrue(set([_attach_1.id, _attach_2.id]).issubset(msg.attachment_ids.ids),
                         'message_post: mail.message attachments duplicated')
         # notifications

addons/mail/tests/test_mail_message.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import base64
 import itertools
 
 from odoo.addons.mail.tests.common import TestMail
@@ -215,7 +215,7 @@ class TestMailMessageAccess(TestMail):
 
     def test_mail_message_access_read_notification(self):
         attachment = self.env['ir.attachment'].create({
-            'datas': 'My attachment'.encode('base64'),
+            'datas': base64.b64encode(b'My attachment'),
             'name': 'doc.txt',
             'datas_fname': 'doc.txt'})
         # attach the attachment to the message

addons/mail/tests/test_mail_template.py

@@ -17,13 +17,13 @@ class TestMailTemplate(TestMail):
             'name': '_Test_First',
             'datas_fname':
             'first.txt',
-            'datas': base64.b64encode('My first attachment'),
+            'datas': base64.b64encode(b'My first attachment'),
             'res_model': 'res.partner',
             'res_id': self.user_admin.partner_id.id
         }, {
             'name': '_Test_Second',
             'datas_fname': 'second.txt',
-            'datas': base64.b64encode('My second attachment'),
+            'datas': base64.b64encode(b'My second attachment'),
             'res_model': 'res.partner',
             'res_id': self.user_admin.partner_id.id
         }]

addons/mail/views/mail_message_views.xml

@@ -148,11 +148,11 @@
                                             <field name="name"/>
                                         </div>
                                         <div t-if="record.type.raw_value == 'binary'">
-                                            <a t-attf-href="/web/content/#{record.id.value}?download=1">
+                                            <a t-attf-href="/web/content/#{record.id.raw_value}?download=1">
                                                 <t t-set="webimage" t-value="new RegExp('image.*(gif|jpeg|jpg|png)').test(record.mimetype.value)"/>
                                                 <div t-if="!webimage" class="o_image" t-att-data-mimetype="record.mimetype.value"/>
                                                 <div t-if="webimage" class="o_attachment_image">
-                                                    <img t-attf-src="/web/image/#{record.id.value}?download=1"/>
+                                                    <img t-attf-src="/web/image/#{record.id.raw_value}?download=1"/>
                                                 </div>
                                             </a>
                                         </div>

addons/mail/wizard/mail_compose_message.py

@@ -140,7 +140,7 @@ class MailComposer(models.TransientModel):
             for mid, rmod, rid in self._cr.fetchall():
                 message_values[mid] = {'model': rmod, 'res_id': rid}
             # remove from the set to check the ids that mail_compose_message accepts
-            author_ids = [mid for mid, message in pycompat.items(message_values)
+            author_ids = [mid for mid, message in message_values.items()
                           if message.get('model') and not message.get('res_id')]
             self = self.browse(list(set(self.ids) - set(author_ids)))  # not sure slef = ...
 
@@ -248,7 +248,7 @@ class MailComposer(models.TransientModel):
             for res_ids in sliced_res_ids:
                 batch_mails = Mail
                 all_mail_values = wizard.get_mail_values(res_ids)
-                for res_id, mail_values in pycompat.items(all_mail_values):
+                for res_id, mail_values in all_mail_values.items():
                     if wizard.composition_mode == 'mass_mail':
                         batch_mails |= Mail.create(mail_values)
                     else:
@@ -295,8 +295,11 @@ class MailComposer(models.TransientModel):
                 'mail_server_id': self.mail_server_id.id,
                 'mail_activity_type_id': self.mail_activity_type_id.id,
             }
+
             # mass mailing: rendering override wizard static values
             if mass_mail_mode and self.model:
+                if self.model in self.env and hasattr(self.env[self.model], 'message_get_email_values'):
+                    mail_values.update(self.env[self.model].browse(res_id).message_get_email_values())
                 # keep a copy unless specifically requested, reset record name (avoid browsing records)
                 mail_values.update(notification=not self.auto_delete_message, model=self.model, res_id=res_id, record_name=False)
                 # auto deletion of mail_mail
@@ -338,7 +341,7 @@ class MailComposer(models.TransientModel):
     def onchange_template_id_wrapper(self):
         self.ensure_one()
         values = self.onchange_template_id(self.template_id.id, self.composition_mode, self.model, self.res_id)['value']
-        for fname, value in pycompat.items(values):
+        for fname, value in values.items():
             setattr(self, fname, value)
 
     @api.multi

addons/marketing_campaign/models/marketing_campaign.py

@@ -9,7 +9,6 @@ import re
 
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError, ValidationError
-from odoo.tools import pycompat
 from odoo.tools.safe_eval import safe_eval
 
 from odoo.addons import decimal_precision as dp
@@ -466,7 +465,7 @@ class MarketingCampaignWorkitem(models.Model):
         matching_workitems = []
         for id, res_id, model in res:
             workitem_map.setdefault(model, {}).setdefault(res_id, set()).add(id)
-        for model, id_map in pycompat.items(workitem_map):
+        for model, id_map in workitem_map.items():
             Model = self.env[model]
             condition_name[0] = Model._rec_name
             condition = [('id', 'in', list(id_map)), condition_name]

addons/mass_mailing/controllers/main.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
+import base64
 
 import werkzeug
 
@@ -37,7 +38,7 @@ class MassMailController(http.Controller):
         request.env['mail.mail.statistics'].sudo().set_opened(mail_mail_ids=[mail_id])
         response = werkzeug.wrappers.Response()
         response.mimetype = 'image/gif'
-        response.data = 'R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=='.decode('base64')
+        response.data = base64.b64decode(b'R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==')
 
         return response
 

addons/mass_mailing/models/mass_mailing.py

@@ -430,7 +430,7 @@ class MassMailing(models.Model):
         secret = self.env["ir.config_parameter"].sudo().get_param(
             "database.secret")
         token = (self.env.cr.dbname, self.id, int(res_id), tools.ustr(email))
-        return hmac.new(str(secret), repr(token), hashlib.sha512).hexdigest()
+        return hmac.new(secret.encode('utf-8'), repr(token).encode('utf-8'), hashlib.sha512).hexdigest()
 
     def _compute_next_departure(self):
         cron_next_call = self.env.ref('mass_mailing.ir_cron_mass_mailing_queue').sudo().nextcall

addons/mass_mailing_event_track/views/event_views.xml

@@ -11,7 +11,6 @@
                         string="Mail Track Speakers"
                         icon="fa-envelope-o"
                         attrs="{'invisible': [('track_count', '=', 0)]}">
-                    Mail Track Speakers
                 </button>
             </button>
         </field>

addons/mrp/models/mrp_workcenter.py

@@ -5,7 +5,6 @@ from dateutil import relativedelta
 import datetime
 
 from odoo import api, exceptions, fields, models, _
-from odoo.tools import pycompat
 
 
 class MrpWorkcenter(models.Model):
@@ -74,7 +73,7 @@ class MrpWorkcenter(models.Model):
             if res_group['state'] in ('pending', 'ready', 'progress'):
                 result_duration_expected[res_group['workcenter_id'][0]] += res_group['duration_expected']
         for workcenter in self:
-            workcenter.workorder_count = sum(count for state, count in pycompat.items(result[workcenter.id]) if state not in ('done', 'cancel'))
+            workcenter.workorder_count = sum(count for state, count in result[workcenter.id].items() if state not in ('done', 'cancel'))
             workcenter.workorder_pending_count = result[workcenter.id].get('pending', 0)
             workcenter.workcenter_load = result_duration_expected[workcenter.id]
             workcenter.workorder_ready_count = result[workcenter.id].get('ready', 0)

addons/mrp/views/ir_attachment_view.xml

@@ -19,11 +19,11 @@
                                 <div class="oe_attachment">
                                     <div class="row">
                                         <div class="col-xs-4 o_kanban_primary_left" t-if="record.type.raw_value == 'binary'">
-                                            <a t-attf-href="/web/content/#{record.id.value}?download=1">
+                                            <a t-attf-href="/web/content/#{record.id.raw_value}?download=1">
                                                 <t t-set="webimage" t-value="new RegExp('image.*(gif|jpeg|jpg|png)').test(record.mimetype.value)"/>
                                                 <div t-if="!webimage" class="o_image" t-att-data-mimetype="record.mimetype.value"/>
                                                 <div t-if="webimage" class="o_attachment_image">
-                                                    <img t-attf-src="/web/image/#{record.id.value}?download=1"/>
+                                                    <img t-attf-src="/web/image/#{record.id.raw_value}?download=1"/>
                                                 </div>
                                             </a>
                                         </div>

addons/mrp/wizard/mrp_product_produce_views.xml

@@ -26,7 +26,7 @@
                                 <field name="product_id" readonly="1"/>
                                 <field name="product_qty" readonly="1"/>
                                 <field name="qty_done"/>
-                                <field name="lot_id" context="{'default_product_id': product_id}"/>
+                                <field name="lot_id" context="{'default_product_id': product_id}" domain="[('product_id', '=', product_id)]"/>
                                 <field name="move_id" invisible="1"/>
                             </tree>
                         </field>

addons/mrp_repair/wizard/mrp_repair_make_invoice.py

@@ -2,7 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from odoo import api, fields, models
-from odoo.tools import pycompat
 
 
 class MakeInvoice(models.TransientModel):
@@ -25,7 +24,7 @@ class MakeInvoice(models.TransientModel):
             # but that second call will not do anything, since the repairs are already invoiced.
             repairs.action_repair_invoice_create()
         return {
-            'domain': [('id', 'in', list(pycompat.values(new_invoice)))],
+            'domain': [('id', 'in', list(new_invoice.values()))],
             'name': 'Invoices',
             'view_type': 'form',
             'view_mode': 'tree,form',

addons/pad/models/pad.py

@@ -10,7 +10,7 @@ import requests
 
 from odoo import api, models, _
 from odoo.exceptions import UserError
-from odoo.tools import html2plaintext, pycompat
+from odoo.tools import html2plaintext
 
 from ..py_etherpad import EtherpadLiteClient
 
@@ -106,7 +106,7 @@ class PadCommon(models.AbstractModel):
 
         # In case the pad is created programmatically, the content is not filled in yet since it is
         # normally initialized by the JS layer
-        for k, field in pycompat.items(self._fields):
+        for k, field in self._fields.items():
             if hasattr(field, 'pad_content_field') and k not in vals:
                 ctx = {
                     'model': self._name,
@@ -121,7 +121,7 @@ class PadCommon(models.AbstractModel):
     def _set_pad_value(self, vals):
 
         # Update the pad if the `pad_content_field` is modified
-        for k, field in pycompat.items(self._fields):
+        for k, field in self._fields.items():
             if hasattr(field, 'pad_content_field') and vals.get(field.pad_content_field) and self[k]:
                 company = self.env.user.sudo().company_id
                 myPad = EtherpadLiteClient(company.pad_key, company.pad_server + '/api')
@@ -129,7 +129,7 @@ class PadCommon(models.AbstractModel):
                 myPad.setText(path, (html2plaintext(vals[field.pad_content_field]).encode('utf-8')))
 
         # Update the `pad_content_field` if the pad is modified
-        for k, v in list(pycompat.items(vals)):
+        for k, v in vals.items():
             field = self._fields[k]
             if hasattr(field, 'pad_content_field'):
                 vals[field.pad_content_field] = self.pad_get_content(v)
@@ -139,7 +139,7 @@ class PadCommon(models.AbstractModel):
         self.ensure_one()
         if not default:
             default = {}
-        for k, field in pycompat.items(self._fields):
+        for k, field in self._fields.items():
             if hasattr(field, 'pad_content_field'):
                 pad = self.pad_generate_url()
                 default[k] = pad.get('url')

addons/pad/static/tests/pad_tests.js

@@ -230,4 +230,37 @@ QUnit.module('pad widget', {
         delete FieldPad.prototype.isPadConfigured;
     });
 
+    QUnit.test('record should be discarded properly even if only pad has changed', function (assert) {
+        assert.expect(1);
+
+        var form = createView({
+            View: FormView,
+            model: 'task',
+            data: this.data,
+            arch:'<form>' +
+                    '<sheet>' +
+                        '<group>' +
+                            '<field name="description" widget="pad"/>' +
+                        '</group>' +
+                    '</sheet>' +
+                '</form>',
+            res_id: 2,
+            mockRPC: function (route, args) {
+                if (!args.method) {
+                    return $.when(true);
+                }
+                return this._super.apply(this, arguments);
+            },
+            session: {
+                userName: "batman",
+            },
+        });
+        form.$buttons.find('.o_form_button_edit').click();
+        form.$buttons.find('.o_form_button_cancel').click();
+        assert.strictEqual(form.$('.oe_pad_readonly').text(), this.data.task.pad_get_content(),
+            "pad content should not have changed");
+        form.destroy();
+        delete FieldPad.prototype.isPadConfigured;
+    });
+
 });

addons/payment/models/payment_acquirer.py

@@ -5,7 +5,7 @@ import logging
 import datetime
 
 from odoo import api, exceptions, fields, models, _
-from odoo.tools import consteq, float_round, image_resize_images, ustr, pycompat
+from odoo.tools import consteq, float_round, image_resize_images, ustr
 from odoo.addons.base.module import module
 from odoo.exceptions import ValidationError
 
@@ -169,7 +169,7 @@ class PaymentAcquirer(models.Model):
         """ If the field has 'required_if_provider="<provider>"' attribute, then it
         required if record.provider is <provider>. """
         for acquirer in self:
-            if any(getattr(f, 'required_if_provider', None) == acquirer.provider and not acquirer[k] for k, f in pycompat.items(self._fields)):
+            if any(getattr(f, 'required_if_provider', None) == acquirer.provider and not acquirer[k] for k, f in self._fields.items()):
                 return False
         return True
 
@@ -583,7 +583,7 @@ class PaymentTransaction(models.Model):
         token = '%s%s%s' % (self.callback_model_id.model,
                             self.callback_res_id,
                             self.sudo().callback_method)
-        return hmac.new(str(secret), token, hashlib.sha256).hexdigest()
+        return hmac.new(secret.encode('utf-8'), token.encode('utf-8'), hashlib.sha256).hexdigest()
 
     # --------------------------------------------------
     # FORM RELATED METHODS

addons/payment_adyen/models/payment.py

@@ -15,8 +15,6 @@ from odoo import api, fields, models, tools, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment_adyen.controllers.main import AdyenController
 
-from odoo.tools import pycompat
-
 _logger = logging.getLogger(__name__)
 
 
@@ -52,9 +50,9 @@ class AcquirerAdyen(models.Model):
         def signParams(parms):
             signing_string = ':'.join(
                 escapeVal(v)
-                for v in chain(pycompat.keys(parms), pycompat.values(parms))
+                for v in chain(parms.keys(), parms.values())
             )
-            hm = hmac.new(hmac_key, signing_string, hashlib.sha256)
+            hm = hmac.new(hmac_key, signing_string.encode('utf-8'), hashlib.sha256)
             return base64.b64encode(hm.digest())
 
         assert inout in ('in', 'out')
@@ -81,7 +79,7 @@ class AcquirerAdyen(models.Model):
 
         hmac_key = binascii.a2b_hex(self.adyen_skin_hmac_key.encode('ascii'))
         raw_values = {k: values.get(k.encode('ascii'), '') for k in keys if k in values}
-        raw_values_ordered = OrderedDict(sorted(pycompat.items(raw_values), key=lambda t: t[0]))
+        raw_values_ordered = OrderedDict(sorted(raw_values.items(), key=lambda t: t[0]))
 
         return signParams(raw_values_ordered)
 

addons/payment_authorize/models/authorize_request.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
+import io
 import requests
 from lxml import etree, objectify
-from StringIO import StringIO
 from xml.etree import ElementTree as ET
 from uuid import uuid4
 
@@ -21,7 +21,7 @@ def strip_ns(xml, ns):
     :rtype: etree._Element
     :return: the parsed xml string with the namespace prefix removed
     """
-    it = ET.iterparse(StringIO(xml))
+    it = ET.iterparse(io.BytesIO(xml))
     ns_prefix = '{%s}' % XMLNS
     for _, el in it:
         if el.tag.startswith(ns_prefix):
@@ -58,7 +58,7 @@ class AuthorizeAPI():
 
         :param etree._Element data: etree data to process
         """
-        data = etree.tostring(data, xml_declaration=True, encoding='utf-8')
+        data = etree.tostring(data, encoding='utf-8')
         r = requests.post(self.url, data=data, headers={'Content-Type': 'text/xml'})
         r.raise_for_status()
         response = strip_ns(r.content, XMLNS)

addons/payment_authorize/models/payment.py

@@ -54,7 +54,7 @@ class PaymentAcquirerAuthorize(models.Model):
             values['x_fp_timestamp'],
             values['x_amount'],
             values['x_currency_code']])
-        return hmac.new(str(values['x_trans_key']), data, hashlib.md5).hexdigest()
+        return hmac.new(values['x_trans_key'].encode('utf-8'), data.encode('utf-8'), hashlib.md5).hexdigest()
 
     @api.multi
     def authorize_form_generate_values(self, values):

addons/payment_authorize/tests/test_authorize.py

@@ -11,7 +11,7 @@ import odoo
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment.tests.common import PaymentAcquirerCommon
 from odoo.addons.payment_authorize.controllers.main import AuthorizeController
-from odoo.tools import mute_logger, pycompat
+from odoo.tools import mute_logger
 
 
 @odoo.tests.common.at_install(True)
@@ -39,7 +39,7 @@ class AuthorizeForm(AuthorizeCommon):
             values['x_fp_timestamp'],
             values['x_amount'],
         ]) + '^'
-        return hmac.new(str(values['x_trans_key']), data, hashlib.md5).hexdigest()
+        return hmac.new(values['x_trans_key'].encode('utf-8'), data.encode('utf-8'), hashlib.md5).hexdigest()
 
     def test_10_Authorize_form_render(self):
         self.assertEqual(self.authorize.environment, 'test', 'test without test environment')
@@ -91,11 +91,11 @@ class AuthorizeForm(AuthorizeCommon):
         tree = objectify.fromstring(res)
         self.assertEqual(tree.get('action'), 'https://test.authorize.net/gateway/transact.dll', 'Authorize: wrong form POST url')
         for el in tree.iterfind('input'):
-            values = list(pycompat.values(el.attrib))
+            values = list(el.attrib.values())
             if values[1] in ['submit', 'x_fp_hash', 'return_url', 'x_state', 'x_ship_to_state']:
                 continue
             self.assertEqual(
-                unicode(values[2], "utf-8"),
+                values[2],
                 form_values[values[1]],
                 'Authorize: wrong value for input %s: received %s instead of %s' % (values[1], values[2], form_values[values[1]])
             )

addons/payment_buckaroo/controllers/main.py

@@ -6,7 +6,6 @@ import werkzeug
 
 from odoo import http
 from odoo.http import request
-from odoo.tools import pycompat
 
 _logger = logging.getLogger(__name__)
 
@@ -27,6 +26,6 @@ class BuckarooController(http.Controller):
         """ Buckaroo."""
         _logger.info('Buckaroo: entering form_feedback with post data %s', pprint.pformat(post))  # debug
         request.env['payment.transaction'].sudo().form_feedback(post, 'buckaroo')
-        post = {key.upper(): value for key, value in pycompat.items(post)}
+        post = {key.upper(): value for key, value in post.items()}
         return_url = post.get('ADD_RETURNDATA') or '/'
         return werkzeug.utils.redirect(return_url)

addons/payment_buckaroo/models/payment.py

@@ -8,7 +8,6 @@ from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment_buckaroo.controllers.main import BuckarooController
 
-from odoo.tools import pycompat
 from odoo.tools.float_utils import float_compare
 
 _logger = logging.getLogger(__name__)
@@ -21,7 +20,7 @@ def normalize_keys_upper(data):
     convert everything to upper case to be able to easily detected the presence
     of a parameter by checking the uppercase key only
     """
-    return {key.upper(): val for key, val in pycompat.items(data)}
+    return {key.upper(): val for key, val in data.items()}
 
 
 class AcquirerBuckaroo(models.Model):
@@ -73,7 +72,7 @@ class AcquirerBuckaroo(models.Model):
                     del values[key]
                     break
 
-            items = sorted(pycompat.items(values), key=lambda pair: pair[0].lower())
+            items = sorted(values.items(), key=lambda pair: pair[0].lower())
             sign = ''.join('%s=%s' % (k, urls.url_unquote_plus(v)) for k, v in items)
         else:
             sign = ''.join('%s=%s' % (k, get_value(k)) for k in keys)

addons/payment_ogone/models/payment.py

@@ -1,4 +1,5 @@
 # coding: utf-8
+import base64
 import datetime
 import logging
 import time
@@ -14,7 +15,7 @@ from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment_ogone.controllers.main import OgoneController
 from odoo.addons.payment_ogone.data import ogone
-from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT, pycompat
+from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT
 from odoo.tools.float_utils import float_compare, float_repr
 from odoo.tools.safe_eval import safe_eval
 
@@ -141,7 +142,7 @@ class PaymentAcquirerOgone(models.Model):
                 ]
                 return key.upper() in keys
 
-        items = sorted((k.upper(), v) for k, v in pycompat.items(values))
+        items = sorted((k.upper(), v) for k, v in values.items())
         sign = ''.join('%s=%s%s' % (k, v, key) for k, v in items if v and filter_key(k))
         sign = sign.encode("utf-8")
         shasign = sha1(sign).hexdigest()
@@ -469,7 +470,7 @@ class PaymentTxOgone(models.Model):
             self.write({
                 'state': new_state,
                 'acquirer_reference': tree.get('PAYID'),
-                'html_3ds': str(tree.HTML_ANSWER).decode('base64')
+                'html_3ds': base64.b64decode(tree.HTML_ANSWER.decode('ascii')),
             })
         elif status in self._ogone_wait_tx_status and tries > 0:
             time.sleep(0.5)

addons/payment_payumoney/models/payment.py

@@ -49,7 +49,7 @@ class PaymentAcquirerPayumoney(models.Model):
             sign = ''.join('%s|' % (values.get(k) or '') for k in keys)
             sign = self.payumoney_merchant_salt + sign + self.payumoney_merchant_key
 
-        shasign = hashlib.sha512(sign).hexdigest()
+        shasign = hashlib.sha512(sign.encode('utf-8')).hexdigest()
         return shasign
 
     @api.multi

addons/payment_sips/models/payment.py

@@ -66,7 +66,7 @@ class AcquirerSips(models.Model):
         if self.environment == 'prod':
             key = getattr(self, 'sips_secret')
 
-        shasign = sha256(data + key)
+        shasign = sha256((data + key).encode('utf-8'))
         return shasign.hexdigest()
 
     @api.multi

addons/payment_stripe/models/payment.py

@@ -17,8 +17,8 @@ STRIPE_HEADERS = {'Stripe-Version': '2016-03-07'}
 
 # The following currencies are integer only, see https://stripe.com/docs/currencies#zero-decimal
 INT_CURRENCIES = [
-    'BIF', 'XAF', 'XPF', 'CLP', 'KMF', 'DJF', 'GNF', 'JPY', 'MGA', 'PYGí', 'RWF', 'KRW', 'VUV',
-    'VND', 'XOF'
+    u'BIF', u'XAF', u'XPF', u'CLP', u'KMF', u'DJF', u'GNF', u'JPY', u'MGA', u'PYGí', u'RWF', u'KRW',
+    u'VUV', u'VND', u'XOF'
 ];
 
 
@@ -154,10 +154,13 @@ class PaymentTransactionStripe(models.Model):
     def _stripe_form_get_tx_from_data(self, data):
         """ Given a data dict coming from stripe, verify it and find the related
         transaction record. """
-        reference = data['metadata']['reference']
+        reference = data.get('metadata', {}).get('reference')
         if not reference:
-            error_msg = _('Stripe: invalid reply received from provider, missing reference')
-            _logger.error(error_msg, data['metadata'])
+            error_msg = _(
+                'Stripe: invalid reply received from provider, missing reference. Additional message: %s'
+                % data.get('error', {}).get('message', '')
+            )
+            _logger.error(error_msg)
             raise ValidationError(error_msg)
         tx = self.search([('reference', '=', reference)])
         if not tx: