From 70599807a163d156f2399c56d698bcf1b876daf3 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 16:50:52 +0200 Subject: [PATCH 01/47] [FIX] P3: round() compatibility shim In Python 3 round: * rounds half to even (banker's rounding) rather than up * returns an int if not given a precision (or given a precision of None) * discards the negative sign when rounding to 0 Add a compatibility shim which implements P2 behaviour and at least allows passing float_utils's tests. Implementing float_round on top of Decimal was attempted but did not work: although Decimal.quantize takes a non-integral Decimal (e.g. ``d.quantize(D('0.001'))``) it only supports integral powers of 10 and does not support quantizing to e.g. 0.05: while that will not fail it will just use the exponent for quantization and 0.05 is thus equivalent to 0.01, which is not what we want if e.g. need to round a value to 5 cents. This is hinted at but maybe not clearly spelled out by the official documentation: > Return a value equal to the first operand after rounding > and **having the exponent of** the second operand. Also convert a bunch of hand-rolled currency rounding to just using the round() method on res.currency. --- addons/account/models/account_analytic_line.py | 2 +- addons/account/models/account_invoice.py | 9 ++++----- addons/purchase/models/account_invoice.py | 7 +++---- addons/resource/models/resource_mixin.py | 3 +-- addons/stock_account/models/account_invoice.py | 2 +- odoo/tools/float_utils.py | 17 +++++++++++++++++ 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/addons/account/models/account_analytic_line.py b/addons/account/models/account_analytic_line.py index 20b44c04dbb..1d212d31bf8 100644 --- a/addons/account/models/account_analytic_line.py +++ b/addons/account/models/account_analytic_line.py @@ -44,7 +44,7 @@ class AccountAnalyticLine(models.Model): # Compute based on pricetype amount_unit = self.product_id.price_compute('standard_price', uom=unit)[self.product_id.id] amount = amount_unit * self.unit_amount or 0.0 - result = round(amount, self.currency_id.decimal_places) * -1 + result = self.currency_id.round(amount) * -1 self.amount = result self.general_account_id = account self.product_uom_id = unit diff --git a/addons/account/models/account_invoice.py b/addons/account/models/account_invoice.py index d5630cdc134..ebaff3d4c9a 100644 --- a/addons/account/models/account_invoice.py +++ b/addons/account/models/account_invoice.py @@ -1550,14 +1550,13 @@ class AccountPaymentTerm(models.Model): currency = self.env['res.currency'].browse(self.env.context['currency_id']) else: currency = self.env.user.company_id.currency_id - prec = currency.decimal_places for line in self.line_ids: if line.value == 'fixed': - amt = round(line.value_amount, prec) + amt = currency.round(line.value_amount) elif line.value == 'percent': - amt = round(value * (line.value_amount / 100.0), prec) + amt = currency.round(value * (line.value_amount / 100.0)) elif line.value == 'balance': - amt = round(amount, prec) + amt = currency.round(amount) if amt: next_date = fields.Date.from_string(date_ref) if line.option == 'day_after_invoice_date': @@ -1572,7 +1571,7 @@ class AccountPaymentTerm(models.Model): result.append((fields.Date.to_string(next_date), amt)) amount -= amt amount = sum(amt for _, amt in result) - dist = round(value - amount, prec) + dist = currency.round(value - amount) if dist: last_date = result and result[-1][0] or fields.Date.today() result.append((last_date, dist)) diff --git a/addons/purchase/models/account_invoice.py b/addons/purchase/models/account_invoice.py index c1c62f32da6..350e8ad8684 100644 --- a/addons/purchase/models/account_invoice.py +++ b/addons/purchase/models/account_invoice.py @@ -146,7 +146,6 @@ class AccountInvoice(models.Model): # reference_account_id is the stock input account reference_account_id = i_line.product_id.product_tmpl_id.get_product_accounts(fiscal_pos=fpos)['stock_input'].id diff_res = [] - account_prec = inv.company_id.currency_id.decimal_places # calculate and write down the possible price difference between invoice price and product price for line in res: if line.get('invl_id', 0) == i_line.id and reference_account_id == line['account_id']: @@ -180,13 +179,13 @@ class AccountInvoice(models.Model): if child.type_tax_use != 'none': tax_ids.append((4, child.id, None)) price_before = line.get('price', 0.0) - line.update({'price': round(valuation_price_unit * line['quantity'], account_prec)}) + line.update({'price': company_currency.round(valuation_price_unit * line['quantity'])}) diff_res.append({ 'type': 'src', 'name': i_line.name[:64], - 'price_unit': round(price_unit - valuation_price_unit, account_prec), + 'price_unit': company_currency.round(price_unit - valuation_price_unit), 'quantity': line['quantity'], - 'price': round(price_before - line.get('price', 0.0), account_prec), + 'price': company_currency.round(price_before - line.get('price', 0.0)), 'account_id': acc, 'product_id': line['product_id'], 'uom_id': line['uom_id'], diff --git a/addons/resource/models/resource_mixin.py b/addons/resource/models/resource_mixin.py index 8b922dd5ff8..9c21f7f935f 100644 --- a/addons/resource/models/resource_mixin.py +++ b/addons/resource/models/resource_mixin.py @@ -5,7 +5,6 @@ from datetime import timedelta from odoo import api, fields, models - class ResourceMixin(models.AbstractModel): _name = "resource.mixin" _description = 'Resource Mixin' @@ -70,7 +69,7 @@ class ResourceMixin(models.AbstractModel): for day_intervals in calendar._iter_leave_intervals(from_datetime, to_datetime, self.resource_id.id): theoric_hours = self.get_day_work_hours_count(day_intervals[0][0].date(), calendar=calendar) leave_time = sum((interval[1] - interval[0] for interval in day_intervals), timedelta()) - days_count += round((leave_time.total_seconds()/3600 / theoric_hours) * 4) / 4 + days_count += round((leave_time.total_seconds() / 3600 / theoric_hours) * 4) / 4 return days_count def iter_leaves(self, from_datetime, to_datetime, calendar=None): diff --git a/addons/stock_account/models/account_invoice.py b/addons/stock_account/models/account_invoice.py index 6291da2922f..281d15de0e3 100644 --- a/addons/stock_account/models/account_invoice.py +++ b/addons/stock_account/models/account_invoice.py @@ -94,7 +94,7 @@ class AccountInvoiceLine(models.Model): price = company_currency.with_context(date=self.invoice_id.date_invoice).compute(price_unit * self.quantity, self.invoice_id.currency_id) else: price = price_unit * self.quantity - return round(price, self.invoice_id.currency_id.decimal_places) + return self.invoice_id.currency_id.round(price) def get_invoice_line_account(self, type, product, fpos, company): if company.anglo_saxon_accounting and type in ('in_invoice', 'in_refund') and product and product.type == 'product': diff --git a/odoo/tools/float_utils.py b/odoo/tools/float_utils.py index 9c37da07fe9..8f95a43ba6f 100644 --- a/odoo/tools/float_utils.py +++ b/odoo/tools/float_utils.py @@ -6,6 +6,23 @@ import math from odoo.tools import pycompat +if not pycompat.PY2: + import builtins + def round(f): + # P3's builtin round differs from P2 in the following manner: + # * it rounds half to even rather than up (away from 0) + # * round(-0.) loses the sign (it returns -0 rather than 0) + # * round(x) returns an int rather than a float + # + # this compatibility shim implements Python 2's round in terms of + # Python 3's so that important rounding error under P3 can be + # trivially fixed, assuming the P2 behaviour to be debugged and + # correct. + roundf = builtins.round(f) + if builtins.round(f + 1) - roundf != 1: + return f + math.copysign(0.5, f) + # copysign ensures round(-0.) -> -0 *and* result is a float + return math.copysign(roundf, f) def _float_check_precision(precision_digits=None, precision_rounding=None): assert (precision_digits is not None or precision_rounding is not None) and \ From a4ada695e17871b27ece6baabe9abe639cf6d93f Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:01:47 +0200 Subject: [PATCH 02/47] [FIX] P3: absolute ordering removed --- addons/account/models/account_move.py | 10 +++++----- addons/account/report/account_partner_ledger.py | 2 +- addons/crm/models/crm_lead.py | 4 ++-- addons/mail/controllers/main.py | 5 ++--- addons/membership/models/account_invoice.py | 6 +++--- addons/survey/tests/test_survey.py | 4 +--- addons/web_editor/models/ir_qweb.py | 4 ++-- addons/website_blog/models/website_blog.py | 2 +- addons/website_customer/controllers/main.py | 2 +- addons/website_event_track/controllers/main.py | 2 +- odoo/addons/test_read_group/tests/test_empty.py | 2 +- 11 files changed, 20 insertions(+), 23 deletions(-) diff --git a/addons/account/models/account_move.py b/addons/account/models/account_move.py index d2369a6292b..384e9b0f8e7 100644 --- a/addons/account/models/account_move.py +++ b/addons/account/models/account_move.py @@ -192,10 +192,10 @@ class AccountMove(models.Model): @api.multi def _check_lock_date(self): for move in self: - lock_date = max(move.company_id.period_lock_date, move.company_id.fiscalyear_lock_date) + lock_date = max(move.company_id.period_lock_date or '0000-00-00', move.company_id.fiscalyear_lock_date or '0000-00-00') if self.user_has_groups('account.group_account_manager'): lock_date = move.company_id.fiscalyear_lock_date - if move.date <= lock_date: + if move.date <= (lock_date or '0000-00-00'): if self.user_has_groups('account.group_account_manager'): message = _("You cannot add/modify entries prior to and inclusive of the lock date %s") % (lock_date) else: @@ -1030,7 +1030,7 @@ class AccountMoveLine(models.Model): partial_rec_set = self.env['account.partial.reconcile'] aml_id = False partial_rec_id = False - maxdate = None + maxdate = '0000-00-00' for aml in self: total_debit += aml.debit total_credit += aml.credit @@ -1492,7 +1492,7 @@ class AccountPartialReconcile(models.Model): # The move date should be the maximum date between payment and invoice # (in case of payment in advance). However, we should make sure the # move date is not recorded after the end of year closing. - if move_date > self.company_id.fiscalyear_lock_date: + if (move_date or '0000-00-00' > self.company_id.fiscalyear_lock_date or '0000-00-00'): res['date'] = move_date return res @@ -1720,7 +1720,7 @@ class AccountPartialReconcile(models.Model): #make sure that all partial reconciliations share the same secondary currency otherwise it's not #possible to compute the exchange difference entry and it has to be done manually. currency = list(partial_rec_set)[0].currency_id - maxdate = None + maxdate = '0000-00-00' for partial_rec in partial_rec_set: if partial_rec.currency_id != currency: diff --git a/addons/account/report/account_partner_ledger.py b/addons/account/report/account_partner_ledger.py index 96e54607e4e..2da6c42abe7 100644 --- a/addons/account/report/account_partner_ledger.py +++ b/addons/account/report/account_partner_ledger.py @@ -109,7 +109,7 @@ class ReportPartnerLedger(models.AbstractModel): self.env.cr.execute(query, tuple(params)) partner_ids = [res['partner_id'] for res in self.env.cr.dictfetchall()] partners = obj_partner.browse(partner_ids) - partners = sorted(partners, key=lambda x: (x.ref, x.name)) + partners = sorted(partners, key=lambda x: (x.ref or '', x.name or '')) return { 'doc_ids': partner_ids, diff --git a/addons/crm/models/crm_lead.py b/addons/crm/models/crm_lead.py index 040ddd097d3..76f8a286205 100644 --- a/addons/crm/models/crm_lead.py +++ b/addons/crm/models/crm_lead.py @@ -298,9 +298,9 @@ class Lead(models.Model): # stage change with new stage: update probability and date_closed if vals.get('stage_id') and 'probability' not in vals: vals.update(self._onchange_stage_id_values(vals.get('stage_id'))) - if vals.get('probability') >= 100 or not vals.get('active', True): + if vals.get('probability', 0) >= 100 or not vals.get('active', True): vals['date_closed'] = fields.Datetime.now() - elif 'probability' in vals and vals['probability'] < 100: + elif 'probability' in vals: vals['date_closed'] = False return super(Lead, self).write(vals) diff --git a/addons/mail/controllers/main.py b/addons/mail/controllers/main.py index 95165441dfd..cfa5654b0cd 100644 --- a/addons/mail/controllers/main.py +++ b/addons/mail/controllers/main.py @@ -7,7 +7,6 @@ import logging import psycopg2 import werkzeug -from operator import itemgetter from werkzeug import url_encode from odoo import api, http, registry, SUPERUSER_ID, _ @@ -154,10 +153,10 @@ class MailController(http.Controller): 'default': subtype.default, 'internal': subtype.internal, 'followed': subtype.id in followers.mapped('subtype_ids').ids, - 'parent_model': subtype.parent_id and subtype.parent_id.res_model or False, + 'parent_model': subtype.parent_id.res_model, 'id': subtype.id } for subtype in subtypes] - subtypes_list = sorted(subtypes_list, key=itemgetter('parent_model', 'res_model', 'internal', 'sequence')) + subtypes_list = sorted(subtypes_list, key=lambda it: (it['parent_model'] or '', it['res_model'] or '', it['internal'], it['sequence'])) return subtypes_list @http.route('/mail/view', type='http', auth='none') diff --git a/addons/membership/models/account_invoice.py b/addons/membership/models/account_invoice.py index 273dd4290ba..73622ce0fb2 100644 --- a/addons/membership/models/account_invoice.py +++ b/addons/membership/models/account_invoice.py @@ -60,9 +60,9 @@ class AccountInvoiceLine(models.Model): invoice_line.product_id.membership and \ not MemberLine.search([('account_invoice_line', '=', invoice_line.id)]): # Product line is a membership product - date_from = invoice_line.product_id.membership_date_from - date_to = invoice_line.product_id.membership_date_to - if invoice_line.invoice_id.date_invoice > date_from and invoice_line.invoice_id.date_invoice < date_to: + date_from = invoice_line.product_id.membership_date_from or '0000-00-00' + date_to = invoice_line.product_id.membership_date_to or '0000-00-00' + if date_from < (invoice_line.invoice_id.date_invoice or '0000-00-00') < date_to: date_from = invoice_line.invoice_id.date_invoice MemberLine.create({ 'partner': invoice_line.invoice_id.partner_id.id, diff --git a/addons/survey/tests/test_survey.py b/addons/survey/tests/test_survey.py index 699aac0cde3..04a65a02562 100644 --- a/addons/survey/tests/test_survey.py +++ b/addons/survey/tests/test_survey.py @@ -226,9 +226,7 @@ class TestSurvey(TransactionCase): lines = [line.value_suggested.id for line in question.user_input_line_ids] answers = [{'text': label.value, 'count': lines.count(label.id), 'answer_id': label.id} for label in question.labels_ids] prp_result = self.env['survey.survey'].prepare_result(question)['answers'] - answers.sort() - prp_result.sort() - self.assertEqual(prp_result, answers, msg="Statistics of simple, multiple choice questions are different from expectation") + self.assertItemsEqual(prp_result, answers, msg="Statistics of simple, multiple choice questions are different from expectation") def test_11_survey_result_matrix(self): question = self.env['survey.question'].sudo(self.survey_manager).create({ diff --git a/addons/web_editor/models/ir_qweb.py b/addons/web_editor/models/ir_qweb.py index fcab2902a3b..4cceec10421 100644 --- a/addons/web_editor/models/ir_qweb.py +++ b/addons/web_editor/models/ir_qweb.py @@ -510,7 +510,7 @@ def _realize_padding(it): requests for at least n newlines of padding. Runs thereof can be collapsed into the largest requests and converted to newlines. """ - padding = None + padding = 0 for item in it: if isinstance(item, int): padding = max(padding, item) @@ -518,7 +518,7 @@ def _realize_padding(it): if padding: yield '\n' * padding - padding = None + padding = 0 yield item # leftover padding irrelevant as the output will be stripped diff --git a/addons/website_blog/models/website_blog.py b/addons/website_blog/models/website_blog.py index 22456302949..979ae14c5d3 100644 --- a/addons/website_blog/models/website_blog.py +++ b/addons/website_blog/models/website_blog.py @@ -209,7 +209,7 @@ class BlogPost(models.Model): def write(self, vals): self.ensure_one() if 'website_published' in vals and 'published_date' not in vals: - if self.published_date <= fields.Datetime.now(): + if (self.published_date or '') <= fields.Datetime.now(): vals['published_date'] = vals['website_published'] and fields.Datetime.now() result = super(BlogPost, self).write(vals) self._check_for_publication(vals) diff --git a/addons/website_customer/controllers/main.py b/addons/website_customer/controllers/main.py index d6b10b37dc1..e7318a64abf 100644 --- a/addons/website_customer/controllers/main.py +++ b/addons/website_customer/controllers/main.py @@ -73,7 +73,7 @@ class WebsiteCustomer(http.Controller): 'country_id_count': 0, 'country_id': (country.id, country.name) }) - countries.sort(key=lambda d: d['country_id'] and d['country_id'][1]) + countries.sort(key=lambda d: (d['country_id'] or (0, ""))[1]) countries.insert(0, { 'country_id_count': country_count, diff --git a/addons/website_event_track/controllers/main.py b/addons/website_event_track/controllers/main.py index 3fb37210589..1864b28379c 100644 --- a/addons/website_event_track/controllers/main.py +++ b/addons/website_event_track/controllers/main.py @@ -68,7 +68,7 @@ class WebsiteEventTrackController(http.Controller): @http.route(['''/event/ Date: Mon, 15 May 2017 16:37:07 +0200 Subject: [PATCH 03/47] [FIX] P3: / has become 'real division' in all cases --- .../views/event_track_templates.xml | 2 +- addons/website_forum/controllers/main.py | 5 ++--- addons/website_sale/controllers/main.py | 14 +++++++------- odoo/addons/base/tests/test_api.py | 2 +- odoo/tools/image.py | 11 +++++------ 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/addons/website_event_track/views/event_track_templates.xml b/addons/website_event_track/views/event_track_templates.xml index c8282180215..14eee014c6f 100644 --- a/addons/website_event_track/views/event_track_templates.xml +++ b/addons/website_event_track/views/event_track_templates.xml @@ -15,7 +15,7 @@

Our Sponsors

-
+
', 1), ('website_published', '=', True)], limit=step, offset=pager['offset'], order='karma DESC') # put the users in block of 3 to display them as a table - users = [[] for i in range(len(user_obj) / 3 + 1)] + users = [[] for i in range(len(user_obj) // 3 + 1)] for index, user in enumerate(user_obj): - users[index / 3].append(user) + users[index // 3].append(user) searches['users'] = 'True' values = self._prepare_forum_values(forum=forum, searches=searches) diff --git a/addons/website_sale/controllers/main.py b/addons/website_sale/controllers/main.py index 8f02d4a0117..51462df1de7 100644 --- a/addons/website_sale/controllers/main.py +++ b/addons/website_sale/controllers/main.py @@ -53,28 +53,28 @@ class TableCompute(object): x = y = 1 pos = minpos - while not self._check_place(pos % PPR, pos / PPR, x, y): + while not self._check_place(pos % PPR, pos // PPR, x, y): pos += 1 # if 21st products (index 20) and the last line is full (PPR products in it), break # (pos + 1.0) / PPR is the line where the product would be inserted # maxy is the number of existing lines # + 1.0 is because pos begins at 0, thus pos 20 is actually the 21st block # and to force python to not round the division operation - if index >= ppg and ((pos + 1.0) / PPR) > maxy: + if index >= ppg and ((pos + 1.0) // PPR) > maxy: break if x == 1 and y == 1: # simple heuristic for CPU optimization - minpos = pos / PPR + minpos = pos // PPR for y2 in range(y): for x2 in range(x): - self.table[(pos / PPR) + y2][(pos % PPR) + x2] = False - self.table[pos / PPR][pos % PPR] = { + self.table[(pos // PPR) + y2][(pos % PPR) + x2] = False + self.table[pos // PPR][pos % PPR] = { 'product': p, 'x': x, 'y': y, 'class': " ".join(x.html_class for x in p.website_style_ids if x.html_class) } if index <= ppg: - maxy = max(maxy, y + (pos / PPR)) + maxy = max(maxy, y + (pos // PPR)) index += 1 # Format table according to HTML needs @@ -910,7 +910,7 @@ class WebsiteSale(http.Controller): @http.route(['/shop/get_unit_price'], type='json', auth="public", methods=['POST'], website=True) def get_unit_price(self, product_ids, add_qty, **kw): products = request.env['product.product'].with_context({'quantity': add_qty}).browse(product_ids) - return {product.id: product.website_price / add_qty for product in products} + return {product.id: product.website_price // add_qty for product in products} # ------------------------------------------------------ # Edit diff --git a/odoo/addons/base/tests/test_api.py b/odoo/addons/base/tests/test_api.py index 55cf1ff26b5..ad2dd1f0e85 100644 --- a/odoo/addons/base/tests/test_api.py +++ b/odoo/addons/base/tests/test_api.py @@ -482,7 +482,7 @@ class TestAPI(common.TransactionCase): ps = self.env['res.partner'].search([]) # sort by model order - qs = ps[:len(ps) / 2] + ps[len(ps) / 2:] + qs = ps[:len(ps) // 2] + ps[len(ps) // 2:] self.assertEqual(qs.sorted().ids, ps.ids) # sort by name, with a function or a field name diff --git a/odoo/tools/image.py b/odoo/tools/image.py index d6dd1488aba..791d1a978d3 100644 --- a/odoo/tools/image.py +++ b/odoo/tools/image.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - try: import cStringIO as StringIO except ImportError: @@ -102,7 +101,7 @@ def image_resize_and_sharpen(image, size, preserve_aspect_ratio=False, factor=2. resized_image = sharpener.enhance(factor) # create a transparent image for background and paste the image on it image = Image.new('RGBA', size, (255, 255, 255, 0)) - image.paste(resized_image, ((size[0] - resized_image.size[0]) / 2, (size[1] - resized_image.size[1]) / 2)) + image.paste(resized_image, ((size[0] - resized_image.size[0]) // 2, (size[1] - resized_image.size[1]) // 2)) if image.mode != origin_mode: image = image.convert(origin_mode) return image @@ -181,17 +180,17 @@ def crop_image(data, type='top', ratio=False, thumbnail_ratio=None, image_format if ratio: w_ratio, h_ratio = ratio - new_h = (w * h_ratio) / w_ratio + new_h = (w * h_ratio) // w_ratio new_w = w if new_h > h: new_h = h - new_w = (h * w_ratio) / h_ratio + new_w = (h * w_ratio) // h_ratio if type == "top": cropped_image = image_stream.crop((0, 0, new_w, new_h)) cropped_image.save(output_stream, format=image_format) elif type == "center": - cropped_image = image_stream.crop(((w - new_w) / 2, (h - new_h) / 2, (w + new_w) / 2, (h + new_h) / 2)) + cropped_image = image_stream.crop(((w - new_w) // 2, (h - new_h) // 2, (w + new_w) // 2, (h + new_h) // 2)) cropped_image.save(output_stream, format=image_format) elif type == "bottom": cropped_image = image_stream.crop((0, h - new_h, new_w, h)) @@ -201,7 +200,7 @@ def crop_image(data, type='top', ratio=False, thumbnail_ratio=None, image_format # TDE FIXME: should not have a ratio, makes no sense -> should have maximum width (std: 64; 256 px) if thumbnail_ratio: thumb_image = Image.open(StringIO.StringIO(output_stream.getvalue())) - thumb_image.thumbnail((new_w / thumbnail_ratio, new_h / thumbnail_ratio), Image.ANTIALIAS) + thumb_image.thumbnail((new_w // thumbnail_ratio, new_h // thumbnail_ratio), Image.ANTIALIAS) thumb_image.save(output_stream, image_format) return output_stream.getvalue().encode('base64') From 4d6d5c8290ec9640ad91e70b3b6836f7232fbb4d Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 13:56:46 +0200 Subject: [PATCH 04/47] [FIX] P3: stupid use of hasattr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Python 2, hasattr swallows *any* exception raised during the access. In Python 3, it only swallows AttributeError. Accessing an attribute on an unbound threadlocal proxy raises RuntimeError, while running test_ir_http_mimetype ``request`` is unbound, in Python 2 ``hasattr`` would return ``False`` on an unbound threadlocal, in Python 3 it lets ``RuntimeError`` pass through blowing up the test. Fix this by just checking if the threadlocal is bound by using it as a boolean… --- odoo/addons/base/ir/ir_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odoo/addons/base/ir/ir_http.py b/odoo/addons/base/ir/ir_http.py index ff4654cf6c9..0aa1abc52ca 100644 --- a/odoo/addons/base/ir/ir_http.py +++ b/odoo/addons/base/ir/ir_http.py @@ -323,7 +323,7 @@ class IrHttp(models.AbstractModel): headers += [('Content-Type', mimetype), ('X-Content-Type-Options', 'nosniff')] # cache - etag = hasattr(request, 'httprequest') and request.httprequest.headers.get('If-None-Match') + etag = bool(request) and request.httprequest.headers.get('If-None-Match') retag = '"%s"' % hashlib.md5(last_update).hexdigest() status = status or (304 if etag == retag else 200) headers.append(('ETag', retag)) From 1edc84dfdf2fd497522ff4fbde5d58f592d0dffe Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 26 May 2017 13:07:33 +0200 Subject: [PATCH 05/47] [FIX] P3: surprise, time.time() isn't falsy https://bugs.python.org/issue13936 That's a pretty sensible change, but it turns out we did have some bits of resource which implicitly depended on the old behaviour. --- addons/resource/models/resource.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/addons/resource/models/resource.py b/addons/resource/models/resource.py index cd26abaabc8..b50faa855fc 100644 --- a/addons/resource/models/resource.py +++ b/addons/resource/models/resource.py @@ -420,8 +420,12 @@ class ResourceCalendar(models.Model): dtstart=start_dt, until=end_dt, byweekday=self._get_weekdays()): - start_time = day.date() == start_dt.date() and start_dt.time() or datetime.time.min - end_time = day.date() == end_dt.date() and end_dt.time() or datetime.time.max + start_time = datetime.time.min + if day.date() == start_dt.date(): + start_time = start_dt.time() + end_time = datetime.time.max + if day.date() == end_dt.date() and end_dt.time() != datetime.time(): + end_time = end_dt.time() intervals = self._get_day_work_intervals( day.date(), @@ -445,8 +449,12 @@ class ResourceCalendar(models.Model): dtstart=start_dt, until=end_dt, byweekday=self._get_weekdays()): - start_time = day.date() == start_dt.date() and start_dt.time() or datetime.time.min - end_time = day.date() == end_dt.date() and end_dt.time() or datetime.time.max + start_time = datetime.time.min + if day.date() == start_dt.date(): + start_time = start_dt.time() + end_time = datetime.time.max + if day.date() == end_dt.date() and end_dt.time() != datetime.time(): + end_time = end_dt.time() intervals = self._get_day_leave_intervals( day.date(), From d634fbd9ab432bf7bb35320fd044c0373ecfda2e Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Thu, 1 Jun 2017 11:38:22 +0200 Subject: [PATCH 06/47] [FIX] P3: reorder finders in meta_path So that was a fun one: mock.patch calls would regularly fail refusing to find the addon in odoo.addon (e.g. essentially getattr(odoo.addon, 'account_budget' deep within the bowels of mock). Turns out the answer is that our import hooks would not be used for many imports: while in Python 2, sys.meta_path is empty and the default finders are run after all meta_path finders fail as noted by the documentation[0]. However when the import system was rewritten in Python 3.3[1] meta_path was "despecialised" and the default finders were moved to meta_path rather than be a hidden part of the import machinery[2]: > sys.meta_path and sys.path_hooks now store all of the meta path > finders and path entry hooks used by import. Previously the finders > were implicit and hidden within the C code of import instead of > being directly exposed. The result of this change is that ``sys.meta_path.append`` means the default finders should take priority and the custom ones should be fallback. This is the exact opposite of what we want. Fix issue by ``sys.meta_path.insert``-ing our finders at the start of the path rather than appending them at the end. This should change nothing in Python 2 but seems to fix the issue in P3. [0] https://docs.python.org/2/library/sys.html?highlight=meta_path#sys.meta_path [1] https://docs.python.org/3/whatsnew/3.3.html#importlib [2] https://docs.python.org/3/whatsnew/3.3.html#visible-changes --- odoo/modules/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/odoo/modules/module.py b/odoo/modules/module.py index e6194a5fb10..f0836cbe041 100644 --- a/odoo/modules/module.py +++ b/odoo/modules/module.py @@ -149,8 +149,8 @@ def initialize_sys_path(): ad_paths.append(ad) if not hooked: - sys.meta_path.append(AddonsHook()) - sys.meta_path.append(OdooHook()) + sys.meta_path.insert(0, OdooHook()) + sys.meta_path.insert(0, AddonsHook()) hooked = True def get_module_path(module, downloaded=False, display_warning=True): From 3824b5dcc13678d3985ad45ac21dcf9ac9202c20 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 28 Apr 2017 15:32:10 +0200 Subject: [PATCH 07/47] [FIX] P3: fix base64 and StringIO uses * StringIO removed from stdlib, replace with io * try to correctly handle BytesIO/StringIO (one is for bytes the other is for text) * fix base64: Python 3 removed bytes-encoding and bytes-bytes codecs (via #encode) so replace all calls to str.encode('base64'), also b64encode is a bytes->bytes conversion so attempt to properly handle that issue #8530 --- addons/base_import/models/base_import.py | 8 +- addons/base_import/tests/test_base_import.py | 8 +- addons/base_import_module/models/ir_module.py | 5 +- addons/calendar/models/calendar.py | 4 +- addons/document/models/ir_attachment.py | 13 ++- addons/hr/models/hr.py | 4 +- .../im_livechat/models/im_livechat_channel.py | 4 +- addons/l10n_fr_fec/wizard/account_fr_fec.py | 4 +- addons/link_tracker/models/link_tracker.py | 6 +- addons/mail/controllers/main.py | 2 +- addons/mail/models/mail_channel.py | 4 +- addons/mail/models/mail_thread.py | 4 +- addons/mail/tests/test_mail_features.py | 7 +- addons/mail/tests/test_mail_message.py | 4 +- addons/mail/tests/test_mail_template.py | 4 +- addons/mass_mailing/controllers/main.py | 3 +- .../models/authorize_request.py | 4 +- addons/payment_ogone/models/payment.py | 3 +- addons/rating/models/rating.py | 4 +- addons/web/controllers/main.py | 11 ++- addons/web_editor/controllers/main.py | 17 ++-- addons/web_editor/models/ir_qweb.py | 11 ++- addons/website/controllers/main.py | 6 +- addons/website_blog/controllers/main.py | 4 +- addons/website_forum/controllers/main.py | 4 +- addons/website_quote/controllers/main.py | 3 +- .../website_sale_digital/controllers/main.py | 4 +- addons/website_slides/controllers/main.py | 2 +- addons/website_slides/models/slides.py | 5 +- doc/python3.rst | 98 +++++++++++++++++++ odoo/addons/base/ir/ir_attachment.py | 10 +- odoo/addons/base/ir/ir_http.py | 2 +- odoo/addons/base/ir/ir_qweb/assetsbundle.py | 9 +- odoo/addons/base/ir/ir_qweb/fields.py | 11 ++- odoo/addons/base/module/module.py | 3 +- odoo/addons/base/res/res_company.py | 4 +- odoo/addons/base/tests/test_ir_attachment.py | 10 +- .../addons/base/tests/test_view_validation.py | 12 +-- odoo/addons/test_converter/tests/test_html.py | 10 +- odoo/addons/test_pylint/tests/test_pylint.py | 7 ++ odoo/fields.py | 7 +- odoo/service/db.py | 6 +- odoo/tools/convert.py | 4 +- odoo/tools/image.py | 31 +++--- odoo/tools/pycompat.py | 8 +- 45 files changed, 258 insertions(+), 136 deletions(-) diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index 85fb49924f6..e707beecc17 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -17,12 +17,6 @@ from odoo.tools.mimetypes import guess_mimetype from odoo.tools.misc import ustr from odoo.tools import DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT, pycompat -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - - FIELDS_RECURSION_LIMIT = 2 ERROR_PREVIEW_BYTES = 200 _logger = logging.getLogger(__name__) @@ -269,7 +263,7 @@ class Import(models.TransientModel): csv_data = csv_data.decode(encoding).encode('utf-8') csv_iterator = csv.reader( - StringIO(csv_data), + io.BytesIO(csv_data), quotechar=str(options['quoting']), delimiter=str(options['separator'])) diff --git a/addons/base_import/tests/test_base_import.py b/addons/base_import/tests/test_base_import.py index 2d1d850d3a9..e3357a5c3a1 100644 --- a/addons/base_import/tests/test_base_import.py +++ b/addons/base_import/tests/test_base_import.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import csv import io import unittest @@ -527,16 +527,16 @@ class test_failures(TransactionCase): Ensure big fields (e.g. b64-encoded image data) can be imported and we're not hitting limits of the default CSV parser config """ - import csv, cStringIO + import csv, io from PIL import Image im = Image.new('RGB', (1920, 1080)) - fout = cStringIO.StringIO() + fout = io.BytesIO() writer = csv.writer(fout, dialect=None) writer.writerows([ ['name', 'db_datas'], - ['foo', im.tobytes().encode('base64')] + ['foo', base64.b64encode(im.tobytes())] ]) import_wizard = self.env['base_import.import'].create({ diff --git a/addons/base_import_module/models/ir_module.py b/addons/base_import_module/models/ir_module.py index ec94bc22d27..d6bb62a4d46 100644 --- a/addons/base_import_module/models/ir_module.py +++ b/addons/base_import_module/models/ir_module.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import base64 import logging import os import sys @@ -63,8 +64,8 @@ class IrModule(models.Model): for root, dirs, files in os.walk(path_static): for static_file in files: full_path = opj(root, static_file) - with open(full_path, 'r') as fp: - data = fp.read().encode('base64') + with open(full_path, 'rb') as fp: + data = base64.b64encode(fp.read()) url_path = '/%s%s' % (module, full_path.split(path)[1].replace(os.path.sep, '/')) url_path = url_path.decode(sys.getfilesystemencoding()) filename = os.path.split(url_path)[1] diff --git a/addons/calendar/models/calendar.py b/addons/calendar/models/calendar.py index 9c8b088aeb1..b0c16c15b5a 100644 --- a/addons/calendar/models/calendar.py +++ b/addons/calendar/models/calendar.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import base64 import babel.dates import collections @@ -173,6 +174,7 @@ class Attendee(models.Model): mails_to_send = self.env['mail.mail'] for attendee in self: if attendee.email or attendee.partner_id.email: + # FIXME: is ics_file text or bytes? ics_file = ics_files.get(attendee.event_id.id) mail_id = invitation_template.send_mail(attendee.id) @@ -181,7 +183,7 @@ class Attendee(models.Model): vals['attachment_ids'] = [(0, 0, {'name': 'invitation.ics', 'mimetype': 'text/calendar', 'datas_fname': 'invitation.ics', - 'datas': str(ics_file).encode('base64')})] + 'datas': base64.b64encode(ics_file)})] vals['model'] = None # We don't want to have the mail in the tchatter while in queue! vals['res_id'] = False current_mail = self.env['mail.mail'].browse(mail_id) diff --git a/addons/document/models/ir_attachment.py b/addons/document/models/ir_attachment.py index 53ae1f1f463..eff40db5df4 100644 --- a/addons/document/models/ir_attachment.py +++ b/addons/document/models/ir_attachment.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import io import logging import PyPDF2 import xml.dom.minidom import zipfile -from StringIO import StringIO - from odoo import api, models _logger = logging.getLogger(__name__) @@ -41,7 +40,7 @@ class IrAttachment(models.Model): def _index_docx(self, bin_data): '''Index Microsoft .docx documents''' buf = u"" - f = StringIO(bin_data) + f = io.BytesIO(bin_data) if zipfile.is_zipfile(f): try: zf = zipfile.ZipFile(f) @@ -57,7 +56,7 @@ class IrAttachment(models.Model): '''Index Microsoft .pptx documents''' buf = u"" - f = StringIO(bin_data) + f = io.BytesIO(bin_data) if zipfile.is_zipfile(f): try: zf = zipfile.ZipFile(f) @@ -75,7 +74,7 @@ class IrAttachment(models.Model): '''Index Microsoft .xlsx documents''' buf = u"" - f = StringIO(bin_data) + f = io.BytesIO(bin_data) if zipfile.is_zipfile(f): try: zf = zipfile.ZipFile(f) @@ -91,7 +90,7 @@ class IrAttachment(models.Model): '''Index OpenDocument documents (.odt, .ods...)''' buf = u"" - f = StringIO(bin_data) + f = io.BytesIO(bin_data) if zipfile.is_zipfile(f): try: zf = zipfile.ZipFile(f) @@ -108,7 +107,7 @@ class IrAttachment(models.Model): buf = u"" if bin_data.startswith('%PDF-'): - f = StringIO(bin_data) + f = io.BytesIO(bin_data) try: pdf = PyPDF2.PdfFileReader(f, overwriteWarnings=False) for page in pdf.pages: diff --git a/addons/hr/models/hr.py b/addons/hr/models/hr.py index ccfe751025b..3e513bb9e5d 100644 --- a/addons/hr/models/hr.py +++ b/addons/hr/models/hr.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import logging from odoo import api, fields, models @@ -103,7 +103,7 @@ class Employee(models.Model): @api.model def _default_image(self): image_path = get_module_resource('hr', 'static/src/img', 'default_image.png') - return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64')) + return tools.image_resize_image_big(base64.b64encode(open(image_path, 'rb').read())) # resource and user name = fields.Char(related='resource_id.name', store=True, oldname='name_related') diff --git a/addons/im_livechat/models/im_livechat_channel.py b/addons/im_livechat/models/im_livechat_channel.py index 1c72ba8725d..a878389913c 100644 --- a/addons/im_livechat/models/im_livechat_channel.py +++ b/addons/im_livechat/models/im_livechat_channel.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import random import re from datetime import datetime, timedelta @@ -21,7 +21,7 @@ class ImLivechatChannel(models.Model): def _default_image(self): image_path = modules.get_module_resource('im_livechat', 'static/src/img', 'default.png') - return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64')) + return tools.image_resize_image_big(base64.b64encode(open(image_path, 'rb').read())) def _default_user_ids(self): return [(6, 0, [self._uid])] diff --git a/addons/l10n_fr_fec/wizard/account_fr_fec.py b/addons/l10n_fr_fec/wizard/account_fr_fec.py index 215ed49cb4e..f82df08dc99 100644 --- a/addons/l10n_fr_fec/wizard/account_fr_fec.py +++ b/addons/l10n_fr_fec/wizard/account_fr_fec.py @@ -5,7 +5,7 @@ import base64 import csv -import StringIO +import io from odoo import api, fields, models, _ from odoo.exceptions import Warning @@ -114,7 +114,7 @@ class AccountFrFec(models.TransientModel): raise Warning( _("FEC is for French companies only !")) - fecfile = StringIO.StringIO() + fecfile = io.BytesIO() w = csv.writer(fecfile, delimiter='|') w.writerow(header) diff --git a/addons/link_tracker/models/link_tracker.py b/addons/link_tracker/models/link_tracker.py index 1d9a7cdf561..da24f66e905 100644 --- a/addons/link_tracker/models/link_tracker.py +++ b/addons/link_tracker/models/link_tracker.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import datetime import random import re @@ -90,7 +90,7 @@ class link_tracker(models.Model): @api.one @api.depends('favicon') def _compute_icon_src(self): - self.icon_src = 'data:image/png;base64,' + self.favicon + self.icon_src = b'data:image/png;base64,' + self.favicon @api.one @api.depends('url') @@ -123,7 +123,7 @@ class link_tracker(models.Model): def _compute_favicon(self): try: icon = requests.get('http://www.google.com/s2/favicons', params={'domain': self.url}, timeout=5).content - icon_base64 = icon.encode('base64').replace("\n", "") + icon_base64 = base64.b64encode(icon).replace(b"\n", b"").decode('ascii') except: icon_base64 = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsSAAALEgHS3X78AAACiElEQVQ4EaVTzU8TURCf2tJuS7tQtlRb6UKBIkQwkRRSEzkQgyEc6lkOKgcOph78Y+CgjXjDs2i44FXY9AMTlQRUELZapVlouy3d7kKtb0Zr0MSLTvL2zb75eL838xtTvV6H/xELBptMJojeXLCXyobnyog4YhzXYvmCFi6qVSfaeRdXdrfaU1areV5KykmX06rcvzumjY/1ggkR3Jh+bNf1mr8v1D5bLuvR3qDgFbvbBJYIrE1mCIoCrKxsHuzK+Rzvsi29+6DEbTZz9unijEYI8ObBgXOzlcrx9OAlXyDYKUCzwwrDQx1wVDGg089Dt+gR3mxmhcUnaWeoxwMbm/vzDFzmDEKMMNhquRqduT1KwXiGt0vre6iSeAUHNDE0d26NBtAXY9BACQyjFusKuL2Ry+IPb/Y9ZglwuVscdHaknUChqLF/O4jn3V5dP4mhgRJgwSYm+gV0Oi3XrvYB30yvhGa7BS70eGFHPoTJyQHhMK+F0ZesRVVznvXw5Ixv7/C10moEo6OZXbWvlFAF9FVZDOqEABUMRIkMd8GnLwVWg9/RkJF9sA4oDfYQAuzzjqzwvnaRUFxn/X2ZlmGLXAE7AL52B4xHgqAUqrC1nSNuoJkQtLkdqReszz/9aRvq90NOKdOS1nch8TpL555WDp49f3uAMXhACRjD5j4ykuCtf5PP7Fm1b0DIsl/VHGezzP1KwOiZQobFF9YyjSRYQETRENSlVzI8iK9mWlzckpSSCQHVALmN9Az1euDho9Xo8vKGd2rqooA8yBcrwHgCqYR0kMkWci08t/R+W4ljDCanWTg9TJGwGNaNk3vYZ7VUdeKsYJGFNkfSzjXNrSX20s4/h6kB81/271ghG17l+rPTAAAAAElFTkSuQmCC' diff --git a/addons/mail/controllers/main.py b/addons/mail/controllers/main.py index cfa5654b0cd..74f326d19fb 100644 --- a/addons/mail/controllers/main.py +++ b/addons/mail/controllers/main.py @@ -101,7 +101,7 @@ class MailController(http.Controller): """ End-point to receive mail from an external SMTP server. """ dbs = req.jsonrequest.get('databases') for db in dbs: - message = dbs[db].decode('base64') + message = base64.b64decode(dbs[db]) try: db_registry = registry(db) with db_registry.cursor() as cr: diff --git a/addons/mail/models/mail_channel.py b/addons/mail/models/mail_channel.py index fac6edfff98..a2ac768b772 100644 --- a/addons/mail/models/mail_channel.py +++ b/addons/mail/models/mail_channel.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 from email.utils import formataddr import re @@ -41,7 +41,7 @@ class Channel(models.Model): def _get_default_image(self): image_path = modules.get_module_resource('mail', 'static/src/img', 'groupdefault.png') - return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64')) + return tools.image_resize_image_big(base64.b64encode(open(image_path, 'rb').read())) @api.model def default_get(self, fields): diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 0d23032d051..35ae6d4ff9e 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -26,6 +26,7 @@ from werkzeug import url_encode from odoo import _, api, exceptions, fields, models, tools from odoo.tools import pycompat +from odoo.tools.pycompat import text_type from odoo.tools.safe_eval import safe_eval @@ -1266,7 +1267,8 @@ class MailThread(models.AbstractModel): # we don't know its encoding until we parse its headers and hence can't # convert it to utf-8 for transport between the mailgate script and here. if isinstance(message, xmlrpclib.Binary): - message = str(message.data) + message = bytes(message.data) + # FIXME: message_from_string parses from a *native string* # Warning: message_from_string doesn't always work correctly on unicode, # we must use utf-8 strings here :-( if isinstance(message, unicode): diff --git a/addons/mail/tests/test_mail_features.py b/addons/mail/tests/test_mail_features.py index 92613f55e51..a6fce59c10a 100644 --- a/addons/mail/tests/test_mail_features.py +++ b/addons/mail/tests/test_mail_features.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import base64 from odoo.addons.mail.tests.common import TestMail from odoo.addons.mail.tests.test_mail_gateway import MAIL_TEMPLATE_PLAINTEXT @@ -89,8 +90,8 @@ class TestMessagePost(TestMail): _body, _body_alt = '

Test Body

', 'Test Body' _subject = 'Test Subject' _attachments = [ - ('List1', 'My first attachment'), - ('List2', 'My second attachment') + ('List1', b'My first attachment'), + ('List2', b'My second attachment') ] _attach_1 = self.env['ir.attachment'].sudo(self.user_employee).create({ 'name': 'Attach1', 'datas_fname': 'Attach1', @@ -129,7 +130,7 @@ class TestMessagePost(TestMail): 'message_post: all atttachments should be linked to the mail.test model') self.assertEqual(set(msg.attachment_ids.mapped('res_id')), set([self.test_pigs.id]), 'message_post: all atttachments should be linked to the pigs group') - self.assertEqual(set([x.decode('base64') for x in msg.attachment_ids.mapped('datas')]), + self.assertEqual(set([base64.b64decode(x) for x in msg.attachment_ids.mapped('datas')]), set(['migration test', _attachments[0][1], _attachments[1][1]])) self.assertTrue(set([_attach_1.id, _attach_2.id]).issubset(msg.attachment_ids.ids), 'message_post: mail.message attachments duplicated') diff --git a/addons/mail/tests/test_mail_message.py b/addons/mail/tests/test_mail_message.py index db6dc9457a5..566a038d9ef 100644 --- a/addons/mail/tests/test_mail_message.py +++ b/addons/mail/tests/test_mail_message.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import itertools from odoo.addons.mail.tests.common import TestMail @@ -215,7 +215,7 @@ class TestMailMessageAccess(TestMail): def test_mail_message_access_read_notification(self): attachment = self.env['ir.attachment'].create({ - 'datas': 'My attachment'.encode('base64'), + 'datas': base64.b64encode(b'My attachment'), 'name': 'doc.txt', 'datas_fname': 'doc.txt'}) # attach the attachment to the message diff --git a/addons/mail/tests/test_mail_template.py b/addons/mail/tests/test_mail_template.py index 60df04f0654..e5da1c628ae 100644 --- a/addons/mail/tests/test_mail_template.py +++ b/addons/mail/tests/test_mail_template.py @@ -17,13 +17,13 @@ class TestMailTemplate(TestMail): 'name': '_Test_First', 'datas_fname': 'first.txt', - 'datas': base64.b64encode('My first attachment'), + 'datas': base64.b64encode(b'My first attachment'), 'res_model': 'res.partner', 'res_id': self.user_admin.partner_id.id }, { 'name': '_Test_Second', 'datas_fname': 'second.txt', - 'datas': base64.b64encode('My second attachment'), + 'datas': base64.b64encode(b'My second attachment'), 'res_model': 'res.partner', 'res_id': self.user_admin.partner_id.id }] diff --git a/addons/mass_mailing/controllers/main.py b/addons/mass_mailing/controllers/main.py index 276056311b8..b2e77c2875e 100644 --- a/addons/mass_mailing/controllers/main.py +++ b/addons/mass_mailing/controllers/main.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import base64 import werkzeug @@ -37,7 +38,7 @@ class MassMailController(http.Controller): request.env['mail.mail.statistics'].sudo().set_opened(mail_mail_ids=[mail_id]) response = werkzeug.wrappers.Response() response.mimetype = 'image/gif' - response.data = 'R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=='.decode('base64') + response.data = base64.b64decode(b'R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==') return response diff --git a/addons/payment_authorize/models/authorize_request.py b/addons/payment_authorize/models/authorize_request.py index fcfab1e3ac2..0a00b1a4c70 100644 --- a/addons/payment_authorize/models/authorize_request.py +++ b/addons/payment_authorize/models/authorize_request.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- +import io import requests from lxml import etree, objectify -from StringIO import StringIO from xml.etree import ElementTree as ET from uuid import uuid4 @@ -21,7 +21,7 @@ def strip_ns(xml, ns): :rtype: etree._Element :return: the parsed xml string with the namespace prefix removed """ - it = ET.iterparse(StringIO(xml)) + it = ET.iterparse(io.BytesIO(xml)) ns_prefix = '{%s}' % XMLNS for _, el in it: if el.tag.startswith(ns_prefix): diff --git a/addons/payment_ogone/models/payment.py b/addons/payment_ogone/models/payment.py index 9ac90f9478f..a5ae46bdcd3 100644 --- a/addons/payment_ogone/models/payment.py +++ b/addons/payment_ogone/models/payment.py @@ -1,4 +1,5 @@ # coding: utf-8 +import base64 import datetime import logging import time @@ -469,7 +470,7 @@ class PaymentTxOgone(models.Model): self.write({ 'state': new_state, 'acquirer_reference': tree.get('PAYID'), - 'html_3ds': str(tree.HTML_ANSWER).decode('base64') + 'html_3ds': base64.b64decode(tree.HTML_ANSWER.decode('ascii')), }) elif status in self._ogone_wait_tx_status and tries > 0: time.sleep(0.5) diff --git a/addons/rating/models/rating.py b/addons/rating/models/rating.py index 78369403d2d..a5b54afb3ac 100644 --- a/addons/rating/models/rating.py +++ b/addons/rating/models/rating.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import uuid from odoo import api, fields, models, tools, _ @@ -70,7 +70,7 @@ class Rating(models.Model): for rating in self: try: image_path = get_resource_path('rating', 'static/src/img', 'rating_%s.png' % (int(rating.rating),)) - rating.rating_image = open(image_path, 'rb').read().encode('base64') + rating.rating_image = base64.b64decode(open(image_path, 'rb').read()) except (IOError, OSError): rating.rating_image = False diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 9a29c2ec3e8..4efc3df9eec 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. + import babel.messages.pofile import base64 import csv @@ -8,6 +9,7 @@ import functools import glob import hashlib import imghdr +import io import itertools import jinja2 import json @@ -22,7 +24,6 @@ import werkzeug.utils import werkzeug.wrappers import zlib from xml.etree import ElementTree -from cStringIO import StringIO from werkzeug import url_decode @@ -1136,8 +1137,8 @@ class Binary(http.Controller): """, (uid,)) row = cr.fetchone() if row and row[0]: - image_base64 = str(row[0]).decode('base64') - image_data = StringIO(image_base64) + image_base64 = base64.b64decode(row[0]) + image_data = io.BytesIO(image_base64) imgext = '.' + (imghdr.what(None, h=image_base64) or 'png') response = http.send_file(image_data, filename=imgname + imgext, mtime=row[1]) else: @@ -1387,7 +1388,7 @@ class CSVExport(ExportFormat, http.Controller): return base + '.csv' def from_data(self, fields, rows): - fp = StringIO() + fp = io.BytesIO() writer = csv.writer(fp, quoting=csv.QUOTE_ALL) writer.writerow([name.encode('utf-8') for name in fields]) @@ -1453,7 +1454,7 @@ class ExcelExport(ExportFormat, http.Controller): cell_style = date_style worksheet.write(row_index + 1, cell_index, cell_value, cell_style) - fp = StringIO() + fp = io.BytesIO() workbook.save(fp) fp.seek(0) data = fp.read() diff --git a/addons/web_editor/controllers/main.py b/addons/web_editor/controllers/main.py index 5a3eca776ef..c501ce769d2 100644 --- a/addons/web_editor/controllers/main.py +++ b/addons/web_editor/controllers/main.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - -import cStringIO +import base64 import io import json import logging @@ -14,11 +13,11 @@ from lxml import etree, html from odoo.http import request from odoo import http, tools +from odoo.tools import pycompat from odoo.modules.module import get_resource_path, get_module_path logger = logging.getLogger(__name__) - class Web_Editor(http.Controller): #------------------------------------------------------ # Backend snippet @@ -96,7 +95,7 @@ class Web_Editor(http.Controller): font_obj = ImageFont.truetype(addons_path + font, size) # if received character is not a number, keep old behaviour (icon is character) - icon = unichr(int(icon)) if icon.isdigit() else icon + icon = pycompat.unichr(int(icon)) if icon.isdigit() else icon # Determine the dimensions of the icon image = Image.new("RGBA", (size, size), color=(0, 0, 0, 0)) @@ -165,7 +164,7 @@ class Web_Editor(http.Controller): for c_file in request.httprequest.files.getlist('upload'): data = c_file.read() try: - image = Image.open(cStringIO.StringIO(data)) + image = Image.open(io.BytesIO(data)) w, h = image.size if w*h > 42e6: # Nokia Lumia 1020 photo resolution raise ValueError( @@ -178,7 +177,7 @@ class Web_Editor(http.Controller): attachment = Attachments.create({ 'name': c_file.filename, - 'datas': data.encode('base64'), + 'datas': base64.b64encode(data), 'datas_fname': c_file.filename, 'public': True, 'res_model': 'ir.ui.view', @@ -316,7 +315,7 @@ class Web_Editor(http.Controller): content = None if url_info["customized"]: # If the file is already customized, the content is found in the corresponding attachment - content = custom_attachments.filtered(lambda a: a.url == url).datas.decode("base64") + content = base64.b64decode(custom_attachments.filtered(lambda a: a.url == url).datas) else: # If the file is not yet customized, the content is found by reading the local less file module = url_info["module"] @@ -354,14 +353,14 @@ class Web_Editor(http.Controller): custom_attachment = IrAttachment.search([("url", "=", custom_url)]) if custom_attachment: # If it was already modified, simply override the corresponding attachment content - custom_attachment.write({"datas": content.encode("utf-8").encode("base64")}) + custom_attachment.write({"datas": base64.b64encode(content.encode("utf-8"))}) else: # If not, create a new attachment to copy the original LESS file content, with its modifications IrAttachment.create(dict( name = custom_url, type = "binary", mimetype = "text/less", - datas = content.encode("utf-8").encode("base64"), + datas = base64.b64encode(content.encode("utf-8")), datas_fname = url.split("/")[-1], url = custom_url, # Having an attachment of "binary" type with an non empty "url" field # is quite of an hack. This allows to fetch the "datas" field by adding diff --git a/addons/web_editor/models/ir_qweb.py b/addons/web_editor/models/ir_qweb.py index 4cceec10421..cd665f38833 100644 --- a/addons/web_editor/models/ir_qweb.py +++ b/addons/web_editor/models/ir_qweb.py @@ -9,7 +9,8 @@ Also, adds methods to convert values back to Odoo models. """ import ast -import cStringIO +import base64 +import io import itertools import json import logging @@ -374,7 +375,7 @@ class Image(models.AbstractModel): image = I.open(f) image.load() f.seek(0) - return f.read().encode('base64') + return base64.b64encode(f.read()) except Exception: logger.exception("Failed to load local image %r", url) return None @@ -390,7 +391,7 @@ class Image(models.AbstractModel): req = requests.get(url, timeout=REMOTE_CONNECTION_TIMEOUT) # PIL needs a seekable file-like image so wrap result in IO buffer - image = I.open(cStringIO.StringIO(req.content)) + image = I.open(io.BytesIO(req.content)) # force a complete load of the image data to validate it image.load() except Exception: @@ -399,9 +400,9 @@ class Image(models.AbstractModel): # don't use original data in case weird stuff was smuggled in, with # luck PIL will remove some of it? - out = cStringIO.StringIO() + out = io.BytesIO() image.save(out, image.format) - return out.getvalue().encode('base64') + return base64.b64encode(out.getvalue()) class Monetary(models.AbstractModel): diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py index e0eb3c34c3a..2d758a3cc1f 100644 --- a/addons/website/controllers/main.py +++ b/addons/website/controllers/main.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import datetime from itertools import islice import json @@ -149,7 +149,7 @@ class Website(Home): def create_sitemap(url, content): return Attachment.create({ - 'datas': content.encode('base64'), + 'datas': base64.b64encode(content), 'mimetype': mimetype, 'type': 'binary', 'name': url, @@ -162,7 +162,7 @@ class Website(Home): create_date = fields.Datetime.from_string(sitemap.create_date) delta = datetime.datetime.now() - create_date if delta < SITEMAP_CACHE_TIME: - content = sitemap.datas.decode('base64') + content = base64.b64decode(sitemap) if not content: # Remove all sitemaps in ir.attachments as we're going to regenerated them diff --git a/addons/website_blog/controllers/main.py b/addons/website_blog/controllers/main.py index c400b8b792f..a7e930d8e9f 100644 --- a/addons/website_blog/controllers/main.py +++ b/addons/website_blog/controllers/main.py @@ -288,8 +288,8 @@ class WebsiteBlog(http.Controller): "id": message.id, "author_name": message.author_id.name, "author_image": message.author_id.image and \ - ("data:image/png;base64,%s" % message.author_id.image) or \ - '/website_blog/static/src/img/anonymous.png', + (b"data:image/png;base64,%s" % message.author_id.image) or \ + b'/website_blog/static/src/img/anonymous.png', "date": message.date, 'body': html2plaintext(message.body), 'website_published' : message.website_published, diff --git a/addons/website_forum/controllers/main.py b/addons/website_forum/controllers/main.py index 703cf517374..b70407775f0 100644 --- a/addons/website_forum/controllers/main.py +++ b/addons/website_forum/controllers/main.py @@ -577,7 +577,7 @@ class WebsiteForum(http.Controller): img_path = modules.get_module_resource('web', 'static/src/img', 'placeholder.png') with open(img_path, 'rb') as f: image = f.read() - content = image.encode('base64') + content = base64.b64encode(image) if status == 304: return werkzeug.wrappers.Response(status=304) image_base64 = base64.b64decode(content) @@ -711,7 +711,7 @@ class WebsiteForum(http.Controller): values['image'] = False elif kwargs.get('ufile'): image = kwargs.get('ufile').read() - values['image'] = image.encode('base64') + values['image'] = base64.b64encode(image) if request.uid == user.id: # the controller allows to edit only its own privacy settings; use partner management for other cases values['website_published'] = kwargs.get('website_published') == 'True' diff --git a/addons/website_quote/controllers/main.py b/addons/website_quote/controllers/main.py index 8e4359ecbec..c9dff4745ad 100644 --- a/addons/website_quote/controllers/main.py +++ b/addons/website_quote/controllers/main.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import base64 import werkzeug @@ -91,7 +92,7 @@ class sale_quote(http.Controller): return request.render('website.404') if Order.state != 'sent': return False - attachments = [('signature.png', sign.decode('base64'))] if sign else [] + attachments = [('signature.png', base64.b64decode(sign))] if sign else [] Order.action_confirm() message = _('Order signed by %s') % (signer,) _message_post_helper(message=message, res_id=order_id, res_model='sale.order', attachments=attachments, **({'token': token, 'token_field': 'access_token'} if token else {})) diff --git a/addons/website_sale_digital/controllers/main.py b/addons/website_sale_digital/controllers/main.py index ec0aeec82a7..67e125ee341 100644 --- a/addons/website_sale_digital/controllers/main.py +++ b/addons/website_sale_digital/controllers/main.py @@ -2,7 +2,7 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import base64 -from cStringIO import StringIO +import io from werkzeug.utils import redirect from odoo import http @@ -99,7 +99,7 @@ class WebsiteSaleDigital(website_account): else: return request.not_found() elif attachment["datas"]: - data = StringIO(base64.standard_b64decode(attachment["datas"])) + data = io.BytesIO(base64.standard_b64decode(attachment["datas"])) return http.send_file(data, filename=attachment['name'], as_attachment=True) else: return request.not_found() diff --git a/addons/website_slides/controllers/main.py b/addons/website_slides/controllers/main.py index 0b7fdb47982..7bf77624fe6 100644 --- a/addons/website_slides/controllers/main.py +++ b/addons/website_slides/controllers/main.py @@ -164,7 +164,7 @@ class WebsiteSlides(http.Controller): @http.route('''/slides/slide//pdf_content''', type='http', auth="public", website=True) def slide_get_pdf_content(self, slide): response = werkzeug.wrappers.Response() - response.data = slide.datas and slide.datas.decode('base64') or '' + response.data = slide.datas and base64.b64decode(slide.datas) or b'' response.mimetype = 'application/pdf' return response diff --git a/addons/website_slides/models/slides.py b/addons/website_slides/models/slides.py index ee65b15734b..dc67ea0de78 100644 --- a/addons/website_slides/models/slides.py +++ b/addons/website_slides/models/slides.py @@ -3,6 +3,7 @@ import requests from PIL import Image +import base64 import datetime import io import json @@ -502,7 +503,7 @@ class Slide(models.Model): if content_type == 'json': result['values'] = json.loads(content) elif content_type in ('image', 'pdf'): - result['values'] = content.encode('base64') + result['values'] = base64.b64encode(content) else: result['values'] = content except requests.exceptions.HTTPError as e: @@ -565,7 +566,7 @@ class Slide(models.Model): # TDE FIXME: WTF ?? slide_type = 'presentation' if vals.get('image'): - image = Image.open(io.BytesIO(vals['image'].decode('base64'))) + image = Image.open(io.BytesIO(base64.b64decode(vals['image']))) width, height = image.size if height > width: return 'document' diff --git a/doc/python3.rst b/doc/python3.rst index 3773d77c4e7..a7d143b85b7 100644 --- a/doc/python3.rst +++ b/doc/python3.rst @@ -61,6 +61,11 @@ features whereas: generator delegation, pathlib, ...), you must *not* use them in Odoo until Python 2 support is dropped +.. note:: + + In the *very rare* cases where you *need* to differentiate between + Python 2 and Python 3, use the :data:`odoo.tools.pycompat.PY2` flag. + Semantics changes ================= @@ -369,8 +374,101 @@ Minor syntax changes error, octal literals now follow the hexadecimal convention with a ``0o`` prefix. +.. _changed-strings: + +Bytes/String/Text: The Big One +============================== + +The most impactful Python 3 change by far is to the text model: for historical +reasons the distinction Python 2's bytestrings (``bytes``/``str``) and text +strings (``unicode``) is fuzzy and it will try to implicitly convert between +one and the other using the ASCII encoding. + +Python 3 changes this, it removes the implicit conversions, removes APIs which +contribute to the fuzz and tends to strictly segregate other to work on either +bytes or text. + +This is fundamentally good and mostly sensible, but it means lots of breakage: + +the builtins +------------ + +Python 3 removes both ``unicode`` and ``basestring``, and ``str`` now +corresponds to *text* strings (the old ``unicode``) with ``bytes`` being +bytestrings in both languages [#bytes]_. + +Both versions have the following prefixes for string literals: + +* ``b'foo'`` is a bytestring (``bytes`` object). + +* ``'foo'`` is that version's ``str`` type, which may be either a bytestring + or a text string [#native-string]_. + +* ``u'foo'`` is that version's text string. + +For best cross-version compatibility you should avoid unprefixed string +literals unless you *specifically* need a "native string" [#native-string]_. + +For easier type-testing, :mod:`odoo.tools.pycompat` provides the following +constants: + +* :data:`~odoo.tools.pycompat.string_types` is an alias/type tuple for testing + string types, essentially a replacement of testing for ``basestring`` or + ``(str, unicode)``. +* :data:`~odoo.tools.pycompat.text_type` is the proper *text* type for the + current version, it should mostly be used for converting non-bytes objects + to text. +* ``bytes`` should be avoided for type conversions, though it can be used to + check if an object is a bytestring. + +``open`` +-------- + +.. important:: + + the ``open`` builtin should always be explicitly used in binary mode + (``rb``, ``wb``, ...) + + To read *text* files, use ``io.open``. + +On both P2 and P3, ``open`` defaults to returning *native strings* in default +("text") mode, however in P3 that means it actually decodes the file's bytes +using whatever encoding was set up (default: UTF-8) while on Python 2 it has +no concept of encoding. + +Using ``open`` in binary mode provides bytestrings on both versions and works +fine. To read *text* files, use ``io.open`` and provide an explicit encoding. + +base64 +------ + +base64 is a bytes->bytes conversion. bytes->bytes codecs were removed from the +"native" encoding/decoding system which is now exclusively for bytes<->text +conversions: text is *encoded* to bytes and bytes are *decoded* to text. + +.. important:: + + both ``bytes.encode('base64')`` and ``bytes.decode('base64')`` must be + migrated to using ``base64.b64encode`` and ``base64.b64decode`` + respectively. + .. _hash randomisation: http://bugs.python.org/issue13703 .. _requests: http://docs.python-requests.org/ .. _werkzeug: http://werkzeug.pocoo.org/docs/urls/ + +.. [#bytes] + + with the caveat that Python 3 makes them less text-y and more byte-y e.g. + in Python 2 ``b"foo"[0]`` is ``b"f"``, but in Python 3 it's ``102`` (the + value of the first byte), you'll want to *slice* bytestrings for + compatibility. + +.. [#native-string] + + this is important because some API/contexts take a *native string* rather + than either bytes or text. The ``csv`` module of the standard library is + one such problematic API (it is also notoriously problematic for its + terrible support of non-ascii-compatible encodings in Python 2). + ``email.message_from_string`` is an other one. diff --git a/odoo/addons/base/ir/ir_attachment.py b/odoo/addons/base/ir/ir_attachment.py index 9d19d076438..f40d842c4f9 100644 --- a/odoo/addons/base/ir/ir_attachment.py +++ b/odoo/addons/base/ir/ir_attachment.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import hashlib import itertools import logging @@ -97,14 +97,14 @@ class IrAttachment(models.Model): if bin_size: r = human_size(os.path.getsize(full_path)) else: - r = open(full_path,'rb').read().encode('base64') + r = base64.b64encode(open(full_path,'rb').read()) except (IOError, OSError): _logger.info("_read_file reading %s", full_path, exc_info=True) return r @api.model def _file_write(self, value, checksum): - bin_value = value.decode('base64') + bin_value = base64.b64decode(value) fname, full_path = self._get_path(bin_value, checksum) if not os.path.exists(full_path): try: @@ -195,7 +195,7 @@ class IrAttachment(models.Model): for attach in self: # compute the fields that depend on datas value = attach.datas - bin_data = value and value.decode('base64') or '' + bin_data = value and base64.b64decode(value) or '' vals = { 'file_size': len(bin_data), 'checksum': self._compute_checksum(bin_data), @@ -235,7 +235,7 @@ class IrAttachment(models.Model): if not mimetype and values.get('url'): mimetype = mimetypes.guess_type(values['url'])[0] if values.get('datas') and (not mimetype or mimetype == 'application/octet-stream'): - mimetype = guess_mimetype(values['datas'].decode('base64')) + mimetype = guess_mimetype(base64.b64decode(values['datas'])) return mimetype or 'application/octet-stream' def _check_contents(self, values): diff --git a/odoo/addons/base/ir/ir_http.py b/odoo/addons/base/ir/ir_http.py index 0aa1abc52ca..83231ebc3e6 100644 --- a/odoo/addons/base/ir/ir_http.py +++ b/odoo/addons/base/ir/ir_http.py @@ -148,7 +148,7 @@ class IrHttp(models.AbstractModel): return response response.mimetype = attach[0]['mimetype'] or 'application/octet-stream' - response.data = datas.decode('base64') + response.data = base64.b64decode(datas) return response @classmethod diff --git a/odoo/addons/base/ir/ir_qweb/assetsbundle.py b/odoo/addons/base/ir/ir_qweb/assetsbundle.py index ac7c04fba38..24bc9bea492 100644 --- a/odoo/addons/base/ir/ir_qweb/assetsbundle.py +++ b/odoo/addons/base/ir/ir_qweb/assetsbundle.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import base64 import os import re import hashlib @@ -207,7 +208,7 @@ class AssetsBundle(object): 'res_id': False, 'type': 'binary', 'public': True, - 'datas': content.encode('utf8').encode('base64'), + 'datas': base64.b64encode(content.encode('utf8')), } attachment = ira.sudo().create(values) @@ -315,7 +316,7 @@ class AssetsBundle(object): outdated = True break if asset._content is None: - asset._content = attachment.datas and attachment.datas.decode('base64').decode('utf8') or '' + asset._content = attachment.datas and base64.b64decode(attachment.datas).decode('utf8') or '' if not asset._content and attachment.file_size > 0: asset._content = None # file missing, force recompile @@ -357,7 +358,7 @@ class AssetsBundle(object): url = asset.html_url with self.env.cr.savepoint(): self.env['ir.attachment'].sudo().create(dict( - datas=asset.content.encode('utf8').encode('base64'), + datas=base64.b64encode(asset.content.encode('utf8')), mimetype='text/css', type='binary', name=url, @@ -501,7 +502,7 @@ class WebAsset(object): with open(self._filename, 'rb') as fp: return fp.read().decode('utf-8') else: - return self._ir_attach['datas'].decode('base64').decode('utf-8') + return base64.b64decode(self._ir_attach['datas']).decode('utf-8') except UnicodeDecodeError: raise AssetError('%s is not utf-8 encoded.' % self.name) except IOError: diff --git a/odoo/addons/base/ir/ir_qweb/fields.py b/odoo/addons/base/ir/ir_qweb/fields.py index 6b9b0536b1a..7e83fced25f 100644 --- a/odoo/addons/base/ir/ir_qweb/fields.py +++ b/odoo/addons/base/ir/ir_qweb/fields.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import base64 import re from collections import OrderedDict from io import BytesIO @@ -269,15 +270,15 @@ class ImageConverter(models.AbstractModel): @api.model def value_to_html(self, value, options): - try: - image = Image.open(BytesIO(value.decode('base64'))) + try: # FIXME: maaaaaybe it could also take raw bytes? + image = Image.open(BytesIO(base64.b64decode(value))) image.verify() except IOError: raise ValueError("Non-image binary fields can not be converted to HTML") except: # image.verify() throws "suitable exceptions", I have no idea what they are raise ValueError("Invalid image content") - return unicodifier('' % (Image.MIME[image.format], value)) + return unicodifier(b'' % (Image.MIME[image.format].encode('ascii'), value)) class MonetaryConverter(models.AbstractModel): @@ -427,8 +428,8 @@ class BarcodeConverter(models.AbstractModel): barcode = self.env['ir.actions.report'].barcode( barcode_type, value, - **dict((key, value) for key, value in options.items() if key in ['width', 'height', 'humanreadable'])) - return unicodifier('' % ('png', barcode.encode('base64'))) + **{key: value for key, value in pycompat.items(options) if key in ['width', 'height', 'humanreadable']}) + return u'' % base64.b64encode(barcode).decode('ascii') @api.model def from_html(self, model, field, element): diff --git a/odoo/addons/base/module/module.py b/odoo/addons/base/module/module.py index 32c875c3fd8..e2f48e444a0 100644 --- a/odoo/addons/base/module/module.py +++ b/odoo/addons/base/module/module.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import base64 from collections import defaultdict from operator import attrgetter import importlib @@ -220,7 +221,7 @@ class Module(models.Model): path = modules.module.get_module_icon(module.name) if path: with tools.file_open(path, 'rb') as image_file: - module.icon_image = image_file.read().encode('base64') + module.icon_image = base64.b64encode(image_file.read()) name = fields.Char('Technical Name', readonly=True, required=True, index=True) category_id = fields.Many2one('ir.module.category', string='Category', readonly=True, index=True) diff --git a/odoo/addons/base/res/res_company.py b/odoo/addons/base/res/res_company.py index 387ba1afe1c..e81af42efcb 100644 --- a/odoo/addons/base/res/res_company.py +++ b/odoo/addons/base/res/res_company.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import os import re @@ -28,7 +28,7 @@ class Company(models.Model): return super(Company, self).copy(default) def _get_logo(self): - return open(os.path.join(tools.config['root_path'], 'addons', 'base', 'res', 'res_company_logo.png'), 'rb') .read().encode('base64') + return base64.b64encode(open(os.path.join(tools.config['root_path'], 'addons', 'base', 'res', 'res_company_logo.png'), 'rb') .read()) @api.model def _get_euro(self): diff --git a/odoo/addons/base/tests/test_ir_attachment.py b/odoo/addons/base/tests/test_ir_attachment.py index 284d8c71da2..8c3769b1178 100644 --- a/odoo/addons/base/tests/test_ir_attachment.py +++ b/odoo/addons/base/tests/test_ir_attachment.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import hashlib import os @@ -16,14 +16,14 @@ class TestIrAttachment(TransactionCase): self.filestore = self.Attachment._filestore() # Blob1 - self.blob1 = 'blob1' - self.blob1_b64 = self.blob1.encode('base64') + self.blob1 = b'blob1' + self.blob1_b64 = base64.b64encode(self.blob1) blob1_hash = hashlib.sha1(self.blob1).hexdigest() self.blob1_fname = blob1_hash[:HASH_SPLIT] + '/' + blob1_hash # Blob2 - self.blob2 = 'blob2' - self.blob2_b64 = self.blob2.encode('base64') + self.blob2 = b'blob2' + self.blob2_b64 = base64.b64encode(self.blob2) def test_01_store_in_db(self): # force storing in database diff --git a/odoo/addons/base/tests/test_view_validation.py b/odoo/addons/base/tests/test_view_validation.py index a408b993605..ff8ce85267a 100644 --- a/odoo/addons/base/tests/test_view_validation.py +++ b/odoo/addons/base/tests/test_view_validation.py @@ -11,7 +11,7 @@ from odoo.tools.view_validation import ( valid_field_in_graph, valid_field_in_tree, ) -invalid_form = etree.parse(io.BytesIO('''\ +invalid_form = etree.parse(io.BytesIO(b'''\
@@ -34,7 +34,7 @@ invalid_form = etree.parse(io.BytesIO('''\ ''')).getroot() -valid_form = etree.parse(io.BytesIO('''\ +valid_form = etree.parse(io.BytesIO(b'''\
@@ -54,7 +54,7 @@ valid_form = etree.parse(io.BytesIO('''\
''')).getroot() -invalid_graph = etree.parse(io.BytesIO('''\ +invalid_graph = etree.parse(io.BytesIO(b'''\ ''')).getroot() -valid_graph = etree.parse(io.BytesIO('''\ +valid_graph = etree.parse(io.BytesIO(b'''\ ''')).getroot() -invalid_tree = etree.parse(io.BytesIO('''\ +invalid_tree = etree.parse(io.BytesIO(b'''\
@@ -84,7 +84,7 @@ invalid_tree = etree.parse(io.BytesIO('''\ ''')).getroot() -valid_tree = etree.parse(io.BytesIO('''\ +valid_tree = etree.parse(io.BytesIO(b'''\ diff --git a/odoo/addons/test_converter/tests/test_html.py b/odoo/addons/test_converter/tests/test_html.py index c9bb6769d08..f03bbe8d710 100644 --- a/odoo/addons/test_converter/tests/test_html.py +++ b/odoo/addons/test_converter/tests/test_html.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import datetime import os @@ -213,11 +213,11 @@ class TestBinaryExport(TestBasicExport): with open(os.path.join(directory, 'test_vectors', 'image'), 'rb') as f: content = f.read() - encoded_content = content.encode('base64') + encoded_content = base64.b64encode(content) value = converter.value_to_html(encoded_content, {}) self.assertEqual( - value, '' % ( + value, b'' % ( encoded_content )) @@ -225,13 +225,13 @@ class TestBinaryExport(TestBasicExport): content = f.read() with self.assertRaises(ValueError): - converter.value_to_html(content.encode('base64'), {}) + converter.value_to_html(base64.b64encode(content), {}) with open(os.path.join(directory, 'test_vectors', 'pptx'), 'rb') as f: content = f.read() with self.assertRaises(ValueError): - converter.value_to_html(content.encode('base64'), {}) + converter.value_to_html(base64.b64encode(content), {}) class TestSelectionExport(TestBasicExport): diff --git a/odoo/addons/test_pylint/tests/test_pylint.py b/odoo/addons/test_pylint/tests/test_pylint.py index c5edead5a0d..cc73ca5a709 100644 --- a/odoo/addons/test_pylint/tests/test_pylint.py +++ b/odoo/addons/test_pylint/tests/test_pylint.py @@ -40,6 +40,7 @@ class TestPyLint(TransactionCase): 'old-ne-operator', 'old-octal-operator', 'parameter-unpacking', + 'invalid-string-codec', 'metaclass-assignment', 'deprecated-module', @@ -67,6 +68,10 @@ class TestPyLint(TransactionCase): 'filter', 'zip', + 'basestring', + 'unichr', + 'unicode', + 'file', 'reduce', ] @@ -74,6 +79,7 @@ class TestPyLint(TransactionCase): BAD_MODULES = [ 'commands', 'cPickle', + 'cStringIO', 'md5', 'urllib', 'urllib2', @@ -84,6 +90,7 @@ class TestPyLint(TransactionCase): 'htmlentitydefs', 'HTMLParser', 'Queue', + 'StringIO', 'UserDict', 'UserString', 'UserList', diff --git a/odoo/fields.py b/odoo/fields.py index 18174f1d145..12702612d48 100644 --- a/odoo/fields.py +++ b/odoo/fields.py @@ -11,6 +11,7 @@ import itertools import logging import pytz + try: from xmlrpc.client import MAXINT except ImportError: @@ -1619,7 +1620,11 @@ class Binary(Field): # unicode in some circumstances, hence the str() cast here. # This str() coercion will only work for pure ASCII unicode strings, # on purpose - non base64 data must be passed as a 8bit byte strings. - return psycopg2.Binary(str(value)) if value else None + if not value: + return None + if isinstance(value, bytes): + return psycopg2.Binary(value) + return psycopg2.Binary(pycompat.text_type(value).encode('ascii')) def convert_to_cache(self, value, record, validate=True): if isinstance(value, buffer): diff --git a/odoo/service/db.py b/odoo/service/db.py index e335e5b685d..861369b5b07 100644 --- a/odoo/service/db.py +++ b/odoo/service/db.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import base64 import json import logging import os @@ -160,7 +160,7 @@ def exp_dump(db_name, format): with tempfile.TemporaryFile() as t: dump_db(db_name, t, format) t.seek(0) - return t.read().encode('base64') + return base64.b64encode(t.read()) def dump_db_manifest(cr): pg_version = "%d.%d" % divmod(cr._obj.connection.server_version / 100, 100) @@ -215,7 +215,7 @@ def dump_db(db_name, stream, backup_format='zip'): def exp_restore(db_name, data, copy=False): data_file = tempfile.NamedTemporaryFile(delete=False) try: - data_file.write(data.decode('base64')) + data_file.write(base64.b64decode(data)) data_file.close() restore_db(db_name, data_file.name, copy=copy) finally: diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py index 779abec9db1..66e72c5d3ff 100644 --- a/odoo/tools/convert.py +++ b/odoo/tools/convert.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - +import base64 import io import csv import logging @@ -156,7 +156,7 @@ def _eval_xml(self, node, env): return data if t == 'base64': - return data.encode('base64') + return base64.b64encode(data) if t == 'int': d = data.strip() diff --git a/odoo/tools/image.py b/odoo/tools/image.py index 791d1a978d3..0f830b0924c 100644 --- a/odoo/tools/image.py +++ b/odoo/tools/image.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -try: - import cStringIO as StringIO -except ImportError: - import StringIO +import base64 +import codecs +import io from PIL import Image from PIL import ImageEnhance @@ -52,7 +51,7 @@ def image_resize_image(base64_source, size=(1024, 1024), encoding='base64', file return False if size == (None, None): return base64_source - image_stream = StringIO.StringIO(base64_source.decode(encoding)) + image_stream = io.BytesIO(codecs.decode(base64_source, encoding)) image = Image.open(image_stream) # store filetype here, as Image.new below will lose image.format filetype = (filetype or image.format).upper() @@ -77,9 +76,9 @@ def image_resize_image(base64_source, size=(1024, 1024), encoding='base64', file if image.mode not in ["1", "L", "P", "RGB", "RGBA"] or (filetype == 'JPEG' and image.mode == 'RGBA'): image = image.convert("RGB") - background_stream = StringIO.StringIO() + background_stream = io.BytesIO() image.save(background_stream, filetype) - return background_stream.getvalue().encode(encoding) + return codecs.encode(background_stream.getvalue(), encoding) def image_resize_and_sharpen(image, size, preserve_aspect_ratio=False, factor=2.0): """ @@ -130,7 +129,7 @@ def image_save_for_web(image, fp=None, format=None): if fp: image.save(fp, **opt) else: - img = StringIO.StringIO() + img = io.BytesIO() image.save(img, **opt) return img.getvalue() @@ -172,8 +171,8 @@ def crop_image(data, type='top', ratio=False, thumbnail_ratio=None, image_format """ if not data: return False - image_stream = Image.open(StringIO.StringIO(data.decode('base64'))) - output_stream = StringIO.StringIO() + image_stream = Image.open(io.BytesIO(base64.b64decode(data))) + output_stream = io.BytesIO() w, h = image_stream.size new_h = h new_w = w @@ -199,10 +198,10 @@ def crop_image(data, type='top', ratio=False, thumbnail_ratio=None, image_format raise ValueError('ERROR: invalid value for crop_type') # TDE FIXME: should not have a ratio, makes no sense -> should have maximum width (std: 64; 256 px) if thumbnail_ratio: - thumb_image = Image.open(StringIO.StringIO(output_stream.getvalue())) + thumb_image = Image.open(io.BytesIO(output_stream.getvalue())) thumb_image.thumbnail((new_w // thumbnail_ratio, new_h // thumbnail_ratio), Image.ANTIALIAS) thumb_image.save(output_stream, image_format) - return output_stream.getvalue().encode('base64') + return base64.b64encode(output_stream.getvalue()) # ---------------------------------------- # Colors @@ -215,7 +214,7 @@ def image_colorize(original, randomize=True, color=(255, 255, 255)): :param color: background-color, if not randomize """ # create a new image, based on the original one - original = Image.open(StringIO.StringIO(original)) + original = Image.open(io.BytesIO(original)) image = Image.new('RGB', original.size) # generate the background color, past it as background if randomize: @@ -223,7 +222,7 @@ def image_colorize(original, randomize=True, color=(255, 255, 255)): image.paste(color, box=(0, 0) + original.size) image.paste(original, mask=original) # return the new image - buffer = StringIO.StringIO() + buffer = io.BytesIO() image.save(buffer, 'PNG') return buffer.getvalue() @@ -286,6 +285,6 @@ if __name__=="__main__": assert len(sys.argv)==3, 'Usage to Test: image.py SRC.png DEST.png' - img = open(sys.argv[1],'rb').read().encode('base64') + img = base64.b64encode(open(sys.argv[1],'rb').read()) new = image_resize_image(img, (128,100)) - open(sys.argv[2], 'wb').write(new.decode('base64')) + open(sys.argv[2], 'wb').write(base64.b64decode(new)) diff --git a/odoo/tools/pycompat.py b/odoo/tools/pycompat.py index 7ce4f45773a..ff229f0562c 100644 --- a/odoo/tools/pycompat.py +++ b/odoo/tools/pycompat.py @@ -6,7 +6,10 @@ import sys PY2 = sys.version_info[0] == 2 if PY2: - # pylint: disable=long-builtin,dict-iter-method + # pylint: disable=long-builtin,unichr-builtin,unicode-builtin + unichr = unichr + text_type = unicode + string_types = (str, unicode) integer_types = (int, long) keys = lambda d: iter(d.iterkeys()) @@ -24,6 +27,9 @@ if PY2: exec ('def reraise(tp, value, tb=None):\n raise tp, value, tb') else: # pylint: disable=bad-functions + unichr = chr + text_type = str + string_types = (str,) integer_types = (int,) keys = lambda d: iter(d.keys()) From 7dd062f83552fa8ff96224c29d60bafc001d73d3 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Tue, 2 May 2017 12:07:30 +0200 Subject: [PATCH 08/47] [FIX] P3: text model types * remove references to basestring & unicode (use relevant pycompat helpers) * remove some str calls (either entirely or replaced by relevant helper, either text or native) * use better API to avoid unnecessary conversions * remove some XML declarations in views --- addons/anonymization/models/anonymization.py | 3 +- addons/base_import/models/base_import.py | 6 +- addons/board/models/board.py | 3 +- addons/bus/controllers/main.py | 6 +- addons/calendar/models/calendar.py | 17 +++--- addons/calendar/models/ir_attachment.py | 6 +- addons/calendar/models/mail_message.py | 4 +- addons/crm/wizard/base_partner_merge.py | 2 +- addons/document/models/ir_attachment.py | 13 ----- addons/hw_escpos/escpos/escpos.py | 4 +- addons/l10n_be_intrastat/wizard/xml_decl.py | 23 ++++---- addons/l10n_lu/scripts/tax2csv.py | 9 ++- addons/mail/controllers/main.py | 4 +- addons/mail/models/ir_model.py | 3 +- addons/mail/models/mail_thread.py | 27 +++++---- .../models/authorize_request.py | 2 +- .../payment_authorize/tests/test_authorize.py | 2 +- addons/product/models/product.py | 2 +- addons/product/models/product_template.py | 2 +- addons/stock/models/product.py | 4 +- addons/survey/controllers/main.py | 2 +- .../wizard/survey_email_compose_message.py | 3 +- addons/utm/models/utm.py | 3 +- addons/web/controllers/main.py | 4 +- addons/web_editor/controllers/main.py | 4 +- addons/web_editor/models/ir_qweb.py | 15 +++-- addons/web_editor/models/ir_ui_view.py | 2 +- addons/website/controllers/main.py | 2 +- addons/website/models/ir_ui_view.py | 2 +- addons/website/models/website.py | 9 +-- doc/_extensions/github_link.py | 4 +- doc/_extensions/odoo_ext/translator.py | 10 ++-- odoo/addons/base/ir/ir_fields.py | 10 ++-- odoo/addons/base/ir/ir_model.py | 9 +-- odoo/addons/base/ir/ir_qweb/__init__.py | 2 +- odoo/addons/base/ir/ir_qweb/fields.py | 27 +++++---- odoo/addons/base/ir/ir_qweb/ir_qweb.py | 2 +- odoo/addons/base/ir/ir_qweb/qweb.py | 55 +++++++++---------- odoo/addons/base/ir/ir_translation.py | 2 +- odoo/addons/base/ir/ir_ui_view.py | 10 ++-- odoo/addons/base/ir/ir_values.py | 6 +- odoo/addons/base/module/module.py | 2 +- odoo/addons/base/res/res_users.py | 6 +- odoo/addons/base/tests/test_func.py | 4 +- odoo/addons/base/tests/test_qweb.py | 2 +- odoo/addons/base/tests/test_res_config.py | 4 +- odoo/addons/test_impex/models.py | 7 ++- odoo/addons/test_impex/tests/test_export.py | 4 +- .../test_new_api/tests/test_new_fields.py | 6 +- odoo/fields.py | 45 ++++++--------- odoo/http.py | 18 +++--- odoo/loglevels.py | 19 ++++--- odoo/models.py | 46 ++++++++-------- odoo/osv/expression.py | 29 ++++------ odoo/osv/orm.py | 2 +- odoo/service/db.py | 3 +- odoo/tools/__init__.py | 2 +- odoo/tools/appdirs.py | 9 +-- odoo/tools/cache.py | 4 +- odoo/tools/config.py | 20 +++---- odoo/tools/convert.py | 5 +- odoo/tools/misc.py | 8 +-- odoo/tools/pycompat.py | 43 +++++++++++++++ odoo/tools/safe_eval.py | 4 +- odoo/tools/test_reports.py | 29 +++++----- odoo/tools/translate.py | 31 +++++------ odoo/tools/yaml_import.py | 19 +++---- 67 files changed, 359 insertions(+), 337 deletions(-) diff --git a/addons/anonymization/models/anonymization.py b/addons/anonymization/models/anonymization.py index 2d4fe1b17d2..a0c90382938 100644 --- a/addons/anonymization/models/anonymization.py +++ b/addons/anonymization/models/anonymization.py @@ -6,6 +6,7 @@ from operator import itemgetter from odoo import api, fields, models, _ from odoo.exceptions import UserError +from odoo.tools import pycompat FIELD_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('not_existing', 'Not Existing'), ('new', 'New')] ANONYMIZATION_HISTORY_STATE = [('started', 'Started'), ('done', 'Done'), ('in_exception', 'Exception occured')] @@ -13,7 +14,7 @@ ANONYMIZATION_DIRECTION = [('clear -> anonymized', 'clear -> anonymized'), ('ano def group(lst, cols): - if isinstance(cols, basestring): + if isinstance(cols, pycompat.string_types): cols = [cols] return dict((k, [v for v in itr]) for k, itr in groupby(sorted(lst, key=itemgetter(*cols)), itemgetter(*cols))) diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index e707beecc17..f2bab8b52b3 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -208,9 +208,9 @@ class Import(models.TransientModel): if cell.ctype is xlrd.XL_CELL_NUMBER: is_float = cell.value % 1 != 0.0 values.append( - unicode(cell.value) + pycompat.text_type(cell.value) if is_float - else unicode(int(cell.value)) + else pycompat.text_type(int(cell.value)) ) elif cell.ctype is xlrd.XL_CELL_DATE: is_datetime = cell.value % 1 != 0.0 @@ -653,7 +653,7 @@ class Import(models.TransientModel): except ValueError as error: return [{ 'type': 'error', - 'message': unicode(error), + 'message': pycompat.text_type(error), 'record': False, }] diff --git a/addons/board/models/board.py b/addons/board/models/board.py index 3acd277826a..81e849af263 100644 --- a/addons/board/models/board.py +++ b/addons/board/models/board.py @@ -2,6 +2,7 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. from odoo import api, models +from odoo.tools import pycompat class Board(models.AbstractModel): @@ -45,7 +46,7 @@ class Board(models.AbstractModel): return node def encode(s): - if isinstance(s, unicode): + if isinstance(s, pycompat.text_type): return s.encode('utf8') return s diff --git a/addons/bus/controllers/main.py b/addons/bus/controllers/main.py index 40118764751..4e82116e7c7 100644 --- a/addons/bus/controllers/main.py +++ b/addons/bus/controllers/main.py @@ -4,6 +4,8 @@ from odoo import exceptions from odoo.http import Controller, request, route from odoo.addons.bus.models.bus import dispatch +from odoo.tools import pycompat + class BusController(Controller): """ Examples: @@ -14,7 +16,7 @@ class BusController(Controller): @route('/longpolling/send', type="json", auth="public") def send(self, channel, message): - if not isinstance(channel, basestring): + if not isinstance(channel, pycompat.string_types): raise Exception("bus.Bus only string channels are allowed.") return request.env['bus.bus'].sendone(channel, message) @@ -33,7 +35,7 @@ class BusController(Controller): options = {} if not dispatch: raise Exception("bus.Bus unavailable") - if [c for c in channels if not isinstance(c, basestring)]: + if [c for c in channels if not isinstance(c, pycompat.string_types)]: raise Exception("bus.Bus only string channels are allowed.") if request.registry.in_test_mode(): raise exceptions.UserError("bus.Bus not available in test mode") diff --git a/addons/calendar/models/calendar.py b/addons/calendar/models/calendar.py index b0c16c15b5a..a6656af1a5b 100644 --- a/addons/calendar/models/calendar.py +++ b/addons/calendar/models/calendar.py @@ -34,7 +34,7 @@ def calendar_id2real_id(calendar_id=None, with_date=False): :param with_date: if a value is passed to this param it will return dates based on value of withdate + calendar_id :return: real event id """ - if calendar_id and isinstance(calendar_id, (basestring)): + if calendar_id and isinstance(calendar_id, pycompat.string_types): res = [bit for bit in calendar_id.split('-') if bit] if len(res) == 2: real_id = res[0] @@ -48,7 +48,7 @@ def calendar_id2real_id(calendar_id=None, with_date=False): def get_real_ids(ids): - if isinstance(ids, (basestring, pycompat.integer_types)): + if isinstance(ids, (pycompat.string_types, pycompat.integer_types)): return calendar_id2real_id(ids) if isinstance(ids, (list, tuple)): @@ -1060,7 +1060,7 @@ class Meeting(models.Model): pile.reverse() new_pile = [] for item in pile: - if not isinstance(item, basestring): + if not isinstance(item, pycompat.string_types): res = item elif str(item) == str('&'): first = new_pile.pop() @@ -1209,7 +1209,7 @@ class Meeting(models.Model): if interval == 'day': # Day number (1-31) - result = unicode(date.day) + result = pycompat.text_type(date.day) elif interval == 'month': # Localized month name and year @@ -1221,6 +1221,7 @@ class Meeting(models.Model): elif interval == 'time': # Localized time + # FIXME: formats are specifically encoded to bytes, maybe use babel? dummy, format_time = self._get_date_formats() result = tools.ustr(date.strftime(format_time + " %Z")) @@ -1316,7 +1317,7 @@ class Meeting(models.Model): @api.returns('self', lambda value: value.id) def message_post(self, **kwargs): thread_id = self.id - if isinstance(self.id, basestring): + if isinstance(self.id, pycompat.string_types): thread_id = get_real_ids(self.id) if self.env.context.get('default_date'): context = dict(self.env.context) @@ -1352,7 +1353,7 @@ class Meeting(models.Model): for arg in args: if arg[0] == 'id': for n, calendar_id in enumerate(arg[2]): - if isinstance(calendar_id, basestring): + if isinstance(calendar_id, pycompat.string_types): arg[2][n] = calendar_id.split('-')[0] return super(Meeting, self)._name_search(name=name, args=args, operator=operator, limit=limit, name_get_uid=name_get_uid) @@ -1390,7 +1391,7 @@ class Meeting(models.Model): # set end_date for calendar searching if any(field in values for field in ['recurrency', 'end_type', 'count', 'rrule_type', 'start', 'stop']): for real_meeting in real_meetings: - if real_meeting.recurrency and real_meeting.end_type in ('count', unicode('count')): + if real_meeting.recurrency and real_meeting.end_type == u'count': final_date = real_meeting._get_recurrency_end_date() super(Meeting, real_meeting).write({'final_date': final_date}) @@ -1474,7 +1475,7 @@ class Meeting(models.Model): for calendar_id, real_id in select: res = real_data[real_id].copy() ls = calendar_id2real_id(calendar_id, with_date=res and res.get('duration', 0) > 0 and res.get('duration') or 1) - if not isinstance(ls, (basestring, pycompat.integer_types)) and len(ls) >= 2: + if not isinstance(ls, (pycompat.string_types, pycompat.integer_types)) and len(ls) >= 2: res['start'] = ls[1] res['stop'] = ls[2] diff --git a/addons/calendar/models/ir_attachment.py b/addons/calendar/models/ir_attachment.py index c98242db7e7..ae734ec59f4 100644 --- a/addons/calendar/models/ir_attachment.py +++ b/addons/calendar/models/ir_attachment.py @@ -5,6 +5,8 @@ from odoo import api, models from odoo.addons.calendar.models.calendar import get_real_ids +from odoo.tools import pycompat + class Attachment(models.Model): @@ -16,13 +18,13 @@ class Attachment(models.Model): args = list(args) if any([leaf for leaf in args if leaf[0] == "res_model" and leaf[2] == 'calendar.event']): for index in range(len(args)): - if args[index][0] == "res_id" and isinstance(args[index][2], basestring): + if args[index][0] == "res_id" and isinstance(args[index][2], pycompat.string_types): args[index] = (args[index][0], args[index][1], get_real_ids(args[index][2])) return super(Attachment, self).search(args, offset=offset, limit=limit, order=order, count=count) @api.multi def write(self, vals): """ When posting an attachment (new or not), convert the virtual ids in real ids. """ - if isinstance(vals.get('res_id'), basestring): + if isinstance(vals.get('res_id'), pycompat.string_types): vals['res_id'] = get_real_ids(vals.get('res_id')) return super(Attachment, self).write(vals) diff --git a/addons/calendar/models/mail_message.py b/addons/calendar/models/mail_message.py index 0dc4d24e364..26be5080ab5 100644 --- a/addons/calendar/models/mail_message.py +++ b/addons/calendar/models/mail_message.py @@ -5,6 +5,8 @@ from odoo import api, models from odoo.addons.calendar.models.calendar import get_real_ids +from odoo.tools import pycompat + class Message(models.Model): @@ -16,7 +18,7 @@ class Message(models.Model): args = list(args) for index in range(len(args)): if args[index][0] == "res_id": - if isinstance(args[index][2], basestring): + if isinstance(args[index][2], pycompat.string_types): args[index] = (args[index][0], args[index][1], get_real_ids(args[index][2])) elif isinstance(args[index][2], list): args[index] = (args[index][0], args[index][1], [get_real_ids(x) for x in args[index][2]]) diff --git a/addons/crm/wizard/base_partner_merge.py b/addons/crm/wizard/base_partner_merge.py index 7751e0106ed..24a573ad871 100644 --- a/addons/crm/wizard/base_partner_merge.py +++ b/addons/crm/wizard/base_partner_merge.py @@ -39,7 +39,7 @@ def html_entity_decode(string): def sanitize_email(email): - assert isinstance(email, basestring) and email + assert isinstance(email, pycompat.string_types) and email result = re.subn(r';|/|:', ',', html_entity_decode(email or ''))[0].split(',') diff --git a/addons/document/models/ir_attachment.py b/addons/document/models/ir_attachment.py index eff40db5df4..7e1afe36e4e 100644 --- a/addons/document/models/ir_attachment.py +++ b/addons/document/models/ir_attachment.py @@ -11,19 +11,6 @@ from odoo import api, models _logger = logging.getLogger(__name__) FTYPES = ['docx', 'pptx', 'xlsx', 'opendoc', 'pdf'] -# Keep function in case it is necessary to do toUnicode(buf.encode('ascii', 'replace')) -def toUnicode(s): - try: - return s.decode('utf-8') - except UnicodeError: - try: - return s.decode('latin') - except UnicodeError: - try: - return s.encode('ascii') - except UnicodeError: - return s - def textToString(element): buff = u"" for node in element.childNodes: diff --git a/addons/hw_escpos/escpos/escpos.py b/addons/hw_escpos/escpos/escpos.py index e50537e0eeb..f076778ca9c 100644 --- a/addons/hw_escpos/escpos/escpos.py +++ b/addons/hw_escpos/escpos/escpos.py @@ -12,6 +12,8 @@ from hashlib import md5 from PIL import Image from xml.etree import ElementTree as ET +from odoo.tools import pycompat + try: import jcconv except ImportError: @@ -27,7 +29,7 @@ from .exceptions import * def utfstr(stuff): """ converts stuff to string and does without failing if stuff is a utf8 string """ - if isinstance(stuff,basestring): + if isinstance(stuff,pycompat.string_types): return stuff else: return str(stuff) diff --git a/addons/l10n_be_intrastat/wizard/xml_decl.py b/addons/l10n_be_intrastat/wizard/xml_decl.py index f7d01db30d1..b5486d963c9 100644 --- a/addons/l10n_be_intrastat/wizard/xml_decl.py +++ b/addons/l10n_be_intrastat/wizard/xml_decl.py @@ -6,6 +6,7 @@ from xml.etree import ElementTree as ET from collections import namedtuple from odoo import api, exceptions, fields, models, _ +from odoo.tools.pycompat import text_type INTRASTAT_XMLNS = 'http://www.onegate.eu/2010-01-01' @@ -264,18 +265,18 @@ class XmlDeclaration(models.TransientModel): continue numlgn += 1 item = ET.SubElement(datas, 'Item') - self._set_Dim(item, 'EXSEQCODE', unicode(numlgn)) - self._set_Dim(item, 'EXTRF', unicode(linekey.EXTRF)) - self._set_Dim(item, 'EXCNT', unicode(linekey.EXCNT)) - self._set_Dim(item, 'EXTTA', unicode(linekey.EXTTA)) - self._set_Dim(item, 'EXREG', unicode(linekey.EXREG)) - self._set_Dim(item, 'EXTGO', unicode(linekey.EXGO)) + self._set_Dim(item, 'EXSEQCODE', text_type(numlgn)) + self._set_Dim(item, 'EXTRF', text_type(linekey.EXTRF)) + self._set_Dim(item, 'EXCNT', text_type(linekey.EXCNT)) + self._set_Dim(item, 'EXTTA', text_type(linekey.EXTTA)) + self._set_Dim(item, 'EXREG', text_type(linekey.EXREG)) + self._set_Dim(item, 'EXTGO', text_type(linekey.EXGO)) if extendedmode: - self._set_Dim(item, 'EXTPC', unicode(linekey.EXTPC)) - self._set_Dim(item, 'EXDELTRM', unicode(linekey.EXDELTRM)) - self._set_Dim(item, 'EXTXVAL', unicode(round(amounts[0], 0)).replace(".", ",")) - self._set_Dim(item, 'EXWEIGHT', unicode(round(amounts[1], 0)).replace(".", ",")) - self._set_Dim(item, 'EXUNITS', unicode(round(amounts[2], 0)).replace(".", ",")) + self._set_Dim(item, 'EXTPC', text_type(linekey.EXTPC)) + self._set_Dim(item, 'EXDELTRM', text_type(linekey.EXDELTRM)) + self._set_Dim(item, 'EXTXVAL', text_type(round(amounts[0], 0)).replace(".", ",")) + self._set_Dim(item, 'EXWEIGHT', text_type(round(amounts[1], 0)).replace(".", ",")) + self._set_Dim(item, 'EXUNITS', text_type(round(amounts[2], 0)).replace(".", ",")) if numlgn == 0: #no datas diff --git a/addons/l10n_lu/scripts/tax2csv.py b/addons/l10n_lu/scripts/tax2csv.py index 9993272573f..013df32e3d3 100644 --- a/addons/l10n_lu/scripts/tax2csv.py +++ b/addons/l10n_lu/scripts/tax2csv.py @@ -5,12 +5,11 @@ import xlrd from odoo.tools import pycompat def _e(s): - if type(s) is unicode: - return s.encode('utf8') - elif s is None: + if s is None: return '' - else: - return str(s) + if pycompat.PY2 and type(s) == pycompat.text_type: + return s.encode('utf8') + return str(s) def _is_true(s): diff --git a/addons/mail/controllers/main.py b/addons/mail/controllers/main.py index 74f326d19fb..52ea655c078 100644 --- a/addons/mail/controllers/main.py +++ b/addons/mail/controllers/main.py @@ -13,7 +13,7 @@ from odoo import api, http, registry, SUPERUSER_ID, _ from odoo.addons.web.controllers.main import binary_content from odoo.exceptions import AccessError from odoo.http import request -from odoo.tools import consteq +from odoo.tools import consteq, pycompat _logger = logging.getLogger(__name__) @@ -182,7 +182,7 @@ class MailController(http.Controller): else: # either a wrong message_id, either someone trying ids -> just go to messaging return self._redirect_to_messaging() - elif res_id and isinstance(res_id, basestring): + elif res_id and isinstance(res_id, pycompat.string_types): res_id = int(res_id) return self._redirect_to_record(model, res_id, access_token) diff --git a/addons/mail/models/ir_model.py b/addons/mail/models/ir_model.py index e2231e38f5d..01862352e96 100644 --- a/addons/mail/models/ir_model.py +++ b/addons/mail/models/ir_model.py @@ -3,6 +3,7 @@ from odoo import _, api, fields, models from odoo.exceptions import UserError +from odoo.tools import pycompat class IrModel(models.Model): @@ -41,7 +42,7 @@ class IrModel(models.Model): model_class = super(IrModel, self)._instanciate(model_data) if model_data.get('is_mail_thread') and model_class._name != 'mail.thread': parents = model_class._inherit or [] - parents = [parents] if isinstance(parents, basestring) else parents + parents = [parents] if isinstance(parents, pycompat.string_types) else parents model_class._inherit = parents + ['mail.thread'] return model_class diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 35ae6d4ff9e..bc7b1d1072d 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -423,7 +423,7 @@ class MailThread(models.AbstractModel): for field_name, (template, post_kwargs) in pycompat.items(templates): if not template: continue - if isinstance(template, basestring): + if isinstance(template, pycompat.string_types): self.message_post_with_view(template, **post_kwargs) else: self.message_post_with_template(template.id, **post_kwargs) @@ -1268,12 +1268,15 @@ class MailThread(models.AbstractModel): # convert it to utf-8 for transport between the mailgate script and here. if isinstance(message, xmlrpclib.Binary): message = bytes(message.data) - # FIXME: message_from_string parses from a *native string* - # Warning: message_from_string doesn't always work correctly on unicode, - # we must use utf-8 strings here :-( - if isinstance(message, unicode): + # message_from_string parses from a *native string*, except apparently + # sometimes message is ISO-8859-1 binary data or some shit and the + # straightforward version (pycompat.to_native) won't work right -> + # always encode message to bytes then use the relevant method + # depending on ~python version + if isinstance(message, pycompat.text_type): message = message.encode('utf-8') - msg_txt = email.message_from_string(message) + extract = getattr(email, 'message_from_bytes', email.message_from_string) + msg_txt = extract(message) # parse the message, verify we are not in a loop by checking message_id is not duplicated msg = self.message_parse(msg_txt, save_original=save_original) @@ -1494,9 +1497,9 @@ class MailThread(models.AbstractModel): 'message_type': 'email', } if not isinstance(message, Message): - if isinstance(message, unicode): - # Warning: message_from_string doesn't always work correctly on unicode, - # we must use utf-8 strings here :-( + # message_from_string works on a native str, so on py2 we need to + # encode incoming unicode strings + if pycompat.PY2 and not isinstance(message, str): message = message.encode('utf-8') message = email.message_from_string(message) @@ -1725,11 +1728,11 @@ class MailThread(models.AbstractModel): cid = info and info.get('cid') else: continue - if isinstance(content, unicode): + if isinstance(content, pycompat.text_type): content = content.encode('utf-8') data_attach = { 'name': name, - 'datas': base64.b64encode(str(content)), + 'datas': base64.b64encode(content), 'type': 'binary', 'datas_fname': name, 'description': name, @@ -1922,7 +1925,7 @@ class MailThread(models.AbstractModel): values['slug'] = slug except ImportError: values['slug'] = lambda self: self.id - if isinstance(views_or_xmlid, basestring): + if isinstance(views_or_xmlid, pycompat.string_types): views = self.env.ref(views_or_xmlid, raise_if_not_found=False) else: views = views_or_xmlid diff --git a/addons/payment_authorize/models/authorize_request.py b/addons/payment_authorize/models/authorize_request.py index 0a00b1a4c70..7388a1a4329 100644 --- a/addons/payment_authorize/models/authorize_request.py +++ b/addons/payment_authorize/models/authorize_request.py @@ -58,7 +58,7 @@ class AuthorizeAPI(): :param etree._Element data: etree data to process """ - data = etree.tostring(data, xml_declaration=True, encoding='utf-8') + data = etree.tostring(data, encoding='utf-8') r = requests.post(self.url, data=data, headers={'Content-Type': 'text/xml'}) r.raise_for_status() response = strip_ns(r.content, XMLNS) diff --git a/addons/payment_authorize/tests/test_authorize.py b/addons/payment_authorize/tests/test_authorize.py index b02d3d81209..668e17d0562 100644 --- a/addons/payment_authorize/tests/test_authorize.py +++ b/addons/payment_authorize/tests/test_authorize.py @@ -95,7 +95,7 @@ class AuthorizeForm(AuthorizeCommon): if values[1] in ['submit', 'x_fp_hash', 'return_url', 'x_state', 'x_ship_to_state']: continue self.assertEqual( - unicode(values[2], "utf-8"), + values[2], form_values[values[1]], 'Authorize: wrong value for input %s: received %s instead of %s' % (values[1], values[2], form_values[values[1]]) ) diff --git a/addons/product/models/product.py b/addons/product/models/product.py index 638e898aed8..c550e063709 100644 --- a/addons/product/models/product.py +++ b/addons/product/models/product.py @@ -158,7 +158,7 @@ class ProductProduct(models.Model): quantity = self._context.get('quantity', 1.0) # Support context pricelists specified as display_name or ID for compatibility - if isinstance(pricelist_id_or_name, basestring): + if isinstance(pricelist_id_or_name, pycompat.string_types): pricelist_name_search = self.env['product.pricelist'].name_search(pricelist_id_or_name, operator='=', limit=1) if pricelist_name_search: pricelist = self.env['product.pricelist'].browse([pricelist_name_search[0][0]]) diff --git a/addons/product/models/product_template.py b/addons/product/models/product_template.py index ccfbcf28b96..c9a2411f173 100644 --- a/addons/product/models/product_template.py +++ b/addons/product/models/product_template.py @@ -171,7 +171,7 @@ class ProductTemplate(models.Model): quantity = self._context.get('quantity', 1.0) # Support context pricelists specified as display_name or ID for compatibility - if isinstance(pricelist_id_or_name, basestring): + if isinstance(pricelist_id_or_name, pycompat.string_types): pricelist_data = self.env['product.pricelist'].name_search(pricelist_id_or_name, operator='=', limit=1) if pricelist_data: pricelist = self.env['product.pricelist'].browse(pricelist_data[0][0]) diff --git a/addons/stock/models/product.py b/addons/stock/models/product.py index 43cbe1e2e4d..693f5c0119e 100644 --- a/addons/stock/models/product.py +++ b/addons/stock/models/product.py @@ -161,7 +161,7 @@ class Product(models.Model): if self.env.context.get('location', False): if isinstance(self.env.context['location'], pycompat.integer_types): location_ids = [self.env.context['location']] - elif isinstance(self.env.context['location'], basestring): + elif isinstance(self.env.context['location'], pycompat.string_types): domain = [('complete_name', 'ilike', self.env.context['location'])] if self.env.context.get('force_company', False): domain += [('company_id', '=', self.env.context['force_company'])] @@ -172,7 +172,7 @@ class Product(models.Model): if self.env.context.get('warehouse', False): if isinstance(self.env.context['warehouse'], pycompat.integer_types): wids = [self.env.context['warehouse']] - elif isinstance(self.env.context['warehouse'], basestring): + elif isinstance(self.env.context['warehouse'], pycompat.string_types): domain = [('name', 'ilike', self.env.context['warehouse'])] if self.env.context.get('force_company', False): domain += [('company_id', '=', self.env.context['force_company'])] diff --git a/addons/survey/controllers/main.py b/addons/survey/controllers/main.py index 553fc5ad4b2..150503f07da 100644 --- a/addons/survey/controllers/main.py +++ b/addons/survey/controllers/main.py @@ -174,7 +174,7 @@ class WebsiteSurvey(http.Controller): answer_tag = "%s_%s" % (answer_tag, 'comment') answer_value = answer.value_text elif answer.answer_type == 'number': - answer_value = answer.value_number.__str__() + answer_value = str(answer.value_number) elif answer.answer_type == 'date': answer_value = answer.value_date elif answer.answer_type == 'suggestion' and not answer.value_suggested_row: diff --git a/addons/survey/wizard/survey_email_compose_message.py b/addons/survey/wizard/survey_email_compose_message.py index fb36d839c57..f23d836bc30 100644 --- a/addons/survey/wizard/survey_email_compose_message.py +++ b/addons/survey/wizard/survey_email_compose_message.py @@ -8,6 +8,7 @@ from werkzeug import urls from odoo import api, fields, models, _ from odoo.exceptions import UserError +from odoo.tools import pycompat emails_split = re.compile(r"[;,\n\r]+") email_validator = re.compile(r"[^@]+@[^@]+\.[^@]+") @@ -119,7 +120,7 @@ class SurveyMailComposeMessage(models.TransientModel): if wizard.public != 'email_private': return None else: - token = uuid.uuid4().__str__() + token = pycompat.text_type(uuid.uuid4()) # create response with token survey_user_input = SurveyUserInput.create({ 'survey_id': wizard.survey_id.id, diff --git a/addons/utm/models/utm.py b/addons/utm/models/utm.py index 5cb04d51425..b73329c1011 100644 --- a/addons/utm/models/utm.py +++ b/addons/utm/models/utm.py @@ -3,6 +3,7 @@ from odoo import api, fields, models, SUPERUSER_ID from odoo.http import request +from odoo.tools import pycompat class UtmMedium(models.Model): @@ -72,7 +73,7 @@ class UtmMixin(models.AbstractModel): # ir_http dispatch saves the url params in a cookie value = request.httprequest.cookies.get(cookie_name) # if we receive a string for a many2one, we search/create the id - if field.type == 'many2one' and isinstance(value, basestring) and value: + if field.type == 'many2one' and isinstance(value, pycompat.string_types) and value: Model = self.env[field.comodel_name] records = Model.search([('name', '=', value)], limit=1) if not records: diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 4efc3df9eec..9ac7b7a2fd8 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -1396,7 +1396,7 @@ class CSVExport(ExportFormat, http.Controller): for data in rows: row = [] for d in data: - if isinstance(d, unicode): + if type(d) != str and isinstance(d, pycompat.string_types): try: d = d.encode('utf-8') except UnicodeError: @@ -1446,7 +1446,7 @@ class ExcelExport(ExportFormat, http.Controller): for row_index, row in enumerate(rows): for cell_index, cell_value in enumerate(row): cell_style = base_style - if isinstance(cell_value, basestring): + if isinstance(cell_value, pycompat.string_types): cell_value = re.sub("\r", " ", cell_value) elif isinstance(cell_value, datetime.datetime): cell_style = datetime_style diff --git a/addons/web_editor/controllers/main.py b/addons/web_editor/controllers/main.py index c501ce769d2..50a9540323a 100644 --- a/addons/web_editor/controllers/main.py +++ b/addons/web_editor/controllers/main.py @@ -39,7 +39,7 @@ class Web_Editor(http.Controller): debug=request.debug) for k in kwargs: - if isinstance(kwargs[k], basestring) and kwargs[k].isdigit(): + if isinstance(kwargs[k], pycompat.string_types) and kwargs[k].isdigit(): kwargs[k] = int(kwargs[k]) trans = dict( @@ -186,7 +186,7 @@ class Web_Editor(http.Controller): uploads += attachments.read(['name', 'mimetype', 'checksum', 'url']) except Exception as e: logger.exception("Failed to upload image to attachment") - message = unicode(e) + message = pycompat.text_type(e) return """' + replacement = u'' view.save(replacement, xpath='/t/p/h1') self.assertIn( - replacement.replace('&', '&'), + replacement.replace(u'&', u'&'), view.arch, 'inline script should be escaped server side' ) self.assertIn( replacement, - view.render(), + view.render().decode('utf-8'), 'inline script should not be escaped when rendering' ) # common text nodes should be be escaped client side - replacement = 'world &amp; &lt;b&gt;cie' + replacement = u'world &amp; &lt;b&gt;cie' view.save(replacement, xpath='/t/p') self.assertIn(replacement, view.arch, 'common text node should not be escaped server side') self.assertIn( replacement, - view.render().replace('&', '&'), + view.render().decode('utf-8').replace(u'&', u'&'), 'text node characters wrongly unescaped when rendering' ) diff --git a/odoo/addons/base/tests/test_qweb.py b/odoo/addons/base/tests/test_qweb.py index 088949b3bd7..cfe65a00f11 100644 --- a/odoo/addons/base/tests/test_qweb.py +++ b/odoo/addons/base/tests/test_qweb.py @@ -484,7 +484,7 @@ class FileSystemLoader(object): if node.get('t-name') == name: root = etree.Element('templates') root.append(deepcopy(node)) - arch = etree.tostring(root, encoding='utf-8') + arch = etree.tostring(root, encoding='unicode') return arch diff --git a/odoo/addons/test_converter/tests/test_html.py b/odoo/addons/test_converter/tests/test_html.py index f03bbe8d710..e915785e333 100644 --- a/odoo/addons/test_converter/tests/test_html.py +++ b/odoo/addons/test_converter/tests/test_html.py @@ -123,7 +123,7 @@ class TestCurrencyExport(TestExport): converted, u'\u20110.12' u'\N{NO-BREAK SPACE}{symbol}'.format( obj=obj, - symbol=currency.symbol.encode('utf-8') + symbol=currency.symbol ),) def test_currency_pre(self): @@ -136,9 +136,9 @@ class TestCurrencyExport(TestExport): self.assertEqual( converted, u'{symbol}\N{NO-BREAK SPACE}' - '0.12'.format( + u'0.12'.format( obj=obj, - symbol=currency.symbol.encode('utf-8') + symbol=currency.symbol ),) def test_currency_precision(self): @@ -151,10 +151,10 @@ class TestCurrencyExport(TestExport): self.assertEqual( converted, - '0.12' + u'0.12' u'\N{NO-BREAK SPACE}{symbol}'.format( obj=obj, - symbol=currency.symbol.encode('utf-8') + symbol=currency.symbol ),) @@ -217,9 +217,7 @@ class TestBinaryExport(TestBasicExport): value = converter.value_to_html(encoded_content, {}) self.assertEqual( - value, b'' % ( - encoded_content - )) + value, u'' % encoded_content.decode('ascii')) with open(os.path.join(directory, 'test_vectors', 'pdf'), 'rb') as f: content = f.read() From a5e21f69e7a283933e83af9e7ab88b9a9260786b Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:29:33 +0200 Subject: [PATCH 21/47] [FIX] P3: view arch should be text-'encoded' XML --- addons/account/models/account_invoice.py | 2 +- addons/account_asset/wizard/asset_modify.py | 2 +- addons/anonymization/wizard/anonymize_wizard.py | 2 +- addons/base_address_city/models/res_partner.py | 2 +- addons/board/controllers/main.py | 2 +- addons/board/models/board.py | 11 +++-------- addons/mail/models/mail_thread.py | 2 +- addons/project/models/project.py | 2 +- addons/website/tests/test_views.py | 9 +++++---- odoo/addons/base/ir/ir_ui_view.py | 10 +++++----- odoo/addons/base/res/res_config.py | 2 +- odoo/addons/base/res/res_partner.py | 2 +- odoo/addons/base/res/res_users.py | 2 +- odoo/addons/base/tests/test_views.py | 2 +- odoo/models.py | 2 +- 15 files changed, 25 insertions(+), 29 deletions(-) diff --git a/addons/account/models/account_invoice.py b/addons/account/models/account_invoice.py index ebaff3d4c9a..87480662784 100644 --- a/addons/account/models/account_invoice.py +++ b/addons/account/models/account_invoice.py @@ -1350,7 +1350,7 @@ class AccountInvoiceLine(models.Model): node.set('domain', "[('purchase_ok', '=', True)]") else: node.set('domain', "[('sale_ok', '=', True)]") - res['arch'] = etree.tostring(doc) + res['arch'] = etree.tostring(doc, encoding='unicode') return res @api.v8 diff --git a/addons/account_asset/wizard/asset_modify.py b/addons/account_asset/wizard/asset_modify.py index 9f8ce7cd497..d31df656455 100644 --- a/addons/account_asset/wizard/asset_modify.py +++ b/addons/account_asset/wizard/asset_modify.py @@ -39,7 +39,7 @@ class AssetModify(models.TransientModel): node = doc.xpath("//field[@name='method_number']")[0] node.set('invisible', '1') setup_modifiers(node, result['fields']['method_number']) - result['arch'] = etree.tostring(doc) + result['arch'] = etree.tostring(doc, encoding='unicode') return result @api.model diff --git a/addons/anonymization/wizard/anonymize_wizard.py b/addons/anonymization/wizard/anonymize_wizard.py index fb514fe778e..a319347f918 100644 --- a/addons/anonymization/wizard/anonymize_wizard.py +++ b/addons/anonymization/wizard/anonymize_wizard.py @@ -120,7 +120,7 @@ class IrModelFieldsAnonymizeWizard(models.TransientModel): else: raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized," " while some fields are not anonymized. You should try to solve this problem before trying to do anything else.")) - res['arch'] = etree.tostring(eview) + res['arch'] = etree.tostring(eview, encoding='unicode') return res @api.multi diff --git a/addons/base_address_city/models/res_partner.py b/addons/base_address_city/models/res_partner.py index f6fcbf61d42..e3e3ad574e0 100644 --- a/addons/base_address_city/models/res_partner.py +++ b/addons/base_address_city/models/res_partner.py @@ -35,5 +35,5 @@ class Partner(models.Model): city_id_node = etree.fromstring(replacement_xml) city_node.getparent().replace(city_node, city_id_node) - arch = etree.tostring(doc) + arch = etree.tostring(doc, encoding='unicode') return arch diff --git a/addons/board/controllers/main.py b/addons/board/controllers/main.py index 2ba236ea109..ebd9b568583 100644 --- a/addons/board/controllers/main.py +++ b/addons/board/controllers/main.py @@ -29,7 +29,7 @@ class Board(Controller): 'domain': str(domain) }) column.insert(0, new_action) - arch = ElementTree.tostring(xml, 'utf-8') + arch = ElementTree.tostring(xml, encoding='unicode') request.env['ir.ui.view.custom'].create({ 'user_id': request.session.uid, 'ref_id': view_id, diff --git a/addons/board/models/board.py b/addons/board/models/board.py index 81e849af263..ebbbec74e2c 100644 --- a/addons/board/models/board.py +++ b/addons/board/models/board.py @@ -42,13 +42,8 @@ class Board(models.AbstractModel): if child.tag == 'action' and child.get('invisible'): node.remove(child) else: - child = remove_unauthorized_children(child) + remove_unauthorized_children(child) return node - def encode(s): - if isinstance(s, pycompat.text_type): - return s.encode('utf8') - return s - - archnode = etree.fromstring(encode(arch)) - return etree.tostring(remove_unauthorized_children(archnode), pretty_print=True) + archnode = etree.fromstring(arch) + return etree.tostring(remove_unauthorized_children(archnode), pretty_print=True, encoding='unicode') diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index 8dfe1ef452c..835fd42b5a6 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -374,7 +374,7 @@ class MailThread(models.AbstractModel): options['display_log_button'] = is_employee # save options on the node node.set('options', repr(options)) - res['arch'] = etree.tostring(doc) + res['arch'] = etree.tostring(doc, encoding='unicode') return res # ------------------------------------------------------ diff --git a/addons/project/models/project.py b/addons/project/models/project.py index cfe3f91aff5..8e20f1bc029 100644 --- a/addons/project/models/project.py +++ b/addons/project/models/project.py @@ -500,7 +500,7 @@ class Task(models.Model): _check_rec(eview) - res['arch'] = etree.tostring(eview) + res['arch'] = etree.tostring(eview, encoding='unicode') # replace reference of 'Hours' to 'Day(s)' for f in res['fields']: diff --git a/addons/website/tests/test_views.py b/addons/website/tests/test_views.py index 00c52f40a8e..af9d362cfb9 100644 --- a/addons/website/tests/test_views.py +++ b/addons/website/tests/test_views.py @@ -48,7 +48,7 @@ class TestViewSaving(common.TransactionCase): self.view_id = self.env['ir.ui.view'].create({ 'name': "Test View", 'type': 'qweb', - 'arch': ET.tostring(self.arch, encoding='utf-8').decode('utf-8') + 'arch': ET.tostring(self.arch, encoding='unicode') }) def test_embedded_extraction(self): @@ -160,7 +160,7 @@ class TestViewSaving(common.TransactionCase): h.LI(h.SPAN("Acme Corporation", attrs(model='res.company', id=1, field='name', expression="bob", type='char'))), h.LI(h.SPAN("+12 3456789", attrs(model='res.company', id=1, field='phone', expression="edmund", type='char'))), ) - ), encoding='utf-8') + ), encoding='unicode') self.view_id.save(value=replacement, xpath='/div/div[2]') # the xml_id of the view should be flagged as 'noupdate' @@ -170,7 +170,7 @@ class TestViewSaving(common.TransactionCase): self.assertEqual(company.name, "Acme Corporation") self.assertEqual(company.phone, "+12 3456789") self.eq( - ET.fromstring(self.view_id.arch.encode('utf-8')), + ET.fromstring(self.view_id.arch), h.DIV( h.DIV( h.H3("Column 1"), @@ -225,7 +225,8 @@ class TestViewSaving(common.TransactionCase): node = html.tostring(h.SPAN( "Acme Corporation", - attrs(model='res.company', id=company_id, field="name", expression='bob', type='char'))) + attrs(model='res.company', id=company_id, field="name", expression='bob', type='char')), + encoding='unicode') View = self.env['ir.ui.view'] View.browse(company_id).save(value=node) self.assertEqual(company.name, "Acme Corporation") diff --git a/odoo/addons/base/ir/ir_ui_view.py b/odoo/addons/base/ir/ir_ui_view.py index 3344d0bfee6..e848d88b093 100644 --- a/odoo/addons/base/ir/ir_ui_view.py +++ b/odoo/addons/base/ir/ir_ui_view.py @@ -110,7 +110,7 @@ def get_view_arch_from_file(filename, xmlid): if node.tag == 'record': field = node.find('field[@name="arch"]') _fix_multiple_roots(field) - inner = ''.join([etree.tostring(child) for child in field.iterchildren()]) + inner = u''.join([etree.tostring(child, encoding='unicode') for child in field.iterchildren()]) return field.text + inner elif node.tag == 'template': # The following dom operations has been copied from convert.py's _tag_template() @@ -120,7 +120,7 @@ def get_view_arch_from_file(filename, xmlid): else: node.tag = 'data' node.attrib.pop('id', None) - return etree.tostring(node) + return etree.tostring(node, encoding='unicode') _logger.warning("Could not find view arch definition in file '%s' for xmlid '%s'", filename, xmlid) return None @@ -702,7 +702,7 @@ actual arch. # and apply inheritance arch = self.apply_view_inheritance(arch_tree, root.id, self.model) - return dict(view_data, arch=etree.tostring(arch, encoding='utf-8')) + return dict(view_data, arch=etree.tostring(arch, encoding='unicode')) def _apply_group(self, model, node, modifiers, fields): """Apply group restrictions, may be set at view level or model level:: @@ -910,7 +910,7 @@ actual arch. not self._context.get(action, True) and is_base_model): node.set(action, 'false') - arch = etree.tostring(node, encoding="utf-8").replace(b'\t', b'') + arch = etree.tostring(node, encoding="unicode").replace('\t', '') for k in list(fields): if k not in fields_def: del fields[k] @@ -942,7 +942,7 @@ actual arch. arch_tree = etree.fromstring(arch) self.distribute_branding(arch_tree) root = E.templates(arch_tree) - arch = etree.tostring(root, encoding='utf-8') + arch = etree.tostring(root, encoding='unicode') return arch @api.model diff --git a/odoo/addons/base/res/res_config.py b/odoo/addons/base/res/res_config.py index d5c1ac59717..ec81d2ce882 100644 --- a/odoo/addons/base/res/res_config.py +++ b/odoo/addons/base/res/res_config.py @@ -405,7 +405,7 @@ class ResConfigSettings(models.TransientModel, ResConfigModuleInstallationMixin) node.set("on_change", "onchange_module(%s, '%s')" % (field, field)) - ret_val['arch'] = etree.tostring(doc) + ret_val['arch'] = etree.tostring(doc, encoding='unicode') return ret_val @api.multi diff --git a/odoo/addons/base/res/res_partner.py b/odoo/addons/base/res/res_partner.py index 7873dfda525..972b74d7f3e 100644 --- a/odoo/addons/base/res/res_partner.py +++ b/odoo/addons/base/res/res_partner.py @@ -61,7 +61,7 @@ class FormatAddressMixin(models.AbstractModel): except ValueError: return arch address_node.getparent().replace(address_node, sub_view_node) - arch = etree.tostring(doc) + arch = etree.tostring(doc, encoding='unicode') return arch class PartnerCategory(models.Model): diff --git a/odoo/addons/base/res/res_users.py b/odoo/addons/base/res/res_users.py index 548e18e6879..24027c6ec00 100644 --- a/odoo/addons/base/res/res_users.py +++ b/odoo/addons/base/res/res_users.py @@ -739,7 +739,7 @@ class GroupsView(models.Model): xml2.append({'class': "o_label_nowrap"}) xml = E.field(E.group(*(xml1), col="2"), E.group(*(xml2), col="4"), name="groups_id", position="replace") xml.addprevious(etree.Comment("GENERATED AUTOMATICALLY BY GROUPS")) - xml_content = etree.tostring(xml, pretty_print=True, encoding="utf-8") + xml_content = etree.tostring(xml, pretty_print=True, encoding="unicode") view.with_context(lang=None).write({'arch': xml_content, 'arch_fs': False}) def get_application_groups(self, domain): diff --git a/odoo/addons/base/tests/test_views.py b/odoo/addons/base/tests/test_views.py index 3f007b5f71b..aba01b0aa32 100644 --- a/odoo/addons/base/tests/test_views.py +++ b/odoo/addons/base/tests/test_views.py @@ -144,7 +144,7 @@ class TestViewInheritance(ViewCase): E.attribute(name, name='string'), position='attributes' ) - return etree.tostring(element) + return etree.tostring(element, encoding='unicode') def makeView(self, name, parent=None, arch=None): """ Generates a basic ir.ui.view with the provided name, parent and arch. diff --git a/odoo/models.py b/odoo/models.py index d8b693322c7..a9cd2ded435 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -1264,7 +1264,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): # fallback on default views methods if no ir.ui.view could be found try: arch_etree = getattr(self, '_get_default_%s_view' % view_type)() - result['arch'] = etree.tostring(arch_etree, encoding='utf-8') + result['arch'] = etree.tostring(arch_etree, encoding='unicode') result['type'] = view_type result['name'] = 'default' except AttributeError: From 57ddba862ad919c33e85f947b3270ba32bc2eda0 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:31:11 +0200 Subject: [PATCH 22/47] [FIX] P3: (ir.values).value is bytes or buffer/memoryview when coming directly from XML --- odoo/addons/base/ir/ir_values.py | 2 +- odoo/addons/base/tests/test_ir_actions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/odoo/addons/base/ir/ir_values.py b/odoo/addons/base/ir/ir_values.py index 11b6eb1a530..f05241a8c05 100644 --- a/odoo/addons/base/ir/ir_values.py +++ b/odoo/addons/base/ir/ir_values.py @@ -407,7 +407,7 @@ class IrValues(models.Model): for id, name, value in self._cr.fetchall(): if not value: continue # skip if undefined - action_model, action_id = value.decode('utf-8').split(',') + action_model, action_id = bytes(value).decode('utf-8').split(',') if action_model not in self.env: continue # unknown model? skip it! action = self.env[action_model].browse(int(action_id)) diff --git a/odoo/addons/base/tests/test_ir_actions.py b/odoo/addons/base/tests/test_ir_actions.py index cb48752a960..a2663964f59 100644 --- a/odoo/addons/base/tests/test_ir_actions.py +++ b/odoo/addons/base/tests/test_ir_actions.py @@ -64,7 +64,7 @@ class TestServerActions(TestServerActionsBase): # Test: ir_values created ir_values = self.env['ir.values'].search([('name', '=', 'Run TestAction')]) self.assertEqual(len(ir_values), 1, 'ir_actions_server: create_action should have created an entry in ir_values') - self.assertEqual(ir_values.value, 'ir.actions.server,%s' % self.action.id, 'ir_actions_server: created ir_values should reference the server action') + self.assertEqual(ir_values.value, b'ir.actions.server,%d' % self.action.id, 'ir_actions_server: created ir_values should reference the server action') self.assertEqual(ir_values.model, 'res.partner', 'ir_actions_server: created ir_values should be linked to the action base model') # Do: remove contextual action From 23299189c45302f30930a2fc411ea85ae975d6b8 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:53:36 +0200 Subject: [PATCH 23/47] [FIX] P3: translations + text model --- odoo/tools/translate.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/odoo/tools/translate.py b/odoo/tools/translate.py index 58bbf6bcec5..0afb04b3972 100644 --- a/odoo/tools/translate.py +++ b/odoo/tools/translate.py @@ -130,8 +130,7 @@ csv.register_dialect("UNIX", UNIX_LINE_TERMINATOR) # FIXME: holy shit this whole thing needs to be cleaned up hard it's a mess def encode(s): - if isinstance(s, unicode): - return s.encode('utf8') + assert isinstance(s, pycompat.text_type) return s # which elements are translated inline @@ -808,9 +807,9 @@ def trans_generate(lang, modules, cr): try: # verify the minimal size without eventual xml tags # wrap to make sure html content like 'bd' is accepted by lxml - wrapped = "
%s
" % sanitized_term + wrapped = u"
%s
" % sanitized_term node = etree.fromstring(wrapped) - sanitized_term = etree.tostring(node, encoding='UTF-8', method='text') + sanitized_term = etree.tostring(node, encoding='unicode', method='text') except etree.ParseError: pass # remove non-alphanumeric chars @@ -845,12 +844,10 @@ def trans_generate(lang, modules, cr): cr.execute(query, query_param) for (xml_name, model, res_id, module) in cr.fetchall(): - module = encode(module) - model = encode(model) - xml_name = "%s.%s" % (module, encode(xml_name)) + xml_name = "%s.%s" % (module, xml_name) if model not in env: - _logger.error("Unable to find object %r", model) + _logger.error(u"Unable to find object %r", model) continue record = env[model].browse(res_id) @@ -859,14 +856,14 @@ def trans_generate(lang, modules, cr): continue if not record.exists(): - _logger.warning("Unable to find object %r with id %d", model, res_id) + _logger.warning(u"Unable to find object %r with id %d", model, res_id) continue - if model=='ir.model.fields': + if model==u'ir.model.fields': try: - field_name = encode(record.name) + field_name = record.name except AttributeError as exc: - _logger.error("name error in %s: %s", xml_name, str(exc)) + _logger.error(u"name error in %s: %s", xml_name, str(exc)) continue field_model = env.get(record.model) if (field_model is None or not field_model._translate or @@ -875,9 +872,9 @@ def trans_generate(lang, modules, cr): field = field_model._fields[field_name] if isinstance(getattr(field, 'selection', None), (list, tuple)): - name = "%s,%s" % (encode(record.model), field_name) + name = "%s,%s" % (record.model, field_name) for dummy, val in field.selection: - push_translation(module, 'selection', name, 0, encode(val)) + push_translation(module, 'selection', name, 0, val) for field_name, field in record._fields.iteritems(): if field.translate: @@ -887,13 +884,13 @@ def trans_generate(lang, modules, cr): except Exception: continue for term in set(field.get_trans_terms(value)): - push_translation(module, 'model', name, xml_name, encode(term)) + push_translation(module, 'model', name, xml_name, term) # End of data for ir.model.data query results def push_constraint_msg(module, term_type, model, msg): if not callable(msg): - push_translation(encode(module), term_type, encode(model), 0, encode(msg)) + push_translation(encode(module), term_type, encode(model), 0, msg) def push_local_constraints(module, model, cons_type='sql_constraints'): """ Climb up the class hierarchy and ignore inherited constraints from other modules. """ From 508926d2b86b81ad2c37e038cbd97bebdacb77ab Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:55:14 +0200 Subject: [PATCH 24/47] [FIX] P3: hex() returns text, don't re-decode it --- odoo/osv/expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odoo/osv/expression.py b/odoo/osv/expression.py index 1f55f58600a..69833f45de0 100644 --- a/odoo/osv/expression.py +++ b/odoo/osv/expression.py @@ -337,7 +337,7 @@ def generate_table_alias(src_table_alias, joined_tables=[]): # We have to fit a crc32 hash and one underscore # into a 63 character alias. The remaining space we can use to add # a human readable prefix. - alias_hash = hex(crc32(alias.encode('utf-8')))[2:].decode('utf-8') + alias_hash = hex(crc32(alias.encode('utf-8')))[2:] ALIAS_PREFIX_LENGTH = 63 - len(alias_hash) - 1 alias = "%s_%s" % ( alias[:ALIAS_PREFIX_LENGTH], alias_hash) From a9d8d5983382b22a80dbcd1d9266c2df816b3929 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:56:21 +0200 Subject: [PATCH 25/47] [FIX] P3: error messages are text, dump XML docs as text --- odoo/tools/convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py index fec641c619c..86697b2b4d2 100644 --- a/odoo/tools/convert.py +++ b/odoo/tools/convert.py @@ -600,7 +600,7 @@ form: module.record_id""" % (xml_id,) ' xmltag: %s\n' \ ' expected value: %r\n' \ ' obtained value: %r\n' \ - % (rec_string, etree.tostring(test), expected_value, expression_value) + % (rec_string, etree.tostring(test, encoding='unicode'), expected_value, expression_value) _logger.error(msg) return else: # all tests were successful for this assertion tag (no break) @@ -788,7 +788,7 @@ form: module.record_id""" % (xml_id,) exc_info = sys.exc_info() pycompat.reraise( ParseError, - ParseError(ustr(e), etree.tostring(rec).rstrip(), rec.getroottree().docinfo.URL, rec.sourceline), + ParseError(ustr(e), etree.tostring(rec, encoding='unicode').rstrip(), rec.getroottree().docinfo.URL, rec.sourceline), exc_info[2] ) return True From 43dedda9eb5c48045a05d8016def8e86537ff056 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:57:13 +0200 Subject: [PATCH 26/47] [FIX] P3: cross-version binary data is bytes, not str --- addons/web_editor/tests/test_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/web_editor/tests/test_converter.py b/addons/web_editor/tests/test_converter.py index 91d4b578383..761de27515f 100644 --- a/addons/web_editor/tests/test_converter.py +++ b/addons/web_editor/tests/test_converter.py @@ -136,7 +136,7 @@ class TestConvertBack(common.TransactionCase): converter = self.env[model] if model in self.env else self.env['ir.qweb.field'] value_back = converter.from_html(model, record._fields[field], element) - if isinstance(expected, str): + if isinstance(expected, bytes): expected = expected.decode('utf-8') self.assertEqual(value_back, expected) From ae866acf08abe91fe87a3d3f4899070962b795cc Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:58:30 +0200 Subject: [PATCH 27/47] [FIX] P3: assets bundle text model * remains is a list of text * sep is text * json.dumps generates text (surprise!) so beware with that --- odoo/addons/base/ir/ir_qweb/assetsbundle.py | 10 +++++----- odoo/addons/base/ir/ir_qweb/ir_qweb.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/odoo/addons/base/ir/ir_qweb/assetsbundle.py b/odoo/addons/base/ir/ir_qweb/assetsbundle.py index fcf9abe4d5d..8f894107b81 100644 --- a/odoo/addons/base/ir/ir_qweb/assetsbundle.py +++ b/odoo/addons/base/ir/ir_qweb/assetsbundle.py @@ -102,7 +102,7 @@ class AssetsBundle(object): def to_html(self, sep=None, css=True, js=True, debug=False, async=False, url_for=(lambda url: url)): if sep is None: - sep = '\n ' + sep = u'\n ' response = [] if debug == 'assets': if css and self.stylesheets: @@ -121,12 +121,12 @@ class AssetsBundle(object): if css and self.stylesheets: css_attachments = self.css() or [] for attachment in css_attachments: - response.append('' % url_for(attachment.url)) + response.append(u'' % url_for(attachment.url)) if self.css_errors: msg = '\n'.join(self.css_errors) response.append(JavascriptAsset(self, inline=self.dialog_message(msg)).to_html()) if js and self.javascripts: - response.append('' % (async and 'async="async"' or '', url_for(self.js().url))) + response.append(u'' % (async and u'async="async"' or '', url_for(self.js().url))) response.extend(self.remains) return sep + sep.join(response) @@ -149,8 +149,8 @@ class AssetsBundle(object): Not really a full checksum. We compute a SHA1 on the rendered bundle + max linked files last_modified date """ - check = json.dumps(self.files) + b",".join(self.remains) + self.last_modified.encode('utf-8') - return hashlib.sha1(check).hexdigest() + check = u"%s%s%s" % (json.dumps(self.files), u",".join(self.remains), self.last_modified) + return hashlib.sha1(check.encode('utf-8')).hexdigest() def clean_attachments(self, type): """ Takes care of deleting any outdated ir.attachment records associated to a bundle before diff --git a/odoo/addons/base/ir/ir_qweb/ir_qweb.py b/odoo/addons/base/ir/ir_qweb/ir_qweb.py index b23ab34f103..6a23054519e 100644 --- a/odoo/addons/base/ir/ir_qweb/ir_qweb.py +++ b/odoo/addons/base/ir/ir_qweb/ir_qweb.py @@ -219,7 +219,7 @@ class IrQWeb(models.AbstractModel, QWeb): remains = [] for el in html.fragments_fromstring(template): if isinstance(el, pycompat.string_types): - remains.append(el) + remains.append(pycompat.to_text(el)) elif isinstance(el, html.HtmlElement): href = el.get('href', '') src = el.get('src', '') @@ -243,10 +243,10 @@ class IrQWeb(models.AbstractModel, QWeb): filename = get_resource_path(*path) if path else None files.append({'atype': atype, 'url': src, 'filename': filename, 'content': el.text, 'media': media}) else: - remains.append(html.tostring(el)) + remains.append(html.tostring(el, encoding='unicode')) else: try: - remains.append(html.tostring(el)) + remains.append(html.tostring(el, encoding='unicode')) except Exception: # notYETimplementederror raise NotImplementedError From 200612ca1ef1a1bf75dbe52feef5a09ae5a58ca6 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:59:41 +0200 Subject: [PATCH 28/47] [FIX] P3: text model in various mail bits No formal decision there, and the stdlib API seems like a PITA as it's based around native strings, not bytes and not text. * try to use cross-model API instead of random encode/decode * unicode all the mail_template things * formataddr *requires* a text tuple now In P2, formataddr((False, False)) "worked" (generated "False "), in P3 it will fail and blow up. * attachments are quite definitely bytes * remove try_coerce_ascii because it's garbage --- addons/mail/models/mail_mail.py | 2 +- addons/mail/models/mail_template.py | 10 +-- addons/mail/tests/test_mail_features.py | 2 +- odoo/addons/base/ir/ir_mail_server.py | 75 ++++++++--------- odoo/addons/base/res/res_partner.py | 2 +- odoo/addons/base/tests/test_mail.py | 14 ++-- odoo/addons/base/tests/test_mail_examples.py | 84 ++++++++++---------- odoo/tools/mail.py | 43 +++++----- 8 files changed, 112 insertions(+), 120 deletions(-) diff --git a/addons/mail/models/mail_mail.py b/addons/mail/models/mail_mail.py index 3f206424e07..e53c24fb176 100644 --- a/addons/mail/models/mail_mail.py +++ b/addons/mail/models/mail_mail.py @@ -169,7 +169,7 @@ class MailMail(models.Model): - else fallback on mail.email_to splitting """ self.ensure_one() if partner: - email_to = [formataddr((partner.name, partner.email))] + email_to = [formataddr((partner.name or 'False', partner.email or 'False'))] else: email_to = tools.email_split_and_format(self.email_to) return email_to diff --git a/addons/mail/models/mail_template.py b/addons/mail/models/mail_template.py index e9bf605c382..f44ee7be09a 100644 --- a/addons/mail/models/mail_template.py +++ b/addons/mail/models/mail_template.py @@ -306,8 +306,8 @@ class MailTemplate(models.Model): # form a tree root = lxml.html.fromstring(html) if not len(root) and root.text is None and root.tail is None: - html = '
%s
' % html - root = lxml.html.fromstring(html) + html = u'
%s
' % html + root = lxml.html.fromstring(html, encoding='unicode') base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url') base = urls.url_parse(base_url) @@ -324,12 +324,12 @@ class MailTemplate(models.Model): for node in root.iter(): if node.tag == 'a' and node.get('href'): node.set('href', _process_link(node.get('href'))) - elif node.tag == 'img' and not node.get('src', 'data').startswith('data'): + elif node.tag == 'img' and not node.get('src', 'data').startswith(u'data'): node.set('src', _process_link(node.get('src'))) - html = lxml.html.tostring(root, pretty_print=False, method='html') + html = lxml.html.tostring(root, pretty_print=False, method='html', encoding='unicode') # this is ugly, but lxml/etree tostring want to put everything in a 'div' that breaks the editor -> remove that - if html.startswith('
') and html.endswith('
'): + if html.startswith(u'
') and html.endswith(u'
'): html = html[5:-6] return html diff --git a/addons/mail/tests/test_mail_features.py b/addons/mail/tests/test_mail_features.py index a6fce59c10a..8e2264ad6d3 100644 --- a/addons/mail/tests/test_mail_features.py +++ b/addons/mail/tests/test_mail_features.py @@ -131,7 +131,7 @@ class TestMessagePost(TestMail): self.assertEqual(set(msg.attachment_ids.mapped('res_id')), set([self.test_pigs.id]), 'message_post: all atttachments should be linked to the pigs group') self.assertEqual(set([base64.b64decode(x) for x in msg.attachment_ids.mapped('datas')]), - set(['migration test', _attachments[0][1], _attachments[1][1]])) + set([b'migration test', _attachments[0][1], _attachments[1][1]])) self.assertTrue(set([_attach_1.id, _attach_2.id]).issubset(msg.attachment_ids.ids), 'message_post: mail.message attachments duplicated') # notifications diff --git a/odoo/addons/base/ir/ir_mail_server.py b/odoo/addons/base/ir/ir_mail_server.py index 91cf0e61e73..eab160b4a84 100644 --- a/odoo/addons/base/ir/ir_mail_server.py +++ b/odoo/addons/base/ir/ir_mail_server.py @@ -39,20 +39,8 @@ class WriteToLogger(object): _logger.debug(s) smtplib.stderr = WriteToLogger() -def try_coerce_ascii(string_utf8): - """Attempts to decode the given utf8-encoded string - as ASCII after coercing it to UTF-8, then return - the confirmed 7-bit ASCII string. - - If the process fails (because the string - contains non-ASCII characters) returns ``None``. - """ - try: - string_utf8.decode('ascii') - except UnicodeDecodeError: - return - return string_utf8 - +def is_ascii(s): + return all(ord(cp) < 128 for cp in s) def encode_header(header_text): """Returns an appropriate representation of the given header value, @@ -70,16 +58,10 @@ def encode_header(header_text): """ if not header_text: return "" - # convert anything to utf-8, suitable for testing ASCIIness, as 7-bit chars are - # encoded as ASCII in utf-8 - header_text_utf8 = ustr(header_text).encode('utf-8') - header_text_ascii = try_coerce_ascii(header_text_utf8) - # if this header contains non-ASCII characters, - # we'll need to wrap it up in a message.header.Header - # that will take care of RFC2047-encoding it as - # 7-bit string. - return header_text_ascii or Header(header_text_utf8, 'utf-8') - + header_text = ustr(header_text) # FIXME: require unicode higher up? + if is_ascii(header_text): + return pycompat.to_native(header_text) + Header(header_text, 'utf-8') def encode_header_param(param_text): """Returns an appropriate RFC2047 encoded representation of the given @@ -97,10 +79,10 @@ def encode_header_param(param_text): # For details see the encode_header() method that uses the same logic if not param_text: return "" - param_text_utf8 = ustr(param_text).encode('utf-8') - param_text_ascii = try_coerce_ascii(param_text_utf8) - return param_text_ascii or Charset('utf8').header_encode(param_text_utf8) - + param_text = ustr(param_text) # FIXME: require unicode higher up? + if is_ascii(param_text): + return pycompat.to_native(param_text) # TODO: is that actually necessary? + return Charset("utf-8").header_encode(param_text) address_pattern = re.compile(r'([^ ,<@]+@[^> ,]+)') @@ -111,8 +93,8 @@ def extract_rfc2822_addresses(text): """ if not text: return [] - candidates = address_pattern.findall(ustr(text).encode('utf-8')) - return [c for c in candidates if try_coerce_ascii(c)] + candidates = address_pattern.findall(ustr(text)) + return [c for c in candidates if is_ascii(c)] def encode_rfc2822_address_header(header_text): @@ -124,11 +106,20 @@ def encode_rfc2822_address_header(header_text): """ def encode_addr(addr): name, email = addr - if not try_coerce_ascii(name): - name = str(Header(name, 'utf-8')) + # If s is a , then charset is a hint specifying the + # character set of the characters in the string. The Unicode string + # will be encoded using the following charsets in order: us-ascii, + # the charset hint, utf-8. The first character set to not provoke a + # UnicodeError is used. + # -> always pass a text string to Header + + # also Header.__str__ in Python 3 "Returns an approximation of the + # Header as a string, using an unlimited line length.", the old one + # was "A synonym for Header.encode()." so call encode() directly? + name = Header(pycompat.to_text(name)).encode() return formataddr((name, email)) - addresses = getaddresses([ustr(header_text).encode('utf-8')]) + addresses = getaddresses([pycompat.to_native(ustr(header_text))]) return COMMASPACE.join(encode_addr(a) for a in addresses) @@ -252,8 +243,8 @@ class IrMailServer(models.Model): # The user/password must be converted to bytestrings in order to be usable for # certain hashing schemes, like HMAC. # See also bug #597143 and python issue #5285 - smtp_user = ustr(smtp_user).encode('utf-8') - smtp_password = ustr(smtp_password).encode('utf-8') + smtp_user = pycompat.to_native(ustr(smtp_user)) + smtp_password = pycompat.to_native(ustr(smtp_password)) connection.login(smtp_user, smtp_password) return connection @@ -301,8 +292,8 @@ class IrMailServer(models.Model): email_bcc = email_bcc or [] body = body or u'' - email_body_utf8 = ustr(body).encode('utf-8') - email_text_part = MIMEText(email_body_utf8, _subtype=subtype, _charset='utf-8') + email_body = ustr(body) + email_text_part = MIMEText(email_body, _subtype=subtype, _charset='utf-8') msg = MIMEMultipart() if not message_id: @@ -328,20 +319,20 @@ class IrMailServer(models.Model): msg['Date'] = formatdate() # Custom headers may override normal headers or provide additional ones for key, value in pycompat.items(headers): - msg[ustr(key).encode('utf-8')] = encode_header(value) + msg[pycompat.to_native(ustr(key))] = encode_header(value) if subtype == 'html' and not body_alternative: # Always provide alternative text body ourselves if possible. - text_utf8 = html2text.html2text(email_body_utf8.decode('utf-8')).encode('utf-8') + text = html2text.html2text(email_body_utf8) alternative_part = MIMEMultipart(_subtype="alternative") - alternative_part.attach(MIMEText(text_utf8, _charset='utf-8', _subtype='plain')) + alternative_part.attach(MIMEText(text, _charset='utf-8', _subtype='plain')) alternative_part.attach(email_text_part) msg.attach(alternative_part) elif body_alternative: # Include both alternatives, as specified, within a multipart/alternative part alternative_part = MIMEMultipart(_subtype="alternative") - body_alternative_utf8 = ustr(body_alternative).encode('utf-8') - alternative_body_part = MIMEText(body_alternative_utf8, _subtype=subtype_alternative, _charset='utf-8') + body_alternative_ = ustr(body_alternative) + alternative_body_part = MIMEText(body_alternative_, _subtype=subtype_alternative, _charset='utf-8') alternative_part.attach(alternative_body_part) alternative_part.attach(email_text_part) msg.attach(alternative_part) diff --git a/odoo/addons/base/res/res_partner.py b/odoo/addons/base/res/res_partner.py index 972b74d7f3e..32772fa19a6 100644 --- a/odoo/addons/base/res/res_partner.py +++ b/odoo/addons/base/res/res_partner.py @@ -337,7 +337,7 @@ class Partner(models.Model): @api.depends('name', 'email') def _compute_email_formatted(self): for partner in self: - partner.email_formatted = formataddr((partner.name, partner.email)) + partner.email_formatted = formataddr((partner.name or u"False", partner.email or u"False")) @api.depends('is_company') def _compute_company_type(self): diff --git a/odoo/addons/base/tests/test_mail.py b/odoo/addons/base/tests/test_mail.py index 1a30ef4690b..84cf892ea30 100644 --- a/odoo/addons/base/tests/test_mail.py +++ b/odoo/addons/base/tests/test_mail.py @@ -171,14 +171,14 @@ class TestSanitizer(unittest.TestCase): for ext in test_mail_examples.QUOTE_BLOCKQUOTE_IN: self.assertIn(ext, html) for ext in test_mail_examples.QUOTE_BLOCKQUOTE_OUT: - self.assertIn('%s' % misc.html_escape(ext.decode('utf-8')), html) + self.assertIn(u'%s' % misc.html_escape(ext), html) def test_quote_thunderbird(self): html = html_sanitize(test_mail_examples.QUOTE_THUNDERBIRD_1) for ext in test_mail_examples.QUOTE_THUNDERBIRD_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.QUOTE_THUNDERBIRD_1_OUT: - self.assertIn('%s' % misc.html_escape(ext.decode('utf-8')), html) + self.assertIn(u'%s' % misc.html_escape(ext), html) def test_quote_hotmail_html(self): html = html_sanitize(test_mail_examples.QUOTE_HOTMAIL_HTML) @@ -225,7 +225,7 @@ class TestSanitizer(unittest.TestCase): for text in in_lst: self.assertIn(text, new_html) for text in out_lst: - self.assertIn('%s' % misc.html_escape(text), new_html) + self.assertIn(u'%s' % misc.html_escape(text), new_html) def test_quote_signature(self): test_data = [ @@ -244,27 +244,27 @@ class TestSanitizer(unittest.TestCase): for ext in test_mail_examples.GMAIL_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.GMAIL_1_OUT: - self.assertIn('%s' % misc.html_escape(ext), html) + self.assertIn(u'%s' % misc.html_escape(ext), html) def test_quote_text(self): html = html_sanitize(test_mail_examples.TEXT_1) for ext in test_mail_examples.TEXT_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.TEXT_1_OUT: - self.assertIn('%s' % misc.html_escape(ext), html) + self.assertIn(u'%s' % misc.html_escape(ext), html) html = html_sanitize(test_mail_examples.TEXT_2) for ext in test_mail_examples.TEXT_2_IN: self.assertIn(ext, html) for ext in test_mail_examples.TEXT_2_OUT: - self.assertIn('%s' % misc.html_escape(ext), html) + self.assertIn(u'%s' % misc.html_escape(ext), html) def test_quote_bugs(self): html = html_sanitize(test_mail_examples.BUG1) for ext in test_mail_examples.BUG_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.BUG_1_OUT: - self.assertIn('%s' % misc.html_escape(ext.decode('utf-8')), html) + self.assertIn(u'%s' % misc.html_escape(ext), html) def test_misc(self): # False / void should not crash diff --git a/odoo/addons/base/tests/test_mail_examples.py b/odoo/addons/base/tests/test_mail_examples.py index dfecfe7c0d4..4f3b5b48300 100644 --- a/odoo/addons/base/tests/test_mail_examples.py +++ b/odoo/addons/base/tests/test_mail_examples.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -MISC_HTML_SOURCE = """ +MISC_HTML_SOURCE = u""" test1
test2
@@ -19,7 +19,7 @@ test12

test link """ -EDI_LIKE_HTML_SOURCE = """
+EDI_LIKE_HTML_SOURCE = u"""

Hello ${object.partner_id.name},

A new invoice is available for you:

@@ -63,7 +63,7 @@ EDI_LIKE_HTML_SOURCE = """

+QUOTE_BLOCKQUOTE = u""" @@ -103,14 +103,14 @@ web: https://opener.am """ -QUOTE_BLOCKQUOTE_IN = ["""
"""] -QUOTE_BLOCKQUOTE_OUT = ["""-- +QUOTE_BLOCKQUOTE_IN = [u"""
"""] +QUOTE_BLOCKQUOTE_OUT = [u"""-- Opener B.V. - Business solutions driven by open source collaboration Stefan Rijnhart - Consultant/developer"""] -QUOTE_THUNDERBIRD_HTML = """ +QUOTE_THUNDERBIRD_HTML = u""" @@ -137,13 +137,13 @@ Raoul Poilvache """ -QUOTE_THUNDERBIRD_HTML_IN = ["""
"""] -QUOTE_THUNDERBIRD_HTML_OUT = ["""
-- 
+QUOTE_THUNDERBIRD_HTML_IN = [u"""
"""]
+QUOTE_THUNDERBIRD_HTML_OUT = [u"""
-- 
 Raoul Poilvache
 
"""]
 
 
-QUOTE_HOTMAIL_HTML = """
+QUOTE_HOTMAIL_HTML = u"""