From b9c3365d21d82bdf289d30065a645052bcd8cbe7 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Tue, 9 May 2017 12:34:26 +0200 Subject: [PATCH 01/11] [FIX] P3: cross-version smtplib/email changes * SMTPLib[P3] uses a debug method on ``SMTP`` rather than a global ``stderr`` * email.Encoders is a deprecated & outdated (since 2006) alias for email.encoders --- odoo/addons/base/ir/ir_mail_server.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/odoo/addons/base/ir/ir_mail_server.py b/odoo/addons/base/ir/ir_mail_server.py index 737e9806d68..35588019ebc 100644 --- a/odoo/addons/base/ir/ir_mail_server.py +++ b/odoo/addons/base/ir/ir_mail_server.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -from email import Encoders +from email import encoders from email.charset import Charset from email.header import Header from email.mime.base import MIMEBase @@ -28,16 +28,16 @@ class MailDeliveryException(except_orm): def __init__(self, name, value): super(MailDeliveryException, self).__init__(name, value) +# Python 3: patch SMTP's internal printer/debugger +def _print_debug(self, *args): + _logger.debug(' '.join(str(a) for a in args)) +smtplib.SMTP._print_debug = _print_debug +# Python 2: replace smtplib's stderr class WriteToLogger(object): - """debugging helper: behave as a fd and pipe to logger at the given level""" - def __init__(self, logger, level=logging.DEBUG): - self.logger = logger - self.level = level - def write(self, s): - self.logger.log(self.level, s) - + _logger.debug(s) +smtplib.stderr = WriteToLogger() def try_coerce_ascii(string_utf8): """Attempts to decode the given utf8-encoded string @@ -159,14 +159,6 @@ class IrMailServer(models.Model): "is used. Default priority is 10 (smaller number = higher priority)") active = fields.Boolean(default=True) - def __init__(self, *args, **kwargs): - # Make sure we pipe the smtplib outputs to our own DEBUG logger - if not isinstance(smtplib.stderr, WriteToLogger): - logpiper = WriteToLogger(_logger) - smtplib.stderr = logpiper - smtplib.stdout = logpiper - super(IrMailServer, self).__init__(*args, **kwargs) - @api.multi def name_get(self): return [(server.id, "(%s)" % server.name) for server in self] @@ -371,7 +363,7 @@ class IrMailServer(models.Model): part.add_header('Content-Disposition', 'attachment', filename=filename_rfc2047) part.set_payload(fcontent) - Encoders.encode_base64(part) + encoders.encode_base64(part) msg.attach(part) return msg From 3833c8997d43b2cc51a30a30a136e345cffb6b6b Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Tue, 9 May 2017 13:11:29 +0200 Subject: [PATCH 02/11] [FIX] P3: removed commands module --- addons/hw_escpos/controllers/main.py | 3 +-- addons/hw_proxy/controllers/main.py | 10 +++++----- odoo/addons/test_pylint/tests/test_pylint.py | 7 +++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/addons/hw_escpos/controllers/main.py b/addons/hw_escpos/controllers/main.py index 6c929ca2f26..35efc5216ea 100644 --- a/addons/hw_escpos/controllers/main.py +++ b/addons/hw_escpos/controllers/main.py @@ -2,7 +2,6 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. from __future__ import print_function -import commands import logging import math import os @@ -196,7 +195,7 @@ class EscposDriver(Thread): hosting_ap = os.system('pgrep hostapd') == 0 ssid = subprocess.check_output('iwconfig 2>&1 | grep \'ESSID:"\' | sed \'s/.*"\\(.*\\)"/\\1/\'', shell=True).rstrip() mac = subprocess.check_output('ifconfig | grep -B 1 \'inet addr\' | grep -o \'HWaddr .*\' | sed \'s/HWaddr //\'', shell=True).rstrip() - ips = [ c.split(':')[1].split(' ')[0] for c in commands.getoutput("/sbin/ifconfig").split('\n') if 'inet addr' in c ] + ips = [ c.split(':')[1].split(' ')[0] for c in subprocess.check_output("/sbin/ifconfig").split('\n') if 'inet addr' in c ] ips = [ ip for ip in ips if ip not in localips ] eprint.text('\n\n') eprint.set(align='center',type='b',height=2,width=2) diff --git a/addons/hw_proxy/controllers/main.py b/addons/hw_proxy/controllers/main.py index f341ab7b9e0..0521b72c89b 100644 --- a/addons/hw_proxy/controllers/main.py +++ b/addons/hw_proxy/controllers/main.py @@ -2,8 +2,8 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. from __future__ import print_function -import commands import logging +import subprocess import time from threading import Lock @@ -16,11 +16,11 @@ _logger = logging.getLogger(__name__) # Those are the builtin raspberry pi USB modules, they should # not appear in the list of connected devices. -BANNED_DEVICES = set([ +BANNED_DEVICES = { "0424:9514", # Standard Microsystem Corp. Builtin Ethernet module "1d6b:0002", # Linux Foundation 2.0 root hub "0424:ec00", # Standard Microsystem Corp. Other Builtin Ethernet module -]) +} # drivers modules must add to drivers an object with a get_status() method @@ -99,14 +99,14 @@ class Proxy(http.Controller): """ if debug is None: resp += """(debug version)""" - devices = commands.getoutput("lsusb").split('\n') + devices = subprocess.check_output("lsusb").split('\n') count = 0 resp += "
\n" for device in devices: device_name = device[device.find('ID')+2:] device_id = device_name.split()[0] if not (device_id in BANNED_DEVICES): - resp+= "
"+device_name+"
\n" + resp += "
"+device_name+"
\n" count += 1 if count == 0: diff --git a/odoo/addons/test_pylint/tests/test_pylint.py b/odoo/addons/test_pylint/tests/test_pylint.py index c2d1f2cf457..d7a6760854c 100644 --- a/odoo/addons/test_pylint/tests/test_pylint.py +++ b/odoo/addons/test_pylint/tests/test_pylint.py @@ -42,6 +42,7 @@ class TestPyLint(TransactionCase): 'parameter-unpacking', 'metaclass-assignment', + 'deprecated-module', 'exception-message-attribute', 'indexing-exception', @@ -70,6 +71,11 @@ class TestPyLint(TransactionCase): 'reduce', ] + BAD_MODULES = [ + 'commands', + 'types', + ] + def _skip_test(self, reason): _logger.warn(reason) self.skipTest(reason) @@ -93,6 +99,7 @@ class TestPyLint(TransactionCase): "--msg-template='{msg} ({msg_id}) at {path}:{line}'", '--load-plugins=pylint.extensions.bad_builtin', '--bad-functions=%s' % ','.join(self.BAD_FUNCTIONS), + '--deprecated-modules=%s' % ','.join(self.BAD_MODULES) ] try: From 54df4b36d7c826de3d07f81297aa17f74d8271ea Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 10 May 2017 15:08:42 +0200 Subject: [PATCH 03/11] [FIX] P3: types.NoneType removed --- odoo/addons/test_pylint/tests/test_pylint.py | 1 - odoo/fields.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/odoo/addons/test_pylint/tests/test_pylint.py b/odoo/addons/test_pylint/tests/test_pylint.py index d7a6760854c..f518bb8d43e 100644 --- a/odoo/addons/test_pylint/tests/test_pylint.py +++ b/odoo/addons/test_pylint/tests/test_pylint.py @@ -73,7 +73,6 @@ class TestPyLint(TransactionCase): BAD_MODULES = [ 'commands', - 'types', ] def _skip_test(self, reason): diff --git a/odoo/fields.py b/odoo/fields.py index 9026789fee6..a4199916b8c 100644 --- a/odoo/fields.py +++ b/odoo/fields.py @@ -7,7 +7,6 @@ from collections import OrderedDict, defaultdict from datetime import date, datetime from functools import partial from operator import attrgetter -from types import NoneType import logging import pytz try: @@ -1396,7 +1395,7 @@ class Char(_String): def _setup_regular_base(self, model): super(Char, self)._setup_regular_base(model) - assert isinstance(self.size, (NoneType, int)), \ + assert self.size is None or isinstance(self.size, int), \ "Char field %s with non-integer size %r" % (self, self.size) def convert_to_column(self, value, record): From 01e35141479b8dfc4a390518f89fc52e7cec5396 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Thu, 11 May 2017 11:55:22 +0200 Subject: [PATCH 04/11] [FIX] P3: urllib, urllib2 and urlparse In Python 3, all of these were "consolidated" under urllib(.request, .parse, .errors) which is inconvenient. Since we already have hard dependencies on requests and werkzeug(.urls, which is a backport of Python 3's unicode-aware urllib.parse) migrate *everything* to that. A sticking point is urllib2.URLError, those were (mostly) replaced by the slightly more general IOError which URLError extends. --- .../anonymization/wizard/anonymize_wizard.py | 5 +- addons/auth_oauth/models/res_users.py | 14 +--- addons/auth_signup/models/res_partner.py | 7 +- addons/base_geolocalize/models/res_partner.py | 8 +-- .../google_account/models/google_service.py | 67 +++++++++---------- .../google_calendar/models/google_calendar.py | 16 +++-- addons/google_drive/models/google_drive.py | 42 ++++++------ .../google_spreadsheet/models/google_drive.py | 12 ++-- addons/link_tracker/models/link_tracker.py | 24 +++---- addons/mail/models/html2text.py | 36 ++-------- addons/mail/models/mail_template.py | 19 +++--- addons/mail/models/res_config.py | 5 +- addons/mail/models/update.py | 13 ++-- addons/mass_mailing/models/mail_mail.py | 11 ++- addons/pad/models/pad.py | 16 +++-- addons/pad/py_etherpad/__init__.py | 25 ++----- addons/payment_adyen/models/payment.py | 7 +- addons/payment_adyen/tests/test_adyen.py | 4 +- addons/payment_authorize/controllers/main.py | 7 +- .../models/authorize_request.py | 9 ++- addons/payment_authorize/models/payment.py | 6 +- .../payment_authorize/tests/test_authorize.py | 6 +- addons/payment_buckaroo/models/payment.py | 17 ++--- .../payment_buckaroo/tests/test_buckaroo.py | 10 +-- addons/payment_ogone/models/payment.py | 36 +++++----- addons/payment_ogone/tests/test_ogone.py | 11 +-- addons/payment_paypal/controllers/main.py | 15 +++-- addons/payment_paypal/models/payment.py | 8 +-- addons/payment_paypal/tests/test_paypal.py | 9 +-- addons/payment_payumoney/models/payment.py | 9 +-- addons/payment_sips/models/payment.py | 7 +- .../models/pos_mercury_transaction.py | 11 +-- addons/survey/models/survey.py | 9 +-- addons/survey/tests/test_survey.py | 5 +- .../wizard/survey_email_compose_message.py | 5 +- addons/web_editor/models/ir_qweb.py | 17 ++--- addons/web_planner/models/web_planner.py | 5 +- addons/website/controllers/main.py | 14 ++-- addons/website/models/ir_actions.py | 5 +- addons/website/models/website.py | 12 ++-- addons/website/tests/test_crawl.py | 18 ++--- addons/website_forum/controllers/main.py | 8 ++- addons/website_forum/models/res_users.py | 5 +- .../models/hr_recruitment.py | 7 +- .../models/mail_channel.py | 4 +- .../website_sale/controllers/website_mail.py | 4 +- addons/website_slides/models/slides.py | 27 +++----- .../website_twitter/models/website_twitter.py | 16 ++--- .../models/website_twitter_config.py | 14 ++-- doc/_extensions/github_link.py | 5 +- doc/_extensions/odoo_ext/translator.py | 7 +- odoo/addons/base/ir/ir_qweb/ir_qweb.py | 8 +-- odoo/addons/base/module/module.py | 18 ++--- odoo/addons/base/res/res_partner.py | 32 ++++----- odoo/http.py | 13 ++-- odoo/sql_db.py | 8 +-- odoo/tests/common.py | 40 +++-------- odoo/tools/misc.py | 24 ------- 58 files changed, 342 insertions(+), 480 deletions(-) diff --git a/addons/anonymization/wizard/anonymize_wizard.py b/addons/anonymization/wizard/anonymize_wizard.py index b9c35483e18..880dd34617c 100644 --- a/addons/anonymization/wizard/anonymize_wizard.py +++ b/addons/anonymization/wizard/anonymize_wizard.py @@ -4,10 +4,7 @@ import base64 import os import random -try: - import cPickle as pickle -except ImportError: - import pickle +import pickle from lxml import etree from operator import itemgetter diff --git a/addons/auth_oauth/models/res_users.py b/addons/auth_oauth/models/res_users.py index fc7aaca4f56..2f9bf22c6ce 100644 --- a/addons/auth_oauth/models/res_users.py +++ b/addons/auth_oauth/models/res_users.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import werkzeug.urls -import urlparse -import urllib2 import json +import requests + from odoo import api, fields, models from odoo.exceptions import AccessDenied from odoo.addons.auth_signup.models.res_users import SignupError @@ -24,14 +23,7 @@ class ResUsers(models.Model): @api.model def _auth_oauth_rpc(self, endpoint, access_token): - params = werkzeug.url_encode({'access_token': access_token}) - if urlparse.urlparse(endpoint)[4]: - url = endpoint + '&' + params - else: - url = endpoint + '?' + params - f = urllib2.urlopen(url) - response = f.read() - return json.loads(response) + return requests.get(endpoint, params={'access_token': access_token}).json() @api.model def _auth_oauth_validate(self, provider, access_token): diff --git a/addons/auth_signup/models/res_partner.py b/addons/auth_signup/models/res_partner.py index 8527ad6e33d..d8111420a3c 100644 --- a/addons/auth_signup/models/res_partner.py +++ b/addons/auth_signup/models/res_partner.py @@ -2,10 +2,9 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import random -import werkzeug +import werkzeug.urls from datetime import datetime, timedelta -from urlparse import urljoin from odoo import api, fields, models, _ from odoo.tools import pycompat @@ -89,9 +88,9 @@ class ResPartner(models.Model): fragment['res_id'] = res_id if fragment: - query['redirect'] = base + werkzeug.url_encode(fragment) + query['redirect'] = base + werkzeug.urls.url_encode(fragment) - res[partner.id] = urljoin(base_url, "/web/%s?%s" % (route, werkzeug.url_encode(query))) + res[partner.id] = werkzeug.urls.url_join(base_url, "/web/%s?%s" % (route, werkzeug.urls.url_encode(query))) return res @api.multi diff --git a/addons/base_geolocalize/models/res_partner.py b/addons/base_geolocalize/models/res_partner.py index 916f6c0d9e0..0331d2c9734 100644 --- a/addons/base_geolocalize/models/res_partner.py +++ b/addons/base_geolocalize/models/res_partner.py @@ -1,18 +1,18 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. import json -import urllib + +import requests from odoo import api, fields, models, tools, _ from odoo.exceptions import UserError def geo_find(addr): - url = 'https://maps.googleapis.com/maps/api/geocode/json?sensor=false&address=' - url += urllib.quote(addr.encode('utf8')) + url = 'https://maps.googleapis.com/maps/api/geocode/json' try: - result = json.load(urllib.urlopen(url)) + result = requests.get(url, params={'sensor': 'false', 'address': addr}).json() except Exception as e: raise UserError(_('Cannot contact geolocation servers. Please make sure that your Internet connection is up and running (%s).') % e) diff --git a/addons/google_account/models/google_service.py b/addons/google_account/models/google_service.py index 9cd47c39179..44972435da5 100644 --- a/addons/google_account/models/google_service.py +++ b/addons/google_account/models/google_service.py @@ -4,8 +4,9 @@ from datetime import datetime import json import logging -import urllib2 -import werkzeug.urls + +import requests +from werkzeug import urls from odoo import api, fields, models, registry, _ from odoo.http import request @@ -38,27 +39,27 @@ class GoogleService(models.TransientModel): # Get the Refresh Token From Google And store it in ir.config_parameter headers = {"Content-type": "application/x-www-form-urlencoded"} - data = werkzeug.url_encode({ + data = { 'code': authorization_code, 'client_id': client_id, 'client_secret': client_secret, 'redirect_uri': redirect_uri, 'grant_type': "authorization_code" - }) + } try: - req = urllib2.Request(GOOGLE_TOKEN_ENDPOINT, data, headers) - content = urllib2.urlopen(req, timeout=TIMEOUT).read() - except urllib2.HTTPError: + req = requests.post(GOOGLE_TOKEN_ENDPOINT, data=data, headers=headers, timeout=TIMEOUT) + req.raise_for_status() + content = req.json() + except IOError: error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid or already expired") raise self.env['res.config.settings'].get_config_warning(error_msg) - content = json.loads(content) return content.get('refresh_token') @api.model def _get_google_token_uri(self, service, scope): get_param = self.env['ir.config_parameter'].sudo().get_param - encoded_params = werkzeug.url_encode({ + encoded_params = urls.url_encode({ 'scope': scope, 'redirect_uri': get_param('google_redirect_uri'), 'client_id': get_param('google_%s_client_id' % service), @@ -81,7 +82,7 @@ class GoogleService(models.TransientModel): base_url = get_param('web.base.url', default='http://www.odoo.com?NoBaseUrl') client_id = get_param('google_%s_client_id' % (service,), default=False) - encoded_params = werkzeug.url_encode({ + encoded_params = urls.url_encode({ 'response_type': 'code', 'client_id': client_id, 'state': json.dumps(state), @@ -103,17 +104,17 @@ class GoogleService(models.TransientModel): client_secret = get_param('google_%s_client_secret' % (service,), default=False) headers = {"content-type": "application/x-www-form-urlencoded"} - data = werkzeug.url_encode({ + data = { 'code': authorize_code, 'client_id': client_id, 'client_secret': client_secret, 'grant_type': 'authorization_code', 'redirect_uri': base_url + '/google_account/authentication' - }) + } try: dummy, response, dummy = self._do_request(GOOGLE_TOKEN_ENDPOINT, params=data, headers=headers, type='POST', preuri='') return response - except urllib2.HTTPError: + except requests.HTTPError: error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid") raise self.env['res.config.settings'].get_config_warning(error_msg) @@ -125,21 +126,21 @@ class GoogleService(models.TransientModel): client_secret = get_param('google_%s_client_secret' % (service,), default=False) headers = {"content-type": "application/x-www-form-urlencoded"} - data = werkzeug.url_encode({ + data = { 'refresh_token': refresh_token, 'client_id': client_id, 'client_secret': client_secret, 'grant_type': 'refresh_token', - }) + } try: dummy, response, dummy = self._do_request(GOOGLE_TOKEN_ENDPOINT, params=data, headers=headers, type='POST', preuri='') return response - except urllib2.HTTPError as error: - if error.code == 400: # invalid grant + except requests.HTTPError as error: + if error.response.status_code == 400: # invalid grant with registry(request.session.db).cursor() as cur: self.env(cur)['res.users'].browse(self.env.uid).write({'google_%s_rtoken' % service: False}) - error_key = json.loads(error.read()).get("error", "nc") + error_key = json.loads(error.response.json()).get("error", "nc") _logger.exception("Bad google request : %s !", error_key) error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid or already expired [%s]") % error_key raise self.env['res.config.settings'].get_config_warning(error_msg) @@ -154,40 +155,34 @@ class GoogleService(models.TransientModel): :param type : the method to use to make the request :param preuri : pre url to prepend to param uri. """ - _logger.debug("Uri: %s - Type : %s - Headers: %s - Params : %s !" % (uri, type, headers, werkzeug.url_encode(params) if type == 'GET' else params)) + _logger.debug("Uri: %s - Type : %s - Headers: %s - Params : %s !", (uri, type, headers, params)) - status = 418 - response = "" ask_time = fields.Datetime.now() try: - if type.upper() == 'GET' or type.upper() == 'DELETE': - data = werkzeug.url_encode(params) - req = urllib2.Request(preuri + uri + "?" + data) - elif type.upper() == 'POST' or type.upper() == 'PATCH' or type.upper() == 'PUT': - req = urllib2.Request(preuri + uri, params, headers) + if type.upper() in ('GET', 'DELETE'): + res = requests.request(type.lower(), preuri + uri, params=params, timeout=TIMEOUT) + elif type.upper() in ('POST', 'PATCH', 'PUT'): + res = requests.request(type.lower(), preuri + uri, data=params, headers=headers, timeout=TIMEOUT) else: raise Exception(_('Method not supported [%s] not in [GET, POST, PUT, PATCH or DELETE]!') % (type)) - req.get_method = lambda: type.upper() - - resp = urllib2.urlopen(req, timeout=TIMEOUT) - status = resp.getcode() + res.raise_for_status() + status = res.status_code if int(status) in (204, 404): # Page not found, no response response = False else: - content = resp.read() - response = json.loads(content) + response = res.json() try: - ask_time = datetime.strptime(resp.headers.get('date'), "%a, %d %b %Y %H:%M:%S %Z") + ask_time = datetime.strptime(res.headers.get('date'), "%a, %d %b %Y %H:%M:%S %Z") except: pass - except urllib2.HTTPError as error: - if error.code in (204, 404): + except request.HTTPError as error: + if error.response.status_code in (204, 404): status = error.code response = "" else: - _logger.exception("Bad google request : %s !", error.read()) + _logger.exception("Bad google request : %s !", error.response.content) if error.code in (400, 401, 410): raise error raise self.env['res.config.settings'].get_config_warning(_("Something went wrong with your request to google")) diff --git a/addons/google_calendar/models/google_calendar.py b/addons/google_calendar/models/google_calendar.py index f60ce78ef3d..50d0e5d6d2d 100644 --- a/addons/google_calendar/models/google_calendar.py +++ b/addons/google_calendar/models/google_calendar.py @@ -2,12 +2,14 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. from datetime import datetime, timedelta + +import requests from dateutil import parser import json import logging import operator import pytz -import urllib2 +from werkzeug import urls from odoo import api, fields, models, tools, _ from odoo.tools import exception_to_unicode, pycompat @@ -258,7 +260,7 @@ class GoogleCalendar(models.AbstractModel): """ data = self.generate_data(event, isCreating=True) - url = "/calendar/v3/calendars/%s/events?fields=%s&access_token=%s" % ('primary', urllib2.quote('id,updated'), self.get_token()) + url = "/calendar/v3/calendars/%s/events?fields=%s&access_token=%s" % ('primary', urls.url_quote('id,updated'), self.get_token()) headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} data_json = json.dumps(data) return self.env['google.service']._do_request(url, data_json, headers, type='POST') @@ -289,8 +291,8 @@ class GoogleCalendar(models.AbstractModel): try: status, content, ask_time = self.env['google.service']._do_request(url, params, headers, type='GET') - except urllib2.HTTPError as e: - if e.code == 401: # Token invalid / Acces unauthorized + except requests.HTTPError as e: + if e.response.status_code == 401: # Token invalid / Acces unauthorized error_msg = _("Your token is invalid or has been revoked !") self.env.user.write({'google_calendar_token': False, 'google_calendar_token_validity': False}) @@ -674,13 +676,13 @@ class GoogleCalendar(models.AbstractModel): if lastSync: try: all_event_from_google = self.get_event_synchro_dict(lastSync=lastSync) - except urllib2.HTTPError as e: - if e.code == 410: # GONE, Google is lost. + except requests.HTTPError as e: + if e.response.code == 410: # GONE, Google is lost. # we need to force the rollback from this cursor, because it locks my res_users but I need to write in this tuple before to raise. self.env.cr.rollback() self.env.user.write({'google_calendar_last_sync_date': False}) self.env.cr.commit() - error_key = json.loads(str(e)) + error_key = e.response.json() error_key = error_key.get('error', {}).get('message', 'nc') error_msg = _("Google is lost... the next synchro will be a full synchro. \n\n %s") % error_key raise self.env['res.config.settings'].get_config_warning(error_msg) diff --git a/addons/google_drive/models/google_drive.py b/addons/google_drive/models/google_drive.py index b1eda64b842..e653985f20a 100644 --- a/addons/google_drive/models/google_drive.py +++ b/addons/google_drive/models/google_drive.py @@ -3,7 +3,8 @@ import logging import json import re -import urllib2 + +import requests import werkzeug.urls from odoo import api, fields, models @@ -62,26 +63,25 @@ class GoogleDrive(models.Model): google_drive_client_id = Config.get_param('google_drive_client_id') google_drive_client_secret = Config.get_param('google_drive_client_secret') #For Getting New Access Token With help of old Refresh Token - data = werkzeug.url_encode({ + data = { 'client_id': google_drive_client_id, 'refresh_token': google_drive_refresh_token, 'client_secret': google_drive_client_secret, 'grant_type': "refresh_token", 'scope': scope or 'https://www.googleapis.com/auth/drive' - }) + } headers = {"Content-type": "application/x-www-form-urlencoded"} try: - req = urllib2.Request(GOOGLE_TOKEN_ENDPOINT, data, headers) - content = urllib2.urlopen(req, timeout=TIMEOUT).read() - except urllib2.HTTPError: + req = requests.post(GOOGLE_TOKEN_ENDPOINT, data=data, headers=headers, timeout=TIMEOUT) + req.raise_for_status() + except requests.HTTPError: if user_is_admin: dummy, action_id = self.env['ir.model.data'].get_object_reference('base_setup', 'action_general_configuration') msg = _("Something went wrong during the token generation. Please request again an authorization code .") raise RedirectWarning(msg, action_id, _('Go to the configuration panel')) else: raise UserError(_("Google Drive is not yet configured. Please contact your administrator.")) - content = json.loads(content) - return content.get('access_token') + return req.json().get('access_token') @api.model def copy_doc(self, res_id, template_id, name_gdocs, res_model): @@ -91,11 +91,11 @@ class GoogleDrive(models.Model): request_url = "https://www.googleapis.com/drive/v2/files/%s?fields=parents/id&access_token=%s" % (template_id, access_token) headers = {"Content-type": "application/x-www-form-urlencoded"} try: - req = urllib2.Request(request_url, None, headers) - parents = urllib2.urlopen(req, timeout=TIMEOUT).read() - except urllib2.HTTPError: + req = requests.post(request_url, headers=headers, timeout=TIMEOUT) + req.raise_for_status() + parents_dict = req.json() + except requests.HTTPError: raise UserError(_("The Google Template cannot be found. Maybe it has been deleted.")) - parents_dict = json.loads(parents) record_url = "Click on link to open Record in Odoo\n %s/?db=%s#id=%s&model=%s" % (google_web_base_url, self._cr.dbname, res_id, res_model) data = { @@ -108,11 +108,10 @@ class GoogleDrive(models.Model): 'Content-type': 'application/json', 'Accept': 'text/plain' } - data_json = json.dumps(data) # resp, content = Http().request(request_url, "POST", data_json, headers) - req = urllib2.Request(request_url, data_json, headers) - content = urllib2.urlopen(req, timeout=TIMEOUT).read() - content = json.loads(content) + req = requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT) + req.raise_for_status() + content = req.json() res = {} if content.get('alternateLink'): res['id'] = self.env["ir.attachment"].create({ @@ -129,16 +128,15 @@ class GoogleDrive(models.Model): request_url = "https://www.googleapis.com/drive/v2/files/%s/permissions?emailMessage=This+is+a+drive+file+created+by+Odoo&sendNotificationEmails=false&access_token=%s" % (key, access_token) data = {'role': 'writer', 'type': 'anyone', 'value': '', 'withLink': True} try: - req = urllib2.Request(request_url, json.dumps(data), headers) - urllib2.urlopen(req, timeout=TIMEOUT) - except urllib2.HTTPError: + req = requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT) + req.raise_for_status() + except requests.HTTPError: raise self.env['res.config.settings'].get_config_warning(_("The permission 'reader' for 'anyone with the link' has not been written on the document")) if self.env.user.email: data = {'role': 'writer', 'type': 'user', 'value': self.env.user.email} try: - req = urllib2.Request(request_url, json.dumps(data), headers) - urllib2.urlopen(req, timeout=TIMEOUT) - except urllib2.HTTPError: + requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT) + except requests.HTTPError: pass return res diff --git a/addons/google_spreadsheet/models/google_drive.py b/addons/google_spreadsheet/models/google_drive.py index 88b75a65ca3..dce4777a25e 100644 --- a/addons/google_spreadsheet/models/google_drive.py +++ b/addons/google_spreadsheet/models/google_drive.py @@ -3,10 +3,11 @@ import cgi import json import logging + +import requests from lxml import etree import re import werkzeug.urls -import urllib2 from odoo import api, models from odoo.addons.google_account import TIMEOUT @@ -72,12 +73,13 @@ class GoogleDrive(models.Model): ''' .format(key=spreadsheet_key, formula=cgi.escape(formula, quote=True), config=cgi.escape(config_formula, quote=True)) try: - req = urllib2.Request( + req = requests.post( 'https://spreadsheets.google.com/feeds/cells/%s/od6/private/full/batch?%s' % (spreadsheet_key, werkzeug.url_encode({'v': 3, 'access_token': access_token})), data=request, - headers={'content-type': 'application/atom+xml', 'If-Match': '*'}) - urllib2.urlopen(req, timeout=TIMEOUT) - except (urllib2.HTTPError, urllib2.URLError): + headers={'content-type': 'application/atom+xml', 'If-Match': '*'}, + timeout=TIMEOUT, + ) + except IOError: _logger.warning("An error occured while writting the formula on the Google Spreadsheet.") description = ''' diff --git a/addons/link_tracker/models/link_tracker.py b/addons/link_tracker/models/link_tracker.py index 12dcd46c140..1d9a7cdf561 100644 --- a/addons/link_tracker/models/link_tracker.py +++ b/addons/link_tracker/models/link_tracker.py @@ -6,11 +6,10 @@ import random import re import string +import requests from lxml import html -from urllib2 import urlopen -from urlparse import urljoin -from urlparse import urlparse -from werkzeug import url_encode, unescape +from werkzeug import urls, utils + from odoo import models, fields, api, _ from odoo.tools import ustr, pycompat @@ -18,7 +17,7 @@ from odoo.tools import ustr, pycompat URL_REGEX = r'(\bhref=[\'"](?!mailto:)([^\'"]+)[\'"])' def VALIDATE_URL(url): - if urlparse(url).scheme not in ('http', 'https', 'ftp', 'ftps'): + if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'): return 'http://' + url return url @@ -56,7 +55,7 @@ class link_tracker(models.Model): href = match[0] long_url = match[1] - vals['url'] = unescape(long_url) + vals['url'] = utils.unescape(long_url) if not blacklist or not [s for s in blacklist if s in long_url] and not long_url.startswith(short_schema): link = self.create(vals) @@ -77,7 +76,7 @@ class link_tracker(models.Model): @api.depends('code') def _compute_short_url(self): base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url') - self.short_url = urljoin(base_url, '/r/%(code)s' % {'code': self.code}) + self.short_url = urls.url_join(base_url, '/r/%(code)s' % {'code': self.code}) @api.one def _compute_short_url_host(self): @@ -96,22 +95,23 @@ class link_tracker(models.Model): @api.one @api.depends('url') def _compute_redirected_url(self): - parsed = urlparse(self.url) + parsed = urls.url_parse(self.url) utms = {} for key, field, cook in self.env['utm.mixin'].tracking_fields(): attr = getattr(self, field).name if attr: utms[key] = attr + utms.update(parsed.decode_query()) - self.redirected_url = '%s://%s%s?%s&%s#%s' % (parsed.scheme, parsed.netloc, parsed.path, url_encode(utms), parsed.query, parsed.fragment) + self.redirected_url = parsed.replace(query=urls.url_encode(utms)).to_url() @api.model @api.depends('url') def _get_title_from_url(self, url): try: - page = urlopen(url, timeout=5) - p = html.fromstring(ustr(page.read()).encode('utf-8'), parser=html.HTMLParser(encoding='utf-8')) + page = requests.get(url, timeout=5) + p = html.fromstring(page.text.encode('utf-8'), parser=html.HTMLParser(encoding='utf-8')) title = p.find('.//title').text except: title = url @@ -122,7 +122,7 @@ class link_tracker(models.Model): @api.depends('url') def _compute_favicon(self): try: - icon = urlopen('http://www.google.com/s2/favicons?domain=' + self.url, timeout=5).read() + icon = requests.get('http://www.google.com/s2/favicons', params={'domain': self.url}, timeout=5).content icon_base64 = icon.encode('base64').replace("\n", "") except: icon_base64 = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsSAAALEgHS3X78AAACiElEQVQ4EaVTzU8TURCf2tJuS7tQtlRb6UKBIkQwkRRSEzkQgyEc6lkOKgcOph78Y+CgjXjDs2i44FXY9AMTlQRUELZapVlouy3d7kKtb0Zr0MSLTvL2zb75eL838xtTvV6H/xELBptMJojeXLCXyobnyog4YhzXYvmCFi6qVSfaeRdXdrfaU1areV5KykmX06rcvzumjY/1ggkR3Jh+bNf1mr8v1D5bLuvR3qDgFbvbBJYIrE1mCIoCrKxsHuzK+Rzvsi29+6DEbTZz9unijEYI8ObBgXOzlcrx9OAlXyDYKUCzwwrDQx1wVDGg089Dt+gR3mxmhcUnaWeoxwMbm/vzDFzmDEKMMNhquRqduT1KwXiGt0vre6iSeAUHNDE0d26NBtAXY9BACQyjFusKuL2Ry+IPb/Y9ZglwuVscdHaknUChqLF/O4jn3V5dP4mhgRJgwSYm+gV0Oi3XrvYB30yvhGa7BS70eGFHPoTJyQHhMK+F0ZesRVVznvXw5Ixv7/C10moEo6OZXbWvlFAF9FVZDOqEABUMRIkMd8GnLwVWg9/RkJF9sA4oDfYQAuzzjqzwvnaRUFxn/X2ZlmGLXAE7AL52B4xHgqAUqrC1nSNuoJkQtLkdqReszz/9aRvq90NOKdOS1nch8TpL555WDp49f3uAMXhACRjD5j4ykuCtf5PP7Fm1b0DIsl/VHGezzP1KwOiZQobFF9YyjSRYQETRENSlVzI8iK9mWlzckpSSCQHVALmN9Az1euDho9Xo8vKGd2rqooA8yBcrwHgCqYR0kMkWci08t/R+W4ljDCanWTg9TJGwGNaNk3vYZ7VUdeKsYJGFNkfSzjXNrSX20s4/h6kB81/271ghG17l+rPTAAAAAElFTkSuQmCC' diff --git a/addons/mail/models/html2text.py b/addons/mail/models/html2text.py index 5e2fb25150f..8830cb0266d 100755 --- a/addons/mail/models/html2text.py +++ b/addons/mail/models/html2text.py @@ -1,5 +1,7 @@ #!/usr/bin/env python """html2text: Turn HTML into equivalent Markdown-structured text.""" +from werkzeug import urls + from odoo.tools import pycompat __version__ = "2.36" @@ -10,9 +12,8 @@ __contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"] # TODO: # Support decoded entities with unifiable. -import re, sys, urllib, htmlentitydefs, codecs +import re, sys, htmlentitydefs, codecs import sgmllib -import urlparse sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]') try: from textwrap import wrap @@ -390,7 +391,7 @@ class _html2text(sgmllib.SGMLParser): newa = [] for link in self.a: if self.outcount > link['outcount']: - self.out(" ["+repr(link['count'])+"]: " + urlparse.urljoin(self.baseurl, link['href'])) + self.out(" ["+repr(link['count'])+"]: " + urls.url_join(self.baseurl, link['href'])) if 'title' in link: self.out(" ("+link['title']+")") self.out("\n") else: @@ -426,32 +427,3 @@ def html2text_file(html, out=wrapwrite, baseurl=''): def html2text(html, baseurl=''): return optwrap(html2text_file(html, None, baseurl)) - -if __name__ == "__main__": - baseurl = '' - if sys.argv[1:]: - arg = sys.argv[1] - if arg.startswith('http://'): - baseurl = arg - j = urllib.urlopen(baseurl) - try: - from feedparser import _getCharacterEncoding as enc - except ImportError: - enc = lambda x, y: ('utf-8', 1) - text = j.read() - encoding = enc(j.headers, text)[0] - if encoding == 'us-ascii': encoding = 'utf-8' - data = text.decode(encoding) - - else: - encoding = 'utf8' - if len(sys.argv) > 2: - encoding = sys.argv[2] - f = open(arg, 'r') - try: - data = f.read().decode(encoding) - finally: - f.close() - else: - data = sys.stdin.read().decode('utf8') - wrapwrite(html2text(data, baseurl)) diff --git a/addons/mail/models/mail_template.py b/addons/mail/models/mail_template.py index ff89ca2d413..be73a374a45 100644 --- a/addons/mail/models/mail_template.py +++ b/addons/mail/models/mail_template.py @@ -10,9 +10,7 @@ import logging import functools import lxml -import urlparse - -from urllib import urlencode, quote as quote +from werkzeug import urls from odoo import _, api, fields, models, tools from odoo.exceptions import UserError @@ -83,8 +81,8 @@ try: ) mako_template_env.globals.update({ 'str': str, - 'quote': quote, - 'urlencode': urlencode, + 'quote': urls.url_quote, + 'urlencode': urls.url_encode, 'datetime': datetime, 'len': len, 'abs': abs, @@ -297,14 +295,13 @@ class MailTemplate(models.Model): root = lxml.html.fromstring(html) base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url') - (base_scheme, base_netloc, bpath, bparams, bquery, bfragment) = urlparse.urlparse(base_url) + base = urls.url_parse(base_url) def _process_link(url): - new_url = url - (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) - if not scheme and not netloc: - new_url = urlparse.urlunparse((base_scheme, base_netloc, path, params, query, fragment)) - return new_url + new_url = urls.url_parse(url) + if new_url.scheme and new_url.netloc: + return url + return new_url.replace(scheme=base.scheme, netloc=base.netloc).to_url() # check all nodes, replace : # - img src -> check URL diff --git a/addons/mail/models/res_config.py b/addons/mail/models/res_config.py index d34a6663a4f..c0e4f6bab62 100644 --- a/addons/mail/models/res_config.py +++ b/addons/mail/models/res_config.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import urlparse import datetime +from werkzeug import urls + from odoo import api, fields, models, tools @@ -29,7 +30,7 @@ class BaseConfiguration(models.TransientModel): if alias_domain is None: domain = self.env["ir.config_parameter"].get_param("web.base.url") try: - alias_domain = urlparse.urlsplit(domain).netloc.split(':')[0] + alias_domain = urls.url_parse(domain).host except Exception: pass return {'alias_domain': alias_domain or False} diff --git a/addons/mail/models/update.py b/addons/mail/models/update.py index 296bbc32ee1..accbc120f13 100644 --- a/addons/mail/models/update.py +++ b/addons/mail/models/update.py @@ -3,8 +3,9 @@ import datetime import logging + +import requests import werkzeug.urls -import urllib2 from ast import literal_eval @@ -71,16 +72,12 @@ class PublisherWarrantyContract(AbstractModel): """ msg = self._get_message() arguments = {'arg0': msg, "action": "update"} - arguments_raw = werkzeug.urls.url_encode(arguments) url = config.get("publisher_warranty_url") - uo = urllib2.urlopen(url, arguments_raw, timeout=30) - try: - submit_result = uo.read() - return literal_eval(submit_result) - finally: - uo.close() + r = requests.post(url, data=arguments, timeout=30) + r.raise_for_status() + return literal_eval(r.text) @api.multi def update_notification(self, cron_mode=True): diff --git a/addons/mass_mailing/models/mail_mail.py b/addons/mass_mailing/models/mail_mail.py index 3a69530db1a..1772cdaf1ff 100644 --- a/addons/mass_mailing/models/mail_mail.py +++ b/addons/mass_mailing/models/mail_mail.py @@ -2,7 +2,6 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import re -import urlparse import werkzeug.urls from odoo import api, fields, models, tools @@ -29,20 +28,20 @@ class MailMail(models.Model): def _get_tracking_url(self, partner=None): base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url') - track_url = urlparse.urljoin( + track_url = werkzeug.urls.url_join( base_url, 'mail/track/%(mail_id)s/blank.gif?%(params)s' % { 'mail_id': self.id, - 'params': werkzeug.url_encode({'db': self.env.cr.dbname}) + 'params': werkzeug.urls.url_encode({'db': self.env.cr.dbname}) } ) return '' % track_url def _get_unsubscribe_url(self, email_to): base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url') - url = urlparse.urljoin( + url = werkzeug.urls.url_join( base_url, 'mail/mailing/%(mailing_id)s/unsubscribe?%(params)s' % { 'mailing_id': self.mailing_id.id, - 'params': werkzeug.url_encode({ + 'params': werkzeug.urls.url_encode({ 'db': self.env.cr.dbname, 'res_id': self.res_id, 'email': email_to, @@ -79,7 +78,7 @@ class MailMail(models.Model): body = tools.append_content_to_html(base, body, plaintext=False, container_tag='div') # resolve relative image url to absolute for outlook.com def _sub_relative2absolute(match): - return match.group(1) + urlparse.urljoin(domain, match.group(2)) + return match.group(1) + werkzeug.urls.url_join(domain, match.group(2)) body = re.sub('(]*\ssrc=")(/[^/][^"]+)', _sub_relative2absolute, body) body = re.sub(r'(<[^>]+\bstyle="[^"]+\burl\(\'?)(/[^/\'][^\'")]+)', _sub_relative2absolute, body) diff --git a/addons/pad/models/pad.py b/addons/pad/models/pad.py index 1c8f08e79bd..8af18fa1b60 100644 --- a/addons/pad/models/pad.py +++ b/addons/pad/models/pad.py @@ -5,7 +5,8 @@ import logging import random import re import string -import urllib2 + +import requests from odoo import api, models, _ from odoo.exceptions import UserError @@ -53,7 +54,7 @@ class PadCommon(models.AbstractModel): myPad = EtherpadLiteClient(pad["key"], pad["server"] + '/api') try: myPad.createPad(path) - except urllib2.URLError: + except IOError: raise UserError(_("Pad creation failed, either there is a problem with your pad server URL or with your connection.")) # get attr on the field model @@ -79,12 +80,15 @@ class PadCommon(models.AbstractModel): content = '' if url: try: - page = urllib2.urlopen('%s/export/html' % url).read() - mo = re.search('(.*)', page, re.DOTALL) - if mo: - content = mo.group(1) + r = requests.get('%s/export/html' % url) + r.raise_for_status() except: _logger.warning("No url found '%s'.", url) + else: + mo = re.search('(.*)', r.content, re.DOTALL) + if mo: + content = mo.group(1) + return content # TODO diff --git a/addons/pad/py_etherpad/__init__.py b/addons/pad/py_etherpad/__init__.py index 0db607fe16f..d1040cec852 100644 --- a/addons/pad/py_etherpad/__init__.py +++ b/addons/pad/py_etherpad/__init__.py @@ -1,8 +1,5 @@ """Module to talk to EtherpadLite API.""" - -import json -import urllib -import urllib2 +import requests class EtherpadLiteClient: @@ -31,23 +28,11 @@ class EtherpadLiteClient: url = '%s/%d/%s' % (self.baseUrl, self.API_VERSION, function) params = arguments or {} - params.update({'apikey': self.apiKey}) - data = urllib.urlencode(params, True) + params['apikey'] = self.apiKey - try: - opener = urllib2.build_opener() - request = urllib2.Request(url=url, data=data) - response = opener.open(request, timeout=self.TIMEOUT) - result = response.read() - response.close() - except urllib2.HTTPError: - raise - - result = json.loads(result) - if result is None: - raise ValueError("JSON response could not be decoded") - - return self.handleResult(result) + r = requests.get(url, data=params, timeout=self.TIMEOUT) + r.raise_for_status() + return self.handleResult(r.json()) def handleResult(self, result): """Handle API call result""" diff --git a/addons/payment_adyen/models/payment.py b/addons/payment_adyen/models/payment.py index 8b2d8944813..87e51e619cb 100644 --- a/addons/payment_adyen/models/payment.py +++ b/addons/payment_adyen/models/payment.py @@ -7,9 +7,10 @@ from collections import OrderedDict import hashlib import hmac import logging -import urlparse from itertools import chain +from werkzeug import urls + from odoo import api, fields, models, tools, _ from odoo.addons.payment.models.payment_acquirer import ValidationError from odoo.addons.payment_adyen.controllers.main import AdyenController @@ -132,7 +133,7 @@ class AcquirerAdyen(models.Model): 'merchantAccount': self.adyen_merchant_account, 'shopperLocale': values.get('partner_lang', ''), 'sessionValidity': tmp_date.isoformat('T')[:19] + "Z", - 'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url), + 'resURL': urls.url_join(base_url, AdyenController._return_url), 'merchantReturnData': json.dumps({'return_url': '%s' % values.pop('return_url')}) if values.get('return_url', '') else False, 'shopperEmail': values.get('partner_email', ''), }) @@ -150,7 +151,7 @@ class AcquirerAdyen(models.Model): 'merchantAccount': self.adyen_merchant_account, 'shopperLocale': values.get('partner_lang'), 'sessionValidity': tmp_date, - 'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url), + 'resURL': urls.url_join(base_url, AdyenController._return_url), 'merchantReturnData': json.dumps({'return_url': '%s' % values.pop('return_url')}) if values.get('return_url') else False, 'merchantSig': self._adyen_generate_merchant_sig('in', values), }) diff --git a/addons/payment_adyen/tests/test_adyen.py b/addons/payment_adyen/tests/test_adyen.py index 2015df6922e..2e1d3eb53f2 100644 --- a/addons/payment_adyen/tests/test_adyen.py +++ b/addons/payment_adyen/tests/test_adyen.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- from lxml import objectify -import urlparse from odoo.addons.payment.tests.common import PaymentAcquirerCommon from odoo.addons.payment_adyen.controllers.main import AdyenController +from werkzeug import urls class AdyenCommon(PaymentAcquirerCommon): @@ -47,7 +47,7 @@ class AdyenForm(AdyenCommon): 'skinCode': 'cbqYWvVL', 'paymentAmount': '1', 'currencyCode': 'EUR', - 'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url), + 'resURL': urls.url_join(base_url, AdyenController._return_url), } # render the button diff --git a/addons/payment_authorize/controllers/main.py b/addons/payment_authorize/controllers/main.py index d88a74554b9..b42efe817bb 100644 --- a/addons/payment_authorize/controllers/main.py +++ b/addons/payment_authorize/controllers/main.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import pprint import logging -import urlparse -import werkzeug +from werkzeug import urls, utils from odoo import http from odoo.http import request @@ -29,7 +28,7 @@ class AuthorizeController(http.Controller): # This response is in the form of a URL that Authorize.Net will pass on to the # client's browser to redirect them to the desired location need javascript. return request.render('payment_authorize.payment_authorize_redirect', { - 'return_url': '%s' % urlparse.urljoin(base_url, return_url) + 'return_url': urls.url_join(base_url, return_url) }) @http.route(['/payment/authorize/s2s/create_json'], type='json', auth='public') @@ -43,4 +42,4 @@ class AuthorizeController(http.Controller): acquirer_id = int(post.get('acquirer_id')) acquirer = request.env['payment.acquirer'].browse(acquirer_id) acquirer.s2s_process(post) - return werkzeug.utils.redirect(post.get('return_url', '/')) + return utils.redirect(post.get('return_url', '/')) diff --git a/addons/payment_authorize/models/authorize_request.py b/addons/payment_authorize/models/authorize_request.py index 75a54d93526..21716ec558f 100644 --- a/addons/payment_authorize/models/authorize_request.py +++ b/addons/payment_authorize/models/authorize_request.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- +import requests from lxml import etree, objectify -from urllib2 import urlopen, Request from StringIO import StringIO import xml.etree.ElementTree as ET from uuid import uuid4 @@ -57,10 +57,9 @@ class AuthorizeAPI(): :param etree._Element data: etree data to process """ data = etree.tostring(data, xml_declaration=True, encoding='utf-8') - request = Request(self.url, data) - request.add_header('Content-Type', 'text/xml') - response = urlopen(request).read() - response = strip_ns(response, XMLNS) + r = requests.post(self.url, data=data, headers={'Content-Type': 'text/xml'}) + r.raise_for_status() + response = strip_ns(r.content, XMLNS) if response.find('messages/resultCode').text == 'Error': messages = [m.text for m in response.findall('messages/message/text')] raise ValidationError('Authorize.net Error Message(s):\n %s' % '\n'.join(messages)) diff --git a/addons/payment_authorize/models/payment.py b/addons/payment_authorize/models/payment.py index 12f5b353062..9134549b393 100644 --- a/addons/payment_authorize/models/payment.py +++ b/addons/payment_authorize/models/payment.py @@ -1,4 +1,5 @@ # coding: utf-8 +from werkzeug import urls from .authorize_request import AuthorizeAPI from datetime import datetime @@ -6,7 +7,6 @@ import hashlib import hmac import logging import time -import urlparse from odoo import api, fields, models from odoo.addons.payment.models.payment_acquirer import ValidationError @@ -72,8 +72,8 @@ class PaymentAcquirerAuthorize(models.Model): 'x_version': '3.1', 'x_relay_response': 'TRUE', 'x_fp_timestamp': str(int(time.time())), - 'x_relay_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._return_url), - 'x_cancel_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._cancel_url), + 'x_relay_url': urls.url_join(base_url, AuthorizeController._return_url), + 'x_cancel_url': urls.url_join(base_url, AuthorizeController._cancel_url), 'x_currency_code': values['currency'] and values['currency'].name or '', 'address': values.get('partner_address'), 'city': values.get('partner_city'), diff --git a/addons/payment_authorize/tests/test_authorize.py b/addons/payment_authorize/tests/test_authorize.py index c16113cd52b..d19690284a8 100644 --- a/addons/payment_authorize/tests/test_authorize.py +++ b/addons/payment_authorize/tests/test_authorize.py @@ -3,9 +3,9 @@ import hashlib import hmac import time -import urlparse import unittest from lxml import objectify +from werkzeug import urls import odoo from odoo.addons.payment.models.payment_acquirer import ValidationError @@ -59,8 +59,8 @@ class AuthorizeForm(AuthorizeCommon): 'x_version': '3.1', 'x_relay_response': 'TRUE', 'x_fp_timestamp': str(int(time.time())), - 'x_relay_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._return_url), - 'x_cancel_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._cancel_url), + 'x_relay_url': urls.url_join(base_url, AuthorizeController._return_url), + 'x_cancel_url': urls.url_join(base_url, AuthorizeController._cancel_url), 'return_url': None, 'x_currency_code': 'USD', 'x_invoice_num': 'SO004', diff --git a/addons/payment_buckaroo/models/payment.py b/addons/payment_buckaroo/models/payment.py index e46f078c134..a07ca83318d 100644 --- a/addons/payment_buckaroo/models/payment.py +++ b/addons/payment_buckaroo/models/payment.py @@ -1,8 +1,8 @@ # coding: utf-8 from hashlib import sha1 import logging -import urllib -import urlparse + +from werkzeug import urls from odoo import api, fields, models, _ from odoo.addons.payment.models.payment_acquirer import ValidationError @@ -74,14 +74,11 @@ class AcquirerBuckaroo(models.Model): break items = sorted(pycompat.items(values), key=lambda pair: pair[0].lower()) - sign = ''.join('%s=%s' % (k, urllib.unquote_plus(v)) for k, v in items) + sign = ''.join('%s=%s' % (k, urls.url_unquote_plus(v)) for k, v in items) else: sign = ''.join('%s=%s' % (k, get_value(k)) for k in keys) # Add the pre-shared secret key at the end of the signature sign = sign + self.brq_secretkey - if isinstance(sign, str): - # TODO: remove me? should not be used - sign = urlparse.parse_qsl(sign) shasign = sha1(sign.encode('utf-8')).hexdigest() return shasign @@ -95,10 +92,10 @@ class AcquirerBuckaroo(models.Model): 'Brq_currency': values['currency'] and values['currency'].name or '', 'Brq_invoicenumber': values['reference'], 'brq_test': False if self.environment == 'prod' else True, - 'Brq_return': '%s' % urlparse.urljoin(base_url, BuckarooController._return_url), - 'Brq_returncancel': '%s' % urlparse.urljoin(base_url, BuckarooController._cancel_url), - 'Brq_returnerror': '%s' % urlparse.urljoin(base_url, BuckarooController._exception_url), - 'Brq_returnreject': '%s' % urlparse.urljoin(base_url, BuckarooController._reject_url), + 'Brq_return': urls.url_join(base_url, BuckarooController._return_url), + 'Brq_returncancel': urls.url_join(base_url, BuckarooController._cancel_url), + 'Brq_returnerror': urls.url_join(base_url, BuckarooController._exception_url), + 'Brq_returnreject': urls.url_join(base_url, BuckarooController._reject_url), 'Brq_culture': (values.get('partner_lang') or 'en_US').replace('_', '-'), 'add_returndata': buckaroo_tx_values.pop('return_url', '') or '', }) diff --git a/addons/payment_buckaroo/tests/test_buckaroo.py b/addons/payment_buckaroo/tests/test_buckaroo.py index 016f8ec193e..0825f67e875 100644 --- a/addons/payment_buckaroo/tests/test_buckaroo.py +++ b/addons/payment_buckaroo/tests/test_buckaroo.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from lxml import objectify -import urlparse +from werkzeug import urls import odoo from odoo.addons.payment.models.payment_acquirer import ValidationError @@ -41,10 +41,10 @@ class BuckarooForm(BuckarooCommon): 'Brq_invoicenumber': 'SO004', 'Brq_signature': '1b8c10074c622d965272a91a9e88b5b3777d2474', # update me 'brq_test': 'True', - 'Brq_return': '%s' % urlparse.urljoin(base_url, BuckarooController._return_url), - 'Brq_returncancel': '%s' % urlparse.urljoin(base_url, BuckarooController._cancel_url), - 'Brq_returnerror': '%s' % urlparse.urljoin(base_url, BuckarooController._exception_url), - 'Brq_returnreject': '%s' % urlparse.urljoin(base_url, BuckarooController._reject_url), + 'Brq_return': urls.url_join(base_url, BuckarooController._return_url), + 'Brq_returncancel': urls.url_join(base_url, BuckarooController._cancel_url), + 'Brq_returnerror': urls.url_join(base_url, BuckarooController._exception_url), + 'Brq_returnreject': urls.url_join(base_url, BuckarooController._reject_url), 'Brq_culture': 'en-US', } diff --git a/addons/payment_ogone/models/payment.py b/addons/payment_ogone/models/payment.py index 34cf982c82e..559f119fbbd 100644 --- a/addons/payment_ogone/models/payment.py +++ b/addons/payment_ogone/models/payment.py @@ -1,22 +1,20 @@ # coding: utf-8 - -from hashlib import sha1 -from lxml import etree, objectify -from pprint import pformat -from unicodedata import normalize -from urllib import urlencode - import datetime import logging import time -import urllib2 -import urlparse +from hashlib import sha1 +from pprint import pformat +from unicodedata import normalize + +import requests +from lxml import etree, objectify +from werkzeug import urls from odoo import api, fields, models, _ from odoo.addons.payment.models.payment_acquirer import ValidationError from odoo.addons.payment_ogone.controllers.main import OgoneController from odoo.addons.payment_ogone.data import ogone -from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT +from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT, pycompat from odoo.tools.float_utils import float_compare, float_repr from odoo.tools.safe_eval import safe_eval @@ -165,10 +163,10 @@ class PaymentAcquirerOgone(models.Model): 'OWNERTOWN': values.get('partner_city'), 'OWNERCTY': values.get('partner_country') and values.get('partner_country').code or '', 'OWNERTELNO': values.get('partner_phone'), - 'ACCEPTURL': '%s' % urlparse.urljoin(base_url, OgoneController._accept_url), - 'DECLINEURL': '%s' % urlparse.urljoin(base_url, OgoneController._decline_url), - 'EXCEPTIONURL': '%s' % urlparse.urljoin(base_url, OgoneController._exception_url), - 'CANCELURL': '%s' % urlparse.urljoin(base_url, OgoneController._cancel_url), + 'ACCEPTURL': urls.url_join(base_url, OgoneController._accept_url), + 'DECLINEURL': urls.url_join(base_url, OgoneController._decline_url), + 'EXCEPTIONURL': urls.url_join(base_url, OgoneController._exception_url), + 'CANCELURL': urls.url_join(base_url, OgoneController._cancel_url), 'PARAMPLUS': 'return_url=%s' % ogone_tx_values.pop('return_url') if ogone_tx_values.get('return_url') else False, } if self.save_token in ['ask', 'always']: @@ -370,8 +368,7 @@ class PaymentTxOgone(models.Model): direct_order_url = 'https://secure.ogone.com/ncol/%s/orderdirect.asp' % (self.acquirer_id.environment) _logger.debug("Ogone data %s", pformat(data)) - request = urllib2.Request(direct_order_url, urlencode(data)) - result = urllib2.urlopen(request).read() + result = requests.post(direct_order_url, data=data).content _logger.debug('Ogone response = %s', result) try: @@ -455,8 +452,7 @@ class PaymentTxOgone(models.Model): query_direct_url = 'https://secure.ogone.com/ncol/%s/querydirect.asp' % (self.acquirer_id.environment) _logger.debug("Ogone data %s", pformat(data)) - request = urllib2.Request(query_direct_url, urlencode(data)) - result = urllib2.urlopen(request).read() + result = requests.post(query_direct_url, data=data).content _logger.debug('Ogone response = %s', result) try: @@ -497,9 +493,7 @@ class PaymentToken(models.Model): } url = 'https://secure.ogone.com/ncol/%s/AFU_agree.asp' % (acquirer.environment,) - request = urllib2.Request(url, urlencode(data)) - - result = urllib2.urlopen(request).read() + result = requests.post(url, data=data).content try: tree = objectify.fromstring(result) diff --git a/addons/payment_ogone/tests/test_ogone.py b/addons/payment_ogone/tests/test_ogone.py index ac32f0c4fe5..b27e7a3c4ce 100644 --- a/addons/payment_ogone/tests/test_ogone.py +++ b/addons/payment_ogone/tests/test_ogone.py @@ -2,11 +2,12 @@ from lxml import objectify import time -import urlparse from odoo.addons.payment.models.payment_acquirer import ValidationError from odoo.addons.payment.tests.common import PaymentAcquirerCommon from odoo.addons.payment_ogone.controllers.main import OgoneController +from werkzeug import urls + from odoo.tools import mute_logger @@ -40,10 +41,10 @@ class OgonePayment(PaymentAcquirerCommon): 'OWNERTOWN': 'Sin City', 'OWNERTELNO': '0032 12 34 56 78', 'SHASIGN': '815f67b8ff70d234ffcf437c13a9fa7f807044cc', - 'ACCEPTURL': '%s' % urlparse.urljoin(base_url, OgoneController._accept_url), - 'DECLINEURL': '%s' % urlparse.urljoin(base_url, OgoneController._decline_url), - 'EXCEPTIONURL': '%s' % urlparse.urljoin(base_url, OgoneController._exception_url), - 'CANCELURL': '%s' % urlparse.urljoin(base_url, OgoneController._cancel_url), + 'ACCEPTURL': urls.url_join(base_url, OgoneController._accept_url), + 'DECLINEURL': urls.url_join(base_url, OgoneController._decline_url), + 'EXCEPTIONURL': urls.url_join(base_url, OgoneController._exception_url), + 'CANCELURL': urls.url_join(base_url, OgoneController._cancel_url), } # render the button diff --git a/addons/payment_paypal/controllers/main.py b/addons/payment_paypal/controllers/main.py index 56023740d27..edd86c96685 100644 --- a/addons/payment_paypal/controllers/main.py +++ b/addons/payment_paypal/controllers/main.py @@ -3,9 +3,10 @@ import json import logging import pprint -import urllib -import urllib2 + +import requests import werkzeug +from werkzeug import urls from odoo import http from odoo.addons.payment.models.payment_acquirer import ValidationError @@ -23,7 +24,7 @@ class PaypalController(http.Controller): """ Extract the return URL from the data coming from paypal. """ return_url = post.pop('return_url', '') if not return_url: - custom = json.loads(urllib.unquote_plus(post.pop('custom', False) or post.pop('cm', False) or '{}')) + custom = json.loads(urls.url_unquote_plus(post.pop('custom', False) or post.pop('cm', False) or '{}')) return_url = custom.get('return_url', '/') return return_url @@ -44,7 +45,7 @@ class PaypalController(http.Controller): for line in lines: split = line.split('=', 1) if len(split) == 2: - pdt_post[split[0]] = urllib.unquote_plus(split[1]).decode('utf8') + pdt_post[split[0]] = urls.url_unquote_plus(split[1]).decode('utf8') else: _logger.warning('Paypal: error processing pdt response: %s', line) @@ -75,9 +76,9 @@ class PaypalController(http.Controller): new_post['at'] = tx and tx.acquirer_id.paypal_pdt_token or '' new_post['cmd'] = '_notify-synch' # command is different in PDT than IPN/DPN validate_url = paypal_urls['paypal_form_url'] - urequest = urllib2.Request(validate_url, werkzeug.url_encode(new_post)) - uopen = urllib2.urlopen(urequest) - resp = uopen.read() + urequest = requests.post(validate_url, new_post) + urequest.raise_for_status() + resp = urequest.content if pdt_request: resp, post = self._parse_pdt_response(resp) if resp in ['VERIFIED', 'SUCCESS']: diff --git a/addons/payment_paypal/models/payment.py b/addons/payment_paypal/models/payment.py index 3219e3cc469..5d8e5d9ed27 100644 --- a/addons/payment_paypal/models/payment.py +++ b/addons/payment_paypal/models/payment.py @@ -2,10 +2,10 @@ import json import logging -import urlparse import dateutil.parser import pytz +from werkzeug import urls from odoo import api, fields, models, _ from odoo.addons.payment.models.payment_acquirer import ValidationError @@ -109,9 +109,9 @@ class AcquirerPaypal(models.Model): 'zip_code': values.get('partner_zip'), 'first_name': values.get('partner_first_name'), 'last_name': values.get('partner_last_name'), - 'paypal_return': '%s' % urlparse.urljoin(base_url, PaypalController._return_url), - 'notify_url': '%s' % urlparse.urljoin(base_url, PaypalController._notify_url), - 'cancel_return': '%s' % urlparse.urljoin(base_url, PaypalController._cancel_url), + 'paypal_return': urls.url_join(base_url, PaypalController._return_url), + 'notify_url': urls.url_join(base_url, PaypalController._notify_url), + 'cancel_return': urls.url_join(base_url, PaypalController._cancel_url), 'handling': '%.2f' % paypal_tx_values.pop('fees', 0.0) if self.fees_active else False, 'custom': json.dumps({'return_url': '%s' % paypal_tx_values.pop('return_url')}) if paypal_tx_values.get('return_url') else False, }) diff --git a/addons/payment_paypal/tests/test_paypal.py b/addons/payment_paypal/tests/test_paypal.py index c6a44dec41f..e93bfd039b1 100644 --- a/addons/payment_paypal/tests/test_paypal.py +++ b/addons/payment_paypal/tests/test_paypal.py @@ -3,10 +3,11 @@ from odoo.addons.payment.models.payment_acquirer import ValidationError from odoo.addons.payment.tests.common import PaymentAcquirerCommon from odoo.addons.payment_paypal.controllers.main import PaypalController +from werkzeug import urls + from odoo.tools import mute_logger from lxml import objectify -import urlparse class PaypalCommon(PaymentAcquirerCommon): @@ -60,9 +61,9 @@ class PaypalForm(PaypalCommon): 'zip': '1000', 'country': 'BE', 'email': 'norbert.buyer@example.com', - 'return': '%s' % urlparse.urljoin(base_url, PaypalController._return_url), - 'notify_url': '%s' % urlparse.urljoin(base_url, PaypalController._notify_url), - 'cancel_return': '%s' % urlparse.urljoin(base_url, PaypalController._cancel_url), + 'return': urls.url_join(base_url, PaypalController._return_url), + 'notify_url': urls.url_join(base_url, PaypalController._notify_url), + 'cancel_return': urls.url_join(base_url, PaypalController._cancel_url), } # check form result diff --git a/addons/payment_payumoney/models/payment.py b/addons/payment_payumoney/models/payment.py index 8f44b1233cb..243ef5cef53 100644 --- a/addons/payment_payumoney/models/payment.py +++ b/addons/payment_payumoney/models/payment.py @@ -2,7 +2,8 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import hashlib -import urlparse + +from werkzeug import urls from odoo import api, fields, models, _ from odoo.addons.payment.models.payment_acquirer import ValidationError @@ -64,9 +65,9 @@ class PaymentAcquirerPayumoney(models.Model): email=values.get('partner_email'), phone=values.get('partner_phone'), service_provider='payu_paisa', - surl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/return'), - furl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/error'), - curl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/cancel') + surl=urls.url_join(base_url, '/payment/payumoney/return'), + furl=urls.url_join(base_url, '/payment/payumoney/error'), + curl=urls.url_join(base_url, '/payment/payumoney/cancel') ) payumoney_values['udf1'] = payumoney_values.pop('return_url', '/') diff --git a/addons/payment_sips/models/payment.py b/addons/payment_sips/models/payment.py index e9cd4d40e6b..51f78007d91 100644 --- a/addons/payment_sips/models/payment.py +++ b/addons/payment_sips/models/payment.py @@ -3,7 +3,8 @@ import json import logging from hashlib import sha256 -import urlparse + +from werkzeug import urls from odoo import models, fields, api from odoo.tools.float_utils import float_compare @@ -89,8 +90,8 @@ class AcquirerSips(models.Model): 'Data': u'amount=%s|' % amount + u'currencyCode=%s|' % currency_code + u'merchantId=%s|' % merchant_id + - u'normalReturnUrl=%s|' % urlparse.urljoin(base_url, SipsController._return_url) + - u'automaticResponseUrl=%s|' % urlparse.urljoin(base_url, SipsController._return_url) + + u'normalReturnUrl=%s|' % urls.url_join(base_url, SipsController._return_url) + + u'automaticResponseUrl=%s|' % urls.url_join(base_url, SipsController._return_url) + u'transactionReference=%s|' % values['reference'] + u'statementReference=%s|' % values['reference'] + u'keyVersion=%s' % key_version, diff --git a/addons/pos_mercury/models/pos_mercury_transaction.py b/addons/pos_mercury/models/pos_mercury_transaction.py index 36748c5ea0c..5efbdaa0ff5 100644 --- a/addons/pos_mercury/models/pos_mercury_transaction.py +++ b/addons/pos_mercury/models/pos_mercury_transaction.py @@ -4,8 +4,9 @@ from datetime import date, timedelta import cgi -import urllib2 import ssl + +import requests import werkzeug from odoo import models, api, service @@ -61,11 +62,11 @@ class MercuryTransaction(models.Model): 'SOAPAction': 'http://www.mercurypay.com/CreditTransaction', } - r = urllib2.Request('https://w1.mercurypay.com/ws/ws.asmx', data=xml_transaction, headers=headers) try: - u = urllib2.urlopen(r, timeout=65) - response = werkzeug.utils.unescape(u.read()) - except (urllib2.URLError, ssl.SSLError): + r = requests.post('https://w1.mercurypay.com/ws/ws.asmx', data=xml_transaction, headers=headers, timeout=65) + r.raise_for_status() + response = werkzeug.utils.unescape(r.content) + except: response = "timeout" return response diff --git a/addons/survey/models/survey.py b/addons/survey/models/survey.py index a35ae5ab84e..84692c63f9d 100644 --- a/addons/survey/models/survey.py +++ b/addons/survey/models/survey.py @@ -5,10 +5,11 @@ import datetime import logging import re import uuid -from urlparse import urljoin from collections import Counter, OrderedDict from itertools import product +from werkzeug import urls + from odoo import api, fields, models, tools, SUPERUSER_ID, _ from odoo.exceptions import UserError, ValidationError @@ -108,9 +109,9 @@ class Survey(models.Model): base_url = '/' if self.env.context.get('relative_url') else \ self.env['ir.config_parameter'].sudo().get_param('web.base.url') for survey in self: - survey.public_url = urljoin(base_url, "survey/start/%s" % (slug(survey))) - survey.print_url = urljoin(base_url, "survey/print/%s" % (slug(survey))) - survey.result_url = urljoin(base_url, "survey/results/%s" % (slug(survey))) + survey.public_url = urls.url_join(base_url, "survey/start/%s" % (slug(survey))) + survey.print_url = urls.url_join(base_url, "survey/print/%s" % (slug(survey))) + survey.result_url = urls.url_join(base_url, "survey/results/%s" % (slug(survey))) survey.public_url_html = '%s' % (survey.public_url, _("Click here to start survey")) @api.model diff --git a/addons/survey/tests/test_survey.py b/addons/survey/tests/test_survey.py index 01786368b72..699aac0cde3 100644 --- a/addons/survey/tests/test_survey.py +++ b/addons/survey/tests/test_survey.py @@ -5,7 +5,8 @@ import random import re from collections import Counter from itertools import product -from urlparse import urljoin + +from werkzeug import urls from odoo import _ from odoo.exceptions import UserError @@ -183,7 +184,7 @@ class TestSurvey(TransactionCase): survey_url_relative = getattr(self.survey1.with_context({'relative_url': True}), urltype + '_url') self.assertTrue(validate_url(survey_url)) url = "survey/%s/%s" % (urltxt, slug(self.survey1)) - full_url = urljoin(base_url, url) + full_url = urls.url_join(base_url, url) self.assertEqual(full_url, survey_url) self.assertEqual('/' + url, survey_url_relative) if urltype == 'public': diff --git a/addons/survey/wizard/survey_email_compose_message.py b/addons/survey/wizard/survey_email_compose_message.py index 2e333822dc1..fb36d839c57 100644 --- a/addons/survey/wizard/survey_email_compose_message.py +++ b/addons/survey/wizard/survey_email_compose_message.py @@ -3,7 +3,8 @@ import re import uuid -import urlparse + +from werkzeug import urls from odoo import api, fields, models, _ from odoo.exceptions import UserError @@ -85,7 +86,7 @@ class SurveyMailComposeMessage(models.TransientModel): #set url url = wizard.survey_id.public_url - url = urlparse.urlparse(url).path[1:] # dirty hack to avoid incorrect urls + url = urls.url_parse(url).path[1:] # dirty hack to avoid incorrect urls if token: url = url + '/' + token diff --git a/addons/web_editor/models/ir_qweb.py b/addons/web_editor/models/ir_qweb.py index e3bd9ec94e2..71445d4919e 100644 --- a/addons/web_editor/models/ir_qweb.py +++ b/addons/web_editor/models/ir_qweb.py @@ -14,15 +14,16 @@ import itertools import json import logging import os -import urllib2 -import urlparse import re import hashlib import pytz +import requests from dateutil import parser from lxml import etree, html from PIL import Image as I +from werkzeug import urls + import odoo.modules from odoo import api, models, fields @@ -319,11 +320,11 @@ class Image(models.AbstractModel): def from_html(self, model, field, element): url = element.find('img').get('src') - url_object = urlparse.urlsplit(url) + url_object = urls.url_parse(url) if url_object.path.startswith('/web/image'): # url might be /web/image//[_]/[/x] fragments = url_object.path.split('/') - query = dict(urlparse.parse_qsl(url_object.query)) + query = url_object.decode_query() if fragments[3].isdigit(): model = 'ir.attachment' oid = fragments[3] @@ -341,7 +342,7 @@ class Image(models.AbstractModel): return self.load_remote_url(url) def load_local_url(self, url): - match = self.local_url_re.match(urlparse.urlsplit(url).path) + match = self.local_url_re.match(urls.url_parse(url).path) rest = match.group('rest') for sep in os.sep, os.altsep: @@ -374,9 +375,9 @@ class Image(models.AbstractModel): # linking to HTTP images # implement drag & drop image upload to mitigate? - req = urllib2.urlopen(url, timeout=REMOTE_CONNECTION_TIMEOUT) - # PIL needs a seekable file-like image, urllib result is not seekable - image = I.open(cStringIO.StringIO(req.read())) + req = requests.get(url, timeout=REMOTE_CONNECTION_TIMEOUT) + # PIL needs a seekable file-like image so wrap result in IO buffer + image = I.open(cStringIO.StringIO(req.content)) # force a complete load of the image data to validate it image.load() except Exception: diff --git a/addons/web_planner/models/web_planner.py b/addons/web_planner/models/web_planner.py index 3fa38c1dc7c..f1cae08d550 100644 --- a/addons/web_planner/models/web_planner.py +++ b/addons/web_planner/models/web_planner.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - -from urllib import urlencode +from werkzeug import urls from odoo import api, models, fields @@ -69,7 +68,7 @@ class Planner(models.Model): params['id'] = module.id else: return "#show_enterprise" - return "/web#%s" % (urlencode(params),) + return "/web#%s" % (urls.url_encode(params),) @api.model def is_module_installed(self, module_name=None): diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py index 5fab12ac600..e67697e2bcc 100644 --- a/addons/website/controllers/main.py +++ b/addons/website/controllers/main.py @@ -7,7 +7,8 @@ import json import xml.etree.ElementTree as ET import logging import re -import urllib2 + +import requests import werkzeug.utils import werkzeug.wrappers @@ -300,12 +301,13 @@ class Website(Home): language = lang.split("_") url = "http://google.com/complete/search" try: - req = urllib2.Request("%s?%s" % (url, werkzeug.url_encode({ - 'ie': 'utf8', 'oe': 'utf8', 'output': 'toolbar', 'q': keywords, 'hl': language[0], 'gl': language[1]}))) - response = urllib2.urlopen(req) - except (urllib2.HTTPError, urllib2.URLError): + req = requests.get(url, params={ + 'ie': 'utf8', 'oe': 'utf8', 'output': 'toolbar', 'q': keywords, 'hl': language[0], 'gl': language[1]}) + req.raise_for_status() + response = req.content + except IOError: return [] - xmlroot = ET.fromstring(response.read()) + xmlroot = ET.fromstring(response) return json.dumps([sugg[0].attrib['data'] for sugg in xmlroot if len(sugg) and sugg[0].attrib['data']]) #------------------------------------------------------ diff --git a/addons/website/models/ir_actions.py b/addons/website/models/ir_actions.py index 176187ea4e0..f26d6263004 100644 --- a/addons/website/models/ir_actions.py +++ b/addons/website/models/ir_actions.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. - -import urlparse +from werkzeug import urls from odoo import api, fields, models from odoo.http import request @@ -32,7 +31,7 @@ class ServerAction(models.Model): link = website_path or xml_id or (self.id and '%d' % self.id) or '' if base_url and link: path = '%s/%s' % ('/website/action', link) - return '%s' % urlparse.urljoin(base_url, path) + return urls.url_join(base_url, path) return '' @api.depends('state', 'website_published', 'website_path', 'xml_id') diff --git a/addons/website/models/website.py b/addons/website/models/website.py index 9968ad8e636..fe79a8ac1a3 100644 --- a/addons/website/models/website.py +++ b/addons/website/models/website.py @@ -6,9 +6,9 @@ import logging import math import unicodedata import re -import urlparse import hashlib -import werkzeug + +from werkzeug import urls from werkzeug.exceptions import NotFound # optional python-slugify import (https://github.com/un33k/python-slugify) @@ -44,10 +44,10 @@ def url_for(path_or_uri, lang=None): current_path = current_path.encode('utf-8') location = path_or_uri.strip() force_lang = lang is not None - url = urlparse.urlparse(location) + url = urls.url_parse(location) if request and not url.netloc and not url.scheme and (url.path or force_lang): - location = urlparse.urljoin(current_path, location) + location = urls.url_join(current_path, location) lang = lang or request.context.get('lang') langs = [lg[0] for lg in request.website.get_languages()] @@ -459,7 +459,7 @@ class Website(models.Model): def get_url(page): _url = "%s/page/%s" % (url, page) if page > 1 else url if url_args: - _url = "%s?%s" % (_url, werkzeug.url_encode(url_args)) + _url = "%s?%s" % (_url, urls.url_encode(url_args)) return _url return { @@ -598,7 +598,7 @@ class Website(models.Model): cdn_filters = (request.website.cdn_filters or '').splitlines() for flt in cdn_filters: if flt and re.match(flt, uri): - return urlparse.urljoin(cdn_url, uri) + return urls.url_join(cdn_url, uri) return uri @api.model diff --git a/addons/website/tests/test_crawl.py b/addons/website/tests/test_crawl.py index a9f1e9b1f84..f5d4bc4ec12 100644 --- a/addons/website/tests/test_crawl.py +++ b/addons/website/tests/test_crawl.py @@ -2,10 +2,10 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import logging -import urlparse import time import lxml.html +from werkzeug import urls import odoo import re @@ -39,23 +39,17 @@ class Crawler(odoo.tests.HttpCase): _logger.info("%s %s", msg, url) r = self.url_open(url) - code = r.getcode() + code = r.status_code self.assertIn(code, range(200, 300), "%s Fetching %s returned error response (%d)" % (msg, url, code)) - if r.info().gettype() == 'text/html': - doc = lxml.html.fromstring(r.read()) + if r.headers['Content-Type'].startswith('text/html'): + doc = lxml.html.fromstring(r.content) for link in doc.xpath('//a[@href]'): href = link.get('href') - parts = urlparse.urlsplit(href) + parts = urls.url_parse(href) # href with any fragment removed - href = urlparse.urlunsplit(( - parts.scheme, - parts.netloc, - parts.path, - parts.query, - '' - )) + href = parts.replace(fragment='').to_url() # FIXME: handle relative link (not parts.path.startswith /) if parts.netloc or \ diff --git a/addons/website_forum/controllers/main.py b/addons/website_forum/controllers/main.py index 0c8535c9b30..6760ad911fe 100644 --- a/addons/website_forum/controllers/main.py +++ b/addons/website_forum/controllers/main.py @@ -4,12 +4,12 @@ import base64 import json import lxml +import requests import werkzeug.exceptions import werkzeug.urls import werkzeug.wrappers from datetime import datetime -from urllib2 import urlopen, URLError from odoo import http, modules, SUPERUSER_ID, tools, _ from odoo.addons.web.controllers.main import binary_content @@ -217,9 +217,11 @@ class WebsiteForum(http.Controller): @http.route('/forum/get_url_title', type='json', auth="user", methods=['POST'], website=True) def get_url_title(self, **kwargs): try: - arch = lxml.html.parse(urlopen(kwargs.get('url'))) + req = requests.get(kwargs.get('url'), stream=True) + req.raise_for_status() + arch = lxml.html.parse(req.raw) return arch.find(".//title").text - except URLError: + except IOError: return False @http.route(['''/forum//question/= 1 new_headers = [] @@ -1619,7 +1614,7 @@ def send_file(filepath_or_fp, mimetype=None, as_attachment=False, filename=None, def content_disposition(filename): filename = odoo.tools.ustr(filename) - escaped = quote(filename.encode('utf8')) + escaped = urls.url_quote(filename.encode('utf8')) browser = request.httprequest.user_agent.browser version = int((request.httprequest.user_agent.version or '0').split('.')[0]) if browser == 'msie' and version < 9: diff --git a/odoo/sql_db.py b/odoo/sql_db.py index a722c3ea41b..ee91704bb72 100644 --- a/odoo/sql_db.py +++ b/odoo/sql_db.py @@ -13,17 +13,13 @@ from functools import wraps import logging import time import uuid -try: - from urllib import parse as urlparse -except ImportError: - #pylint: disable=bad-python3-import - import urlparse import psycopg2 import psycopg2.extras import psycopg2.extensions from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_REPEATABLE_READ from psycopg2.pool import PoolError +from werkzeug import urls from .tools import pycompat @@ -656,7 +652,7 @@ def connection_info_for(db_or_uri): """ if db_or_uri.startswith(('postgresql://', 'postgres://')): # extract db from uri - us = urlparse.urlsplit(db_or_uri) + us = urls.url_parse(db_or_uri) if len(us.path) > 1: db_name = us.path[1:] elif us.username: diff --git a/odoo/tests/common.py b/odoo/tests/common.py index a491c23ef76..823a56c6190 100644 --- a/odoo/tests/common.py +++ b/odoo/tests/common.py @@ -19,16 +19,15 @@ import unittest from contextlib import contextmanager from datetime import datetime, timedelta from pprint import pformat + +import requests + try: - from urllib import request as urllib2 from xmlrpc import client as xmlrpclib except ImportError: # pylint: disable=bad-python3-import - import urllib2 import xmlrpclib -import werkzeug - import odoo from odoo import api @@ -215,26 +214,6 @@ class SavepointCase(SingleTransactionCase): self.registry.clear_caches() -class RedirectHandler(urllib2.HTTPRedirectHandler): - """ - HTTPRedirectHandler is predicated upon HTTPErrorProcessor being used and - works by intercepting 3xy "errors". - - Inherit from it to handle 3xy non-error responses instead, as we're not - using the error processor - """ - - def http_response(self, request, response): - code, msg, hdrs = response.code, response.msg, response.info() - - if 300 <= code < 400: - return self.parent.error( - 'http', request, response, code, msg, hdrs) - - return response - - https_response = http_response - class HttpCase(TransactionCase): """ Transactional HTTP TestCase with url_open and phantomjs helpers. """ @@ -259,18 +238,15 @@ class HttpCase(TransactionCase): self.session.db = get_db_name() odoo.http.root.session_store.save(self.session) # setup an url opener helper - self.opener = urllib2.OpenerDirector() - self.opener.add_handler(urllib2.UnknownHandler()) - self.opener.add_handler(urllib2.HTTPHandler()) - self.opener.add_handler(urllib2.HTTPSHandler()) - self.opener.add_handler(urllib2.HTTPCookieProcessor()) - self.opener.add_handler(RedirectHandler()) - self.opener.addheaders.append(('Cookie', 'session_id=%s' % self.session_id)) + self.opener = requests.Session() + self.opener.cookies['session_id'] = self.session_id def url_open(self, url, data=None, timeout=10): if url.startswith('/'): url = "http://%s:%s%s" % (HOST, PORT, url) - return self.opener.open(url, data, timeout) + if data: + return self.opener.post(url, data=data, timeout=timeout) + return self.opener.get(url, timeout=timeout) def authenticate(self, user, password): # stay non-authenticated diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py index 81c27105ff1..293ad148632 100644 --- a/odoo/tools/misc.py +++ b/odoo/tools/misc.py @@ -26,7 +26,6 @@ from itertools import islice, groupby, repeat from lxml import etree from .which import which -from threading import local import traceback import csv from operator import itemgetter @@ -677,29 +676,6 @@ def split_every(n, iterable, piece_maker=tuple): yield piece piece = piece_maker(islice(iterator, n)) -if __name__ == '__main__': - import doctest - doctest.testmod() - -class upload_data_thread(threading.Thread): - def __init__(self, email, data, type): - self.args = [('email',email),('type',type),('data',data)] - super(upload_data_thread,self).__init__() - def run(self): - try: - import urllib - args = urllib.urlencode(self.args) - fp = urllib.urlopen('http://www.openerp.com/scripts/survey.php', args) - fp.read() - fp.close() - except Exception: - pass - -def upload_data(email, data, type='SURVEY'): - a = upload_data_thread(email, data, type) - a.start() - return True - def get_and_group_by_field(cr, uid, obj, ids, field, context=None): """ Read the values of ``field´´ for the given ``ids´´ and group ids by value. From b5983a485d3bac6ffaa59e4d867b85b9e7bc09bc Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Thu, 11 May 2017 17:39:49 +0200 Subject: [PATCH 05/11] [FIX] P3: remove cPickle use In Python 3 the native acceleration module was merged into pickle directly. For cross-version simplicity, just use the base module in Python 2 as well, we can reintroduce the accelerator somehow if it turns out too much of a performance hit. --- addons/mail/models/html2text.py | 2 +- odoo/tools/misc.py | 44 +++++++++++++-------------------- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/addons/mail/models/html2text.py b/addons/mail/models/html2text.py index 8830cb0266d..fa2a8507af9 100755 --- a/addons/mail/models/html2text.py +++ b/addons/mail/models/html2text.py @@ -13,7 +13,7 @@ __contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"] # Support decoded entities with unifiable. import re, sys, htmlentitydefs, codecs -import sgmllib +import sgmllib # pylint: disable=deprecated-module sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]') try: from textwrap import wrap diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py index 293ad148632..3eaf476fc8a 100644 --- a/odoo/tools/misc.py +++ b/odoo/tools/misc.py @@ -5,7 +5,6 @@ """ Miscellaneous tools used by OpenERP. """ - from functools import wraps import babel from contextlib import contextmanager @@ -14,11 +13,13 @@ import subprocess import io import os import passlib.utils +import pickle as pickle_ import re import socket import sys import threading import time +import types import werkzeug.utils import zipfile from collections import defaultdict, Iterable, Mapping, MutableSet, OrderedDict @@ -36,12 +37,6 @@ try: except ImportError: import profile as cProfile -try: - # pylint: disable=bad-python3-import - import cPickle as pickle_ -except ImportError: - import pickle as pickle_ - try: from html2text import html2text except ImportError: @@ -1104,23 +1099,18 @@ def _consteq(str1, str2): consteq = getattr(passlib.utils, 'consteq', _consteq) -class Pickle(object): - @classmethod - def load(cls, stream, errors=False): - unpickler = pickle_.Unpickler(stream) - # pickle builtins: str/unicode, int/long, float, bool, tuple, list, dict, None - unpickler.find_global = None - try: - return unpickler.load() - except Exception: - _logger.warning('Failed unpickling data, returning default: %r', errors, exc_info=True) - return errors - - @classmethod - def loads(cls, text): - return cls.load(io.BytesIO(text)) - - dumps = pickle_.dumps - dump = pickle_.dump - -pickle = Pickle +def _pickle_load(stream, errors=False): + unpickler = pickle_.Unpickler(stream) + # pickle builtins: str/unicode, int/long, float, bool, tuple, list, dict, None + unpickler.find_global = None + try: + return unpickler.load() + except Exception: + _logger.warning('Failed unpickling data, returning default: %r', + errors, exc_info=True) + return errors +pickle = types.ModuleType(__name__ + '.pickle') +pickle.load = _pickle_load +pickle.loads = lambda text: _pickle_load(io.BytesIO(text)) +pickle.dump = pickle_.dump +pickle.dumps = pickle_.dumps From 67c17cb37f6ed0fde5e4bffa0d363e2502100f5b Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Thu, 11 May 2017 20:32:33 +0200 Subject: [PATCH 06/11] [REM] P3: outdated html2text vendoring The vendored html2text is *really* old and absolutely not P3-compatible. There is a maintained version on Pypy which is, so use that and stop vendoring html2text. --- addons/mail/models/__init__.py | 1 - addons/mail/models/html2text.py | 429 -------------------------- odoo/addons/base/ir/ir_mail_server.py | 8 +- odoo/tools/misc.py | 4 - requirements.txt | 1 + setup.py | 1 + 6 files changed, 6 insertions(+), 438 deletions(-) delete mode 100755 addons/mail/models/html2text.py diff --git a/addons/mail/models/__init__.py b/addons/mail/models/__init__.py index 89442c66689..f0fbe714fbf 100644 --- a/addons/mail/models/__init__.py +++ b/addons/mail/models/__init__.py @@ -19,4 +19,3 @@ from . import update from . import ir_actions from . import ir_autovacuum from . import ir_model -from . import html2text diff --git a/addons/mail/models/html2text.py b/addons/mail/models/html2text.py deleted file mode 100755 index fa2a8507af9..00000000000 --- a/addons/mail/models/html2text.py +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/env python -"""html2text: Turn HTML into equivalent Markdown-structured text.""" -from werkzeug import urls - -from odoo.tools import pycompat - -__version__ = "2.36" -__author__ = "Aaron Swartz (me@aaronsw.com)" -__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3." -__contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"] - -# TODO: -# Support decoded entities with unifiable. - -import re, sys, htmlentitydefs, codecs -import sgmllib # pylint: disable=deprecated-module -sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]') - -try: from textwrap import wrap -except: pass - -# Use Unicode characters instead of their ascii psuedo-replacements -UNICODE_SNOB = 0 - -# Put the links after each paragraph instead of at the end. -LINKS_EACH_PARAGRAPH = 0 - -# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.) -BODY_WIDTH = 78 - -# Don't show internal links (href="#local-anchor") -- corresponding link targets -# won't be visible in the plain text file anyway. -SKIP_INTERNAL_LINKS = False - -### Entity Nonsense ### - -def name2cp(k): - if k == 'apos': return ord("'") - if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3 - return htmlentitydefs.name2codepoint[k] - else: - k = htmlentitydefs.entitydefs[k] - if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1 - return ord(codecs.latin_1_decode(k)[0]) - -unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"', -'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*', -'ndash':'-', 'oelig':'oe', 'aelig':'ae', -'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a', -'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e', -'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i', -'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o', -'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'} - -unifiable_n = {} - -for k in unifiable: - unifiable_n[name2cp(k)] = unifiable[k] - -def charref(name): - if name[0] in ['x','X']: - c = int(name[1:], 16) - else: - c = int(name) - - if not UNICODE_SNOB and c in unifiable_n: - return unifiable_n[c] - else: - return unichr(c) - -def entityref(c): - if not UNICODE_SNOB and c in unifiable: - return unifiable[c] - else: - try: name2cp(c) - except KeyError: return "&" + c - else: return unichr(name2cp(c)) - -def replaceEntities(s): - s = s.group(1) - if s[0] == "#": - return charref(s[1:]) - else: return entityref(s) - -r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));") -def unescape(s): - return r_unescape.sub(replaceEntities, s) - -def fixattrs(attrs): - # Fix bug in sgmllib.py - if not attrs: return attrs - newattrs = [] - for attr in attrs: - newattrs.append((attr[0], unescape(attr[1]))) - return newattrs - -### End Entity Nonsense ### - -def onlywhite(line): - """Return true if the line does only consist of whitespace characters.""" - for c in line: - if c is not ' ' and c is not ' ': - return c is ' ' - return line - -def optwrap(text): - """Wrap all paragraphs in the provided text.""" - if not BODY_WIDTH: - return text - - assert wrap, "Requires Python 2.3." - result = '' - newlines = 0 - for para in text.split("\n"): - if len(para) > 0: - if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*': - for line in wrap(para, BODY_WIDTH): - result += line + "\n" - result += "\n" - newlines = 2 - else: - if not onlywhite(para): - result += para + "\n" - newlines = 1 - else: - if newlines < 2: - result += "\n" - newlines += 1 - return result - -def hn(tag): - if tag[0] == 'h' and len(tag) == 2: - try: - n = int(tag[1]) - if n in range(1, 10): return n - except ValueError: return 0 - -class _html2text(sgmllib.SGMLParser): - def __init__(self, out=sys.stdout.write, baseurl=''): - sgmllib.SGMLParser.__init__(self) - - if out is None: self.out = self.outtextf - else: self.out = out - self.outtext = u'' - self.quiet = 0 - self.p_p = 0 - self.outcount = 0 - self.start = 1 - self.space = 0 - self.a = [] - self.astack = [] - self.acount = 0 - self.list = [] - self.blockquote = 0 - self.pre = 0 - self.startpre = 0 - self.lastWasNL = 0 - self.abbr_title = None # current abbreviation definition - self.abbr_data = None # last inner HTML (for abbr being defined) - self.abbr_list = {} # stack of abbreviations to write later - self.baseurl = baseurl - - def outtextf(self, s): - self.outtext += s - - def close(self): - sgmllib.SGMLParser.close(self) - - self.pbr() - self.o('', 0, 'end') - - return self.outtext - - def handle_charref(self, c): - self.o(charref(c)) - - def handle_entityref(self, c): - self.o(entityref(c)) - - def unknown_starttag(self, tag, attrs): - self.handle_tag(tag, attrs, 1) - - def unknown_endtag(self, tag): - self.handle_tag(tag, None, 0) - - def previousIndex(self, attrs): - """ returns the index of certain set of attributes (of a link) in the - self.a list - - If the set of attributes is not found, returns None - """ - if 'href' not in attrs: return None - - i = -1 - for a in self.a: - i += 1 - match = 0 - - if 'href' in a and a['href'] == attrs['href']: - if 'title' in a or 'title' in attrs: - if ('title' in a and 'title' in attrs and - a['title'] == attrs['title']): - match = True - else: - match = True - - if match: return i - - def handle_tag(self, tag, attrs, start): - attrs = fixattrs(attrs) - - if hn(tag): - self.p() - if start: self.o(hn(tag)*"#" + ' ') - - if tag in ['p', 'div']: self.p() - - if tag == "br" and start: self.o(" \n") - - if tag == "hr" and start: - self.p() - self.o("* * *") - self.p() - - if tag in ["head", "style", 'script']: - if start: self.quiet += 1 - else: self.quiet -= 1 - - if tag in ["body"]: - self.quiet = 0 # sites like 9rules.com never close - - if tag == "blockquote": - if start: - self.p(); self.o('> ', 0, 1); self.start = 1 - self.blockquote += 1 - else: - self.blockquote -= 1 - self.p() - - if tag in ['em', 'i', 'u']: self.o("_") - if tag in ['strong', 'b']: self.o("**") - if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` `` - if tag == "abbr": - if start: - attrsD = {} - for (x, y) in attrs: attrsD[x] = y - attrs = attrsD - - self.abbr_title = None - self.abbr_data = '' - if 'title' in attrs: - self.abbr_title = attrs['title'] - else: - if self.abbr_title != None: - self.abbr_list[self.abbr_data] = self.abbr_title - self.abbr_title = None - self.abbr_data = '' - - if tag == "a": - if start: - attrsD = {} - for (x, y) in attrs: attrsD[x] = y - attrs = attrsD - if 'href' in attrs and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')): - self.astack.append(attrs) - self.o("[") - else: - self.astack.append(None) - else: - if self.astack: - a = self.astack.pop() - if a: - i = self.previousIndex(a) - if i is not None: - a = self.a[i] - else: - self.acount += 1 - a['count'] = self.acount - a['outcount'] = self.outcount - self.a.append(a) - self.o("][%r]" % a['count']) - - if tag == "img" and start: - attrsD = {} - for (x, y) in attrs: attrsD[x] = y - attrs = attrsD - if 'src' in attrs: - attrs['href'] = attrs['src'] - alt = attrs.get('alt', '') - i = self.previousIndex(attrs) - if i is not None: - attrs = self.a[i] - else: - self.acount += 1 - attrs['count'] = self.acount - attrs['outcount'] = self.outcount - self.a.append(attrs) - self.o("![") - self.o(alt) - self.o("][%r]" % attrs['count']) - - if tag == 'dl' and start: self.p() - if tag == 'dt' and not start: self.pbr() - if tag == 'dd' and start: self.o(' ') - if tag == 'dd' and not start: self.pbr() - - if tag in ["ol", "ul"]: - if start: - self.list.append({'name':tag, 'num':0}) - else: - if self.list: self.list.pop() - - self.p() - - if tag == 'li': - if start: - self.pbr() - if self.list: li = self.list[-1] - else: li = {'name':'ul', 'num':0} - self.o(" "*len(self.list)) #TODO: line up
  1. s > 9 correctly. - if li['name'] == "ul": self.o("* ") - elif li['name'] == "ol": - li['num'] += 1 - self.o("%r. " % li['num']) - self.start = 1 - else: - self.pbr() - - if tag in ["table", "tr"] and start: self.p() - if tag == 'td': self.pbr() - - if tag == "pre": - if start: - self.startpre = 1 - self.pre = 1 - else: - self.pre = 0 - self.p() - - def pbr(self): - if self.p_p == 0: self.p_p = 1 - - def p(self): - self.p_p = 2 - - def o(self, data, puredata=0, force=0): - if self.abbr_data is not None: self.abbr_data += data - - if not self.quiet: - if puredata and not self.pre: - data = re.sub('\s+', ' ', data) - if data and data[0] == ' ': - self.space = 1 - data = data[1:] - if not data and not force: return - - if self.startpre: - #self.out(" :") #TODO: not output when already one there - self.startpre = 0 - - bq = (">" * self.blockquote) - if not (force and data and data[0] == ">") and self.blockquote: bq += " " - - if self.pre: - bq += " " - data = data.replace("\n", "\n"+bq) - - if self.start: - self.space = 0 - self.p_p = 0 - self.start = 0 - - if force == 'end': - # It's the end. - self.p_p = 0 - self.out("\n") - self.space = 0 - - - if self.p_p: - self.out(('\n'+bq)*self.p_p) - self.space = 0 - - if self.space: - if not self.lastWasNL: self.out(' ') - self.space = 0 - - if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"): - if force == "end": self.out("\n") - - newa = [] - for link in self.a: - if self.outcount > link['outcount']: - self.out(" ["+repr(link['count'])+"]: " + urls.url_join(self.baseurl, link['href'])) - if 'title' in link: self.out(" ("+link['title']+")") - self.out("\n") - else: - newa.append(link) - - if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done. - - self.a = newa - - if self.abbr_list and force == "end": - for abbr, definition in pycompat.items(self.abbr_list): - self.out(" *[" + abbr + "]: " + definition + "\n") - - self.p_p = 0 - self.out(data) - self.lastWasNL = data and data[-1] == '\n' - self.outcount += 1 - - def handle_data(self, data): - if r'\/script>' in data: self.quiet -= 1 - self.o(data, 1) - - def unknown_decl(self, data): - pass - -def wrapwrite(text): sys.stdout.write(text.encode('utf8')) - -def html2text_file(html, out=wrapwrite, baseurl=''): - h = _html2text(out, baseurl) - h.feed(html) - h.feed("") - return h.close() - -def html2text(html, baseurl=''): - return optwrap(html2text_file(html, None, baseurl)) diff --git a/odoo/addons/base/ir/ir_mail_server.py b/odoo/addons/base/ir/ir_mail_server.py index 35588019ebc..825c755bc79 100644 --- a/odoo/addons/base/ir/ir_mail_server.py +++ b/odoo/addons/base/ir/ir_mail_server.py @@ -13,11 +13,11 @@ import re import smtplib import threading -import itertools +import html2text from odoo import api, fields, models, tools, _ from odoo.exceptions import except_orm, UserError -from odoo.tools import html2text, ustr, pycompat +from odoo.tools import ustr, pycompat _logger = logging.getLogger(__name__) _test_logger = logging.getLogger('odoo.tests') @@ -330,9 +330,9 @@ class IrMailServer(models.Model): for key, value in pycompat.items(headers): msg[ustr(key).encode('utf-8')] = encode_header(value) - if subtype == 'html' and not body_alternative and html2text: + if subtype == 'html' and not body_alternative: # Always provide alternative text body ourselves if possible. - text_utf8 = tools.html2text(email_body_utf8.decode('utf-8')).encode('utf-8') + text_utf8 = html2text.html2text(email_body_utf8.decode('utf-8')).encode('utf-8') alternative_part = MIMEMultipart(_subtype="alternative") alternative_part.attach(MIMEText(text_utf8, _charset='utf-8', _subtype='plain')) alternative_part.attach(email_text_part) diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py index 3eaf476fc8a..a945ce9631a 100644 --- a/odoo/tools/misc.py +++ b/odoo/tools/misc.py @@ -37,10 +37,6 @@ try: except ImportError: import profile as cProfile -try: - from html2text import html2text -except ImportError: - html2text = None from .config import config from .cache import * diff --git a/requirements.txt b/requirements.txt index f0696276978..94893f524e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ feedparser==5.2.1 gevent==1.1.2 greenlet==0.4.10 jcconv==0.2.3 +html2text==2016.9.19 Jinja2==2.8 lxml==3.5.0 Mako==1.0.4 diff --git a/setup.py b/setup.py index 5d6c75b8842..435b955827b 100644 --- a/setup.py +++ b/setup.py @@ -140,6 +140,7 @@ setup( 'docutils', 'feedparser', 'gevent', + 'html2text', 'Jinja2', 'lxml', # windows binary http://www.lfd.uci.edu/~gohlke/pythonlibs/ 'mako', From 5617d4614b4644f0eaf2feb402f0066f260200c8 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 12 May 2017 16:40:25 +0200 Subject: [PATCH 07/11] [FIX] P3: itertools.iziplongest renamed --- addons/website/tests/test_views.py | 9 ++++++--- odoo/addons/base/tests/test_views.py | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/addons/website/tests/test_views.py b/addons/website/tests/test_views.py index 415e1668ce1..6eb5040f539 100644 --- a/addons/website/tests/test_views.py +++ b/addons/website/tests/test_views.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import itertools +try: + from itertools import zip_longest +except ImportError: + from itertools import izip_longest as zip_longest import unittest from lxml import etree as ET, html @@ -22,7 +25,7 @@ class TestViewSaving(common.TransactionCase): self.assertEqual(a.attrib, b.attrib) self.assertEqual((a.text or '').strip(), (b.text or '').strip()) self.assertEqual((a.tail or '').strip(), (b.tail or '').strip()) - for ca, cb in itertools.izip_longest(a, b): + for ca, cb in zip_longest(a, b): self.eq(ca, cb) def setUp(self): @@ -55,7 +58,7 @@ class TestViewSaving(common.TransactionCase): h.SPAN("My Company", attrs(model='res.company', id=1, field='name', type='char')), h.SPAN("+00 00 000 00 0 000", attrs(model='res.company', id=1, field='phone', type='char')), ] - for actual, expected in itertools.izip_longest(fields, expect): + for actual, expected in zip_longest(fields, expect): self.eq(actual, expected) def test_embedded_save(self): diff --git a/odoo/addons/base/tests/test_views.py b/odoo/addons/base/tests/test_views.py index 40cd922340c..b9b8cd0fe39 100644 --- a/odoo/addons/base/tests/test_views.py +++ b/odoo/addons/base/tests/test_views.py @@ -2,7 +2,10 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. from functools import partial -from itertools import izip_longest +try: + from itertools import zip_longest as izip_longuest +except ImportError: + from itertools import izip_longest from lxml import etree from lxml.builder import E From e2f1af78c465916551f12419f37d4e63a6cfd7f2 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 12 May 2017 17:14:52 +0200 Subject: [PATCH 08/11] [FIX] P3: remove cgi.escape uses cgi.escape is unsafe (quote=False by default) and deprecated in Python 3. We already have an openerp.tools.misc.html_escape version which forwards to the (modern and safe) werkzeug.utils.escape, just use that everywhere, and convert extant uses of werkzeug.utils.escape to utils.html_escape as well so that we do the same thing everywhere. --- addons/base_import/controllers/main.py | 4 ++-- .../google_spreadsheet/models/google_drive.py | 4 ++-- addons/hw_posbox_homepage/controllers/main.py | 3 ++- .../models/pos_mercury_transaction.py | 7 ++----- odoo/addons/base/ir/ir_qweb/assetsbundle.py | 3 +-- odoo/addons/base/tests/test_mail.py | 21 +++++++++---------- odoo/addons/base/tests/test_qweb.py | 8 +++---- odoo/tools/mail.py | 13 ++++++------ 8 files changed, 28 insertions(+), 35 deletions(-) diff --git a/addons/base_import/controllers/main.py b/addons/base_import/controllers/main.py index 23febc086e0..7eff0aa420d 100644 --- a/addons/base_import/controllers/main.py +++ b/addons/base_import/controllers/main.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import cgi import json from odoo import http from odoo.http import request +from odoo.tools import misc class ImportController(http.Controller): @@ -20,4 +20,4 @@ class ImportController(http.Controller): 'file_type': file.content_type, }) - return 'window.top.%s(%s)' % (cgi.escape(jsonp), json.dumps({'result': written})) + return 'window.top.%s(%s)' % (misc.html_escape(jsonp), json.dumps({'result': written})) diff --git a/addons/google_spreadsheet/models/google_drive.py b/addons/google_spreadsheet/models/google_drive.py index dce4777a25e..08c0774cc83 100644 --- a/addons/google_spreadsheet/models/google_drive.py +++ b/addons/google_spreadsheet/models/google_drive.py @@ -1,6 +1,5 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. -import cgi import json import logging @@ -10,6 +9,7 @@ import re import werkzeug.urls from odoo import api, models +from odoo.tools import misc from odoo.addons.google_account import TIMEOUT _logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ class GoogleDrive(models.Model): href="https://spreadsheets.google.com/feeds/cells/{key}/od6/private/full/R60C15"/> -''' .format(key=spreadsheet_key, formula=cgi.escape(formula, quote=True), config=cgi.escape(config_formula, quote=True)) +''' .format(key=spreadsheet_key, formula=misc.html_escape(formula), config=misc.html_escape(config_formula)) try: req = requests.post( diff --git a/addons/hw_posbox_homepage/controllers/main.py b/addons/hw_posbox_homepage/controllers/main.py index f43e585fcba..bdcafdae706 100644 --- a/addons/hw_posbox_homepage/controllers/main.py +++ b/addons/hw_posbox_homepage/controllers/main.py @@ -8,6 +8,7 @@ import werkzeug import odoo from odoo import http +from odoo.tools import misc _logger = logging.getLogger(__name__) @@ -106,7 +107,7 @@ class PosboxHomepage(odoo.addons.web.controllers.main.Home): f = open('/tmp/scanned_networks.txt', 'r') for line in f: line = line.rstrip() - line = werkzeug.utils.escape(line) + line = misc.html_escape(line) wifi_template += '\n' f.close() except IOError: diff --git a/addons/pos_mercury/models/pos_mercury_transaction.py b/addons/pos_mercury/models/pos_mercury_transaction.py index 5efbdaa0ff5..bd190e82f0d 100644 --- a/addons/pos_mercury/models/pos_mercury_transaction.py +++ b/addons/pos_mercury/models/pos_mercury_transaction.py @@ -3,15 +3,12 @@ from datetime import date, timedelta -import cgi -import ssl - import requests import werkzeug from odoo import models, api, service from odoo.exceptions import UserError -from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT +from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT, misc class MercuryTransaction(models.Model): @@ -53,7 +50,7 @@ class MercuryTransaction(models.Model): soap_header = '' soap_footer = '' + data['merchant_pwd'] + '' - xml_transaction = soap_header + cgi.escape(xml_transaction) + soap_footer + xml_transaction = soap_header + misc.escape_html(xml_transaction) + soap_footer response = '' diff --git a/odoo/addons/base/ir/ir_qweb/assetsbundle.py b/odoo/addons/base/ir/ir_qweb/assetsbundle.py index 6a8ad84c9dd..415be062a10 100644 --- a/odoo/addons/base/ir/ir_qweb/assetsbundle.py +++ b/odoo/addons/base/ir/ir_qweb/assetsbundle.py @@ -12,7 +12,6 @@ from odoo import fields, tools from odoo.http import request from odoo.modules.module import get_resource_path import psycopg2 -import werkzeug from odoo.tools import func, misc, pycompat import logging @@ -587,7 +586,7 @@ class StylesheetAsset(WebAsset): return self.with_header(content) def to_html(self): - media = (' media="%s"' % werkzeug.utils.escape(self.media)) if self.media else '' + media = (' media="%s"' % misc.html_escape(self.media)) if self.media else '' if self.url: href = self.html_url return '' % (href, media) diff --git a/odoo/addons/base/tests/test_mail.py b/odoo/addons/base/tests/test_mail.py index 795d6824886..1a30ef4690b 100644 --- a/odoo/addons/base/tests/test_mail.py +++ b/odoo/addons/base/tests/test_mail.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import cgi import unittest -from odoo.tools import html_sanitize, append_content_to_html, plaintext2html, email_split +from odoo.tools import html_sanitize, append_content_to_html, plaintext2html, email_split, misc from . import test_mail_examples @@ -99,7 +98,7 @@ class TestSanitizer(unittest.TestCase): "Div nico " ] for email in emails: - self.assertIn(cgi.escape(email), html_sanitize(email), 'html_sanitize stripped emails of original html') + self.assertIn(misc.html_escape(email), html_sanitize(email), 'html_sanitize stripped emails of original html') def test_sanitize_unescape_emails(self): not_emails = [ @@ -108,7 +107,7 @@ class TestSanitizer(unittest.TestCase): for email in not_emails: sanitized = html_sanitize(email) left_part = email.split('>')[0] # take only left part, as the sanitizer could add data information on node - self.assertNotIn(cgi.escape(email), sanitized, 'html_sanitize stripped emails of original html') + self.assertNotIn(misc.html_escape(email), sanitized, 'html_sanitize stripped emails of original html') self.assertIn(left_part, sanitized) def test_style_parsing(self): @@ -172,14 +171,14 @@ class TestSanitizer(unittest.TestCase): for ext in test_mail_examples.QUOTE_BLOCKQUOTE_IN: self.assertIn(ext, html) for ext in test_mail_examples.QUOTE_BLOCKQUOTE_OUT: - self.assertIn('%s' % cgi.escape(ext.decode('utf-8')), html) + self.assertIn('%s' % misc.html_escape(ext.decode('utf-8')), html) def test_quote_thunderbird(self): html = html_sanitize(test_mail_examples.QUOTE_THUNDERBIRD_1) for ext in test_mail_examples.QUOTE_THUNDERBIRD_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.QUOTE_THUNDERBIRD_1_OUT: - self.assertIn('%s' % cgi.escape(ext.decode('utf-8')), html) + self.assertIn('%s' % misc.html_escape(ext.decode('utf-8')), html) def test_quote_hotmail_html(self): html = html_sanitize(test_mail_examples.QUOTE_HOTMAIL_HTML) @@ -226,7 +225,7 @@ class TestSanitizer(unittest.TestCase): for text in in_lst: self.assertIn(text, new_html) for text in out_lst: - self.assertIn('%s' % cgi.escape(text), new_html) + self.assertIn('%s' % misc.html_escape(text), new_html) def test_quote_signature(self): test_data = [ @@ -245,27 +244,27 @@ class TestSanitizer(unittest.TestCase): for ext in test_mail_examples.GMAIL_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.GMAIL_1_OUT: - self.assertIn('%s' % cgi.escape(ext), html) + self.assertIn('%s' % misc.html_escape(ext), html) def test_quote_text(self): html = html_sanitize(test_mail_examples.TEXT_1) for ext in test_mail_examples.TEXT_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.TEXT_1_OUT: - self.assertIn('%s' % cgi.escape(ext), html) + self.assertIn('%s' % misc.html_escape(ext), html) html = html_sanitize(test_mail_examples.TEXT_2) for ext in test_mail_examples.TEXT_2_IN: self.assertIn(ext, html) for ext in test_mail_examples.TEXT_2_OUT: - self.assertIn('%s' % cgi.escape(ext), html) + self.assertIn('%s' % misc.html_escape(ext), html) def test_quote_bugs(self): html = html_sanitize(test_mail_examples.BUG1) for ext in test_mail_examples.BUG_1_IN: self.assertIn(ext, html) for ext in test_mail_examples.BUG_1_OUT: - self.assertIn('%s' % cgi.escape(ext.decode('utf-8')), html) + self.assertIn('%s' % misc.html_escape(ext.decode('utf-8')), html) def test_misc(self): # False / void should not crash diff --git a/odoo/addons/base/tests/test_qweb.py b/odoo/addons/base/tests/test_qweb.py index cd459fe7669..c331f426748 100644 --- a/odoo/addons/base/tests/test_qweb.py +++ b/odoo/addons/base/tests/test_qweb.py @@ -1,19 +1,17 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import cgi import collections import json import os.path import re from lxml import etree -from itertools import chain from odoo.modules import get_module_resource from odoo.tests.common import TransactionCase from odoo.addons.base.ir.ir_qweb import QWebException -from odoo.tools import pycompat +from odoo.tools import pycompat, misc, ustr def dedent_and_strip(string): @@ -48,12 +46,12 @@ class TestQWebTField(TransactionCase): result = self.engine.render(field, {'company': company}) self.assertEqual( - result, + ustr(result), '%s' % ( company.id, - cgi.escape(s.encode('utf-8')), + misc.html_escape(s), ), ) diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py index 9689512df00..f3396f9137b 100644 --- a/odoo/tools/mail.py +++ b/odoo/tools/mail.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import cgi import logging import lxml.html.clean as clean import random @@ -16,7 +15,7 @@ from lxml import etree import odoo from odoo.loglevels import ustr -from odoo.tools import pycompat +from odoo.tools import pycompat, misc _logger = logging.getLogger(__name__) @@ -178,10 +177,10 @@ def html_sanitize(src, silent=True, sanitize_tags=True, sanitize_attributes=Fals part = re.compile(r"(<(([^a<>]|a[^<>\s])[^<>]*)@[^<>]+>)", re.IGNORECASE | re.DOTALL) # remove results containing cite="mid:email_like@address" (ex: blockquote cite) # cite_except = re.compile(r"^((?!cite[\s]*=['\"]).)*$", re.IGNORECASE) - src = part.sub(lambda m: ('cite=' not in m.group(1) and 'alt=' not in m.group(1)) and cgi.escape(m.group(1)) or m.group(1), src) + src = part.sub(lambda m: ('cite=' not in m.group(1) and 'alt=' not in m.group(1)) and misc.html_escape(m.group(1)) or m.group(1), src) # html encode mako tags <% ... %> to decode them later and keep them alive, otherwise they are stripped by the cleaner - src = src.replace('<%', cgi.escape('<%')) - src = src.replace('%>', cgi.escape('%>')) + src = src.replace('<%', misc.html_escape('<%')) + src = src.replace('%>', misc.html_escape('%>')) kwargs = { 'page_structure': True, @@ -334,7 +333,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'): def plaintext2html(text, container_tag=False): """ Convert plaintext into html. Content of the text is escaped to manage - html entities, using cgi.escape(). + html entities, using misc.html_escape(). - all \n,\r are replaced by
    - enclose content into

    - convert url into clickable link @@ -343,7 +342,7 @@ def plaintext2html(text, container_tag=False): :param string container_tag: container of the html; by default the content is embedded into a

    """ - text = cgi.escape(ustr(text)) + text = misc.html_escape(ustr(text)) # 1. replace \n and \r text = text.replace('\n', '
    ') From 7c6c43d910e303e4c76198d59fc9ccfb25b645b5 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 12 May 2017 17:25:52 +0200 Subject: [PATCH 09/11] [FIX] P3: Queue module renamed to queue --- addons/hw_escpos/controllers/main.py | 5 ++++- addons/hw_scanner/controllers/main.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/addons/hw_escpos/controllers/main.py b/addons/hw_escpos/controllers/main.py index 35efc5216ea..7e4302d7dc0 100644 --- a/addons/hw_escpos/controllers/main.py +++ b/addons/hw_escpos/controllers/main.py @@ -17,7 +17,10 @@ try: except ImportError: escpos = printer = None -from Queue import Queue +try: + from queue import Queue +except ImportError: + from Queue import Queue # pylint: disable=deprecated-module from threading import Thread, Lock try: diff --git a/addons/hw_scanner/controllers/main.py b/addons/hw_scanner/controllers/main.py index a5a3136cec8..89f1318362e 100644 --- a/addons/hw_scanner/controllers/main.py +++ b/addons/hw_scanner/controllers/main.py @@ -5,7 +5,10 @@ import logging import time from os import listdir from os.path import join -from Queue import Queue, Empty +try: + from queue import Queue, Empty +except ImportError: + from Queue import Queue, Empty # pylint: disable=deprecated-module from select import select from threading import Thread, Lock From 822bea14d0eb0a891cdeda9600c4dc9f7649a680 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 12 May 2017 17:32:06 +0200 Subject: [PATCH 10/11] [FIX] P3: htmlentitydefs renamed to html.entities --- addons/crm/wizard/base_partner_merge.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/addons/crm/wizard/base_partner_merge.py b/addons/crm/wizard/base_partner_merge.py index ccfa6d3e692..7751e0106ed 100644 --- a/addons/crm/wizard/base_partner_merge.py +++ b/addons/crm/wizard/base_partner_merge.py @@ -4,7 +4,10 @@ from ast import literal_eval from email.utils import parseaddr import functools -import htmlentitydefs +try: + from html.entities import entitydefs +except ImportError: + from htmlentitydefs import entitydefs # pylint: disable=deprecated-module import itertools import logging import operator @@ -23,7 +26,7 @@ _logger = logging.getLogger('base.partner.merge') # http://www.php2python.com/wiki/function.html-entity-decode/ -def html_entity_decode_char(m, defs=htmlentitydefs.entitydefs): +def html_entity_decode_char(m, defs=entitydefs): try: return defs[m.group(1)] except KeyError: From 4550f45f471d7fe567de95302a2d6adc5ac06974 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 15 May 2017 11:31:04 +0200 Subject: [PATCH 11/11] [FIX] P3: document stdlib changes Also lint away verboten stdlib modules. --- doc/python3.rst | 23 +++++++++++++++++--- odoo/addons/test_pylint/tests/test_pylint.py | 14 ++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/doc/python3.rst b/doc/python3.rst index ba368572c13..ea14b6e4515 100644 --- a/doc/python3.rst +++ b/doc/python3.rst @@ -74,9 +74,22 @@ library: ``io.StringIO`` to replace them in a cross-version manner (``io.BytesIO`` for binary data, ``io.StringIO`` for text/unicode data). * ``urllib``, ``urllib2`` and ``urlparse`` were redistributed across - ``urllib.parse`` and ``urllib.request``, you may want to use conditional - imports e.g. try to import the Python 3 version and fallback on the Python - 2 version. + ``urllib.parse`` and ``urllib.request``. + + Since `requests`_ and `werkzeug`_ are already hard dependencies of Odoo, + replace ``urllib[2].urlopen``/``urllib2.Request`` uses by `requests`_, and + ``urlparse`` and a few utilty functions (``urllib.quote``, + ``urllib.urlencode``) are available through ``werkzeug.urls``, a backport + of Python 3's ``urllib.parse``. + + .. warning:: `requests`_ does not raise by default on non-200 responses + +* ``cgi.escape`` (HTML escaping) is deprecated in Python 3, prefer Odoo's own + :func:`odoo.tools.misc.html_encode`. +* Most of ``types``'s content has been stripped out in Python 3: only + "internal" interpreter types (e.g. CodeType, FrameType, ...) have been left + in, other types can be obtained directly from the corresponding builtin or + by getting the ``type()`` of a literal value. Absolute Imports (:pep:`328`) ----------------------------- @@ -326,3 +339,7 @@ Minor syntax changes In Python 3, leading zeroes followed by neither a 0 nor a period is an error, octal literals now follow the hexadecimal convention with a ``0o`` prefix. + +.. _requests: http://docs.python-requests.org/ + +.. _werkzeug: http://werkzeug.pocoo.org/docs/urls/ diff --git a/odoo/addons/test_pylint/tests/test_pylint.py b/odoo/addons/test_pylint/tests/test_pylint.py index f518bb8d43e..26a8e534686 100644 --- a/odoo/addons/test_pylint/tests/test_pylint.py +++ b/odoo/addons/test_pylint/tests/test_pylint.py @@ -73,6 +73,20 @@ class TestPyLint(TransactionCase): BAD_MODULES = [ 'commands', + 'cPickle', + 'md5', + 'urllib', + 'urllib2', + 'urlparse', + 'sgmllib', + 'sha', + 'cgi', + 'htmlentitydefs', + 'HTMLParser', + 'Queue', + 'UserDict', + 'UserString', + 'UserList', ] def _skip_test(self, reason):