[FIX] P3: urllib, urllib2 and urlparse

In Python 3, all of these were "consolidated" under urllib(.request,
.parse, .errors) which is inconvenient.

Since we already have hard dependencies on requests and
werkzeug(.urls, which is a backport of Python 3's unicode-aware
urllib.parse) migrate *everything* to that.

A sticking point is urllib2.URLError, those were (mostly) replaced by
the slightly more general IOError which URLError extends.

addons/anonymization/wizard/anonymize_wizard.py

@@ -4,10 +4,7 @@
 import base64
 import os
 import random
-try:
+import pickle
-    import cPickle as pickle
-except ImportError:
-    import pickle
 from lxml import etree
 from operator import itemgetter
 

addons/auth_oauth/models/res_users.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import werkzeug.urls
-import urlparse
-import urllib2
 import json
 
+import requests
+
 from odoo import api, fields, models
 from odoo.exceptions import AccessDenied
 from odoo.addons.auth_signup.models.res_users import SignupError
@@ -24,14 +23,7 @@ class ResUsers(models.Model):
 
     @api.model
     def _auth_oauth_rpc(self, endpoint, access_token):
-        params = werkzeug.url_encode({'access_token': access_token})
+        return requests.get(endpoint, params={'access_token': access_token}).json()
-        if urlparse.urlparse(endpoint)[4]:
-            url = endpoint + '&' + params
-        else:
-            url = endpoint + '?' + params
-        f = urllib2.urlopen(url)
-        response = f.read()
-        return json.loads(response)
 
     @api.model
     def _auth_oauth_validate(self, provider, access_token):

addons/auth_signup/models/res_partner.py

@@ -2,10 +2,9 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import random
-import werkzeug
+import werkzeug.urls
 
 from datetime import datetime, timedelta
-from urlparse import urljoin
 
 from odoo import api, fields, models, _
 from odoo.tools import pycompat
@@ -89,9 +88,9 @@ class ResPartner(models.Model):
                 fragment['res_id'] = res_id
 
             if fragment:
-                query['redirect'] = base + werkzeug.url_encode(fragment)
+                query['redirect'] = base + werkzeug.urls.url_encode(fragment)
 
-            res[partner.id] = urljoin(base_url, "/web/%s?%s" % (route, werkzeug.url_encode(query)))
+            res[partner.id] = werkzeug.urls.url_join(base_url, "/web/%s?%s" % (route, werkzeug.urls.url_encode(query)))
         return res
 
     @api.multi

addons/base_geolocalize/models/res_partner.py

@@ -1,18 +1,18 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import json
-import urllib
+
+import requests
 
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import UserError
 
 
 def geo_find(addr):
-    url = 'https://maps.googleapis.com/maps/api/geocode/json?sensor=false&address='
+    url = 'https://maps.googleapis.com/maps/api/geocode/json'
-    url += urllib.quote(addr.encode('utf8'))
 
     try:
-        result = json.load(urllib.urlopen(url))
+        result = requests.get(url, params={'sensor': 'false', 'address': addr}).json()
     except Exception as e:
         raise UserError(_('Cannot contact geolocation servers. Please make sure that your Internet connection is up and running (%s).') % e)
 

addons/google_account/models/google_service.py

@@ -4,8 +4,9 @@
 from datetime import datetime
 import json
 import logging
-import urllib2
+
-import werkzeug.urls
+import requests
+from werkzeug import urls
 
 from odoo import api, fields, models, registry, _
 from odoo.http import request
@@ -38,27 +39,27 @@ class GoogleService(models.TransientModel):
 
         # Get the Refresh Token From Google And store it in ir.config_parameter
         headers = {"Content-type": "application/x-www-form-urlencoded"}
-        data = werkzeug.url_encode({
+        data = {
             'code': authorization_code,
             'client_id': client_id,
             'client_secret': client_secret,
             'redirect_uri': redirect_uri,
             'grant_type': "authorization_code"
-        })
+        }
         try:
-            req = urllib2.Request(GOOGLE_TOKEN_ENDPOINT, data, headers)
+            req = requests.post(GOOGLE_TOKEN_ENDPOINT, data=data, headers=headers, timeout=TIMEOUT)
-            content = urllib2.urlopen(req, timeout=TIMEOUT).read()
+            req.raise_for_status()
-        except urllib2.HTTPError:
+            content = req.json()
+        except IOError:
             error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid or already expired")
             raise self.env['res.config.settings'].get_config_warning(error_msg)
 
-        content = json.loads(content)
         return content.get('refresh_token')
 
     @api.model
     def _get_google_token_uri(self, service, scope):
         get_param = self.env['ir.config_parameter'].sudo().get_param
-        encoded_params = werkzeug.url_encode({
+        encoded_params = urls.url_encode({
             'scope': scope,
             'redirect_uri': get_param('google_redirect_uri'),
             'client_id': get_param('google_%s_client_id' % service),
@@ -81,7 +82,7 @@ class GoogleService(models.TransientModel):
         base_url = get_param('web.base.url', default='http://www.odoo.com?NoBaseUrl')
         client_id = get_param('google_%s_client_id' % (service,), default=False)
 
-        encoded_params = werkzeug.url_encode({
+        encoded_params = urls.url_encode({
             'response_type': 'code',
             'client_id': client_id,
             'state': json.dumps(state),
@@ -103,17 +104,17 @@ class GoogleService(models.TransientModel):
         client_secret = get_param('google_%s_client_secret' % (service,), default=False)
 
         headers = {"content-type": "application/x-www-form-urlencoded"}
-        data = werkzeug.url_encode({
+        data = {
             'code': authorize_code,
             'client_id': client_id,
             'client_secret': client_secret,
             'grant_type': 'authorization_code',
             'redirect_uri': base_url + '/google_account/authentication'
-        })
+        }
         try:
             dummy, response, dummy = self._do_request(GOOGLE_TOKEN_ENDPOINT, params=data, headers=headers, type='POST', preuri='')
             return response
-        except urllib2.HTTPError:
+        except requests.HTTPError:
             error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid")
             raise self.env['res.config.settings'].get_config_warning(error_msg)
 
@@ -125,21 +126,21 @@ class GoogleService(models.TransientModel):
         client_secret = get_param('google_%s_client_secret' % (service,), default=False)
 
         headers = {"content-type": "application/x-www-form-urlencoded"}
-        data = werkzeug.url_encode({
+        data = {
             'refresh_token': refresh_token,
             'client_id': client_id,
             'client_secret': client_secret,
             'grant_type': 'refresh_token',
-        })
+        }
 
         try:
             dummy, response, dummy = self._do_request(GOOGLE_TOKEN_ENDPOINT, params=data, headers=headers, type='POST', preuri='')
             return response
-        except urllib2.HTTPError as error:
+        except requests.HTTPError as error:
-            if error.code == 400:  # invalid grant
+            if error.response.status_code == 400:  # invalid grant
                 with registry(request.session.db).cursor() as cur:
                     self.env(cur)['res.users'].browse(self.env.uid).write({'google_%s_rtoken' % service: False})
-            error_key = json.loads(error.read()).get("error", "nc")
+            error_key = json.loads(error.response.json()).get("error", "nc")
             _logger.exception("Bad google request : %s !", error_key)
             error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid or already expired [%s]") % error_key
             raise self.env['res.config.settings'].get_config_warning(error_msg)
@@ -154,40 +155,34 @@ class GoogleService(models.TransientModel):
             :param type : the method to use to make the request
             :param preuri : pre url to prepend to param uri.
         """
-        _logger.debug("Uri: %s - Type : %s - Headers: %s - Params : %s !" % (uri, type, headers, werkzeug.url_encode(params) if type == 'GET' else params))
+        _logger.debug("Uri: %s - Type : %s - Headers: %s - Params : %s !", (uri, type, headers, params))
 
-        status = 418
-        response = ""
         ask_time = fields.Datetime.now()
         try:
-            if type.upper() == 'GET' or type.upper() == 'DELETE':
+            if type.upper() in ('GET', 'DELETE'):
-                data = werkzeug.url_encode(params)
+                res = requests.request(type.lower(), preuri + uri, params=params, timeout=TIMEOUT)
-                req = urllib2.Request(preuri + uri + "?" + data)
+            elif type.upper() in ('POST', 'PATCH', 'PUT'):
-            elif type.upper() == 'POST' or type.upper() == 'PATCH' or type.upper() == 'PUT':
+                res = requests.request(type.lower(), preuri + uri, data=params, headers=headers, timeout=TIMEOUT)
-                req = urllib2.Request(preuri + uri, params, headers)
             else:
                 raise Exception(_('Method not supported [%s] not in [GET, POST, PUT, PATCH or DELETE]!') % (type))
-            req.get_method = lambda: type.upper()
+            res.raise_for_status()
-
+            status = res.status_code
-            resp = urllib2.urlopen(req, timeout=TIMEOUT)
-            status = resp.getcode()
 
             if int(status) in (204, 404):  # Page not found, no response
                 response = False
             else:
-                content = resp.read()
+                response = res.json()
-                response = json.loads(content)
 
             try:
-                ask_time = datetime.strptime(resp.headers.get('date'), "%a, %d %b %Y %H:%M:%S %Z")
+                ask_time = datetime.strptime(res.headers.get('date'), "%a, %d %b %Y %H:%M:%S %Z")
             except:
                 pass
-        except urllib2.HTTPError as error:
+        except request.HTTPError as error:
-            if error.code in (204, 404):
+            if error.response.status_code in (204, 404):
                 status = error.code
                 response = ""
             else:
-                _logger.exception("Bad google request : %s !", error.read())
+                _logger.exception("Bad google request : %s !", error.response.content)
                 if error.code in (400, 401, 410):
                     raise error
                 raise self.env['res.config.settings'].get_config_warning(_("Something went wrong with your request to google"))

addons/google_calendar/models/google_calendar.py

@@ -2,12 +2,14 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from datetime import datetime, timedelta
+
+import requests
 from dateutil import parser
 import json
 import logging
 import operator
 import pytz
-import urllib2
+from werkzeug import urls
 
 from odoo import api, fields, models, tools, _
 from odoo.tools import exception_to_unicode, pycompat
@@ -258,7 +260,7 @@ class GoogleCalendar(models.AbstractModel):
         """
         data = self.generate_data(event, isCreating=True)
 
-        url = "/calendar/v3/calendars/%s/events?fields=%s&access_token=%s" % ('primary', urllib2.quote('id,updated'), self.get_token())
+        url = "/calendar/v3/calendars/%s/events?fields=%s&access_token=%s" % ('primary', urls.url_quote('id,updated'), self.get_token())
         headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
         data_json = json.dumps(data)
         return self.env['google.service']._do_request(url, data_json, headers, type='POST')
@@ -289,8 +291,8 @@ class GoogleCalendar(models.AbstractModel):
 
         try:
             status, content, ask_time = self.env['google.service']._do_request(url, params, headers, type='GET')
-        except urllib2.HTTPError as e:
+        except requests.HTTPError as e:
-            if e.code == 401:  # Token invalid / Acces unauthorized
+            if e.response.status_code == 401:  # Token invalid / Acces unauthorized
                 error_msg = _("Your token is invalid or has been revoked !")
 
                 self.env.user.write({'google_calendar_token': False, 'google_calendar_token_validity': False})
@@ -674,13 +676,13 @@ class GoogleCalendar(models.AbstractModel):
         if lastSync:
             try:
                 all_event_from_google = self.get_event_synchro_dict(lastSync=lastSync)
-            except urllib2.HTTPError as e:
+            except requests.HTTPError as e:
-                if e.code == 410:  # GONE, Google is lost.
+                if e.response.code == 410:  # GONE, Google is lost.
                     # we need to force the rollback from this cursor, because it locks my res_users but I need to write in this tuple before to raise.
                     self.env.cr.rollback()
                     self.env.user.write({'google_calendar_last_sync_date': False})
                     self.env.cr.commit()
-                error_key = json.loads(str(e))
+                error_key = e.response.json()
                 error_key = error_key.get('error', {}).get('message', 'nc')
                 error_msg = _("Google is lost... the next synchro will be a full synchro. \n\n %s") % error_key
                 raise self.env['res.config.settings'].get_config_warning(error_msg)

addons/google_drive/models/google_drive.py

@@ -3,7 +3,8 @@
 import logging
 import json
 import re
-import urllib2
+
+import requests
 import werkzeug.urls
 
 from odoo import api, fields, models
@@ -62,26 +63,25 @@ class GoogleDrive(models.Model):
         google_drive_client_id = Config.get_param('google_drive_client_id')
         google_drive_client_secret = Config.get_param('google_drive_client_secret')
         #For Getting New Access Token With help of old Refresh Token
-        data = werkzeug.url_encode({
+        data = {
             'client_id': google_drive_client_id,
             'refresh_token': google_drive_refresh_token,
             'client_secret': google_drive_client_secret,
             'grant_type': "refresh_token",
             'scope': scope or 'https://www.googleapis.com/auth/drive'
-        })
+        }
         headers = {"Content-type": "application/x-www-form-urlencoded"}
         try:
-            req = urllib2.Request(GOOGLE_TOKEN_ENDPOINT, data, headers)
+            req = requests.post(GOOGLE_TOKEN_ENDPOINT, data=data, headers=headers, timeout=TIMEOUT)
-            content = urllib2.urlopen(req, timeout=TIMEOUT).read()
+            req.raise_for_status()
-        except urllib2.HTTPError:
+        except requests.HTTPError:
             if user_is_admin:
                 dummy, action_id = self.env['ir.model.data'].get_object_reference('base_setup', 'action_general_configuration')
                 msg = _("Something went wrong during the token generation. Please request again an authorization code .")
                 raise RedirectWarning(msg, action_id, _('Go to the configuration panel'))
             else:
                 raise UserError(_("Google Drive is not yet configured. Please contact your administrator."))
-        content = json.loads(content)
+        return req.json().get('access_token')
-        return content.get('access_token')
 
     @api.model
     def copy_doc(self, res_id, template_id, name_gdocs, res_model):
@@ -91,11 +91,11 @@ class GoogleDrive(models.Model):
         request_url = "https://www.googleapis.com/drive/v2/files/%s?fields=parents/id&access_token=%s" % (template_id, access_token)
         headers = {"Content-type": "application/x-www-form-urlencoded"}
         try:
-            req = urllib2.Request(request_url, None, headers)
+            req = requests.post(request_url, headers=headers, timeout=TIMEOUT)
-            parents = urllib2.urlopen(req, timeout=TIMEOUT).read()
+            req.raise_for_status()
-        except urllib2.HTTPError:
+            parents_dict = req.json()
+        except requests.HTTPError:
             raise UserError(_("The Google Template cannot be found. Maybe it has been deleted."))
-        parents_dict = json.loads(parents)
 
         record_url = "Click on link to open Record in Odoo\n %s/?db=%s#id=%s&model=%s" % (google_web_base_url, self._cr.dbname, res_id, res_model)
         data = {
@@ -108,11 +108,10 @@ class GoogleDrive(models.Model):
             'Content-type': 'application/json',
             'Accept': 'text/plain'
         }
-        data_json = json.dumps(data)
         # resp, content = Http().request(request_url, "POST", data_json, headers)
-        req = urllib2.Request(request_url, data_json, headers)
+        req = requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT)
-        content = urllib2.urlopen(req, timeout=TIMEOUT).read()
+        req.raise_for_status()
-        content = json.loads(content)
+        content = req.json()
         res = {}
         if content.get('alternateLink'):
             res['id'] = self.env["ir.attachment"].create({
@@ -129,16 +128,15 @@ class GoogleDrive(models.Model):
             request_url = "https://www.googleapis.com/drive/v2/files/%s/permissions?emailMessage=This+is+a+drive+file+created+by+Odoo&sendNotificationEmails=false&access_token=%s" % (key, access_token)
             data = {'role': 'writer', 'type': 'anyone', 'value': '', 'withLink': True}
             try:
-                req = urllib2.Request(request_url, json.dumps(data), headers)
+                req = requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT)
-                urllib2.urlopen(req, timeout=TIMEOUT)
+                req.raise_for_status()
-            except urllib2.HTTPError:
+            except requests.HTTPError:
                 raise self.env['res.config.settings'].get_config_warning(_("The permission 'reader' for 'anyone with the link' has not been written on the document"))
             if self.env.user.email:
                 data = {'role': 'writer', 'type': 'user', 'value': self.env.user.email}
                 try:
-                    req = urllib2.Request(request_url, json.dumps(data), headers)
+                    requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT)
-                    urllib2.urlopen(req, timeout=TIMEOUT)
+                except requests.HTTPError:
-                except urllib2.HTTPError:
                     pass
         return res
 

addons/google_spreadsheet/models/google_drive.py

@@ -3,10 +3,11 @@
 import cgi
 import json
 import logging
+
+import requests
 from lxml import etree
 import re
 import werkzeug.urls
-import urllib2
 
 from odoo import api, models
 from odoo.addons.google_account import TIMEOUT
@@ -72,12 +73,13 @@ class GoogleDrive(models.Model):
 </feed>''' .format(key=spreadsheet_key, formula=cgi.escape(formula, quote=True), config=cgi.escape(config_formula, quote=True))
 
         try:
-            req = urllib2.Request(
+            req = requests.post(
                 'https://spreadsheets.google.com/feeds/cells/%s/od6/private/full/batch?%s' % (spreadsheet_key, werkzeug.url_encode({'v': 3, 'access_token': access_token})),
                 data=request,
-                headers={'content-type': 'application/atom+xml', 'If-Match': '*'})
+                headers={'content-type': 'application/atom+xml', 'If-Match': '*'},
-            urllib2.urlopen(req, timeout=TIMEOUT)
+                timeout=TIMEOUT,
-        except (urllib2.HTTPError, urllib2.URLError):
+            )
+        except IOError:
             _logger.warning("An error occured while writting the formula on the Google Spreadsheet.")
 
         description = '''

addons/link_tracker/models/link_tracker.py

@@ -6,11 +6,10 @@ import random
 import re
 import string
 
+import requests
 from lxml import html
-from urllib2 import urlopen
+from werkzeug import urls, utils
-from urlparse import urljoin
+
-from urlparse import urlparse
-from werkzeug import url_encode, unescape
 
 from odoo import models, fields, api, _
 from odoo.tools import ustr, pycompat
@@ -18,7 +17,7 @@ from odoo.tools import ustr, pycompat
 URL_REGEX = r'(\bhref=[\'"](?!mailto:)([^\'"]+)[\'"])'
 
 def VALIDATE_URL(url):
-    if urlparse(url).scheme not in ('http', 'https', 'ftp', 'ftps'):
+    if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'):
         return 'http://' + url
 
     return url
@@ -56,7 +55,7 @@ class link_tracker(models.Model):
             href = match[0]
             long_url = match[1]
 
-            vals['url'] = unescape(long_url)
+            vals['url'] = utils.unescape(long_url)
 
             if not blacklist or not [s for s in blacklist if s in long_url] and not long_url.startswith(short_schema):
                 link = self.create(vals)
@@ -77,7 +76,7 @@ class link_tracker(models.Model):
     @api.depends('code')
     def _compute_short_url(self):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        self.short_url = urljoin(base_url, '/r/%(code)s' % {'code': self.code})
+        self.short_url = urls.url_join(base_url, '/r/%(code)s' % {'code': self.code})
 
     @api.one
     def _compute_short_url_host(self):
@@ -96,22 +95,23 @@ class link_tracker(models.Model):
     @api.one
     @api.depends('url')
     def _compute_redirected_url(self):
-        parsed = urlparse(self.url)
+        parsed = urls.url_parse(self.url)
 
         utms = {}
         for key, field, cook in self.env['utm.mixin'].tracking_fields():
             attr = getattr(self, field).name
             if attr:
                 utms[key] = attr
+        utms.update(parsed.decode_query())
 
-        self.redirected_url = '%s://%s%s?%s&%s#%s' % (parsed.scheme, parsed.netloc, parsed.path, url_encode(utms), parsed.query, parsed.fragment)
+        self.redirected_url = parsed.replace(query=urls.url_encode(utms)).to_url()
 
     @api.model
     @api.depends('url')
     def _get_title_from_url(self, url):
         try:
-            page = urlopen(url, timeout=5)
+            page = requests.get(url, timeout=5)
-            p = html.fromstring(ustr(page.read()).encode('utf-8'), parser=html.HTMLParser(encoding='utf-8'))
+            p = html.fromstring(page.text.encode('utf-8'), parser=html.HTMLParser(encoding='utf-8'))
             title = p.find('.//title').text
         except:
             title = url
@@ -122,7 +122,7 @@ class link_tracker(models.Model):
     @api.depends('url')
     def _compute_favicon(self):
         try:
-            icon = urlopen('http://www.google.com/s2/favicons?domain=' + self.url, timeout=5).read()
+            icon = requests.get('http://www.google.com/s2/favicons', params={'domain': self.url}, timeout=5).content
             icon_base64 = icon.encode('base64').replace("\n", "")
         except:
             icon_base64 = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsSAAALEgHS3X78AAACiElEQVQ4EaVTzU8TURCf2tJuS7tQtlRb6UKBIkQwkRRSEzkQgyEc6lkOKgcOph78Y+CgjXjDs2i44FXY9AMTlQRUELZapVlouy3d7kKtb0Zr0MSLTvL2zb75eL838xtTvV6H/xELBptMJojeXLCXyobnyog4YhzXYvmCFi6qVSfaeRdXdrfaU1areV5KykmX06rcvzumjY/1ggkR3Jh+bNf1mr8v1D5bLuvR3qDgFbvbBJYIrE1mCIoCrKxsHuzK+Rzvsi29+6DEbTZz9unijEYI8ObBgXOzlcrx9OAlXyDYKUCzwwrDQx1wVDGg089Dt+gR3mxmhcUnaWeoxwMbm/vzDFzmDEKMMNhquRqduT1KwXiGt0vre6iSeAUHNDE0d26NBtAXY9BACQyjFusKuL2Ry+IPb/Y9ZglwuVscdHaknUChqLF/O4jn3V5dP4mhgRJgwSYm+gV0Oi3XrvYB30yvhGa7BS70eGFHPoTJyQHhMK+F0ZesRVVznvXw5Ixv7/C10moEo6OZXbWvlFAF9FVZDOqEABUMRIkMd8GnLwVWg9/RkJF9sA4oDfYQAuzzjqzwvnaRUFxn/X2ZlmGLXAE7AL52B4xHgqAUqrC1nSNuoJkQtLkdqReszz/9aRvq90NOKdOS1nch8TpL555WDp49f3uAMXhACRjD5j4ykuCtf5PP7Fm1b0DIsl/VHGezzP1KwOiZQobFF9YyjSRYQETRENSlVzI8iK9mWlzckpSSCQHVALmN9Az1euDho9Xo8vKGd2rqooA8yBcrwHgCqYR0kMkWci08t/R+W4ljDCanWTg9TJGwGNaNk3vYZ7VUdeKsYJGFNkfSzjXNrSX20s4/h6kB81/271ghG17l+rPTAAAAAElFTkSuQmCC'

addons/mail/models/html2text.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 """html2text: Turn HTML into equivalent Markdown-structured text."""
+from werkzeug import urls
+
 from odoo.tools import pycompat
 
 __version__ = "2.36"
@@ -10,9 +12,8 @@ __contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
 # TODO:
 #   Support decoded entities with unifiable.
 
-import re, sys, urllib, htmlentitydefs, codecs
+import re, sys, htmlentitydefs, codecs
 import sgmllib
-import urlparse
 sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]')
 
 try: from textwrap import wrap
@@ -390,7 +391,7 @@ class _html2text(sgmllib.SGMLParser):
                 newa = []
                 for link in self.a:
                     if self.outcount > link['outcount']:
-                        self.out("   ["+repr(link['count'])+"]: " + urlparse.urljoin(self.baseurl, link['href']))
+                        self.out("   ["+repr(link['count'])+"]: " + urls.url_join(self.baseurl, link['href']))
                         if 'title' in link: self.out(" ("+link['title']+")")
                         self.out("\n")
                     else:
@@ -426,32 +427,3 @@ def html2text_file(html, out=wrapwrite, baseurl=''):
 
 def html2text(html, baseurl=''):
     return optwrap(html2text_file(html, None, baseurl))
-
-if __name__ == "__main__":
-    baseurl = ''
-    if sys.argv[1:]:
-        arg = sys.argv[1]
-        if arg.startswith('http://'):
-            baseurl = arg
-            j = urllib.urlopen(baseurl)
-            try:
-                from feedparser import _getCharacterEncoding as enc
-            except ImportError:
-                   enc = lambda x, y: ('utf-8', 1)
-            text = j.read()
-            encoding = enc(j.headers, text)[0]
-            if encoding == 'us-ascii': encoding = 'utf-8'
-            data = text.decode(encoding)
-
-        else:
-            encoding = 'utf8'
-            if len(sys.argv) > 2:
-                encoding = sys.argv[2]
-            f = open(arg, 'r')
-            try:
-                    data = f.read().decode(encoding)
-            finally:
-                    f.close()
-    else:
-        data = sys.stdin.read().decode('utf8')
-    wrapwrite(html2text(data, baseurl))

addons/mail/models/mail_template.py

@@ -10,9 +10,7 @@ import logging
 
 import functools
 import lxml
-import urlparse
+from werkzeug import urls
-
-from urllib import urlencode, quote as quote
 
 from odoo import _, api, fields, models, tools
 from odoo.exceptions import UserError
@@ -83,8 +81,8 @@ try:
     )
     mako_template_env.globals.update({
         'str': str,
-        'quote': quote,
+        'quote': urls.url_quote,
-        'urlencode': urlencode,
+        'urlencode': urls.url_encode,
         'datetime': datetime,
         'len': len,
         'abs': abs,
@@ -297,14 +295,13 @@ class MailTemplate(models.Model):
             root = lxml.html.fromstring(html)
 
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        (base_scheme, base_netloc, bpath, bparams, bquery, bfragment) = urlparse.urlparse(base_url)
+        base = urls.url_parse(base_url)
 
         def _process_link(url):
-            new_url = url
+            new_url = urls.url_parse(url)
-            (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url)
+            if new_url.scheme and new_url.netloc:
-            if not scheme and not netloc:
+                return url
-                new_url = urlparse.urlunparse((base_scheme, base_netloc, path, params, query, fragment))
+            return new_url.replace(scheme=base.scheme, netloc=base.netloc).to_url()
-            return new_url
 
         # check all nodes, replace :
         # - img src -> check URL

addons/mail/models/res_config.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import urlparse
 import datetime
 
+from werkzeug import urls
+
 from odoo import api, fields, models, tools
 
 
@@ -29,7 +30,7 @@ class BaseConfiguration(models.TransientModel):
         if alias_domain is None:
             domain = self.env["ir.config_parameter"].get_param("web.base.url")
             try:
-                alias_domain = urlparse.urlsplit(domain).netloc.split(':')[0]
+                alias_domain = urls.url_parse(domain).host
             except Exception:
                 pass
         return {'alias_domain': alias_domain or False}

addons/mail/models/update.py

@@ -3,8 +3,9 @@
 
 import datetime
 import logging
+
+import requests
 import werkzeug.urls
-import urllib2
 
 from ast import literal_eval
 
@@ -71,16 +72,12 @@ class PublisherWarrantyContract(AbstractModel):
         """
         msg = self._get_message()
         arguments = {'arg0': msg, "action": "update"}
-        arguments_raw = werkzeug.urls.url_encode(arguments)
 
         url = config.get("publisher_warranty_url")
 
-        uo = urllib2.urlopen(url, arguments_raw, timeout=30)
+        r = requests.post(url, data=arguments, timeout=30)
-        try:
+        r.raise_for_status()
-            submit_result = uo.read()
+        return literal_eval(r.text)
-            return literal_eval(submit_result)
-        finally:
-            uo.close()
 
     @api.multi
     def update_notification(self, cron_mode=True):

addons/mass_mailing/models/mail_mail.py

@@ -2,7 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import re
-import urlparse
 import werkzeug.urls
 
 from odoo import api, fields, models, tools
@@ -29,20 +28,20 @@ class MailMail(models.Model):
 
     def _get_tracking_url(self, partner=None):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        track_url = urlparse.urljoin(
+        track_url = werkzeug.urls.url_join(
             base_url, 'mail/track/%(mail_id)s/blank.gif?%(params)s' % {
                 'mail_id': self.id,
-                'params': werkzeug.url_encode({'db': self.env.cr.dbname})
+                'params': werkzeug.urls.url_encode({'db': self.env.cr.dbname})
             }
         )
         return '<img src="%s" alt=""/>' % track_url
 
     def _get_unsubscribe_url(self, email_to):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        url = urlparse.urljoin(
+        url = werkzeug.urls.url_join(
             base_url, 'mail/mailing/%(mailing_id)s/unsubscribe?%(params)s' % {
                 'mailing_id': self.mailing_id.id,
-                'params': werkzeug.url_encode({
+                'params': werkzeug.urls.url_encode({
                     'db': self.env.cr.dbname,
                     'res_id': self.res_id,
                     'email': email_to,
@@ -79,7 +78,7 @@ class MailMail(models.Model):
         body = tools.append_content_to_html(base, body, plaintext=False, container_tag='div')
         # resolve relative image url to absolute for outlook.com
         def _sub_relative2absolute(match):
-            return match.group(1) + urlparse.urljoin(domain, match.group(2))
+            return match.group(1) + werkzeug.urls.url_join(domain, match.group(2))
         body = re.sub('(<img(?=\s)[^>]*\ssrc=")(/[^/][^"]+)', _sub_relative2absolute, body)
         body = re.sub(r'(<[^>]+\bstyle="[^"]+\burl\(\'?)(/[^/\'][^\'")]+)', _sub_relative2absolute, body)
 

addons/pad/models/pad.py

@@ -5,7 +5,8 @@ import logging
 import random
 import re
 import string
-import urllib2
+
+import requests
 
 from odoo import api, models, _
 from odoo.exceptions import UserError
@@ -53,7 +54,7 @@ class PadCommon(models.AbstractModel):
             myPad = EtherpadLiteClient(pad["key"], pad["server"] + '/api')
             try:
                 myPad.createPad(path)
-            except urllib2.URLError:
+            except IOError:
                 raise UserError(_("Pad creation failed, either there is a problem with your pad server URL or with your connection."))
 
             # get attr on the field model
@@ -79,12 +80,15 @@ class PadCommon(models.AbstractModel):
         content = ''
         if url:
             try:
-                page = urllib2.urlopen('%s/export/html' % url).read()
+                r = requests.get('%s/export/html' % url)
-                mo = re.search('<body>(.*)</body>', page, re.DOTALL)
+                r.raise_for_status()
-                if mo:
-                    content = mo.group(1)
             except:
                 _logger.warning("No url found '%s'.", url)
+            else:
+                mo = re.search('<body>(.*)</body>', r.content, re.DOTALL)
+                if mo:
+                    content = mo.group(1)
+
         return content
 
     # TODO

addons/pad/py_etherpad/__init__.py

@@ -1,8 +1,5 @@
 """Module to talk to EtherpadLite API."""
-
+import requests
-import json
-import urllib
-import urllib2
 
 
 class EtherpadLiteClient:
@@ -31,23 +28,11 @@ class EtherpadLiteClient:
         url = '%s/%d/%s' % (self.baseUrl, self.API_VERSION, function)
 
         params = arguments or {}
-        params.update({'apikey': self.apiKey})
+        params['apikey'] = self.apiKey
-        data = urllib.urlencode(params, True)
 
-        try:
+        r = requests.get(url, data=params, timeout=self.TIMEOUT)
-            opener = urllib2.build_opener()
+        r.raise_for_status()
-            request = urllib2.Request(url=url, data=data)
+        return self.handleResult(r.json())
-            response = opener.open(request, timeout=self.TIMEOUT)
-            result = response.read()
-            response.close()
-        except urllib2.HTTPError:
-            raise
-
-        result = json.loads(result)
-        if result is None:
-            raise ValueError("JSON response could not be decoded")
-
-        return self.handleResult(result)
 
     def handleResult(self, result):
         """Handle API call result"""

addons/payment_adyen/models/payment.py

@@ -7,9 +7,10 @@ from collections import OrderedDict
 import hashlib
 import hmac
 import logging
-import urlparse
 from itertools import chain
 
+from werkzeug import urls
+
 from odoo import api, fields, models, tools, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment_adyen.controllers.main import AdyenController
@@ -132,7 +133,7 @@ class AcquirerAdyen(models.Model):
                 'merchantAccount': self.adyen_merchant_account,
                 'shopperLocale': values.get('partner_lang', ''),
                 'sessionValidity': tmp_date.isoformat('T')[:19] + "Z",
-                'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url),
+                'resURL': urls.url_join(base_url, AdyenController._return_url),
                 'merchantReturnData': json.dumps({'return_url': '%s' % values.pop('return_url')}) if values.get('return_url', '') else False,
                 'shopperEmail': values.get('partner_email', ''),
             })
@@ -150,7 +151,7 @@ class AcquirerAdyen(models.Model):
                 'merchantAccount': self.adyen_merchant_account,
                 'shopperLocale': values.get('partner_lang'),
                 'sessionValidity': tmp_date,
-                'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url),
+                'resURL': urls.url_join(base_url, AdyenController._return_url),
                 'merchantReturnData': json.dumps({'return_url': '%s' % values.pop('return_url')}) if values.get('return_url') else False,
                 'merchantSig': self._adyen_generate_merchant_sig('in', values),
             })

addons/payment_adyen/tests/test_adyen.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from lxml import objectify
-import urlparse
 
 from odoo.addons.payment.tests.common import PaymentAcquirerCommon
 from odoo.addons.payment_adyen.controllers.main import AdyenController
+from werkzeug import urls
 
 
 class AdyenCommon(PaymentAcquirerCommon):
@@ -47,7 +47,7 @@ class AdyenForm(AdyenCommon):
             'skinCode': 'cbqYWvVL',
             'paymentAmount': '1',
             'currencyCode': 'EUR',
-            'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url),
+            'resURL': urls.url_join(base_url, AdyenController._return_url),
         }
 
         # render the button

addons/payment_authorize/controllers/main.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 import pprint
 import logging
-import urlparse
+from werkzeug import urls, utils
-import werkzeug
 
 from odoo import http
 from odoo.http import request
@@ -29,7 +28,7 @@ class AuthorizeController(http.Controller):
         # This response is in the form of a URL that Authorize.Net will pass on to the
         # client's browser to redirect them to the desired location need javascript.
         return request.render('payment_authorize.payment_authorize_redirect', {
-            'return_url': '%s' % urlparse.urljoin(base_url, return_url)
+            'return_url': urls.url_join(base_url, return_url)
         })
 
     @http.route(['/payment/authorize/s2s/create_json'], type='json', auth='public')
@@ -43,4 +42,4 @@ class AuthorizeController(http.Controller):
         acquirer_id = int(post.get('acquirer_id'))
         acquirer = request.env['payment.acquirer'].browse(acquirer_id)
         acquirer.s2s_process(post)
-        return werkzeug.utils.redirect(post.get('return_url', '/'))
+        return utils.redirect(post.get('return_url', '/'))

addons/payment_authorize/models/authorize_request.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
+import requests
 from lxml import etree, objectify
-from urllib2 import urlopen, Request
 from StringIO import StringIO
 import xml.etree.ElementTree as ET
 from uuid import uuid4
@@ -57,10 +57,9 @@ class AuthorizeAPI():
         :param etree._Element data: etree data to process
         """
         data = etree.tostring(data, xml_declaration=True, encoding='utf-8')
-        request = Request(self.url, data)
+        r = requests.post(self.url, data=data, headers={'Content-Type': 'text/xml'})
-        request.add_header('Content-Type', 'text/xml')
+        r.raise_for_status()
-        response = urlopen(request).read()
+        response = strip_ns(r.content, XMLNS)
-        response = strip_ns(response, XMLNS)
         if response.find('messages/resultCode').text == 'Error':
             messages = [m.text for m in response.findall('messages/message/text')]
             raise ValidationError('Authorize.net Error Message(s):\n %s' % '\n'.join(messages))

addons/payment_authorize/models/payment.py

@@ -1,4 +1,5 @@
 # coding: utf-8
+from werkzeug import urls
 
 from .authorize_request import AuthorizeAPI
 from datetime import datetime
@@ -6,7 +7,6 @@ import hashlib
 import hmac
 import logging
 import time
-import urlparse
 
 from odoo import api, fields, models
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -72,8 +72,8 @@ class PaymentAcquirerAuthorize(models.Model):
             'x_version': '3.1',
             'x_relay_response': 'TRUE',
             'x_fp_timestamp': str(int(time.time())),
-            'x_relay_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._return_url),
+            'x_relay_url': urls.url_join(base_url, AuthorizeController._return_url),
-            'x_cancel_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._cancel_url),
+            'x_cancel_url': urls.url_join(base_url, AuthorizeController._cancel_url),
             'x_currency_code': values['currency'] and values['currency'].name or '',
             'address': values.get('partner_address'),
             'city': values.get('partner_city'),

addons/payment_authorize/tests/test_authorize.py

@@ -3,9 +3,9 @@
 import hashlib
 import hmac
 import time
-import urlparse
 import unittest
 from lxml import objectify
+from werkzeug import urls
 
 import odoo
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -59,8 +59,8 @@ class AuthorizeForm(AuthorizeCommon):
             'x_version': '3.1',
             'x_relay_response': 'TRUE',
             'x_fp_timestamp': str(int(time.time())),
-            'x_relay_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._return_url),
+            'x_relay_url': urls.url_join(base_url, AuthorizeController._return_url),
-            'x_cancel_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._cancel_url),
+            'x_cancel_url': urls.url_join(base_url, AuthorizeController._cancel_url),
             'return_url': None,
             'x_currency_code': 'USD',
             'x_invoice_num': 'SO004',

addons/payment_buckaroo/models/payment.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 from hashlib import sha1
 import logging
-import urllib
+
-import urlparse
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -74,14 +74,11 @@ class AcquirerBuckaroo(models.Model):
                     break
 
             items = sorted(pycompat.items(values), key=lambda pair: pair[0].lower())
-            sign = ''.join('%s=%s' % (k, urllib.unquote_plus(v)) for k, v in items)
+            sign = ''.join('%s=%s' % (k, urls.url_unquote_plus(v)) for k, v in items)
         else:
             sign = ''.join('%s=%s' % (k, get_value(k)) for k in keys)
         # Add the pre-shared secret key at the end of the signature
         sign = sign + self.brq_secretkey
-        if isinstance(sign, str):
-            # TODO: remove me? should not be used
-            sign = urlparse.parse_qsl(sign)
         shasign = sha1(sign.encode('utf-8')).hexdigest()
         return shasign
 
@@ -95,10 +92,10 @@ class AcquirerBuckaroo(models.Model):
             'Brq_currency': values['currency'] and values['currency'].name or '',
             'Brq_invoicenumber': values['reference'],
             'brq_test': False if self.environment == 'prod' else True,
-            'Brq_return': '%s' % urlparse.urljoin(base_url, BuckarooController._return_url),
+            'Brq_return': urls.url_join(base_url, BuckarooController._return_url),
-            'Brq_returncancel': '%s' % urlparse.urljoin(base_url, BuckarooController._cancel_url),
+            'Brq_returncancel': urls.url_join(base_url, BuckarooController._cancel_url),
-            'Brq_returnerror': '%s' % urlparse.urljoin(base_url, BuckarooController._exception_url),
+            'Brq_returnerror': urls.url_join(base_url, BuckarooController._exception_url),
-            'Brq_returnreject': '%s' % urlparse.urljoin(base_url, BuckarooController._reject_url),
+            'Brq_returnreject': urls.url_join(base_url, BuckarooController._reject_url),
             'Brq_culture': (values.get('partner_lang') or 'en_US').replace('_', '-'),
             'add_returndata': buckaroo_tx_values.pop('return_url', '') or '',
         })

addons/payment_buckaroo/tests/test_buckaroo.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from lxml import objectify
-import urlparse
+from werkzeug import urls
 
 import odoo
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -41,10 +41,10 @@ class BuckarooForm(BuckarooCommon):
             'Brq_invoicenumber': 'SO004',
             'Brq_signature': '1b8c10074c622d965272a91a9e88b5b3777d2474',  # update me
             'brq_test': 'True',
-            'Brq_return': '%s' % urlparse.urljoin(base_url, BuckarooController._return_url),
+            'Brq_return': urls.url_join(base_url, BuckarooController._return_url),
-            'Brq_returncancel': '%s' % urlparse.urljoin(base_url, BuckarooController._cancel_url),
+            'Brq_returncancel': urls.url_join(base_url, BuckarooController._cancel_url),
-            'Brq_returnerror': '%s' % urlparse.urljoin(base_url, BuckarooController._exception_url),
+            'Brq_returnerror': urls.url_join(base_url, BuckarooController._exception_url),
-            'Brq_returnreject': '%s' % urlparse.urljoin(base_url, BuckarooController._reject_url),
+            'Brq_returnreject': urls.url_join(base_url, BuckarooController._reject_url),
             'Brq_culture': 'en-US',
         }
 

addons/payment_ogone/models/payment.py

@@ -1,22 +1,20 @@
 # coding: utf-8
-
-from hashlib import sha1
-from lxml import etree, objectify
-from pprint import pformat
-from unicodedata import normalize
-from urllib import urlencode
-
 import datetime
 import logging
 import time
-import urllib2
+from hashlib import sha1
-import urlparse
+from pprint import pformat
+from unicodedata import normalize
+
+import requests
+from lxml import etree, objectify
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment_ogone.controllers.main import OgoneController
 from odoo.addons.payment_ogone.data import ogone
-from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT
+from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT, pycompat
 from odoo.tools.float_utils import float_compare, float_repr
 from odoo.tools.safe_eval import safe_eval
 
@@ -165,10 +163,10 @@ class PaymentAcquirerOgone(models.Model):
             'OWNERTOWN': values.get('partner_city'),
             'OWNERCTY': values.get('partner_country') and values.get('partner_country').code or '',
             'OWNERTELNO': values.get('partner_phone'),
-            'ACCEPTURL': '%s' % urlparse.urljoin(base_url, OgoneController._accept_url),
+            'ACCEPTURL': urls.url_join(base_url, OgoneController._accept_url),
-            'DECLINEURL': '%s' % urlparse.urljoin(base_url, OgoneController._decline_url),
+            'DECLINEURL': urls.url_join(base_url, OgoneController._decline_url),
-            'EXCEPTIONURL': '%s' % urlparse.urljoin(base_url, OgoneController._exception_url),
+            'EXCEPTIONURL': urls.url_join(base_url, OgoneController._exception_url),
-            'CANCELURL': '%s' % urlparse.urljoin(base_url, OgoneController._cancel_url),
+            'CANCELURL': urls.url_join(base_url, OgoneController._cancel_url),
             'PARAMPLUS': 'return_url=%s' % ogone_tx_values.pop('return_url') if ogone_tx_values.get('return_url') else False,
         }
         if self.save_token in ['ask', 'always']:
@@ -370,8 +368,7 @@ class PaymentTxOgone(models.Model):
         direct_order_url = 'https://secure.ogone.com/ncol/%s/orderdirect.asp' % (self.acquirer_id.environment)
 
         _logger.debug("Ogone data %s", pformat(data))
-        request = urllib2.Request(direct_order_url, urlencode(data))
+        result = requests.post(direct_order_url, data=data).content
-        result = urllib2.urlopen(request).read()
         _logger.debug('Ogone response = %s', result)
 
         try:
@@ -455,8 +452,7 @@ class PaymentTxOgone(models.Model):
         query_direct_url = 'https://secure.ogone.com/ncol/%s/querydirect.asp' % (self.acquirer_id.environment)
 
         _logger.debug("Ogone data %s", pformat(data))
-        request = urllib2.Request(query_direct_url, urlencode(data))
+        result = requests.post(query_direct_url, data=data).content
-        result = urllib2.urlopen(request).read()
         _logger.debug('Ogone response = %s', result)
 
         try:
@@ -497,9 +493,7 @@ class PaymentToken(models.Model):
             }
 
             url = 'https://secure.ogone.com/ncol/%s/AFU_agree.asp' % (acquirer.environment,)
-            request = urllib2.Request(url, urlencode(data))
+            result = requests.post(url, data=data).content
-
-            result = urllib2.urlopen(request).read()
 
             try:
                 tree = objectify.fromstring(result)

addons/payment_ogone/tests/test_ogone.py

@@ -2,11 +2,12 @@
 
 from lxml import objectify
 import time
-import urlparse
 
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment.tests.common import PaymentAcquirerCommon
 from odoo.addons.payment_ogone.controllers.main import OgoneController
+from werkzeug import urls
+
 from odoo.tools import mute_logger
 
 
@@ -40,10 +41,10 @@ class OgonePayment(PaymentAcquirerCommon):
             'OWNERTOWN': 'Sin City',
             'OWNERTELNO': '0032 12 34 56 78',
             'SHASIGN': '815f67b8ff70d234ffcf437c13a9fa7f807044cc',
-            'ACCEPTURL': '%s' % urlparse.urljoin(base_url, OgoneController._accept_url),
+            'ACCEPTURL': urls.url_join(base_url, OgoneController._accept_url),
-            'DECLINEURL': '%s' % urlparse.urljoin(base_url, OgoneController._decline_url),
+            'DECLINEURL': urls.url_join(base_url, OgoneController._decline_url),
-            'EXCEPTIONURL': '%s' % urlparse.urljoin(base_url, OgoneController._exception_url),
+            'EXCEPTIONURL': urls.url_join(base_url, OgoneController._exception_url),
-            'CANCELURL': '%s' % urlparse.urljoin(base_url, OgoneController._cancel_url),
+            'CANCELURL': urls.url_join(base_url, OgoneController._cancel_url),
         }
 
         # render the button

addons/payment_paypal/controllers/main.py

@@ -3,9 +3,10 @@
 import json
 import logging
 import pprint
-import urllib
+
-import urllib2
+import requests
 import werkzeug
+from werkzeug import urls
 
 from odoo import http
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -23,7 +24,7 @@ class PaypalController(http.Controller):
         """ Extract the return URL from the data coming from paypal. """
         return_url = post.pop('return_url', '')
         if not return_url:
-            custom = json.loads(urllib.unquote_plus(post.pop('custom', False) or post.pop('cm', False) or '{}'))
+            custom = json.loads(urls.url_unquote_plus(post.pop('custom', False) or post.pop('cm', False) or '{}'))
             return_url = custom.get('return_url', '/')
         return return_url
 
@@ -44,7 +45,7 @@ class PaypalController(http.Controller):
         for line in lines:
             split = line.split('=', 1)
             if len(split) == 2:
-                pdt_post[split[0]] = urllib.unquote_plus(split[1]).decode('utf8')
+                pdt_post[split[0]] = urls.url_unquote_plus(split[1]).decode('utf8')
             else:
                 _logger.warning('Paypal: error processing pdt response: %s', line)
 
@@ -75,9 +76,9 @@ class PaypalController(http.Controller):
             new_post['at'] = tx and tx.acquirer_id.paypal_pdt_token or ''
             new_post['cmd'] = '_notify-synch'  # command is different in PDT than IPN/DPN
         validate_url = paypal_urls['paypal_form_url']
-        urequest = urllib2.Request(validate_url, werkzeug.url_encode(new_post))
+        urequest = requests.post(validate_url, new_post)
-        uopen = urllib2.urlopen(urequest)
+        urequest.raise_for_status()
-        resp = uopen.read()
+        resp = urequest.content
         if pdt_request:
             resp, post = self._parse_pdt_response(resp)
         if resp in ['VERIFIED', 'SUCCESS']:

addons/payment_paypal/models/payment.py

@@ -2,10 +2,10 @@
 
 import json
 import logging
-import urlparse
 
 import dateutil.parser
 import pytz
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -109,9 +109,9 @@ class AcquirerPaypal(models.Model):
             'zip_code': values.get('partner_zip'),
             'first_name': values.get('partner_first_name'),
             'last_name': values.get('partner_last_name'),
-            'paypal_return': '%s' % urlparse.urljoin(base_url, PaypalController._return_url),
+            'paypal_return': urls.url_join(base_url, PaypalController._return_url),
-            'notify_url': '%s' % urlparse.urljoin(base_url, PaypalController._notify_url),
+            'notify_url': urls.url_join(base_url, PaypalController._notify_url),
-            'cancel_return': '%s' % urlparse.urljoin(base_url, PaypalController._cancel_url),
+            'cancel_return': urls.url_join(base_url, PaypalController._cancel_url),
             'handling': '%.2f' % paypal_tx_values.pop('fees', 0.0) if self.fees_active else False,
             'custom': json.dumps({'return_url': '%s' % paypal_tx_values.pop('return_url')}) if paypal_tx_values.get('return_url') else False,
         })

addons/payment_paypal/tests/test_paypal.py

@@ -3,10 +3,11 @@
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment.tests.common import PaymentAcquirerCommon
 from odoo.addons.payment_paypal.controllers.main import PaypalController
+from werkzeug import urls
+
 from odoo.tools import mute_logger
 
 from lxml import objectify
-import urlparse
 
 
 class PaypalCommon(PaymentAcquirerCommon):
@@ -60,9 +61,9 @@ class PaypalForm(PaypalCommon):
             'zip': '1000',
             'country': 'BE',
             'email': 'norbert.buyer@example.com',
-            'return': '%s' % urlparse.urljoin(base_url, PaypalController._return_url),
+            'return': urls.url_join(base_url, PaypalController._return_url),
-            'notify_url': '%s' % urlparse.urljoin(base_url, PaypalController._notify_url),
+            'notify_url': urls.url_join(base_url, PaypalController._notify_url),
-            'cancel_return': '%s' % urlparse.urljoin(base_url, PaypalController._cancel_url),
+            'cancel_return': urls.url_join(base_url, PaypalController._cancel_url),
         }
 
         # check form result

addons/payment_payumoney/models/payment.py

@@ -2,7 +2,8 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import hashlib
-import urlparse
+
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -64,9 +65,9 @@ class PaymentAcquirerPayumoney(models.Model):
                                 email=values.get('partner_email'),
                                 phone=values.get('partner_phone'),
                                 service_provider='payu_paisa',
-                                surl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/return'),
+                                surl=urls.url_join(base_url, '/payment/payumoney/return'),
-                                furl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/error'),
+                                furl=urls.url_join(base_url, '/payment/payumoney/error'),
-                                curl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/cancel')
+                                curl=urls.url_join(base_url, '/payment/payumoney/cancel')
                                 )
 
         payumoney_values['udf1'] = payumoney_values.pop('return_url', '/')

addons/payment_sips/models/payment.py

@@ -3,7 +3,8 @@
 import json
 import logging
 from hashlib import sha256
-import urlparse
+
+from werkzeug import urls
 
 from odoo import models, fields, api
 from odoo.tools.float_utils import float_compare
@@ -89,8 +90,8 @@ class AcquirerSips(models.Model):
             'Data': u'amount=%s|' % amount +
                     u'currencyCode=%s|' % currency_code +
                     u'merchantId=%s|' % merchant_id +
-                    u'normalReturnUrl=%s|' % urlparse.urljoin(base_url, SipsController._return_url) +
+                    u'normalReturnUrl=%s|' % urls.url_join(base_url, SipsController._return_url) +
-                    u'automaticResponseUrl=%s|' % urlparse.urljoin(base_url, SipsController._return_url) +
+                    u'automaticResponseUrl=%s|' % urls.url_join(base_url, SipsController._return_url) +
                     u'transactionReference=%s|' % values['reference'] +
                     u'statementReference=%s|' % values['reference'] +
                     u'keyVersion=%s' % key_version,

addons/pos_mercury/models/pos_mercury_transaction.py

@@ -4,8 +4,9 @@
 from datetime import date, timedelta
 
 import cgi
-import urllib2
 import ssl
+
+import requests
 import werkzeug
 
 from odoo import models, api, service
@@ -61,11 +62,11 @@ class MercuryTransaction(models.Model):
             'SOAPAction': 'http://www.mercurypay.com/CreditTransaction',
         }
 
-        r = urllib2.Request('https://w1.mercurypay.com/ws/ws.asmx', data=xml_transaction, headers=headers)
         try:
-            u = urllib2.urlopen(r, timeout=65)
+            r = requests.post('https://w1.mercurypay.com/ws/ws.asmx', data=xml_transaction, headers=headers, timeout=65)
-            response = werkzeug.utils.unescape(u.read())
+            r.raise_for_status()
-        except (urllib2.URLError, ssl.SSLError):
+            response = werkzeug.utils.unescape(r.content)
+        except:
             response = "timeout"
 
         return response

addons/survey/models/survey.py

@@ -5,10 +5,11 @@ import datetime
 import logging
 import re
 import uuid
-from urlparse import urljoin
 from collections import Counter, OrderedDict
 from itertools import product
 
+from werkzeug import urls
+
 from odoo import api, fields, models, tools, SUPERUSER_ID, _
 from odoo.exceptions import UserError, ValidationError
 
@@ -108,9 +109,9 @@ class Survey(models.Model):
         base_url = '/' if self.env.context.get('relative_url') else \
                    self.env['ir.config_parameter'].sudo().get_param('web.base.url')
         for survey in self:
-            survey.public_url = urljoin(base_url, "survey/start/%s" % (slug(survey)))
+            survey.public_url = urls.url_join(base_url, "survey/start/%s" % (slug(survey)))
-            survey.print_url = urljoin(base_url, "survey/print/%s" % (slug(survey)))
+            survey.print_url = urls.url_join(base_url, "survey/print/%s" % (slug(survey)))
-            survey.result_url = urljoin(base_url, "survey/results/%s" % (slug(survey)))
+            survey.result_url = urls.url_join(base_url, "survey/results/%s" % (slug(survey)))
             survey.public_url_html = '<a href="%s">%s</a>' % (survey.public_url, _("Click here to start survey"))
 
     @api.model

addons/survey/tests/test_survey.py

@@ -5,7 +5,8 @@ import random
 import re
 from collections import Counter
 from itertools import product
-from urlparse import urljoin
+
+from werkzeug import urls
 
 from odoo import _
 from odoo.exceptions import UserError
@@ -183,7 +184,7 @@ class TestSurvey(TransactionCase):
             survey_url_relative = getattr(self.survey1.with_context({'relative_url': True}), urltype + '_url')
             self.assertTrue(validate_url(survey_url))
             url = "survey/%s/%s" % (urltxt, slug(self.survey1))
-            full_url = urljoin(base_url, url)
+            full_url = urls.url_join(base_url, url)
             self.assertEqual(full_url, survey_url)
             self.assertEqual('/' + url, survey_url_relative)
             if urltype == 'public':

addons/survey/wizard/survey_email_compose_message.py

@@ -3,7 +3,8 @@
 
 import re
 import uuid
-import urlparse
+
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
@@ -85,7 +86,7 @@ class SurveyMailComposeMessage(models.TransientModel):
             #set url
             url = wizard.survey_id.public_url
 
-            url = urlparse.urlparse(url).path[1:]  # dirty hack to avoid incorrect urls
+            url = urls.url_parse(url).path[1:]  # dirty hack to avoid incorrect urls
 
             if token:
                 url = url + '/' + token

addons/web_editor/models/ir_qweb.py

@@ -14,15 +14,16 @@ import itertools
 import json
 import logging
 import os
-import urllib2
-import urlparse
 import re
 import hashlib
 
 import pytz
+import requests
 from dateutil import parser
 from lxml import etree, html
 from PIL import Image as I
+from werkzeug import urls
+
 import odoo.modules
 
 from odoo import api, models, fields
@@ -319,11 +320,11 @@ class Image(models.AbstractModel):
     def from_html(self, model, field, element):
         url = element.find('img').get('src')
 
-        url_object = urlparse.urlsplit(url)
+        url_object = urls.url_parse(url)
         if url_object.path.startswith('/web/image'):
             # url might be /web/image/<model>/<id>[_<checksum>]/<field>[/<width>x<height>]
             fragments = url_object.path.split('/')
-            query = dict(urlparse.parse_qsl(url_object.query))
+            query = url_object.decode_query()
             if fragments[3].isdigit():
                 model = 'ir.attachment'
                 oid = fragments[3]
@@ -341,7 +342,7 @@ class Image(models.AbstractModel):
         return self.load_remote_url(url)
 
     def load_local_url(self, url):
-        match = self.local_url_re.match(urlparse.urlsplit(url).path)
+        match = self.local_url_re.match(urls.url_parse(url).path)
 
         rest = match.group('rest')
         for sep in os.sep, os.altsep:
@@ -374,9 +375,9 @@ class Image(models.AbstractModel):
             #   linking to HTTP images
             # implement drag & drop image upload to mitigate?
 
-            req = urllib2.urlopen(url, timeout=REMOTE_CONNECTION_TIMEOUT)
+            req = requests.get(url, timeout=REMOTE_CONNECTION_TIMEOUT)
-            # PIL needs a seekable file-like image, urllib result is not seekable
+            # PIL needs a seekable file-like image so wrap result in IO buffer
-            image = I.open(cStringIO.StringIO(req.read()))
+            image = I.open(cStringIO.StringIO(req.content))
             # force a complete load of the image data to validate it
             image.load()
         except Exception:

addons/web_planner/models/web_planner.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+from werkzeug import urls
-from urllib import urlencode
 
 from odoo import api, models, fields
 
@@ -69,7 +68,7 @@ class Planner(models.Model):
                 params['id'] = module.id
             else:
                 return "#show_enterprise"
-        return "/web#%s" % (urlencode(params),)
+        return "/web#%s" % (urls.url_encode(params),)
 
     @api.model
     def is_module_installed(self, module_name=None):

addons/website/controllers/main.py

@@ -7,7 +7,8 @@ import json
 import xml.etree.ElementTree as ET
 import logging
 import re
-import urllib2
+
+import requests
 import werkzeug.utils
 import werkzeug.wrappers
 
@@ -300,12 +301,13 @@ class Website(Home):
         language = lang.split("_")
         url = "http://google.com/complete/search"
         try:
-            req = urllib2.Request("%s?%s" % (url, werkzeug.url_encode({
+            req = requests.get(url, params={
-                'ie': 'utf8', 'oe': 'utf8', 'output': 'toolbar', 'q': keywords, 'hl': language[0], 'gl': language[1]})))
+                'ie': 'utf8', 'oe': 'utf8', 'output': 'toolbar', 'q': keywords, 'hl': language[0], 'gl': language[1]})
-            response = urllib2.urlopen(req)
+            req.raise_for_status()
-        except (urllib2.HTTPError, urllib2.URLError):
+            response = req.content
+        except IOError:
             return []
-        xmlroot = ET.fromstring(response.read())
+        xmlroot = ET.fromstring(response)
         return json.dumps([sugg[0].attrib['data'] for sugg in xmlroot if len(sugg) and sugg[0].attrib['data']])
 
     #------------------------------------------------------

addons/website/models/ir_actions.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+from werkzeug import urls
-import urlparse
 
 from odoo import api, fields, models
 from odoo.http import request
@@ -32,7 +31,7 @@ class ServerAction(models.Model):
         link = website_path or xml_id or (self.id and '%d' % self.id) or ''
         if base_url and link:
             path = '%s/%s' % ('/website/action', link)
-            return '%s' % urlparse.urljoin(base_url, path)
+            return urls.url_join(base_url, path)
         return ''
 
     @api.depends('state', 'website_published', 'website_path', 'xml_id')

addons/website/models/website.py

@@ -6,9 +6,9 @@ import logging
 import math
 import unicodedata
 import re
-import urlparse
 import hashlib
-import werkzeug
+
+from werkzeug import urls
 from werkzeug.exceptions import NotFound
 
 # optional python-slugify import (https://github.com/un33k/python-slugify)
@@ -44,10 +44,10 @@ def url_for(path_or_uri, lang=None):
         current_path = current_path.encode('utf-8')
     location = path_or_uri.strip()
     force_lang = lang is not None
-    url = urlparse.urlparse(location)
+    url = urls.url_parse(location)
 
     if request and not url.netloc and not url.scheme and (url.path or force_lang):
-        location = urlparse.urljoin(current_path, location)
+        location = urls.url_join(current_path, location)
 
         lang = lang or request.context.get('lang')
         langs = [lg[0] for lg in request.website.get_languages()]
@@ -459,7 +459,7 @@ class Website(models.Model):
         def get_url(page):
             _url = "%s/page/%s" % (url, page) if page > 1 else url
             if url_args:
-                _url = "%s?%s" % (_url, werkzeug.url_encode(url_args))
+                _url = "%s?%s" % (_url, urls.url_encode(url_args))
             return _url
 
         return {
@@ -598,7 +598,7 @@ class Website(models.Model):
             cdn_filters = (request.website.cdn_filters or '').splitlines()
             for flt in cdn_filters:
                 if flt and re.match(flt, uri):
-                    return urlparse.urljoin(cdn_url, uri)
+                    return urls.url_join(cdn_url, uri)
         return uri
 
     @api.model

addons/website/tests/test_crawl.py

@@ -2,10 +2,10 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import logging
-import urlparse
 import time
 
 import lxml.html
+from werkzeug import urls
 
 import odoo
 import re
@@ -39,23 +39,17 @@ class Crawler(odoo.tests.HttpCase):
 
         _logger.info("%s %s", msg, url)
         r = self.url_open(url)
-        code = r.getcode()
+        code = r.status_code
         self.assertIn(code, range(200, 300), "%s Fetching %s returned error response (%d)" % (msg, url, code))
 
-        if r.info().gettype() == 'text/html':
+        if r.headers['Content-Type'].startswith('text/html'):
-            doc = lxml.html.fromstring(r.read())
+            doc = lxml.html.fromstring(r.content)
             for link in doc.xpath('//a[@href]'):
                 href = link.get('href')
 
-                parts = urlparse.urlsplit(href)
+                parts = urls.url_parse(href)
                 # href with any fragment removed
-                href = urlparse.urlunsplit((
+                href = parts.replace(fragment='').to_url()
-                    parts.scheme,
-                    parts.netloc,
-                    parts.path,
-                    parts.query,
-                    ''
-                ))
 
                 # FIXME: handle relative link (not parts.path.startswith /)
                 if parts.netloc or \

addons/website_forum/controllers/main.py

@@ -4,12 +4,12 @@
 import base64
 import json
 import lxml
+import requests
 import werkzeug.exceptions
 import werkzeug.urls
 import werkzeug.wrappers
 
 from datetime import datetime
-from urllib2 import urlopen, URLError
 
 from odoo import http, modules, SUPERUSER_ID, tools, _
 from odoo.addons.web.controllers.main import binary_content
@@ -217,9 +217,11 @@ class WebsiteForum(http.Controller):
     @http.route('/forum/get_url_title', type='json', auth="user", methods=['POST'], website=True)
     def get_url_title(self, **kwargs):
         try:
-            arch = lxml.html.parse(urlopen(kwargs.get('url')))
+            req = requests.get(kwargs.get('url'), stream=True)
+            req.raise_for_status()
+            arch = lxml.html.parse(req.raw)
             return arch.find(".//title").text
-        except URLError:
+        except IOError:
             return False
 
     @http.route(['''/forum/<model("forum.forum"):forum>/question/<model("forum.post", "[('forum_id','=',forum[0]),('parent_id','=',False),('can_view', '=', True)]"):question>'''], type='http', auth="public", website=True)

addons/website_forum/models/res_users.py

@@ -4,7 +4,8 @@
 import hashlib
 
 from datetime import datetime
-from urllib import urlencode
+
+from werkzeug import urls
 
 from odoo import api, fields, models
 
@@ -85,7 +86,7 @@ class Users(models.Model):
             if forum_id:
                 params['forum_id'] = forum_id
             base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-            token_url = base_url + '/forum/validate_email?%s' % urlencode(params)
+            token_url = base_url + '/forum/validate_email?%s' % urls.url_encode(params)
             activation_template.sudo().with_context(token_url=token_url).send_mail(self.id, force_send=True)
         return True
 

addons/website_hr_recruitment/models/hr_recruitment.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-from urlparse import urljoin
+from werkzeug import urls
-from werkzeug import url_encode
 
 from odoo import api, fields, models
 from odoo.addons.website.models.website import slug
@@ -19,8 +18,8 @@ class RecruitmentSource(models.Model):
     def _compute_url(self):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
         for source in self:
-            source.url = urljoin(base_url, "%s?%s" % (source.job_id.website_url,
+            source.url = urls.url_join(base_url, "%s?%s" % (source.job_id.website_url,
-                url_encode({
+                urls.url_encode({
                     'utm_campaign': self.env.ref('hr_recruitment.utm_campaign_job').name,
                     'utm_medium': self.env.ref('utm.utm_medium_website').name,
                     'utm_source': source.source_id.name

addons/website_mail_channel/models/mail_channel.py

@@ -3,7 +3,7 @@
 
 import hmac
 
-from urlparse import urljoin
+from werkzeug import urls
 
 from odoo import api, models
 from odoo.tools.safe_eval import safe_eval
@@ -47,7 +47,7 @@ class MailGroup(models.Model):
             # generate a new token per subscriber
             token = self._generate_action_token(partner_id, action=action)
 
-            token_url = urljoin(base_url, route % {
+            token_url = urls.url_join(base_url, route % {
                 'action': action,
                 'channel': self.id,
                 'partner': partner_id,

addons/website_sale/controllers/website_mail.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-import urlparse
+from werkzeug import urls
 
 from odoo import http
 from odoo.http import request
@@ -37,7 +37,7 @@ class WebsiteMailController(WebsiteMail):
         response = super(WebsiteMailController, self).chatter_post(res_model=res_model, res_id=res_id, message=message, redirect=redirect, **params)
         if kw.get('rating') and res_model == 'product.template':  # restrict rating only for product template
             try:
-                fragment = urlparse.urlparse(response.location).fragment
+                fragment = urls.url_parse(response.location).fragment
                 message_id = int(fragment.replace('message-', ''))
                 res_model_id = request.env.ref('product.model_product_template').id
                 request.env['rating.rating'].create({

addons/website_slides/models/slides.py

@@ -1,15 +1,14 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import requests
 from PIL import Image
-from urllib import urlencode
-from urlparse import urlparse
 
 import datetime
 import io
 import json
 import re
-import urllib2
+
+from werkzeug import urls
 
 from odoo import api, fields, models, SUPERUSER_ID, _
 from odoo.tools import image, pycompat
@@ -224,8 +223,7 @@ class EmbeddedSlide(models.Model):
     count_views = fields.Integer('# Views', default=1)
 
     def add_embed_url(self, slide_id, url):
-        schema = urlparse(url)
+        baseurl = urls.url_parse(url).netloc
-        baseurl = schema.netloc
         embeds = self.search([('url', '=', baseurl), ('slide_id', '=', int(slide_id))], limit=1)
         if embeds:
             embeds.count_views += 1
@@ -498,22 +496,19 @@ class Slide(models.Model):
     def _fetch_data(self, base_url, data, content_type=False, extra_params=False):
         result = {'values': dict()}
         try:
-            if data:
+            response = requests.get(base_url, params=data)
-                sep = '?' if not extra_params else '&'
+            response.raise_for_status()
-                base_url = base_url + '%s%s' % (sep, urlencode(data))
+            content = response.content
-            req = urllib2.Request(base_url)
-            content = urllib2.urlopen(req).read()
             if content_type == 'json':
                 result['values'] = json.loads(content)
             elif content_type in ('image', 'pdf'):
                 result['values'] = content.encode('base64')
             else:
                 result['values'] = content
-        except urllib2.HTTPError as e:
+        except requests.exceptions.HTTPError as e:
-            result['error'] = e.read()
+            result['error'] = e.response.content
-            e.close()
+        except requests.exceptions.ConnectionError as e:
-        except urllib2.URLError as e:
+            result['error'] = str(e)
-            result['error'] = e.reason
         return result
 
     def _find_document_data_from_url(self, url):

addons/website_twitter/models/website_twitter.py

@@ -4,8 +4,9 @@
 import base64
 import json
 import logging
+
+import requests
 import werkzeug
-from urllib2 import urlopen, Request, HTTPError
 from odoo import api, fields, models
 
 API_ENDPOINT = 'https://api.twitter.com'
@@ -28,16 +29,13 @@ class WebsiteTwitter(models.Model):
     def _request(self, website, url, params=None):
         """Send an authenticated request to the Twitter API."""
         access_token = self._get_access_token(website)
-        if params:
-            params = werkzeug.url_encode(params)
-            url = url + '?' + params
         try:
-            request = Request(url)
+            request = requests.get(url, params=params, headers={'Authorization': 'Bearer %s' % access_token}, timeout=URLOPEN_TIMEOUT)
-            request.add_header('Authorization', 'Bearer %s' % access_token)
+            request.raise_for_status()
-            return json.load(urlopen(request, timeout=URLOPEN_TIMEOUT))
+            return request.json()
-        except HTTPError as e:
+        except requests.HTTPError as e:
             _logger.debug("Twitter API request failed with code: %r, msg: %r, content: %r",
-                          e.code, e.msg, e.fp.read())
+                          e.response.status_code, e.response.reason, e.response.content)
             raise
 
     @api.model

addons/website_twitter/models/website_twitter_config.py

@@ -2,7 +2,9 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import logging
-from urllib2 import URLError, HTTPError
+
+import requests
+
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
 
@@ -49,13 +51,13 @@ class WebsiteTwitterConfig(models.TransientModel):
         try:
             self.website_id.fetch_favorite_tweets()
 
-        except HTTPError as e:
+        except requests.HTTPError as e:
-            _logger.info("%s - %s" % (e.code, e.reason), exc_info=True)
+            _logger.info("%s - %s" % (e.response.status_code, e.response.reason), exc_info=True)
-            raise UserError("%s - %s" % (e.code, e.reason) + ':' + self._get_twitter_exception_message(e.code))
+            raise UserError("%s - %s" % (e.response.status_code, e.response.reason) + ':' + self._get_twitter_exception_message(e.response.status_code))
-        except URLError as e:
+        except IOError:
             _logger.info(_('We failed to reach a twitter server.'), exc_info=True)
             raise UserError(_('Internet connection refused') + ' ' + _('We failed to reach a twitter server.'))
-        except Exception as e:
+        except Exception:
             _logger.info(_('Please double-check your Twitter API Key and Secret!'), exc_info=True)
             raise UserError(_('Twitter authorization error!') + ' ' + _('Please double-check your Twitter API Key and Secret!'))
 

doc/_extensions/github_link.py

@@ -1,7 +1,8 @@
 import inspect
 import importlib
 import os.path
-from urlparse import urlunsplit
+
+from werkzeug import urls
 
 """
 * adds github_link(mode) context variable: provides URL (in relevant mode) of
@@ -81,7 +82,7 @@ def make_github_link(app, path, line=None, mode="blob"):
         path=path,
         mode=mode,
     )
-    return urlunsplit((
+    return urls.url_unparse((
         'https',
         'github.com',
         urlpath,

doc/_extensions/odoo_ext/translator.py

@@ -2,7 +2,10 @@
 import os.path
 import posixpath
 import re
-import urllib
+try:
+    from urllib.request import url2pathname  # pylint: disable=deprecated-module
+except ImportError:
+    from urllib import url2pathname  # pylint: disable=deprecated-module
 
 from docutils import nodes
 from sphinx import addnodes, util
@@ -636,7 +639,7 @@ class BootstrapTranslator(nodes.NodeVisitor, object):
                     banner = '_static/' + cover
                     base, ext = os.path.splitext(banner)
                     small = "{}.small{}".format(base, ext)
-                    if os.path.isfile(urllib.url2pathname(small)):
+                    if os.path.isfile(url2pathname(small)):
                         banner = small
                     style = u"background-image: url('{}')".format(
                         util.relative_uri(baseuri, banner) or '#')

odoo/addons/base/ir/ir_qweb/ir_qweb.py

@@ -5,14 +5,10 @@ import json
 import logging
 from collections import OrderedDict
 from time import time
-try:
-    from urllib.parse import urlparse
-except ImportError:
-    # pylint: disable=bad-python3-import
-    from urlparse import urlparse
 
 from lxml import html
 from lxml import etree
+from werkzeug import urls
 
 from odoo.tools import pycompat
 
@@ -230,7 +226,7 @@ class IrQWeb(models.AbstractModel, QWeb):
                 atype = el.get('type')
                 media = el.get('media')
 
-                can_aggregate = not urlparse(href).netloc and not href.startswith('/web/content')
+                can_aggregate = not urls.url_parse(href).netloc and not href.startswith('/web/content')
                 if el.tag == 'style' or (el.tag == 'link' and el.get('rel') == 'stylesheet' and can_aggregate):
                     if href.endswith('.sass'):
                         atype = 'text/sass'

odoo/addons/base/module/module.py

@@ -10,15 +10,9 @@ import shutil
 import tempfile
 import zipfile
 
-from odoo.tools import pycompat
+import requests
 
-try:
+from odoo.tools import pycompat
-    from urllib import parse as urlparse
-    from urllib.request import urlopen
-except ImportError:
-    # pylint: disable=bad-python3-import
-    import urlparse
-    from urllib2 import urlopen
 
 from docutils import nodes
 from docutils.core import publish_string
@@ -663,7 +657,7 @@ class Module(models.Model):
             _logger.warning(msg)
             raise UserError(msg)
 
-        apps_server = urlparse.urlparse(self.get_apps_server())
+        apps_server = urls.url_parse(self.get_apps_server())
 
         OPENERP = odoo.release.product_name.lower()
         tmp = tempfile.mkdtemp()
@@ -674,13 +668,15 @@ class Module(models.Model):
                 if not url:
                     continue    # nothing to download, local version is already the last one
 
-                up = urlparse.urlparse(url)
+                up = urls.url_parse(url)
                 if up.scheme != apps_server.scheme or up.netloc != apps_server.netloc:
                     raise AccessDenied()
 
                 try:
                     _logger.info('Downloading module `%s` from OpenERP Apps', module_name)
-                    content = urlopen(url).read()
+                    response = requests.get(url)
+                    response.raise_for_status()
+                    content = response.content
                 except Exception:
                     _logger.exception('Failed to fetch module %s', module_name)
                     raise UserError(_('The `%s` module appears to be unavailable at the moment, please try again later.') % module_name)

odoo/addons/base/res/res_partner.py

@@ -7,16 +7,11 @@ import hashlib
 import pytz
 import threading
 
-try:
-    from urllib import parse as urlparse
-    from urllib.request import urlopen
-except ImportError:
-    # pylint: disable=bad-python3-import
-    import urlparse
-    from urllib2 import urlopen
-
 from email.utils import formataddr
+
+import requests
 from lxml import etree
+from werkzeug import urls
 
 from odoo import api, fields, models, tools, SUPERUSER_ID, _
 from odoo.modules import get_module_resource
@@ -481,11 +476,11 @@ class Partner(models.Model):
             parent.update_address(addr_vals)
 
     def _clean_website(self, website):
-        (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(website)
+        url = urls.url_parse(website)
-        if not scheme:
+        if not url.scheme:
-            if not netloc:
+            if not url.netloc:
-                netloc, path = path, ''
+                url = url.replace(netloc=url.path, path='')
-            website = urlparse.urlunparse(('http', netloc, path, params, query, fragment))
+            website = url.replace(scheme='http').to_url()
         return website
 
     @api.multi
@@ -703,15 +698,12 @@ class Partner(models.Model):
         return partners.id or self.name_create(email)[0]
 
     def _get_gravatar_image(self, email):
-        gravatar_image = False
         email_hash = hashlib.md5(email.lower()).hexdigest()
         url = "https://www.gravatar.com/avatar/" + email_hash
-        try:
+        res = requests.get(url, params={'d': '404', 's': '128'}, timeout=5)
-            image_content = urlopen(url + "?d=404&s=128", timeout=5).read()
+        if res.status_code != requests.codes.ok:
-            gravatar_image = base64.b64encode(image_content)
+            return False
-        except Exception:
+        return base64.b64encode(res.content)
-            pass
-        return gravatar_image
 
     @api.multi
     def _email_send(self, email_from, subject, body, on_error=None):

odoo/http.py

@@ -20,12 +20,6 @@ import sys
 import threading
 import time
 import traceback
-try:
-    from urllib.parse import parse_qs, urlparse, quote
-except ImportError:
-    # pylint: disable=bad-python3-import
-    from urllib2 import quote
-    from urlparse import parse_qs, urlparse
 import warnings
 from os.path import join as opj
 from zlib import adler32
@@ -41,6 +35,7 @@ import werkzeug.local
 import werkzeug.routing
 import werkzeug.wrappers
 import werkzeug.wsgi
+from werkzeug import urls
 from werkzeug.wsgi import wrap_file
 
 try:
@@ -351,7 +346,7 @@ class WebRequest(object):
             debug = self.httprequest.environ.get('HTTP_X_DEBUG_MODE')
 
         if not debug and self.httprequest.referrer:
-            debug = bool(parse_qs(urlparse(self.httprequest.referrer).query, keep_blank_values=True).get('debug'))
+            debug = 'debug' in urls.url_parse(self.httprequest.referrer).decode_query()
         return debug
 
     @contextlib.contextmanager
@@ -1254,7 +1249,7 @@ class DisableCacheMiddleware(object):
     def __call__(self, environ, start_response):
         def start_wrapped(status, headers):
             referer = environ.get('HTTP_REFERER', '')
-            parsed = urlparse(referer)
+            parsed = urls.url_parse(referer)
             debug = parsed.query.count('debug') >= 1
 
             new_headers = []
@@ -1619,7 +1614,7 @@ def send_file(filepath_or_fp, mimetype=None, as_attachment=False, filename=None,
 
 def content_disposition(filename):
     filename = odoo.tools.ustr(filename)
-    escaped = quote(filename.encode('utf8'))
+    escaped = urls.url_quote(filename.encode('utf8'))
     browser = request.httprequest.user_agent.browser
     version = int((request.httprequest.user_agent.version or '0').split('.')[0])
     if browser == 'msie' and version < 9:

odoo/sql_db.py

@@ -13,17 +13,13 @@ from functools import wraps
 import logging
 import time
 import uuid
-try:
-    from urllib import parse as urlparse
-except ImportError:
-    #pylint: disable=bad-python3-import
-    import urlparse
 
 import psycopg2
 import psycopg2.extras
 import psycopg2.extensions
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_REPEATABLE_READ
 from psycopg2.pool import PoolError
+from werkzeug import urls
 
 from .tools import pycompat
 
@@ -656,7 +652,7 @@ def connection_info_for(db_or_uri):
     """
     if db_or_uri.startswith(('postgresql://', 'postgres://')):
         # extract db from uri
-        us = urlparse.urlsplit(db_or_uri)
+        us = urls.url_parse(db_or_uri)
         if len(us.path) > 1:
             db_name = us.path[1:]
         elif us.username:

odoo/tests/common.py

@@ -19,16 +19,15 @@ import unittest
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 from pprint import pformat
+
+import requests
+
 try:
-    from urllib import request as urllib2
     from xmlrpc import client as xmlrpclib
 except ImportError:
     # pylint: disable=bad-python3-import
-    import urllib2
     import xmlrpclib
 
-import werkzeug
-
 import odoo
 from odoo import api
 
@@ -215,26 +214,6 @@ class SavepointCase(SingleTransactionCase):
         self.registry.clear_caches()
 
 
-class RedirectHandler(urllib2.HTTPRedirectHandler):
-    """
-    HTTPRedirectHandler is predicated upon HTTPErrorProcessor being used and
-    works by intercepting 3xy "errors".
-
-    Inherit from it to handle 3xy non-error responses instead, as we're not
-    using the error processor
-    """
-
-    def http_response(self, request, response):
-        code, msg, hdrs = response.code, response.msg, response.info()
-
-        if 300 <= code < 400:
-            return self.parent.error(
-                'http', request, response, code, msg, hdrs)
-
-        return response
-
-    https_response = http_response
-
 class HttpCase(TransactionCase):
     """ Transactional HTTP TestCase with url_open and phantomjs helpers.
     """
@@ -259,18 +238,15 @@ class HttpCase(TransactionCase):
         self.session.db = get_db_name()
         odoo.http.root.session_store.save(self.session)
         # setup an url opener helper
-        self.opener = urllib2.OpenerDirector()
+        self.opener = requests.Session()
-        self.opener.add_handler(urllib2.UnknownHandler())
+        self.opener.cookies['session_id'] = self.session_id
-        self.opener.add_handler(urllib2.HTTPHandler())
-        self.opener.add_handler(urllib2.HTTPSHandler())
-        self.opener.add_handler(urllib2.HTTPCookieProcessor())
-        self.opener.add_handler(RedirectHandler())
-        self.opener.addheaders.append(('Cookie', 'session_id=%s' % self.session_id))
 
     def url_open(self, url, data=None, timeout=10):
         if url.startswith('/'):
             url = "http://%s:%s%s" % (HOST, PORT, url)
-        return self.opener.open(url, data, timeout)
+        if data:
+            return self.opener.post(url, data=data, timeout=timeout)
+        return self.opener.get(url, timeout=timeout)
 
     def authenticate(self, user, password):
         # stay non-authenticated

odoo/tools/misc.py

@@ -26,7 +26,6 @@ from itertools import islice, groupby, repeat
 from lxml import etree
 
 from .which import which
-from threading import local
 import traceback
 import csv
 from operator import itemgetter
@@ -677,29 +676,6 @@ def split_every(n, iterable, piece_maker=tuple):
         yield piece
         piece = piece_maker(islice(iterator, n))
 
-if __name__ == '__main__':
-    import doctest
-    doctest.testmod()
-
-class upload_data_thread(threading.Thread):
-    def __init__(self, email, data, type):
-        self.args = [('email',email),('type',type),('data',data)]
-        super(upload_data_thread,self).__init__()
-    def run(self):
-        try:
-            import urllib
-            args = urllib.urlencode(self.args)
-            fp = urllib.urlopen('http://www.openerp.com/scripts/survey.php', args)
-            fp.read()
-            fp.close()
-        except Exception:
-            pass
-
-def upload_data(email, data, type='SURVEY'):
-    a = upload_data_thread(email, data, type)
-    a.start()
-    return True
-
 def get_and_group_by_field(cr, uid, obj, ids, field, context=None):
     """ Read the values of ``field´´ for the given ``ids´´ and group ids by value.