[FIX] P3: stdlib compatibility fixes

addons/anonymization/wizard/anonymize_wizard.py

@@ -4,10 +4,7 @@
 import base64
 import os
 import random
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
+import pickle
 from lxml import etree
 from operator import itemgetter
 

addons/auth_oauth/models/res_users.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import werkzeug.urls
-import urlparse
-import urllib2
 import json
 
+import requests
+
 from odoo import api, fields, models
 from odoo.exceptions import AccessDenied
 from odoo.addons.auth_signup.models.res_users import SignupError
@@ -24,14 +23,7 @@ class ResUsers(models.Model):
 
     @api.model
     def _auth_oauth_rpc(self, endpoint, access_token):
-        params = werkzeug.url_encode({'access_token': access_token})
-        if urlparse.urlparse(endpoint)[4]:
-            url = endpoint + '&' + params
-        else:
-            url = endpoint + '?' + params
-        f = urllib2.urlopen(url)
-        response = f.read()
-        return json.loads(response)
+        return requests.get(endpoint, params={'access_token': access_token}).json()
 
     @api.model
     def _auth_oauth_validate(self, provider, access_token):

addons/auth_signup/models/res_partner.py

@@ -2,10 +2,9 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import random
-import werkzeug
+import werkzeug.urls
 
 from datetime import datetime, timedelta
-from urlparse import urljoin
 
 from odoo import api, fields, models, _
 from odoo.tools import pycompat
@@ -89,9 +88,9 @@ class ResPartner(models.Model):
                 fragment['res_id'] = res_id
 
             if fragment:
-                query['redirect'] = base + werkzeug.url_encode(fragment)
+                query['redirect'] = base + werkzeug.urls.url_encode(fragment)
 
-            res[partner.id] = urljoin(base_url, "/web/%s?%s" % (route, werkzeug.url_encode(query)))
+            res[partner.id] = werkzeug.urls.url_join(base_url, "/web/%s?%s" % (route, werkzeug.urls.url_encode(query)))
         return res
 
     @api.multi

addons/base_geolocalize/models/res_partner.py

@@ -1,18 +1,18 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 import json
-import urllib
+
+import requests
 
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import UserError
 
 
 def geo_find(addr):
-    url = 'https://maps.googleapis.com/maps/api/geocode/json?sensor=false&address='
-    url += urllib.quote(addr.encode('utf8'))
+    url = 'https://maps.googleapis.com/maps/api/geocode/json'
 
     try:
-        result = json.load(urllib.urlopen(url))
+        result = requests.get(url, params={'sensor': 'false', 'address': addr}).json()
     except Exception as e:
         raise UserError(_('Cannot contact geolocation servers. Please make sure that your Internet connection is up and running (%s).') % e)
 

addons/base_import/controllers/main.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import cgi
 import json
 
 from odoo import http
 from odoo.http import request
+from odoo.tools import misc
 
 
 class ImportController(http.Controller):
@@ -20,4 +20,4 @@ class ImportController(http.Controller):
             'file_type': file.content_type,
         })
 
-        return 'window.top.%s(%s)' % (cgi.escape(jsonp), json.dumps({'result': written}))
+        return 'window.top.%s(%s)' % (misc.html_escape(jsonp), json.dumps({'result': written}))

addons/crm/wizard/base_partner_merge.py

@@ -4,7 +4,10 @@
 from ast import literal_eval
 from email.utils import parseaddr
 import functools
-import htmlentitydefs
+try:
+    from html.entities import entitydefs
+except ImportError:
+    from htmlentitydefs import entitydefs # pylint: disable=deprecated-module
 import itertools
 import logging
 import operator
@@ -23,7 +26,7 @@ _logger = logging.getLogger('base.partner.merge')
 
 
 # http://www.php2python.com/wiki/function.html-entity-decode/
-def html_entity_decode_char(m, defs=htmlentitydefs.entitydefs):
+def html_entity_decode_char(m, defs=entitydefs):
     try:
         return defs[m.group(1)]
     except KeyError:

addons/google_account/models/google_service.py

@@ -4,8 +4,9 @@
 from datetime import datetime
 import json
 import logging
-import urllib2
-import werkzeug.urls
+
+import requests
+from werkzeug import urls
 
 from odoo import api, fields, models, registry, _
 from odoo.http import request
@@ -38,27 +39,27 @@ class GoogleService(models.TransientModel):
 
         # Get the Refresh Token From Google And store it in ir.config_parameter
         headers = {"Content-type": "application/x-www-form-urlencoded"}
-        data = werkzeug.url_encode({
+        data = {
             'code': authorization_code,
             'client_id': client_id,
             'client_secret': client_secret,
             'redirect_uri': redirect_uri,
             'grant_type': "authorization_code"
-        })
+        }
         try:
-            req = urllib2.Request(GOOGLE_TOKEN_ENDPOINT, data, headers)
-            content = urllib2.urlopen(req, timeout=TIMEOUT).read()
-        except urllib2.HTTPError:
+            req = requests.post(GOOGLE_TOKEN_ENDPOINT, data=data, headers=headers, timeout=TIMEOUT)
+            req.raise_for_status()
+            content = req.json()
+        except IOError:
             error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid or already expired")
             raise self.env['res.config.settings'].get_config_warning(error_msg)
 
-        content = json.loads(content)
         return content.get('refresh_token')
 
     @api.model
     def _get_google_token_uri(self, service, scope):
         get_param = self.env['ir.config_parameter'].sudo().get_param
-        encoded_params = werkzeug.url_encode({
+        encoded_params = urls.url_encode({
             'scope': scope,
             'redirect_uri': get_param('google_redirect_uri'),
             'client_id': get_param('google_%s_client_id' % service),
@@ -81,7 +82,7 @@ class GoogleService(models.TransientModel):
         base_url = get_param('web.base.url', default='http://www.odoo.com?NoBaseUrl')
         client_id = get_param('google_%s_client_id' % (service,), default=False)
 
-        encoded_params = werkzeug.url_encode({
+        encoded_params = urls.url_encode({
             'response_type': 'code',
             'client_id': client_id,
             'state': json.dumps(state),
@@ -103,17 +104,17 @@ class GoogleService(models.TransientModel):
         client_secret = get_param('google_%s_client_secret' % (service,), default=False)
 
         headers = {"content-type": "application/x-www-form-urlencoded"}
-        data = werkzeug.url_encode({
+        data = {
             'code': authorize_code,
             'client_id': client_id,
             'client_secret': client_secret,
             'grant_type': 'authorization_code',
             'redirect_uri': base_url + '/google_account/authentication'
-        })
+        }
         try:
             dummy, response, dummy = self._do_request(GOOGLE_TOKEN_ENDPOINT, params=data, headers=headers, type='POST', preuri='')
             return response
-        except urllib2.HTTPError:
+        except requests.HTTPError:
             error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid")
             raise self.env['res.config.settings'].get_config_warning(error_msg)
 
@@ -125,21 +126,21 @@ class GoogleService(models.TransientModel):
         client_secret = get_param('google_%s_client_secret' % (service,), default=False)
 
         headers = {"content-type": "application/x-www-form-urlencoded"}
-        data = werkzeug.url_encode({
+        data = {
             'refresh_token': refresh_token,
             'client_id': client_id,
             'client_secret': client_secret,
             'grant_type': 'refresh_token',
-        })
+        }
 
         try:
             dummy, response, dummy = self._do_request(GOOGLE_TOKEN_ENDPOINT, params=data, headers=headers, type='POST', preuri='')
             return response
-        except urllib2.HTTPError as error:
-            if error.code == 400:  # invalid grant
+        except requests.HTTPError as error:
+            if error.response.status_code == 400:  # invalid grant
                 with registry(request.session.db).cursor() as cur:
                     self.env(cur)['res.users'].browse(self.env.uid).write({'google_%s_rtoken' % service: False})
-            error_key = json.loads(error.read()).get("error", "nc")
+            error_key = json.loads(error.response.json()).get("error", "nc")
             _logger.exception("Bad google request : %s !", error_key)
             error_msg = _("Something went wrong during your token generation. Maybe your Authorization Code is invalid or already expired [%s]") % error_key
             raise self.env['res.config.settings'].get_config_warning(error_msg)
@@ -154,40 +155,34 @@ class GoogleService(models.TransientModel):
             :param type : the method to use to make the request
             :param preuri : pre url to prepend to param uri.
         """
-        _logger.debug("Uri: %s - Type : %s - Headers: %s - Params : %s !" % (uri, type, headers, werkzeug.url_encode(params) if type == 'GET' else params))
+        _logger.debug("Uri: %s - Type : %s - Headers: %s - Params : %s !", (uri, type, headers, params))
 
-        status = 418
-        response = ""
         ask_time = fields.Datetime.now()
         try:
-            if type.upper() == 'GET' or type.upper() == 'DELETE':
-                data = werkzeug.url_encode(params)
-                req = urllib2.Request(preuri + uri + "?" + data)
-            elif type.upper() == 'POST' or type.upper() == 'PATCH' or type.upper() == 'PUT':
-                req = urllib2.Request(preuri + uri, params, headers)
+            if type.upper() in ('GET', 'DELETE'):
+                res = requests.request(type.lower(), preuri + uri, params=params, timeout=TIMEOUT)
+            elif type.upper() in ('POST', 'PATCH', 'PUT'):
+                res = requests.request(type.lower(), preuri + uri, data=params, headers=headers, timeout=TIMEOUT)
             else:
                 raise Exception(_('Method not supported [%s] not in [GET, POST, PUT, PATCH or DELETE]!') % (type))
-            req.get_method = lambda: type.upper()
-
-            resp = urllib2.urlopen(req, timeout=TIMEOUT)
-            status = resp.getcode()
+            res.raise_for_status()
+            status = res.status_code
 
             if int(status) in (204, 404):  # Page not found, no response
                 response = False
             else:
-                content = resp.read()
-                response = json.loads(content)
+                response = res.json()
 
             try:
-                ask_time = datetime.strptime(resp.headers.get('date'), "%a, %d %b %Y %H:%M:%S %Z")
+                ask_time = datetime.strptime(res.headers.get('date'), "%a, %d %b %Y %H:%M:%S %Z")
             except:
                 pass
-        except urllib2.HTTPError as error:
-            if error.code in (204, 404):
+        except request.HTTPError as error:
+            if error.response.status_code in (204, 404):
                 status = error.code
                 response = ""
             else:
-                _logger.exception("Bad google request : %s !", error.read())
+                _logger.exception("Bad google request : %s !", error.response.content)
                 if error.code in (400, 401, 410):
                     raise error
                 raise self.env['res.config.settings'].get_config_warning(_("Something went wrong with your request to google"))

addons/google_calendar/models/google_calendar.py

@@ -2,12 +2,14 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from datetime import datetime, timedelta
+
+import requests
 from dateutil import parser
 import json
 import logging
 import operator
 import pytz
-import urllib2
+from werkzeug import urls
 
 from odoo import api, fields, models, tools, _
 from odoo.tools import exception_to_unicode, pycompat
@@ -258,7 +260,7 @@ class GoogleCalendar(models.AbstractModel):
         """
         data = self.generate_data(event, isCreating=True)
 
-        url = "/calendar/v3/calendars/%s/events?fields=%s&access_token=%s" % ('primary', urllib2.quote('id,updated'), self.get_token())
+        url = "/calendar/v3/calendars/%s/events?fields=%s&access_token=%s" % ('primary', urls.url_quote('id,updated'), self.get_token())
         headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
         data_json = json.dumps(data)
         return self.env['google.service']._do_request(url, data_json, headers, type='POST')
@@ -289,8 +291,8 @@ class GoogleCalendar(models.AbstractModel):
 
         try:
             status, content, ask_time = self.env['google.service']._do_request(url, params, headers, type='GET')
-        except urllib2.HTTPError as e:
-            if e.code == 401:  # Token invalid / Acces unauthorized
+        except requests.HTTPError as e:
+            if e.response.status_code == 401:  # Token invalid / Acces unauthorized
                 error_msg = _("Your token is invalid or has been revoked !")
 
                 self.env.user.write({'google_calendar_token': False, 'google_calendar_token_validity': False})
@@ -674,13 +676,13 @@ class GoogleCalendar(models.AbstractModel):
         if lastSync:
             try:
                 all_event_from_google = self.get_event_synchro_dict(lastSync=lastSync)
-            except urllib2.HTTPError as e:
-                if e.code == 410:  # GONE, Google is lost.
+            except requests.HTTPError as e:
+                if e.response.code == 410:  # GONE, Google is lost.
                     # we need to force the rollback from this cursor, because it locks my res_users but I need to write in this tuple before to raise.
                     self.env.cr.rollback()
                     self.env.user.write({'google_calendar_last_sync_date': False})
                     self.env.cr.commit()
-                error_key = json.loads(str(e))
+                error_key = e.response.json()
                 error_key = error_key.get('error', {}).get('message', 'nc')
                 error_msg = _("Google is lost... the next synchro will be a full synchro. \n\n %s") % error_key
                 raise self.env['res.config.settings'].get_config_warning(error_msg)

addons/google_drive/models/google_drive.py

@@ -3,7 +3,8 @@
 import logging
 import json
 import re
-import urllib2
+
+import requests
 import werkzeug.urls
 
 from odoo import api, fields, models
@@ -62,26 +63,25 @@ class GoogleDrive(models.Model):
         google_drive_client_id = Config.get_param('google_drive_client_id')
         google_drive_client_secret = Config.get_param('google_drive_client_secret')
         #For Getting New Access Token With help of old Refresh Token
-        data = werkzeug.url_encode({
+        data = {
             'client_id': google_drive_client_id,
             'refresh_token': google_drive_refresh_token,
             'client_secret': google_drive_client_secret,
             'grant_type': "refresh_token",
             'scope': scope or 'https://www.googleapis.com/auth/drive'
-        })
+        }
         headers = {"Content-type": "application/x-www-form-urlencoded"}
         try:
-            req = urllib2.Request(GOOGLE_TOKEN_ENDPOINT, data, headers)
-            content = urllib2.urlopen(req, timeout=TIMEOUT).read()
-        except urllib2.HTTPError:
+            req = requests.post(GOOGLE_TOKEN_ENDPOINT, data=data, headers=headers, timeout=TIMEOUT)
+            req.raise_for_status()
+        except requests.HTTPError:
             if user_is_admin:
                 dummy, action_id = self.env['ir.model.data'].get_object_reference('base_setup', 'action_general_configuration')
                 msg = _("Something went wrong during the token generation. Please request again an authorization code .")
                 raise RedirectWarning(msg, action_id, _('Go to the configuration panel'))
             else:
                 raise UserError(_("Google Drive is not yet configured. Please contact your administrator."))
-        content = json.loads(content)
-        return content.get('access_token')
+        return req.json().get('access_token')
 
     @api.model
     def copy_doc(self, res_id, template_id, name_gdocs, res_model):
@@ -91,11 +91,11 @@ class GoogleDrive(models.Model):
         request_url = "https://www.googleapis.com/drive/v2/files/%s?fields=parents/id&access_token=%s" % (template_id, access_token)
         headers = {"Content-type": "application/x-www-form-urlencoded"}
         try:
-            req = urllib2.Request(request_url, None, headers)
-            parents = urllib2.urlopen(req, timeout=TIMEOUT).read()
-        except urllib2.HTTPError:
+            req = requests.post(request_url, headers=headers, timeout=TIMEOUT)
+            req.raise_for_status()
+            parents_dict = req.json()
+        except requests.HTTPError:
             raise UserError(_("The Google Template cannot be found. Maybe it has been deleted."))
-        parents_dict = json.loads(parents)
 
         record_url = "Click on link to open Record in Odoo\n %s/?db=%s#id=%s&model=%s" % (google_web_base_url, self._cr.dbname, res_id, res_model)
         data = {
@@ -108,11 +108,10 @@ class GoogleDrive(models.Model):
             'Content-type': 'application/json',
             'Accept': 'text/plain'
         }
-        data_json = json.dumps(data)
         # resp, content = Http().request(request_url, "POST", data_json, headers)
-        req = urllib2.Request(request_url, data_json, headers)
-        content = urllib2.urlopen(req, timeout=TIMEOUT).read()
-        content = json.loads(content)
+        req = requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT)
+        req.raise_for_status()
+        content = req.json()
         res = {}
         if content.get('alternateLink'):
             res['id'] = self.env["ir.attachment"].create({
@@ -129,16 +128,15 @@ class GoogleDrive(models.Model):
             request_url = "https://www.googleapis.com/drive/v2/files/%s/permissions?emailMessage=This+is+a+drive+file+created+by+Odoo&sendNotificationEmails=false&access_token=%s" % (key, access_token)
             data = {'role': 'writer', 'type': 'anyone', 'value': '', 'withLink': True}
             try:
-                req = urllib2.Request(request_url, json.dumps(data), headers)
-                urllib2.urlopen(req, timeout=TIMEOUT)
-            except urllib2.HTTPError:
+                req = requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT)
+                req.raise_for_status()
+            except requests.HTTPError:
                 raise self.env['res.config.settings'].get_config_warning(_("The permission 'reader' for 'anyone with the link' has not been written on the document"))
             if self.env.user.email:
                 data = {'role': 'writer', 'type': 'user', 'value': self.env.user.email}
                 try:
-                    req = urllib2.Request(request_url, json.dumps(data), headers)
-                    urllib2.urlopen(req, timeout=TIMEOUT)
-                except urllib2.HTTPError:
+                    requests.post(request_url, data=json.dumps(data), headers=headers, timeout=TIMEOUT)
+                except requests.HTTPError:
                     pass
         return res
 

addons/google_spreadsheet/models/google_drive.py

@@ -1,14 +1,15 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import cgi
 import json
 import logging
+
+import requests
 from lxml import etree
 import re
 import werkzeug.urls
-import urllib2
 
 from odoo import api, models
+from odoo.tools import misc
 from odoo.addons.google_account import TIMEOUT
 
 _logger = logging.getLogger(__name__)
@@ -69,15 +70,16 @@ class GoogleDrive(models.Model):
       href="https://spreadsheets.google.com/feeds/cells/{key}/od6/private/full/R60C15"/>
     <gs:cell row="60" col="15" inputValue="{config}"/>
   </entry>
-</feed>''' .format(key=spreadsheet_key, formula=cgi.escape(formula, quote=True), config=cgi.escape(config_formula, quote=True))
+</feed>''' .format(key=spreadsheet_key, formula=misc.html_escape(formula), config=misc.html_escape(config_formula))
 
         try:
-            req = urllib2.Request(
+            req = requests.post(
                 'https://spreadsheets.google.com/feeds/cells/%s/od6/private/full/batch?%s' % (spreadsheet_key, werkzeug.url_encode({'v': 3, 'access_token': access_token})),
                 data=request,
-                headers={'content-type': 'application/atom+xml', 'If-Match': '*'})
-            urllib2.urlopen(req, timeout=TIMEOUT)
-        except (urllib2.HTTPError, urllib2.URLError):
+                headers={'content-type': 'application/atom+xml', 'If-Match': '*'},
+                timeout=TIMEOUT,
+            )
+        except IOError:
             _logger.warning("An error occured while writting the formula on the Google Spreadsheet.")
 
         description = '''

addons/hw_escpos/controllers/main.py

@@ -2,7 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from __future__ import print_function
-import commands
 import logging
 import math
 import os
@@ -18,7 +17,10 @@ try:
 except ImportError:
     escpos = printer = None
 
-from Queue import Queue
+try:
+    from queue import Queue
+except ImportError:
+    from Queue import Queue # pylint: disable=deprecated-module
 from threading import Thread, Lock
 
 try:
@@ -196,7 +198,7 @@ class EscposDriver(Thread):
         hosting_ap = os.system('pgrep hostapd') == 0
         ssid = subprocess.check_output('iwconfig 2>&1 | grep \'ESSID:"\' | sed \'s/.*"\\(.*\\)"/\\1/\'', shell=True).rstrip()
         mac = subprocess.check_output('ifconfig | grep -B 1 \'inet addr\' | grep -o \'HWaddr .*\' | sed \'s/HWaddr //\'', shell=True).rstrip()
-        ips =  [ c.split(':')[1].split(' ')[0] for c in commands.getoutput("/sbin/ifconfig").split('\n') if 'inet addr' in c ]
+        ips =  [ c.split(':')[1].split(' ')[0] for c in subprocess.check_output("/sbin/ifconfig").split('\n') if 'inet addr' in c ]
         ips =  [ ip for ip in ips if ip not in localips ] 
         eprint.text('\n\n')
         eprint.set(align='center',type='b',height=2,width=2)

addons/hw_posbox_homepage/controllers/main.py

@@ -8,6 +8,7 @@ import werkzeug
 
 import odoo
 from odoo import http
+from odoo.tools import misc
 
 _logger = logging.getLogger(__name__)
 
@@ -106,7 +107,7 @@ class PosboxHomepage(odoo.addons.web.controllers.main.Home):
             f = open('/tmp/scanned_networks.txt', 'r')
             for line in f:
                 line = line.rstrip()
-                line = werkzeug.utils.escape(line)
+                line = misc.html_escape(line)
                 wifi_template += '<option value="' + line + '">' + line + '</option>\n'
             f.close()
         except IOError:

addons/hw_proxy/controllers/main.py

@@ -2,8 +2,8 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from __future__ import print_function
-import commands
 import logging
+import subprocess
 import time
 from threading import Lock
 
@@ -16,11 +16,11 @@ _logger = logging.getLogger(__name__)
 
 # Those are the builtin raspberry pi USB modules, they should
 # not appear in the list of connected devices.
-BANNED_DEVICES = set([
+BANNED_DEVICES = {
 	"0424:9514",	# Standard Microsystem Corp. Builtin Ethernet module
 	"1d6b:0002",	# Linux Foundation 2.0 root hub
 	"0424:ec00",	# Standard Microsystem Corp. Other Builtin Ethernet module
-])
+}
 
 
 # drivers modules must add to drivers an object with a get_status() method 
@@ -99,14 +99,14 @@ class Proxy(http.Controller):
         """
         if debug is None:
             resp += """(<a href="/hw_proxy/status?debug">debug version</a>)"""
-        devices = commands.getoutput("lsusb").split('\n')
+        devices = subprocess.check_output("lsusb").split('\n')
         count   = 0
         resp += "<div class='devices'>\n"
         for device in devices:
             device_name = device[device.find('ID')+2:]
             device_id   = device_name.split()[0]
             if not (device_id in BANNED_DEVICES):
-            	resp+= "<div class='device' data-device='"+device+"'>"+device_name+"</div>\n"
+                resp += "<div class='device' data-device='"+device+"'>"+device_name+"</div>\n"
                 count += 1
         
         if count == 0:

addons/hw_scanner/controllers/main.py

@@ -5,7 +5,10 @@ import logging
 import time
 from os import listdir
 from os.path import join
-from Queue import Queue, Empty
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty # pylint: disable=deprecated-module
 from select import select
 from threading import Thread, Lock
 

addons/link_tracker/models/link_tracker.py

@@ -6,11 +6,10 @@ import random
 import re
 import string
 
+import requests
 from lxml import html
-from urllib2 import urlopen
-from urlparse import urljoin
-from urlparse import urlparse
-from werkzeug import url_encode, unescape
+from werkzeug import urls, utils
+
 
 from odoo import models, fields, api, _
 from odoo.tools import ustr, pycompat
@@ -18,7 +17,7 @@ from odoo.tools import ustr, pycompat
 URL_REGEX = r'(\bhref=[\'"](?!mailto:)([^\'"]+)[\'"])'
 
 def VALIDATE_URL(url):
-    if urlparse(url).scheme not in ('http', 'https', 'ftp', 'ftps'):
+    if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'):
         return 'http://' + url
 
     return url
@@ -56,7 +55,7 @@ class link_tracker(models.Model):
             href = match[0]
             long_url = match[1]
 
-            vals['url'] = unescape(long_url)
+            vals['url'] = utils.unescape(long_url)
 
             if not blacklist or not [s for s in blacklist if s in long_url] and not long_url.startswith(short_schema):
                 link = self.create(vals)
@@ -77,7 +76,7 @@ class link_tracker(models.Model):
     @api.depends('code')
     def _compute_short_url(self):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        self.short_url = urljoin(base_url, '/r/%(code)s' % {'code': self.code})
+        self.short_url = urls.url_join(base_url, '/r/%(code)s' % {'code': self.code})
 
     @api.one
     def _compute_short_url_host(self):
@@ -96,22 +95,23 @@ class link_tracker(models.Model):
     @api.one
     @api.depends('url')
     def _compute_redirected_url(self):
-        parsed = urlparse(self.url)
+        parsed = urls.url_parse(self.url)
 
         utms = {}
         for key, field, cook in self.env['utm.mixin'].tracking_fields():
             attr = getattr(self, field).name
             if attr:
                 utms[key] = attr
+        utms.update(parsed.decode_query())
 
-        self.redirected_url = '%s://%s%s?%s&%s#%s' % (parsed.scheme, parsed.netloc, parsed.path, url_encode(utms), parsed.query, parsed.fragment)
+        self.redirected_url = parsed.replace(query=urls.url_encode(utms)).to_url()
 
     @api.model
     @api.depends('url')
     def _get_title_from_url(self, url):
         try:
-            page = urlopen(url, timeout=5)
-            p = html.fromstring(ustr(page.read()).encode('utf-8'), parser=html.HTMLParser(encoding='utf-8'))
+            page = requests.get(url, timeout=5)
+            p = html.fromstring(page.text.encode('utf-8'), parser=html.HTMLParser(encoding='utf-8'))
             title = p.find('.//title').text
         except:
             title = url
@@ -122,7 +122,7 @@ class link_tracker(models.Model):
     @api.depends('url')
     def _compute_favicon(self):
         try:
-            icon = urlopen('http://www.google.com/s2/favicons?domain=' + self.url, timeout=5).read()
+            icon = requests.get('http://www.google.com/s2/favicons', params={'domain': self.url}, timeout=5).content
             icon_base64 = icon.encode('base64').replace("\n", "")
         except:
             icon_base64 = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsSAAALEgHS3X78AAACiElEQVQ4EaVTzU8TURCf2tJuS7tQtlRb6UKBIkQwkRRSEzkQgyEc6lkOKgcOph78Y+CgjXjDs2i44FXY9AMTlQRUELZapVlouy3d7kKtb0Zr0MSLTvL2zb75eL838xtTvV6H/xELBptMJojeXLCXyobnyog4YhzXYvmCFi6qVSfaeRdXdrfaU1areV5KykmX06rcvzumjY/1ggkR3Jh+bNf1mr8v1D5bLuvR3qDgFbvbBJYIrE1mCIoCrKxsHuzK+Rzvsi29+6DEbTZz9unijEYI8ObBgXOzlcrx9OAlXyDYKUCzwwrDQx1wVDGg089Dt+gR3mxmhcUnaWeoxwMbm/vzDFzmDEKMMNhquRqduT1KwXiGt0vre6iSeAUHNDE0d26NBtAXY9BACQyjFusKuL2Ry+IPb/Y9ZglwuVscdHaknUChqLF/O4jn3V5dP4mhgRJgwSYm+gV0Oi3XrvYB30yvhGa7BS70eGFHPoTJyQHhMK+F0ZesRVVznvXw5Ixv7/C10moEo6OZXbWvlFAF9FVZDOqEABUMRIkMd8GnLwVWg9/RkJF9sA4oDfYQAuzzjqzwvnaRUFxn/X2ZlmGLXAE7AL52B4xHgqAUqrC1nSNuoJkQtLkdqReszz/9aRvq90NOKdOS1nch8TpL555WDp49f3uAMXhACRjD5j4ykuCtf5PP7Fm1b0DIsl/VHGezzP1KwOiZQobFF9YyjSRYQETRENSlVzI8iK9mWlzckpSSCQHVALmN9Az1euDho9Xo8vKGd2rqooA8yBcrwHgCqYR0kMkWci08t/R+W4ljDCanWTg9TJGwGNaNk3vYZ7VUdeKsYJGFNkfSzjXNrSX20s4/h6kB81/271ghG17l+rPTAAAAAElFTkSuQmCC'

addons/mail/models/__init__.py

@@ -19,4 +19,3 @@ from . import update
 from . import ir_actions
 from . import ir_autovacuum
 from . import ir_model
-from . import html2text

addons/mail/models/html2text.py

@@ -1,457 +0,0 @@
-#!/usr/bin/env python
-"""html2text: Turn HTML into equivalent Markdown-structured text."""
-from odoo.tools import pycompat
-
-__version__ = "2.36"
-__author__ = "Aaron Swartz (me@aaronsw.com)"
-__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3."
-__contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
-
-# TODO:
-#   Support decoded entities with unifiable.
-
-import re, sys, urllib, htmlentitydefs, codecs
-import sgmllib
-import urlparse
-sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]')
-
-try: from textwrap import wrap
-except: pass
-
-# Use Unicode characters instead of their ascii psuedo-replacements
-UNICODE_SNOB = 0
-
-# Put the links after each paragraph instead of at the end.
-LINKS_EACH_PARAGRAPH = 0
-
-# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
-BODY_WIDTH = 78
-
-# Don't show internal links (href="#local-anchor") -- corresponding link targets
-# won't be visible in the plain text file anyway.
-SKIP_INTERNAL_LINKS = False
-
-### Entity Nonsense ###
-
-def name2cp(k):
-    if k == 'apos': return ord("'")
-    if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3
-        return htmlentitydefs.name2codepoint[k]
-    else:
-        k = htmlentitydefs.entitydefs[k]
-        if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1
-        return ord(codecs.latin_1_decode(k)[0])
-
-unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"',
-'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*',
-'ndash':'-', 'oelig':'oe', 'aelig':'ae',
-'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a',
-'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e',
-'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i',
-'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o',
-'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'}
-
-unifiable_n = {}
-
-for k in unifiable:
-    unifiable_n[name2cp(k)] = unifiable[k]
-
-def charref(name):
-    if name[0] in ['x','X']:
-        c = int(name[1:], 16)
-    else:
-        c = int(name)
-
-    if not UNICODE_SNOB and c in unifiable_n:
-        return unifiable_n[c]
-    else:
-        return unichr(c)
-
-def entityref(c):
-    if not UNICODE_SNOB and c in unifiable:
-        return unifiable[c]
-    else:
-        try: name2cp(c)
-        except KeyError: return "&" + c
-        else: return unichr(name2cp(c))
-
-def replaceEntities(s):
-    s = s.group(1)
-    if s[0] == "#":
-        return charref(s[1:])
-    else: return entityref(s)
-
-r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
-def unescape(s):
-    return r_unescape.sub(replaceEntities, s)
-
-def fixattrs(attrs):
-    # Fix bug in sgmllib.py
-    if not attrs: return attrs
-    newattrs = []
-    for attr in attrs:
-        newattrs.append((attr[0], unescape(attr[1])))
-    return newattrs
-
-### End Entity Nonsense ###
-
-def onlywhite(line):
-    """Return true if the line does only consist of whitespace characters."""
-    for c in line:
-        if c is not ' ' and c is not '  ':
-            return c is ' '
-    return line
-
-def optwrap(text):
-    """Wrap all paragraphs in the provided text."""
-    if not BODY_WIDTH:
-        return text
-
-    assert wrap, "Requires Python 2.3."
-    result = ''
-    newlines = 0
-    for para in text.split("\n"):
-        if len(para) > 0:
-            if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
-                for line in wrap(para, BODY_WIDTH):
-                    result += line + "\n"
-                result += "\n"
-                newlines = 2
-            else:
-                if not onlywhite(para):
-                    result += para + "\n"
-                    newlines = 1
-        else:
-            if newlines < 2:
-                result += "\n"
-                newlines += 1
-    return result
-
-def hn(tag):
-    if tag[0] == 'h' and len(tag) == 2:
-        try:
-            n = int(tag[1])
-            if n in range(1, 10): return n
-        except ValueError: return 0
-
-class _html2text(sgmllib.SGMLParser):
-    def __init__(self, out=sys.stdout.write, baseurl=''):
-        sgmllib.SGMLParser.__init__(self)
-
-        if out is None: self.out = self.outtextf
-        else: self.out = out
-        self.outtext = u''
-        self.quiet = 0
-        self.p_p = 0
-        self.outcount = 0
-        self.start = 1
-        self.space = 0
-        self.a = []
-        self.astack = []
-        self.acount = 0
-        self.list = []
-        self.blockquote = 0
-        self.pre = 0
-        self.startpre = 0
-        self.lastWasNL = 0
-        self.abbr_title = None # current abbreviation definition
-        self.abbr_data = None # last inner HTML (for abbr being defined)
-        self.abbr_list = {} # stack of abbreviations to write later
-        self.baseurl = baseurl
-
-    def outtextf(self, s):
-        self.outtext += s
-
-    def close(self):
-        sgmllib.SGMLParser.close(self)
-
-        self.pbr()
-        self.o('', 0, 'end')
-
-        return self.outtext
-
-    def handle_charref(self, c):
-        self.o(charref(c))
-
-    def handle_entityref(self, c):
-        self.o(entityref(c))
-
-    def unknown_starttag(self, tag, attrs):
-        self.handle_tag(tag, attrs, 1)
-
-    def unknown_endtag(self, tag):
-        self.handle_tag(tag, None, 0)
-
-    def previousIndex(self, attrs):
-        """ returns the index of certain set of attributes (of a link) in the
-            self.a list
-
-            If the set of attributes is not found, returns None
-        """
-        if 'href' not in attrs: return None
-
-        i = -1
-        for a in self.a:
-            i += 1
-            match = 0
-
-            if 'href' in a and a['href'] == attrs['href']:
-                if 'title' in a or 'title' in attrs:
-                        if ('title' in a and 'title' in attrs and
-                            a['title'] == attrs['title']):
-                            match = True
-                else:
-                    match = True
-
-            if match: return i
-
-    def handle_tag(self, tag, attrs, start):
-        attrs = fixattrs(attrs)
-
-        if hn(tag):
-            self.p()
-            if start: self.o(hn(tag)*"#" + ' ')
-
-        if tag in ['p', 'div']: self.p()
-
-        if tag == "br" and start: self.o("  \n")
-
-        if tag == "hr" and start:
-            self.p()
-            self.o("* * *")
-            self.p()
-
-        if tag in ["head", "style", 'script']:
-            if start: self.quiet += 1
-            else: self.quiet -= 1
-
-        if tag in ["body"]:
-            self.quiet = 0 # sites like 9rules.com never close <head>
-
-        if tag == "blockquote":
-            if start:
-                self.p(); self.o('> ', 0, 1); self.start = 1
-                self.blockquote += 1
-            else:
-                self.blockquote -= 1
-                self.p()
-
-        if tag in ['em', 'i', 'u']: self.o("_")
-        if tag in ['strong', 'b']: self.o("**")
-        if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` ``
-        if tag == "abbr":
-            if start:
-                attrsD = {}
-                for (x, y) in attrs: attrsD[x] = y
-                attrs = attrsD
-
-                self.abbr_title = None
-                self.abbr_data = ''
-                if 'title' in attrs:
-                    self.abbr_title = attrs['title']
-            else:
-                if self.abbr_title != None:
-                    self.abbr_list[self.abbr_data] = self.abbr_title
-                    self.abbr_title = None
-                self.abbr_data = ''
-
-        if tag == "a":
-            if start:
-                attrsD = {}
-                for (x, y) in attrs: attrsD[x] = y
-                attrs = attrsD
-                if 'href' in attrs and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')):
-                    self.astack.append(attrs)
-                    self.o("[")
-                else:
-                    self.astack.append(None)
-            else:
-                if self.astack:
-                    a = self.astack.pop()
-                    if a:
-                        i = self.previousIndex(a)
-                        if i is not None:
-                            a = self.a[i]
-                        else:
-                            self.acount += 1
-                            a['count'] = self.acount
-                            a['outcount'] = self.outcount
-                            self.a.append(a)
-                        self.o("][%r]" % a['count'])
-
-        if tag == "img" and start:
-            attrsD = {}
-            for (x, y) in attrs: attrsD[x] = y
-            attrs = attrsD
-            if 'src' in attrs:
-                attrs['href'] = attrs['src']
-                alt = attrs.get('alt', '')
-                i = self.previousIndex(attrs)
-                if i is not None:
-                    attrs = self.a[i]
-                else:
-                    self.acount += 1
-                    attrs['count'] = self.acount
-                    attrs['outcount'] = self.outcount
-                    self.a.append(attrs)
-                self.o("![")
-                self.o(alt)
-                self.o("][%r]" % attrs['count'])
-
-        if tag == 'dl' and start: self.p()
-        if tag == 'dt' and not start: self.pbr()
-        if tag == 'dd' and start: self.o('    ')
-        if tag == 'dd' and not start: self.pbr()
-
-        if tag in ["ol", "ul"]:
-            if start:
-                self.list.append({'name':tag, 'num':0})
-            else:
-                if self.list: self.list.pop()
-
-            self.p()
-
-        if tag == 'li':
-            if start:
-                self.pbr()
-                if self.list: li = self.list[-1]
-                else: li = {'name':'ul', 'num':0}
-                self.o("  "*len(self.list)) #TODO: line up <ol><li>s > 9 correctly.
-                if li['name'] == "ul": self.o("* ")
-                elif li['name'] == "ol":
-                    li['num'] += 1
-                    self.o("%r. " % li['num'])
-                self.start = 1
-            else:
-                self.pbr()
-
-        if tag in ["table", "tr"] and start: self.p()
-        if tag == 'td': self.pbr()
-
-        if tag == "pre":
-            if start:
-                self.startpre = 1
-                self.pre = 1
-            else:
-                self.pre = 0
-            self.p()
-
-    def pbr(self):
-        if self.p_p == 0: self.p_p = 1
-
-    def p(self):
-        self.p_p = 2
-
-    def o(self, data, puredata=0, force=0):
-        if self.abbr_data is not None: self.abbr_data += data
-
-        if not self.quiet:
-            if puredata and not self.pre:
-                data = re.sub('\s+', ' ', data)
-                if data and data[0] == ' ':
-                    self.space = 1
-                    data = data[1:]
-            if not data and not force: return
-
-            if self.startpre:
-                #self.out(" :") #TODO: not output when already one there
-                self.startpre = 0
-
-            bq = (">" * self.blockquote)
-            if not (force and data and data[0] == ">") and self.blockquote: bq += " "
-
-            if self.pre:
-                bq += "    "
-                data = data.replace("\n", "\n"+bq)
-
-            if self.start:
-                self.space = 0
-                self.p_p = 0
-                self.start = 0
-
-            if force == 'end':
-                # It's the end.
-                self.p_p = 0
-                self.out("\n")
-                self.space = 0
-
-
-            if self.p_p:
-                self.out(('\n'+bq)*self.p_p)
-                self.space = 0
-
-            if self.space:
-                if not self.lastWasNL: self.out(' ')
-                self.space = 0
-
-            if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"):
-                if force == "end": self.out("\n")
-
-                newa = []
-                for link in self.a:
-                    if self.outcount > link['outcount']:
-                        self.out("   ["+repr(link['count'])+"]: " + urlparse.urljoin(self.baseurl, link['href']))
-                        if 'title' in link: self.out(" ("+link['title']+")")
-                        self.out("\n")
-                    else:
-                        newa.append(link)
-
-                if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done.
-
-                self.a = newa
-
-            if self.abbr_list and force == "end":
-                for abbr, definition in pycompat.items(self.abbr_list):
-                    self.out("  *[" + abbr + "]: " + definition + "\n")
-
-            self.p_p = 0
-            self.out(data)
-            self.lastWasNL = data and data[-1] == '\n'
-            self.outcount += 1
-
-    def handle_data(self, data):
-        if r'\/script>' in data: self.quiet -= 1
-        self.o(data, 1)
-
-    def unknown_decl(self, data):
-        pass
-
-def wrapwrite(text): sys.stdout.write(text.encode('utf8'))
-
-def html2text_file(html, out=wrapwrite, baseurl=''):
-    h = _html2text(out, baseurl)
-    h.feed(html)
-    h.feed("")
-    return h.close()
-
-def html2text(html, baseurl=''):
-    return optwrap(html2text_file(html, None, baseurl))
-
-if __name__ == "__main__":
-    baseurl = ''
-    if sys.argv[1:]:
-        arg = sys.argv[1]
-        if arg.startswith('http://'):
-            baseurl = arg
-            j = urllib.urlopen(baseurl)
-            try:
-                from feedparser import _getCharacterEncoding as enc
-            except ImportError:
-                   enc = lambda x, y: ('utf-8', 1)
-            text = j.read()
-            encoding = enc(j.headers, text)[0]
-            if encoding == 'us-ascii': encoding = 'utf-8'
-            data = text.decode(encoding)
-
-        else:
-            encoding = 'utf8'
-            if len(sys.argv) > 2:
-                encoding = sys.argv[2]
-            f = open(arg, 'r')
-            try:
-                    data = f.read().decode(encoding)
-            finally:
-                    f.close()
-    else:
-        data = sys.stdin.read().decode('utf8')
-    wrapwrite(html2text(data, baseurl))

addons/mail/models/mail_template.py

@@ -10,9 +10,7 @@ import logging
 
 import functools
 import lxml
-import urlparse
-
-from urllib import urlencode, quote as quote
+from werkzeug import urls
 
 from odoo import _, api, fields, models, tools
 from odoo.exceptions import UserError
@@ -83,8 +81,8 @@ try:
     )
     mako_template_env.globals.update({
         'str': str,
-        'quote': quote,
-        'urlencode': urlencode,
+        'quote': urls.url_quote,
+        'urlencode': urls.url_encode,
         'datetime': datetime,
         'len': len,
         'abs': abs,
@@ -297,14 +295,13 @@ class MailTemplate(models.Model):
             root = lxml.html.fromstring(html)
 
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        (base_scheme, base_netloc, bpath, bparams, bquery, bfragment) = urlparse.urlparse(base_url)
+        base = urls.url_parse(base_url)
 
         def _process_link(url):
-            new_url = url
-            (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url)
-            if not scheme and not netloc:
-                new_url = urlparse.urlunparse((base_scheme, base_netloc, path, params, query, fragment))
-            return new_url
+            new_url = urls.url_parse(url)
+            if new_url.scheme and new_url.netloc:
+                return url
+            return new_url.replace(scheme=base.scheme, netloc=base.netloc).to_url()
 
         # check all nodes, replace :
         # - img src -> check URL

addons/mail/models/res_config.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import urlparse
 import datetime
 
+from werkzeug import urls
+
 from odoo import api, fields, models, tools
 
 
@@ -29,7 +30,7 @@ class BaseConfiguration(models.TransientModel):
         if alias_domain is None:
             domain = self.env["ir.config_parameter"].get_param("web.base.url")
             try:
-                alias_domain = urlparse.urlsplit(domain).netloc.split(':')[0]
+                alias_domain = urls.url_parse(domain).host
             except Exception:
                 pass
         return {'alias_domain': alias_domain or False}

addons/mail/models/update.py

@@ -3,8 +3,9 @@
 
 import datetime
 import logging
+
+import requests
 import werkzeug.urls
-import urllib2
 
 from ast import literal_eval
 
@@ -71,16 +72,12 @@ class PublisherWarrantyContract(AbstractModel):
         """
         msg = self._get_message()
         arguments = {'arg0': msg, "action": "update"}
-        arguments_raw = werkzeug.urls.url_encode(arguments)
 
         url = config.get("publisher_warranty_url")
 
-        uo = urllib2.urlopen(url, arguments_raw, timeout=30)
-        try:
-            submit_result = uo.read()
-            return literal_eval(submit_result)
-        finally:
-            uo.close()
+        r = requests.post(url, data=arguments, timeout=30)
+        r.raise_for_status()
+        return literal_eval(r.text)
 
     @api.multi
     def update_notification(self, cron_mode=True):

addons/mass_mailing/models/mail_mail.py

@@ -2,7 +2,6 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import re
-import urlparse
 import werkzeug.urls
 
 from odoo import api, fields, models, tools
@@ -29,20 +28,20 @@ class MailMail(models.Model):
 
     def _get_tracking_url(self, partner=None):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        track_url = urlparse.urljoin(
+        track_url = werkzeug.urls.url_join(
             base_url, 'mail/track/%(mail_id)s/blank.gif?%(params)s' % {
                 'mail_id': self.id,
-                'params': werkzeug.url_encode({'db': self.env.cr.dbname})
+                'params': werkzeug.urls.url_encode({'db': self.env.cr.dbname})
             }
         )
         return '<img src="%s" alt=""/>' % track_url
 
     def _get_unsubscribe_url(self, email_to):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-        url = urlparse.urljoin(
+        url = werkzeug.urls.url_join(
             base_url, 'mail/mailing/%(mailing_id)s/unsubscribe?%(params)s' % {
                 'mailing_id': self.mailing_id.id,
-                'params': werkzeug.url_encode({
+                'params': werkzeug.urls.url_encode({
                     'db': self.env.cr.dbname,
                     'res_id': self.res_id,
                     'email': email_to,
@@ -79,7 +78,7 @@ class MailMail(models.Model):
         body = tools.append_content_to_html(base, body, plaintext=False, container_tag='div')
         # resolve relative image url to absolute for outlook.com
         def _sub_relative2absolute(match):
-            return match.group(1) + urlparse.urljoin(domain, match.group(2))
+            return match.group(1) + werkzeug.urls.url_join(domain, match.group(2))
         body = re.sub('(<img(?=\s)[^>]*\ssrc=")(/[^/][^"]+)', _sub_relative2absolute, body)
         body = re.sub(r'(<[^>]+\bstyle="[^"]+\burl\(\'?)(/[^/\'][^\'")]+)', _sub_relative2absolute, body)
 

addons/pad/models/pad.py

@@ -5,7 +5,8 @@ import logging
 import random
 import re
 import string
-import urllib2
+
+import requests
 
 from odoo import api, models, _
 from odoo.exceptions import UserError
@@ -53,7 +54,7 @@ class PadCommon(models.AbstractModel):
             myPad = EtherpadLiteClient(pad["key"], pad["server"] + '/api')
             try:
                 myPad.createPad(path)
-            except urllib2.URLError:
+            except IOError:
                 raise UserError(_("Pad creation failed, either there is a problem with your pad server URL or with your connection."))
 
             # get attr on the field model
@@ -79,12 +80,15 @@ class PadCommon(models.AbstractModel):
         content = ''
         if url:
             try:
-                page = urllib2.urlopen('%s/export/html' % url).read()
-                mo = re.search('<body>(.*)</body>', page, re.DOTALL)
-                if mo:
-                    content = mo.group(1)
+                r = requests.get('%s/export/html' % url)
+                r.raise_for_status()
             except:
                 _logger.warning("No url found '%s'.", url)
+            else:
+                mo = re.search('<body>(.*)</body>', r.content, re.DOTALL)
+                if mo:
+                    content = mo.group(1)
+
         return content
 
     # TODO

addons/pad/py_etherpad/__init__.py

@@ -1,8 +1,5 @@
 """Module to talk to EtherpadLite API."""
-
-import json
-import urllib
-import urllib2
+import requests
 
 
 class EtherpadLiteClient:
@@ -31,23 +28,11 @@ class EtherpadLiteClient:
         url = '%s/%d/%s' % (self.baseUrl, self.API_VERSION, function)
 
         params = arguments or {}
-        params.update({'apikey': self.apiKey})
-        data = urllib.urlencode(params, True)
+        params['apikey'] = self.apiKey
 
-        try:
-            opener = urllib2.build_opener()
-            request = urllib2.Request(url=url, data=data)
-            response = opener.open(request, timeout=self.TIMEOUT)
-            result = response.read()
-            response.close()
-        except urllib2.HTTPError:
-            raise
-
-        result = json.loads(result)
-        if result is None:
-            raise ValueError("JSON response could not be decoded")
-
-        return self.handleResult(result)
+        r = requests.get(url, data=params, timeout=self.TIMEOUT)
+        r.raise_for_status()
+        return self.handleResult(r.json())
 
     def handleResult(self, result):
         """Handle API call result"""

addons/payment_adyen/models/payment.py

@@ -7,9 +7,10 @@ from collections import OrderedDict
 import hashlib
 import hmac
 import logging
-import urlparse
 from itertools import chain
 
+from werkzeug import urls
+
 from odoo import api, fields, models, tools, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment_adyen.controllers.main import AdyenController
@@ -132,7 +133,7 @@ class AcquirerAdyen(models.Model):
                 'merchantAccount': self.adyen_merchant_account,
                 'shopperLocale': values.get('partner_lang', ''),
                 'sessionValidity': tmp_date.isoformat('T')[:19] + "Z",
-                'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url),
+                'resURL': urls.url_join(base_url, AdyenController._return_url),
                 'merchantReturnData': json.dumps({'return_url': '%s' % values.pop('return_url')}) if values.get('return_url', '') else False,
                 'shopperEmail': values.get('partner_email', ''),
             })
@@ -150,7 +151,7 @@ class AcquirerAdyen(models.Model):
                 'merchantAccount': self.adyen_merchant_account,
                 'shopperLocale': values.get('partner_lang'),
                 'sessionValidity': tmp_date,
-                'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url),
+                'resURL': urls.url_join(base_url, AdyenController._return_url),
                 'merchantReturnData': json.dumps({'return_url': '%s' % values.pop('return_url')}) if values.get('return_url') else False,
                 'merchantSig': self._adyen_generate_merchant_sig('in', values),
             })

addons/payment_adyen/tests/test_adyen.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from lxml import objectify
-import urlparse
 
 from odoo.addons.payment.tests.common import PaymentAcquirerCommon
 from odoo.addons.payment_adyen.controllers.main import AdyenController
+from werkzeug import urls
 
 
 class AdyenCommon(PaymentAcquirerCommon):
@@ -47,7 +47,7 @@ class AdyenForm(AdyenCommon):
             'skinCode': 'cbqYWvVL',
             'paymentAmount': '1',
             'currencyCode': 'EUR',
-            'resURL': '%s' % urlparse.urljoin(base_url, AdyenController._return_url),
+            'resURL': urls.url_join(base_url, AdyenController._return_url),
         }
 
         # render the button

addons/payment_authorize/controllers/main.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 import pprint
 import logging
-import urlparse
-import werkzeug
+from werkzeug import urls, utils
 
 from odoo import http
 from odoo.http import request
@@ -29,7 +28,7 @@ class AuthorizeController(http.Controller):
         # This response is in the form of a URL that Authorize.Net will pass on to the
         # client's browser to redirect them to the desired location need javascript.
         return request.render('payment_authorize.payment_authorize_redirect', {
-            'return_url': '%s' % urlparse.urljoin(base_url, return_url)
+            'return_url': urls.url_join(base_url, return_url)
         })
 
     @http.route(['/payment/authorize/s2s/create_json'], type='json', auth='public')
@@ -43,4 +42,4 @@ class AuthorizeController(http.Controller):
         acquirer_id = int(post.get('acquirer_id'))
         acquirer = request.env['payment.acquirer'].browse(acquirer_id)
         acquirer.s2s_process(post)
-        return werkzeug.utils.redirect(post.get('return_url', '/'))
+        return utils.redirect(post.get('return_url', '/'))

addons/payment_authorize/models/authorize_request.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
+import requests
 from lxml import etree, objectify
-from urllib2 import urlopen, Request
 from StringIO import StringIO
 import xml.etree.ElementTree as ET
 from uuid import uuid4
@@ -57,10 +57,9 @@ class AuthorizeAPI():
         :param etree._Element data: etree data to process
         """
         data = etree.tostring(data, xml_declaration=True, encoding='utf-8')
-        request = Request(self.url, data)
-        request.add_header('Content-Type', 'text/xml')
-        response = urlopen(request).read()
-        response = strip_ns(response, XMLNS)
+        r = requests.post(self.url, data=data, headers={'Content-Type': 'text/xml'})
+        r.raise_for_status()
+        response = strip_ns(r.content, XMLNS)
         if response.find('messages/resultCode').text == 'Error':
             messages = [m.text for m in response.findall('messages/message/text')]
             raise ValidationError('Authorize.net Error Message(s):\n %s' % '\n'.join(messages))

addons/payment_authorize/models/payment.py

@@ -1,4 +1,5 @@
 # coding: utf-8
+from werkzeug import urls
 
 from .authorize_request import AuthorizeAPI
 from datetime import datetime
@@ -6,7 +7,6 @@ import hashlib
 import hmac
 import logging
 import time
-import urlparse
 
 from odoo import api, fields, models
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -72,8 +72,8 @@ class PaymentAcquirerAuthorize(models.Model):
             'x_version': '3.1',
             'x_relay_response': 'TRUE',
             'x_fp_timestamp': str(int(time.time())),
-            'x_relay_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._return_url),
-            'x_cancel_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._cancel_url),
+            'x_relay_url': urls.url_join(base_url, AuthorizeController._return_url),
+            'x_cancel_url': urls.url_join(base_url, AuthorizeController._cancel_url),
             'x_currency_code': values['currency'] and values['currency'].name or '',
             'address': values.get('partner_address'),
             'city': values.get('partner_city'),

addons/payment_authorize/tests/test_authorize.py

@@ -3,9 +3,9 @@
 import hashlib
 import hmac
 import time
-import urlparse
 import unittest
 from lxml import objectify
+from werkzeug import urls
 
 import odoo
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -59,8 +59,8 @@ class AuthorizeForm(AuthorizeCommon):
             'x_version': '3.1',
             'x_relay_response': 'TRUE',
             'x_fp_timestamp': str(int(time.time())),
-            'x_relay_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._return_url),
-            'x_cancel_url': '%s' % urlparse.urljoin(base_url, AuthorizeController._cancel_url),
+            'x_relay_url': urls.url_join(base_url, AuthorizeController._return_url),
+            'x_cancel_url': urls.url_join(base_url, AuthorizeController._cancel_url),
             'return_url': None,
             'x_currency_code': 'USD',
             'x_invoice_num': 'SO004',

addons/payment_buckaroo/models/payment.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 from hashlib import sha1
 import logging
-import urllib
-import urlparse
+
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -74,14 +74,11 @@ class AcquirerBuckaroo(models.Model):
                     break
 
             items = sorted(pycompat.items(values), key=lambda pair: pair[0].lower())
-            sign = ''.join('%s=%s' % (k, urllib.unquote_plus(v)) for k, v in items)
+            sign = ''.join('%s=%s' % (k, urls.url_unquote_plus(v)) for k, v in items)
         else:
             sign = ''.join('%s=%s' % (k, get_value(k)) for k in keys)
         # Add the pre-shared secret key at the end of the signature
         sign = sign + self.brq_secretkey
-        if isinstance(sign, str):
-            # TODO: remove me? should not be used
-            sign = urlparse.parse_qsl(sign)
         shasign = sha1(sign.encode('utf-8')).hexdigest()
         return shasign
 
@@ -95,10 +92,10 @@ class AcquirerBuckaroo(models.Model):
             'Brq_currency': values['currency'] and values['currency'].name or '',
             'Brq_invoicenumber': values['reference'],
             'brq_test': False if self.environment == 'prod' else True,
-            'Brq_return': '%s' % urlparse.urljoin(base_url, BuckarooController._return_url),
-            'Brq_returncancel': '%s' % urlparse.urljoin(base_url, BuckarooController._cancel_url),
-            'Brq_returnerror': '%s' % urlparse.urljoin(base_url, BuckarooController._exception_url),
-            'Brq_returnreject': '%s' % urlparse.urljoin(base_url, BuckarooController._reject_url),
+            'Brq_return': urls.url_join(base_url, BuckarooController._return_url),
+            'Brq_returncancel': urls.url_join(base_url, BuckarooController._cancel_url),
+            'Brq_returnerror': urls.url_join(base_url, BuckarooController._exception_url),
+            'Brq_returnreject': urls.url_join(base_url, BuckarooController._reject_url),
             'Brq_culture': (values.get('partner_lang') or 'en_US').replace('_', '-'),
             'add_returndata': buckaroo_tx_values.pop('return_url', '') or '',
         })

addons/payment_buckaroo/tests/test_buckaroo.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from lxml import objectify
-import urlparse
+from werkzeug import urls
 
 import odoo
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -41,10 +41,10 @@ class BuckarooForm(BuckarooCommon):
             'Brq_invoicenumber': 'SO004',
             'Brq_signature': '1b8c10074c622d965272a91a9e88b5b3777d2474',  # update me
             'brq_test': 'True',
-            'Brq_return': '%s' % urlparse.urljoin(base_url, BuckarooController._return_url),
-            'Brq_returncancel': '%s' % urlparse.urljoin(base_url, BuckarooController._cancel_url),
-            'Brq_returnerror': '%s' % urlparse.urljoin(base_url, BuckarooController._exception_url),
-            'Brq_returnreject': '%s' % urlparse.urljoin(base_url, BuckarooController._reject_url),
+            'Brq_return': urls.url_join(base_url, BuckarooController._return_url),
+            'Brq_returncancel': urls.url_join(base_url, BuckarooController._cancel_url),
+            'Brq_returnerror': urls.url_join(base_url, BuckarooController._exception_url),
+            'Brq_returnreject': urls.url_join(base_url, BuckarooController._reject_url),
             'Brq_culture': 'en-US',
         }
 

addons/payment_ogone/models/payment.py

@@ -1,22 +1,20 @@
 # coding: utf-8
-
-from hashlib import sha1
-from lxml import etree, objectify
-from pprint import pformat
-from unicodedata import normalize
-from urllib import urlencode
-
 import datetime
 import logging
 import time
-import urllib2
-import urlparse
+from hashlib import sha1
+from pprint import pformat
+from unicodedata import normalize
+
+import requests
+from lxml import etree, objectify
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment_ogone.controllers.main import OgoneController
 from odoo.addons.payment_ogone.data import ogone
-from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT
+from odoo.tools import float_round, DEFAULT_SERVER_DATE_FORMAT, pycompat
 from odoo.tools.float_utils import float_compare, float_repr
 from odoo.tools.safe_eval import safe_eval
 
@@ -165,10 +163,10 @@ class PaymentAcquirerOgone(models.Model):
             'OWNERTOWN': values.get('partner_city'),
             'OWNERCTY': values.get('partner_country') and values.get('partner_country').code or '',
             'OWNERTELNO': values.get('partner_phone'),
-            'ACCEPTURL': '%s' % urlparse.urljoin(base_url, OgoneController._accept_url),
-            'DECLINEURL': '%s' % urlparse.urljoin(base_url, OgoneController._decline_url),
-            'EXCEPTIONURL': '%s' % urlparse.urljoin(base_url, OgoneController._exception_url),
-            'CANCELURL': '%s' % urlparse.urljoin(base_url, OgoneController._cancel_url),
+            'ACCEPTURL': urls.url_join(base_url, OgoneController._accept_url),
+            'DECLINEURL': urls.url_join(base_url, OgoneController._decline_url),
+            'EXCEPTIONURL': urls.url_join(base_url, OgoneController._exception_url),
+            'CANCELURL': urls.url_join(base_url, OgoneController._cancel_url),
             'PARAMPLUS': 'return_url=%s' % ogone_tx_values.pop('return_url') if ogone_tx_values.get('return_url') else False,
         }
         if self.save_token in ['ask', 'always']:
@@ -370,8 +368,7 @@ class PaymentTxOgone(models.Model):
         direct_order_url = 'https://secure.ogone.com/ncol/%s/orderdirect.asp' % (self.acquirer_id.environment)
 
         _logger.debug("Ogone data %s", pformat(data))
-        request = urllib2.Request(direct_order_url, urlencode(data))
-        result = urllib2.urlopen(request).read()
+        result = requests.post(direct_order_url, data=data).content
         _logger.debug('Ogone response = %s', result)
 
         try:
@@ -455,8 +452,7 @@ class PaymentTxOgone(models.Model):
         query_direct_url = 'https://secure.ogone.com/ncol/%s/querydirect.asp' % (self.acquirer_id.environment)
 
         _logger.debug("Ogone data %s", pformat(data))
-        request = urllib2.Request(query_direct_url, urlencode(data))
-        result = urllib2.urlopen(request).read()
+        result = requests.post(query_direct_url, data=data).content
         _logger.debug('Ogone response = %s', result)
 
         try:
@@ -497,9 +493,7 @@ class PaymentToken(models.Model):
             }
 
             url = 'https://secure.ogone.com/ncol/%s/AFU_agree.asp' % (acquirer.environment,)
-            request = urllib2.Request(url, urlencode(data))
-
-            result = urllib2.urlopen(request).read()
+            result = requests.post(url, data=data).content
 
             try:
                 tree = objectify.fromstring(result)

addons/payment_ogone/tests/test_ogone.py

@@ -2,11 +2,12 @@
 
 from lxml import objectify
 import time
-import urlparse
 
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment.tests.common import PaymentAcquirerCommon
 from odoo.addons.payment_ogone.controllers.main import OgoneController
+from werkzeug import urls
+
 from odoo.tools import mute_logger
 
 
@@ -40,10 +41,10 @@ class OgonePayment(PaymentAcquirerCommon):
             'OWNERTOWN': 'Sin City',
             'OWNERTELNO': '0032 12 34 56 78',
             'SHASIGN': '815f67b8ff70d234ffcf437c13a9fa7f807044cc',
-            'ACCEPTURL': '%s' % urlparse.urljoin(base_url, OgoneController._accept_url),
-            'DECLINEURL': '%s' % urlparse.urljoin(base_url, OgoneController._decline_url),
-            'EXCEPTIONURL': '%s' % urlparse.urljoin(base_url, OgoneController._exception_url),
-            'CANCELURL': '%s' % urlparse.urljoin(base_url, OgoneController._cancel_url),
+            'ACCEPTURL': urls.url_join(base_url, OgoneController._accept_url),
+            'DECLINEURL': urls.url_join(base_url, OgoneController._decline_url),
+            'EXCEPTIONURL': urls.url_join(base_url, OgoneController._exception_url),
+            'CANCELURL': urls.url_join(base_url, OgoneController._cancel_url),
         }
 
         # render the button

addons/payment_paypal/controllers/main.py

@@ -3,9 +3,10 @@
 import json
 import logging
 import pprint
-import urllib
-import urllib2
+
+import requests
 import werkzeug
+from werkzeug import urls
 
 from odoo import http
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -23,7 +24,7 @@ class PaypalController(http.Controller):
         """ Extract the return URL from the data coming from paypal. """
         return_url = post.pop('return_url', '')
         if not return_url:
-            custom = json.loads(urllib.unquote_plus(post.pop('custom', False) or post.pop('cm', False) or '{}'))
+            custom = json.loads(urls.url_unquote_plus(post.pop('custom', False) or post.pop('cm', False) or '{}'))
             return_url = custom.get('return_url', '/')
         return return_url
 
@@ -44,7 +45,7 @@ class PaypalController(http.Controller):
         for line in lines:
             split = line.split('=', 1)
             if len(split) == 2:
-                pdt_post[split[0]] = urllib.unquote_plus(split[1]).decode('utf8')
+                pdt_post[split[0]] = urls.url_unquote_plus(split[1]).decode('utf8')
             else:
                 _logger.warning('Paypal: error processing pdt response: %s', line)
 
@@ -75,9 +76,9 @@ class PaypalController(http.Controller):
             new_post['at'] = tx and tx.acquirer_id.paypal_pdt_token or ''
             new_post['cmd'] = '_notify-synch'  # command is different in PDT than IPN/DPN
         validate_url = paypal_urls['paypal_form_url']
-        urequest = urllib2.Request(validate_url, werkzeug.url_encode(new_post))
-        uopen = urllib2.urlopen(urequest)
-        resp = uopen.read()
+        urequest = requests.post(validate_url, new_post)
+        urequest.raise_for_status()
+        resp = urequest.content
         if pdt_request:
             resp, post = self._parse_pdt_response(resp)
         if resp in ['VERIFIED', 'SUCCESS']:

addons/payment_paypal/models/payment.py

@@ -2,10 +2,10 @@
 
 import json
 import logging
-import urlparse
 
 import dateutil.parser
 import pytz
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -109,9 +109,9 @@ class AcquirerPaypal(models.Model):
             'zip_code': values.get('partner_zip'),
             'first_name': values.get('partner_first_name'),
             'last_name': values.get('partner_last_name'),
-            'paypal_return': '%s' % urlparse.urljoin(base_url, PaypalController._return_url),
-            'notify_url': '%s' % urlparse.urljoin(base_url, PaypalController._notify_url),
-            'cancel_return': '%s' % urlparse.urljoin(base_url, PaypalController._cancel_url),
+            'paypal_return': urls.url_join(base_url, PaypalController._return_url),
+            'notify_url': urls.url_join(base_url, PaypalController._notify_url),
+            'cancel_return': urls.url_join(base_url, PaypalController._cancel_url),
             'handling': '%.2f' % paypal_tx_values.pop('fees', 0.0) if self.fees_active else False,
             'custom': json.dumps({'return_url': '%s' % paypal_tx_values.pop('return_url')}) if paypal_tx_values.get('return_url') else False,
         })

addons/payment_paypal/tests/test_paypal.py

@@ -3,10 +3,11 @@
 from odoo.addons.payment.models.payment_acquirer import ValidationError
 from odoo.addons.payment.tests.common import PaymentAcquirerCommon
 from odoo.addons.payment_paypal.controllers.main import PaypalController
+from werkzeug import urls
+
 from odoo.tools import mute_logger
 
 from lxml import objectify
-import urlparse
 
 
 class PaypalCommon(PaymentAcquirerCommon):
@@ -60,9 +61,9 @@ class PaypalForm(PaypalCommon):
             'zip': '1000',
             'country': 'BE',
             'email': 'norbert.buyer@example.com',
-            'return': '%s' % urlparse.urljoin(base_url, PaypalController._return_url),
-            'notify_url': '%s' % urlparse.urljoin(base_url, PaypalController._notify_url),
-            'cancel_return': '%s' % urlparse.urljoin(base_url, PaypalController._cancel_url),
+            'return': urls.url_join(base_url, PaypalController._return_url),
+            'notify_url': urls.url_join(base_url, PaypalController._notify_url),
+            'cancel_return': urls.url_join(base_url, PaypalController._cancel_url),
         }
 
         # check form result

addons/payment_payumoney/models/payment.py

@@ -2,7 +2,8 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import hashlib
-import urlparse
+
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.addons.payment.models.payment_acquirer import ValidationError
@@ -64,9 +65,9 @@ class PaymentAcquirerPayumoney(models.Model):
                                 email=values.get('partner_email'),
                                 phone=values.get('partner_phone'),
                                 service_provider='payu_paisa',
-                                surl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/return'),
-                                furl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/error'),
-                                curl='%s' % urlparse.urljoin(base_url, '/payment/payumoney/cancel')
+                                surl=urls.url_join(base_url, '/payment/payumoney/return'),
+                                furl=urls.url_join(base_url, '/payment/payumoney/error'),
+                                curl=urls.url_join(base_url, '/payment/payumoney/cancel')
                                 )
 
         payumoney_values['udf1'] = payumoney_values.pop('return_url', '/')

addons/payment_sips/models/payment.py

@@ -3,7 +3,8 @@
 import json
 import logging
 from hashlib import sha256
-import urlparse
+
+from werkzeug import urls
 
 from odoo import models, fields, api
 from odoo.tools.float_utils import float_compare
@@ -89,8 +90,8 @@ class AcquirerSips(models.Model):
             'Data': u'amount=%s|' % amount +
                     u'currencyCode=%s|' % currency_code +
                     u'merchantId=%s|' % merchant_id +
-                    u'normalReturnUrl=%s|' % urlparse.urljoin(base_url, SipsController._return_url) +
-                    u'automaticResponseUrl=%s|' % urlparse.urljoin(base_url, SipsController._return_url) +
+                    u'normalReturnUrl=%s|' % urls.url_join(base_url, SipsController._return_url) +
+                    u'automaticResponseUrl=%s|' % urls.url_join(base_url, SipsController._return_url) +
                     u'transactionReference=%s|' % values['reference'] +
                     u'statementReference=%s|' % values['reference'] +
                     u'keyVersion=%s' % key_version,

addons/pos_mercury/models/pos_mercury_transaction.py

@@ -3,14 +3,12 @@
 
 from datetime import date, timedelta
 
-import cgi
-import urllib2
-import ssl
+import requests
 import werkzeug
 
 from odoo import models, api, service
 from odoo.exceptions import UserError
-from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
+from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT, misc
 
 
 class MercuryTransaction(models.Model):
@@ -52,7 +50,7 @@ class MercuryTransaction(models.Model):
 
         soap_header = '<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:mer="http://www.mercurypay.com"><soapenv:Header/><soapenv:Body><mer:CreditTransaction><mer:tran>'
         soap_footer = '</mer:tran><mer:pw>' + data['merchant_pwd'] + '</mer:pw></mer:CreditTransaction></soapenv:Body></soapenv:Envelope>'
-        xml_transaction = soap_header + cgi.escape(xml_transaction) + soap_footer
+        xml_transaction = soap_header + misc.escape_html(xml_transaction) + soap_footer
 
         response = ''
 
@@ -61,11 +59,11 @@ class MercuryTransaction(models.Model):
             'SOAPAction': 'http://www.mercurypay.com/CreditTransaction',
         }
 
-        r = urllib2.Request('https://w1.mercurypay.com/ws/ws.asmx', data=xml_transaction, headers=headers)
         try:
-            u = urllib2.urlopen(r, timeout=65)
-            response = werkzeug.utils.unescape(u.read())
-        except (urllib2.URLError, ssl.SSLError):
+            r = requests.post('https://w1.mercurypay.com/ws/ws.asmx', data=xml_transaction, headers=headers, timeout=65)
+            r.raise_for_status()
+            response = werkzeug.utils.unescape(r.content)
+        except:
             response = "timeout"
 
         return response

addons/survey/models/survey.py

@@ -5,10 +5,11 @@ import datetime
 import logging
 import re
 import uuid
-from urlparse import urljoin
 from collections import Counter, OrderedDict
 from itertools import product
 
+from werkzeug import urls
+
 from odoo import api, fields, models, tools, SUPERUSER_ID, _
 from odoo.exceptions import UserError, ValidationError
 
@@ -108,9 +109,9 @@ class Survey(models.Model):
         base_url = '/' if self.env.context.get('relative_url') else \
                    self.env['ir.config_parameter'].sudo().get_param('web.base.url')
         for survey in self:
-            survey.public_url = urljoin(base_url, "survey/start/%s" % (slug(survey)))
-            survey.print_url = urljoin(base_url, "survey/print/%s" % (slug(survey)))
-            survey.result_url = urljoin(base_url, "survey/results/%s" % (slug(survey)))
+            survey.public_url = urls.url_join(base_url, "survey/start/%s" % (slug(survey)))
+            survey.print_url = urls.url_join(base_url, "survey/print/%s" % (slug(survey)))
+            survey.result_url = urls.url_join(base_url, "survey/results/%s" % (slug(survey)))
             survey.public_url_html = '<a href="%s">%s</a>' % (survey.public_url, _("Click here to start survey"))
 
     @api.model

addons/survey/tests/test_survey.py

@@ -5,7 +5,8 @@ import random
 import re
 from collections import Counter
 from itertools import product
-from urlparse import urljoin
+
+from werkzeug import urls
 
 from odoo import _
 from odoo.exceptions import UserError
@@ -183,7 +184,7 @@ class TestSurvey(TransactionCase):
             survey_url_relative = getattr(self.survey1.with_context({'relative_url': True}), urltype + '_url')
             self.assertTrue(validate_url(survey_url))
             url = "survey/%s/%s" % (urltxt, slug(self.survey1))
-            full_url = urljoin(base_url, url)
+            full_url = urls.url_join(base_url, url)
             self.assertEqual(full_url, survey_url)
             self.assertEqual('/' + url, survey_url_relative)
             if urltype == 'public':

addons/survey/wizard/survey_email_compose_message.py

@@ -3,7 +3,8 @@
 
 import re
 import uuid
-import urlparse
+
+from werkzeug import urls
 
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
@@ -85,7 +86,7 @@ class SurveyMailComposeMessage(models.TransientModel):
             #set url
             url = wizard.survey_id.public_url
 
-            url = urlparse.urlparse(url).path[1:]  # dirty hack to avoid incorrect urls
+            url = urls.url_parse(url).path[1:]  # dirty hack to avoid incorrect urls
 
             if token:
                 url = url + '/' + token

addons/web_editor/models/ir_qweb.py

@@ -14,15 +14,16 @@ import itertools
 import json
 import logging
 import os
-import urllib2
-import urlparse
 import re
 import hashlib
 
 import pytz
+import requests
 from dateutil import parser
 from lxml import etree, html
 from PIL import Image as I
+from werkzeug import urls
+
 import odoo.modules
 
 from odoo import api, models, fields
@@ -319,11 +320,11 @@ class Image(models.AbstractModel):
     def from_html(self, model, field, element):
         url = element.find('img').get('src')
 
-        url_object = urlparse.urlsplit(url)
+        url_object = urls.url_parse(url)
         if url_object.path.startswith('/web/image'):
             # url might be /web/image/<model>/<id>[_<checksum>]/<field>[/<width>x<height>]
             fragments = url_object.path.split('/')
-            query = dict(urlparse.parse_qsl(url_object.query))
+            query = url_object.decode_query()
             if fragments[3].isdigit():
                 model = 'ir.attachment'
                 oid = fragments[3]
@@ -341,7 +342,7 @@ class Image(models.AbstractModel):
         return self.load_remote_url(url)
 
     def load_local_url(self, url):
-        match = self.local_url_re.match(urlparse.urlsplit(url).path)
+        match = self.local_url_re.match(urls.url_parse(url).path)
 
         rest = match.group('rest')
         for sep in os.sep, os.altsep:
@@ -374,9 +375,9 @@ class Image(models.AbstractModel):
             #   linking to HTTP images
             # implement drag & drop image upload to mitigate?
 
-            req = urllib2.urlopen(url, timeout=REMOTE_CONNECTION_TIMEOUT)
-            # PIL needs a seekable file-like image, urllib result is not seekable
-            image = I.open(cStringIO.StringIO(req.read()))
+            req = requests.get(url, timeout=REMOTE_CONNECTION_TIMEOUT)
+            # PIL needs a seekable file-like image so wrap result in IO buffer
+            image = I.open(cStringIO.StringIO(req.content))
             # force a complete load of the image data to validate it
             image.load()
         except Exception:

addons/web_planner/models/web_planner.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
-from urllib import urlencode
+from werkzeug import urls
 
 from odoo import api, models, fields
 
@@ -69,7 +68,7 @@ class Planner(models.Model):
                 params['id'] = module.id
             else:
                 return "#show_enterprise"
-        return "/web#%s" % (urlencode(params),)
+        return "/web#%s" % (urls.url_encode(params),)
 
     @api.model
     def is_module_installed(self, module_name=None):

addons/website/controllers/main.py

@@ -7,7 +7,8 @@ import json
 import xml.etree.ElementTree as ET
 import logging
 import re
-import urllib2
+
+import requests
 import werkzeug.utils
 import werkzeug.wrappers
 
@@ -300,12 +301,13 @@ class Website(Home):
         language = lang.split("_")
         url = "http://google.com/complete/search"
         try:
-            req = urllib2.Request("%s?%s" % (url, werkzeug.url_encode({
-                'ie': 'utf8', 'oe': 'utf8', 'output': 'toolbar', 'q': keywords, 'hl': language[0], 'gl': language[1]})))
-            response = urllib2.urlopen(req)
-        except (urllib2.HTTPError, urllib2.URLError):
+            req = requests.get(url, params={
+                'ie': 'utf8', 'oe': 'utf8', 'output': 'toolbar', 'q': keywords, 'hl': language[0], 'gl': language[1]})
+            req.raise_for_status()
+            response = req.content
+        except IOError:
             return []
-        xmlroot = ET.fromstring(response.read())
+        xmlroot = ET.fromstring(response)
         return json.dumps([sugg[0].attrib['data'] for sugg in xmlroot if len(sugg) and sugg[0].attrib['data']])
 
     #------------------------------------------------------

addons/website/models/ir_actions.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
-import urlparse
+from werkzeug import urls
 
 from odoo import api, fields, models
 from odoo.http import request
@@ -32,7 +31,7 @@ class ServerAction(models.Model):
         link = website_path or xml_id or (self.id and '%d' % self.id) or ''
         if base_url and link:
             path = '%s/%s' % ('/website/action', link)
-            return '%s' % urlparse.urljoin(base_url, path)
+            return urls.url_join(base_url, path)
         return ''
 
     @api.depends('state', 'website_published', 'website_path', 'xml_id')

addons/website/models/website.py

@@ -6,9 +6,9 @@ import logging
 import math
 import unicodedata
 import re
-import urlparse
 import hashlib
-import werkzeug
+
+from werkzeug import urls
 from werkzeug.exceptions import NotFound
 
 # optional python-slugify import (https://github.com/un33k/python-slugify)
@@ -44,10 +44,10 @@ def url_for(path_or_uri, lang=None):
         current_path = current_path.encode('utf-8')
     location = path_or_uri.strip()
     force_lang = lang is not None
-    url = urlparse.urlparse(location)
+    url = urls.url_parse(location)
 
     if request and not url.netloc and not url.scheme and (url.path or force_lang):
-        location = urlparse.urljoin(current_path, location)
+        location = urls.url_join(current_path, location)
 
         lang = lang or request.context.get('lang')
         langs = [lg[0] for lg in request.website.get_languages()]
@@ -459,7 +459,7 @@ class Website(models.Model):
         def get_url(page):
             _url = "%s/page/%s" % (url, page) if page > 1 else url
             if url_args:
-                _url = "%s?%s" % (_url, werkzeug.url_encode(url_args))
+                _url = "%s?%s" % (_url, urls.url_encode(url_args))
             return _url
 
         return {
@@ -598,7 +598,7 @@ class Website(models.Model):
             cdn_filters = (request.website.cdn_filters or '').splitlines()
             for flt in cdn_filters:
                 if flt and re.match(flt, uri):
-                    return urlparse.urljoin(cdn_url, uri)
+                    return urls.url_join(cdn_url, uri)
         return uri
 
     @api.model

addons/website/tests/test_crawl.py

@@ -2,10 +2,10 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import logging
-import urlparse
 import time
 
 import lxml.html
+from werkzeug import urls
 
 import odoo
 import re
@@ -39,23 +39,17 @@ class Crawler(odoo.tests.HttpCase):
 
         _logger.info("%s %s", msg, url)
         r = self.url_open(url)
-        code = r.getcode()
+        code = r.status_code
         self.assertIn(code, range(200, 300), "%s Fetching %s returned error response (%d)" % (msg, url, code))
 
-        if r.info().gettype() == 'text/html':
-            doc = lxml.html.fromstring(r.read())
+        if r.headers['Content-Type'].startswith('text/html'):
+            doc = lxml.html.fromstring(r.content)
             for link in doc.xpath('//a[@href]'):
                 href = link.get('href')
 
-                parts = urlparse.urlsplit(href)
+                parts = urls.url_parse(href)
                 # href with any fragment removed
-                href = urlparse.urlunsplit((
-                    parts.scheme,
-                    parts.netloc,
-                    parts.path,
-                    parts.query,
-                    ''
-                ))
+                href = parts.replace(fragment='').to_url()
 
                 # FIXME: handle relative link (not parts.path.startswith /)
                 if parts.netloc or \

addons/website/tests/test_views.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import itertools
+try:
+    from itertools import zip_longest
+except ImportError:
+    from itertools import izip_longest as zip_longest
 
 import unittest
 from lxml import etree as ET, html
@@ -22,7 +25,7 @@ class TestViewSaving(common.TransactionCase):
         self.assertEqual(a.attrib, b.attrib)
         self.assertEqual((a.text or '').strip(), (b.text or '').strip())
         self.assertEqual((a.tail or '').strip(), (b.tail or '').strip())
-        for ca, cb in itertools.izip_longest(a, b):
+        for ca, cb in zip_longest(a, b):
             self.eq(ca, cb)
 
     def setUp(self):
@@ -55,7 +58,7 @@ class TestViewSaving(common.TransactionCase):
             h.SPAN("My Company", attrs(model='res.company', id=1, field='name', type='char')),
             h.SPAN("+00 00 000 00 0 000", attrs(model='res.company', id=1, field='phone', type='char')),
         ]
-        for actual, expected in itertools.izip_longest(fields, expect):
+        for actual, expected in zip_longest(fields, expect):
             self.eq(actual, expected)
 
     def test_embedded_save(self):

addons/website_forum/controllers/main.py

@@ -4,12 +4,12 @@
 import base64
 import json
 import lxml
+import requests
 import werkzeug.exceptions
 import werkzeug.urls
 import werkzeug.wrappers
 
 from datetime import datetime
-from urllib2 import urlopen, URLError
 
 from odoo import http, modules, SUPERUSER_ID, tools, _
 from odoo.addons.web.controllers.main import binary_content
@@ -217,9 +217,11 @@ class WebsiteForum(http.Controller):
     @http.route('/forum/get_url_title', type='json', auth="user", methods=['POST'], website=True)
     def get_url_title(self, **kwargs):
         try:
-            arch = lxml.html.parse(urlopen(kwargs.get('url')))
+            req = requests.get(kwargs.get('url'), stream=True)
+            req.raise_for_status()
+            arch = lxml.html.parse(req.raw)
             return arch.find(".//title").text
-        except URLError:
+        except IOError:
             return False
 
     @http.route(['''/forum/<model("forum.forum"):forum>/question/<model("forum.post", "[('forum_id','=',forum[0]),('parent_id','=',False),('can_view', '=', True)]"):question>'''], type='http', auth="public", website=True)

addons/website_forum/models/res_users.py

@@ -4,7 +4,8 @@
 import hashlib
 
 from datetime import datetime
-from urllib import urlencode
+
+from werkzeug import urls
 
 from odoo import api, fields, models
 
@@ -85,7 +86,7 @@ class Users(models.Model):
             if forum_id:
                 params['forum_id'] = forum_id
             base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
-            token_url = base_url + '/forum/validate_email?%s' % urlencode(params)
+            token_url = base_url + '/forum/validate_email?%s' % urls.url_encode(params)
             activation_template.sudo().with_context(token_url=token_url).send_mail(self.id, force_send=True)
         return True
 

addons/website_hr_recruitment/models/hr_recruitment.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-from urlparse import urljoin
-from werkzeug import url_encode
+from werkzeug import urls
 
 from odoo import api, fields, models
 from odoo.addons.website.models.website import slug
@@ -19,8 +18,8 @@ class RecruitmentSource(models.Model):
     def _compute_url(self):
         base_url = self.env['ir.config_parameter'].sudo().get_param('web.base.url')
         for source in self:
-            source.url = urljoin(base_url, "%s?%s" % (source.job_id.website_url,
-                url_encode({
+            source.url = urls.url_join(base_url, "%s?%s" % (source.job_id.website_url,
+                urls.url_encode({
                     'utm_campaign': self.env.ref('hr_recruitment.utm_campaign_job').name,
                     'utm_medium': self.env.ref('utm.utm_medium_website').name,
                     'utm_source': source.source_id.name

addons/website_mail_channel/models/mail_channel.py

@@ -3,7 +3,7 @@
 
 import hmac
 
-from urlparse import urljoin
+from werkzeug import urls
 
 from odoo import api, models
 from odoo.tools.safe_eval import safe_eval
@@ -47,7 +47,7 @@ class MailGroup(models.Model):
             # generate a new token per subscriber
             token = self._generate_action_token(partner_id, action=action)
 
-            token_url = urljoin(base_url, route % {
+            token_url = urls.url_join(base_url, route % {
                 'action': action,
                 'channel': self.id,
                 'partner': partner_id,

addons/website_sale/controllers/website_mail.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-import urlparse
+from werkzeug import urls
 
 from odoo import http
 from odoo.http import request
@@ -37,7 +37,7 @@ class WebsiteMailController(WebsiteMail):
         response = super(WebsiteMailController, self).chatter_post(res_model=res_model, res_id=res_id, message=message, redirect=redirect, **params)
         if kw.get('rating') and res_model == 'product.template':  # restrict rating only for product template
             try:
-                fragment = urlparse.urlparse(response.location).fragment
+                fragment = urls.url_parse(response.location).fragment
                 message_id = int(fragment.replace('message-', ''))
                 res_model_id = request.env.ref('product.model_product_template').id
                 request.env['rating.rating'].create({

addons/website_slides/models/slides.py

@@ -1,15 +1,14 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
-
+import requests
 from PIL import Image
-from urllib import urlencode
-from urlparse import urlparse
 
 import datetime
 import io
 import json
 import re
-import urllib2
+
+from werkzeug import urls
 
 from odoo import api, fields, models, SUPERUSER_ID, _
 from odoo.tools import image, pycompat
@@ -224,8 +223,7 @@ class EmbeddedSlide(models.Model):
     count_views = fields.Integer('# Views', default=1)
 
     def add_embed_url(self, slide_id, url):
-        schema = urlparse(url)
-        baseurl = schema.netloc
+        baseurl = urls.url_parse(url).netloc
         embeds = self.search([('url', '=', baseurl), ('slide_id', '=', int(slide_id))], limit=1)
         if embeds:
             embeds.count_views += 1
@@ -498,22 +496,19 @@ class Slide(models.Model):
     def _fetch_data(self, base_url, data, content_type=False, extra_params=False):
         result = {'values': dict()}
         try:
-            if data:
-                sep = '?' if not extra_params else '&'
-                base_url = base_url + '%s%s' % (sep, urlencode(data))
-            req = urllib2.Request(base_url)
-            content = urllib2.urlopen(req).read()
+            response = requests.get(base_url, params=data)
+            response.raise_for_status()
+            content = response.content
             if content_type == 'json':
                 result['values'] = json.loads(content)
             elif content_type in ('image', 'pdf'):
                 result['values'] = content.encode('base64')
             else:
                 result['values'] = content
-        except urllib2.HTTPError as e:
-            result['error'] = e.read()
-            e.close()
-        except urllib2.URLError as e:
-            result['error'] = e.reason
+        except requests.exceptions.HTTPError as e:
+            result['error'] = e.response.content
+        except requests.exceptions.ConnectionError as e:
+            result['error'] = str(e)
         return result
 
     def _find_document_data_from_url(self, url):

addons/website_twitter/models/website_twitter.py

@@ -4,8 +4,9 @@
 import base64
 import json
 import logging
+
+import requests
 import werkzeug
-from urllib2 import urlopen, Request, HTTPError
 from odoo import api, fields, models
 
 API_ENDPOINT = 'https://api.twitter.com'
@@ -28,16 +29,13 @@ class WebsiteTwitter(models.Model):
     def _request(self, website, url, params=None):
         """Send an authenticated request to the Twitter API."""
         access_token = self._get_access_token(website)
-        if params:
-            params = werkzeug.url_encode(params)
-            url = url + '?' + params
         try:
-            request = Request(url)
-            request.add_header('Authorization', 'Bearer %s' % access_token)
-            return json.load(urlopen(request, timeout=URLOPEN_TIMEOUT))
-        except HTTPError as e:
+            request = requests.get(url, params=params, headers={'Authorization': 'Bearer %s' % access_token}, timeout=URLOPEN_TIMEOUT)
+            request.raise_for_status()
+            return request.json()
+        except requests.HTTPError as e:
             _logger.debug("Twitter API request failed with code: %r, msg: %r, content: %r",
-                          e.code, e.msg, e.fp.read())
+                          e.response.status_code, e.response.reason, e.response.content)
             raise
 
     @api.model

addons/website_twitter/models/website_twitter_config.py

@@ -2,7 +2,9 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import logging
-from urllib2 import URLError, HTTPError
+
+import requests
+
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
 
@@ -49,13 +51,13 @@ class WebsiteTwitterConfig(models.TransientModel):
         try:
             self.website_id.fetch_favorite_tweets()
 
-        except HTTPError as e:
-            _logger.info("%s - %s" % (e.code, e.reason), exc_info=True)
-            raise UserError("%s - %s" % (e.code, e.reason) + ':' + self._get_twitter_exception_message(e.code))
-        except URLError as e:
+        except requests.HTTPError as e:
+            _logger.info("%s - %s" % (e.response.status_code, e.response.reason), exc_info=True)
+            raise UserError("%s - %s" % (e.response.status_code, e.response.reason) + ':' + self._get_twitter_exception_message(e.response.status_code))
+        except IOError:
             _logger.info(_('We failed to reach a twitter server.'), exc_info=True)
             raise UserError(_('Internet connection refused') + ' ' + _('We failed to reach a twitter server.'))
-        except Exception as e:
+        except Exception:
             _logger.info(_('Please double-check your Twitter API Key and Secret!'), exc_info=True)
             raise UserError(_('Twitter authorization error!') + ' ' + _('Please double-check your Twitter API Key and Secret!'))
 

doc/_extensions/github_link.py

@@ -1,7 +1,8 @@
 import inspect
 import importlib
 import os.path
-from urlparse import urlunsplit
+
+from werkzeug import urls
 
 """
 * adds github_link(mode) context variable: provides URL (in relevant mode) of
@@ -81,7 +82,7 @@ def make_github_link(app, path, line=None, mode="blob"):
         path=path,
         mode=mode,
     )
-    return urlunsplit((
+    return urls.url_unparse((
         'https',
         'github.com',
         urlpath,

doc/_extensions/odoo_ext/translator.py

@@ -2,7 +2,10 @@
 import os.path
 import posixpath
 import re
-import urllib
+try:
+    from urllib.request import url2pathname  # pylint: disable=deprecated-module
+except ImportError:
+    from urllib import url2pathname  # pylint: disable=deprecated-module
 
 from docutils import nodes
 from sphinx import addnodes, util
@@ -636,7 +639,7 @@ class BootstrapTranslator(nodes.NodeVisitor, object):
                     banner = '_static/' + cover
                     base, ext = os.path.splitext(banner)
                     small = "{}.small{}".format(base, ext)
-                    if os.path.isfile(urllib.url2pathname(small)):
+                    if os.path.isfile(url2pathname(small)):
                         banner = small
                     style = u"background-image: url('{}')".format(
                         util.relative_uri(baseuri, banner) or '#')

doc/python3.rst

@@ -74,9 +74,22 @@ library:
   ``io.StringIO`` to replace them in a cross-version manner (``io.BytesIO``
   for binary data, ``io.StringIO`` for text/unicode data).
 * ``urllib``, ``urllib2`` and ``urlparse`` were redistributed across
-  ``urllib.parse`` and ``urllib.request``, you may want to use conditional
-  imports e.g. try to import the Python 3 version and fallback on the Python
-  2 version.
+  ``urllib.parse`` and ``urllib.request``.
+
+  Since `requests`_ and `werkzeug`_ are already hard dependencies of Odoo,
+  replace ``urllib[2].urlopen``/``urllib2.Request`` uses by `requests`_, and
+  ``urlparse`` and a few utilty functions (``urllib.quote``,
+  ``urllib.urlencode``) are available through ``werkzeug.urls``, a backport
+  of Python 3's ``urllib.parse``.
+
+  .. warning:: `requests`_ does not raise by default on non-200 responses
+
+* ``cgi.escape`` (HTML escaping) is deprecated in Python 3, prefer Odoo's own
+  :func:`odoo.tools.misc.html_encode`.
+* Most of ``types``'s content has been stripped out in Python 3: only
+  "internal" interpreter types (e.g. CodeType, FrameType, ...) have been left
+  in, other types can be obtained directly from the corresponding builtin or
+  by getting the ``type()`` of a literal value.
 
 Absolute Imports (:pep:`328`)
 -----------------------------
@@ -326,3 +339,7 @@ Minor syntax changes
   In Python 3, leading zeroes followed by neither a 0 nor a period is an
   error, octal literals now follow the hexadecimal convention with a ``0o``
   prefix.
+
+.. _requests: http://docs.python-requests.org/
+
+.. _werkzeug: http://werkzeug.pocoo.org/docs/urls/

odoo/addons/base/ir/ir_mail_server.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-from email import Encoders
+from email import encoders
 from email.charset import Charset
 from email.header import Header
 from email.mime.base import MIMEBase
@@ -13,11 +13,11 @@ import re
 import smtplib
 import threading
 
-import itertools
+import html2text
 
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import except_orm, UserError
-from odoo.tools import html2text, ustr, pycompat
+from odoo.tools import ustr, pycompat
 
 _logger = logging.getLogger(__name__)
 _test_logger = logging.getLogger('odoo.tests')
@@ -28,16 +28,16 @@ class MailDeliveryException(except_orm):
     def __init__(self, name, value):
         super(MailDeliveryException, self).__init__(name, value)
 
+# Python 3: patch SMTP's internal printer/debugger
+def _print_debug(self, *args):
+    _logger.debug(' '.join(str(a) for a in args))
+smtplib.SMTP._print_debug = _print_debug
 
+# Python 2: replace smtplib's stderr
 class WriteToLogger(object):
-    """debugging helper: behave as a fd and pipe to logger at the given level"""
-    def __init__(self, logger, level=logging.DEBUG):
-        self.logger = logger
-        self.level = level
-
     def write(self, s):
-        self.logger.log(self.level, s)
-
+        _logger.debug(s)
+smtplib.stderr = WriteToLogger()
 
 def try_coerce_ascii(string_utf8):
     """Attempts to decode the given utf8-encoded string
@@ -159,14 +159,6 @@ class IrMailServer(models.Model):
                                                                   "is used. Default priority is 10 (smaller number = higher priority)")
     active = fields.Boolean(default=True)
 
-    def __init__(self, *args, **kwargs):
-        # Make sure we pipe the smtplib outputs to our own DEBUG logger
-        if not isinstance(smtplib.stderr, WriteToLogger):
-            logpiper = WriteToLogger(_logger)
-            smtplib.stderr = logpiper
-            smtplib.stdout = logpiper
-        super(IrMailServer, self).__init__(*args, **kwargs)
-
     @api.multi
     def name_get(self):
         return [(server.id, "(%s)" % server.name) for server in self]
@@ -338,9 +330,9 @@ class IrMailServer(models.Model):
         for key, value in pycompat.items(headers):
             msg[ustr(key).encode('utf-8')] = encode_header(value)
 
-        if subtype == 'html' and not body_alternative and html2text:
+        if subtype == 'html' and not body_alternative:
             # Always provide alternative text body ourselves if possible.
-            text_utf8 = tools.html2text(email_body_utf8.decode('utf-8')).encode('utf-8')
+            text_utf8 = html2text.html2text(email_body_utf8.decode('utf-8')).encode('utf-8')
             alternative_part = MIMEMultipart(_subtype="alternative")
             alternative_part.attach(MIMEText(text_utf8, _charset='utf-8', _subtype='plain'))
             alternative_part.attach(email_text_part)
@@ -371,7 +363,7 @@ class IrMailServer(models.Model):
                 part.add_header('Content-Disposition', 'attachment', filename=filename_rfc2047)
 
                 part.set_payload(fcontent)
-                Encoders.encode_base64(part)
+                encoders.encode_base64(part)
                 msg.attach(part)
         return msg
 

odoo/addons/base/ir/ir_qweb/assetsbundle.py

@@ -12,7 +12,6 @@ from odoo import fields, tools
 from odoo.http import request
 from odoo.modules.module import get_resource_path
 import psycopg2
-import werkzeug
 from odoo.tools import func, misc, pycompat
 
 import logging
@@ -587,7 +586,7 @@ class StylesheetAsset(WebAsset):
         return self.with_header(content)
 
     def to_html(self):
-        media = (' media="%s"' % werkzeug.utils.escape(self.media)) if self.media else ''
+        media = (' media="%s"' % misc.html_escape(self.media)) if self.media else ''
         if self.url:
             href = self.html_url
             return '<link rel="stylesheet" href="%s" type="text/css"%s/>' % (href, media)

odoo/addons/base/ir/ir_qweb/ir_qweb.py

@@ -5,14 +5,10 @@ import json
 import logging
 from collections import OrderedDict
 from time import time
-try:
-    from urllib.parse import urlparse
-except ImportError:
-    # pylint: disable=bad-python3-import
-    from urlparse import urlparse
 
 from lxml import html
 from lxml import etree
+from werkzeug import urls
 
 from odoo.tools import pycompat
 
@@ -230,7 +226,7 @@ class IrQWeb(models.AbstractModel, QWeb):
                 atype = el.get('type')
                 media = el.get('media')
 
-                can_aggregate = not urlparse(href).netloc and not href.startswith('/web/content')
+                can_aggregate = not urls.url_parse(href).netloc and not href.startswith('/web/content')
                 if el.tag == 'style' or (el.tag == 'link' and el.get('rel') == 'stylesheet' and can_aggregate):
                     if href.endswith('.sass'):
                         atype = 'text/sass'

odoo/addons/base/module/module.py

@@ -10,15 +10,9 @@ import shutil
 import tempfile
 import zipfile
 
-from odoo.tools import pycompat
+import requests
 
-try:
-    from urllib import parse as urlparse
-    from urllib.request import urlopen
-except ImportError:
-    # pylint: disable=bad-python3-import
-    import urlparse
-    from urllib2 import urlopen
+from odoo.tools import pycompat
 
 from docutils import nodes
 from docutils.core import publish_string
@@ -663,7 +657,7 @@ class Module(models.Model):
             _logger.warning(msg)
             raise UserError(msg)
 
-        apps_server = urlparse.urlparse(self.get_apps_server())
+        apps_server = urls.url_parse(self.get_apps_server())
 
         OPENERP = odoo.release.product_name.lower()
         tmp = tempfile.mkdtemp()
@@ -674,13 +668,15 @@ class Module(models.Model):
                 if not url:
                     continue    # nothing to download, local version is already the last one
 
-                up = urlparse.urlparse(url)
+                up = urls.url_parse(url)
                 if up.scheme != apps_server.scheme or up.netloc != apps_server.netloc:
                     raise AccessDenied()
 
                 try:
                     _logger.info('Downloading module `%s` from OpenERP Apps', module_name)
-                    content = urlopen(url).read()
+                    response = requests.get(url)
+                    response.raise_for_status()
+                    content = response.content
                 except Exception:
                     _logger.exception('Failed to fetch module %s', module_name)
                     raise UserError(_('The `%s` module appears to be unavailable at the moment, please try again later.') % module_name)

odoo/addons/base/res/res_partner.py

@@ -7,16 +7,11 @@ import hashlib
 import pytz
 import threading
 
-try:
-    from urllib import parse as urlparse
-    from urllib.request import urlopen
-except ImportError:
-    # pylint: disable=bad-python3-import
-    import urlparse
-    from urllib2 import urlopen
-
 from email.utils import formataddr
+
+import requests
 from lxml import etree
+from werkzeug import urls
 
 from odoo import api, fields, models, tools, SUPERUSER_ID, _
 from odoo.modules import get_module_resource
@@ -481,11 +476,11 @@ class Partner(models.Model):
             parent.update_address(addr_vals)
 
     def _clean_website(self, website):
-        (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(website)
-        if not scheme:
-            if not netloc:
-                netloc, path = path, ''
-            website = urlparse.urlunparse(('http', netloc, path, params, query, fragment))
+        url = urls.url_parse(website)
+        if not url.scheme:
+            if not url.netloc:
+                url = url.replace(netloc=url.path, path='')
+            website = url.replace(scheme='http').to_url()
         return website
 
     @api.multi
@@ -703,15 +698,12 @@ class Partner(models.Model):
         return partners.id or self.name_create(email)[0]
 
     def _get_gravatar_image(self, email):
-        gravatar_image = False
         email_hash = hashlib.md5(email.lower()).hexdigest()
         url = "https://www.gravatar.com/avatar/" + email_hash
-        try:
-            image_content = urlopen(url + "?d=404&s=128", timeout=5).read()
-            gravatar_image = base64.b64encode(image_content)
-        except Exception:
-            pass
-        return gravatar_image
+        res = requests.get(url, params={'d': '404', 's': '128'}, timeout=5)
+        if res.status_code != requests.codes.ok:
+            return False
+        return base64.b64encode(res.content)
 
     @api.multi
     def _email_send(self, email_from, subject, body, on_error=None):

odoo/addons/base/tests/test_mail.py

@@ -1,10 +1,9 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import cgi
 import unittest
 
-from odoo.tools import html_sanitize, append_content_to_html, plaintext2html, email_split
+from odoo.tools import html_sanitize, append_content_to_html, plaintext2html, email_split, misc
 from . import test_mail_examples
 
 
@@ -99,7 +98,7 @@ class TestSanitizer(unittest.TestCase):
             "Div nico <div-nico@open.com>"
         ]
         for email in emails:
-            self.assertIn(cgi.escape(email), html_sanitize(email), 'html_sanitize stripped emails of original html')
+            self.assertIn(misc.html_escape(email), html_sanitize(email), 'html_sanitize stripped emails of original html')
 
     def test_sanitize_unescape_emails(self):
         not_emails = [
@@ -108,7 +107,7 @@ class TestSanitizer(unittest.TestCase):
         for email in not_emails:
             sanitized = html_sanitize(email)
             left_part = email.split('>')[0]  # take only left part, as the sanitizer could add data information on node
-            self.assertNotIn(cgi.escape(email), sanitized, 'html_sanitize stripped emails of original html')
+            self.assertNotIn(misc.html_escape(email), sanitized, 'html_sanitize stripped emails of original html')
             self.assertIn(left_part, sanitized)
 
     def test_style_parsing(self):
@@ -172,14 +171,14 @@ class TestSanitizer(unittest.TestCase):
         for ext in test_mail_examples.QUOTE_BLOCKQUOTE_IN:
             self.assertIn(ext, html)
         for ext in test_mail_examples.QUOTE_BLOCKQUOTE_OUT:
-            self.assertIn('<span data-o-mail-quote="1">%s' % cgi.escape(ext.decode('utf-8')), html)
+            self.assertIn('<span data-o-mail-quote="1">%s' % misc.html_escape(ext.decode('utf-8')), html)
 
     def test_quote_thunderbird(self):
         html = html_sanitize(test_mail_examples.QUOTE_THUNDERBIRD_1)
         for ext in test_mail_examples.QUOTE_THUNDERBIRD_1_IN:
             self.assertIn(ext, html)
         for ext in test_mail_examples.QUOTE_THUNDERBIRD_1_OUT:
-            self.assertIn('<span data-o-mail-quote="1">%s</span>' % cgi.escape(ext.decode('utf-8')), html)
+            self.assertIn('<span data-o-mail-quote="1">%s</span>' % misc.html_escape(ext.decode('utf-8')), html)
 
     def test_quote_hotmail_html(self):
         html = html_sanitize(test_mail_examples.QUOTE_HOTMAIL_HTML)
@@ -226,7 +225,7 @@ class TestSanitizer(unittest.TestCase):
             for text in in_lst:
                 self.assertIn(text, new_html)
             for text in out_lst:
-                self.assertIn('<span data-o-mail-quote="1">%s</span>' % cgi.escape(text), new_html)
+                self.assertIn('<span data-o-mail-quote="1">%s</span>' % misc.html_escape(text), new_html)
 
     def test_quote_signature(self):
         test_data = [
@@ -245,27 +244,27 @@ class TestSanitizer(unittest.TestCase):
         for ext in test_mail_examples.GMAIL_1_IN:
             self.assertIn(ext, html)
         for ext in test_mail_examples.GMAIL_1_OUT:
-            self.assertIn('<span data-o-mail-quote="1">%s</span>' % cgi.escape(ext), html)
+            self.assertIn('<span data-o-mail-quote="1">%s</span>' % misc.html_escape(ext), html)
 
     def test_quote_text(self):
         html = html_sanitize(test_mail_examples.TEXT_1)
         for ext in test_mail_examples.TEXT_1_IN:
             self.assertIn(ext, html)
         for ext in test_mail_examples.TEXT_1_OUT:
-            self.assertIn('<span data-o-mail-quote="1">%s</span>' % cgi.escape(ext), html)
+            self.assertIn('<span data-o-mail-quote="1">%s</span>' % misc.html_escape(ext), html)
 
         html = html_sanitize(test_mail_examples.TEXT_2)
         for ext in test_mail_examples.TEXT_2_IN:
             self.assertIn(ext, html)
         for ext in test_mail_examples.TEXT_2_OUT:
-            self.assertIn('<span data-o-mail-quote="1">%s</span>' % cgi.escape(ext), html)
+            self.assertIn('<span data-o-mail-quote="1">%s</span>' % misc.html_escape(ext), html)
 
     def test_quote_bugs(self):
         html = html_sanitize(test_mail_examples.BUG1)
         for ext in test_mail_examples.BUG_1_IN:
             self.assertIn(ext, html)
         for ext in test_mail_examples.BUG_1_OUT:
-            self.assertIn('<span data-o-mail-quote="1">%s</span>' % cgi.escape(ext.decode('utf-8')), html)
+            self.assertIn('<span data-o-mail-quote="1">%s</span>' % misc.html_escape(ext.decode('utf-8')), html)
 
     def test_misc(self):
         # False / void should not crash

odoo/addons/base/tests/test_qweb.py

@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import cgi
 import collections
 import json
 import os.path
 import re
 
 from lxml import etree
-from itertools import chain
 
 from odoo.modules import get_module_resource
 from odoo.tests.common import TransactionCase
 from odoo.addons.base.ir.ir_qweb import QWebException
-from odoo.tools import pycompat
+from odoo.tools import pycompat, misc, ustr
 
 
 def dedent_and_strip(string):
@@ -48,12 +46,12 @@ class TestQWebTField(TransactionCase):
 
         result = self.engine.render(field, {'company': company})
         self.assertEqual(
-            result,
+            ustr(result),
             '<span data-oe-model="res.company" data-oe-id="%d" '
                   'data-oe-field="name" data-oe-type="char" '
                   'data-oe-expression="company.name">%s</span>' % (
                 company.id,
-                cgi.escape(s.encode('utf-8')),
+                misc.html_escape(s),
             ),
         )
 

odoo/addons/base/tests/test_views.py

@@ -2,7 +2,10 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 from functools import partial
-from itertools import izip_longest
+try:
+    from itertools import zip_longest as izip_longuest
+except ImportError:
+    from itertools import izip_longest
 
 from lxml import etree
 from lxml.builder import E

odoo/addons/test_pylint/tests/test_pylint.py

@@ -42,6 +42,7 @@ class TestPyLint(TransactionCase):
         'parameter-unpacking',
 
         'metaclass-assignment',
+        'deprecated-module',
 
         'exception-message-attribute',
         'indexing-exception',
@@ -70,6 +71,24 @@ class TestPyLint(TransactionCase):
         'reduce',
     ]
 
+    BAD_MODULES = [
+        'commands',
+        'cPickle',
+        'md5',
+        'urllib',
+        'urllib2',
+        'urlparse',
+        'sgmllib',
+        'sha',
+        'cgi',
+        'htmlentitydefs',
+        'HTMLParser',
+        'Queue',
+        'UserDict',
+        'UserString',
+        'UserList',
+    ]
+
     def _skip_test(self, reason):
         _logger.warn(reason)
         self.skipTest(reason)
@@ -93,6 +112,7 @@ class TestPyLint(TransactionCase):
             "--msg-template='{msg} ({msg_id}) at {path}:{line}'",
             '--load-plugins=pylint.extensions.bad_builtin',
             '--bad-functions=%s' % ','.join(self.BAD_FUNCTIONS),
+            '--deprecated-modules=%s' % ','.join(self.BAD_MODULES)
         ]
 
         try:

odoo/fields.py

@@ -7,7 +7,6 @@ from collections import OrderedDict, defaultdict
 from datetime import date, datetime
 from functools import partial
 from operator import attrgetter
-from types import NoneType
 import logging
 import pytz
 try:
@@ -1396,7 +1395,7 @@ class Char(_String):
 
     def _setup_regular_base(self, model):
         super(Char, self)._setup_regular_base(model)
-        assert isinstance(self.size, (NoneType, int)), \
+        assert self.size is None or isinstance(self.size, int), \
             "Char field %s with non-integer size %r" % (self, self.size)
 
     def convert_to_column(self, value, record):

odoo/http.py

@@ -20,12 +20,6 @@ import sys
 import threading
 import time
 import traceback
-try:
-    from urllib.parse import parse_qs, urlparse, quote
-except ImportError:
-    # pylint: disable=bad-python3-import
-    from urllib2 import quote
-    from urlparse import parse_qs, urlparse
 import warnings
 from os.path import join as opj
 from zlib import adler32
@@ -41,6 +35,7 @@ import werkzeug.local
 import werkzeug.routing
 import werkzeug.wrappers
 import werkzeug.wsgi
+from werkzeug import urls
 from werkzeug.wsgi import wrap_file
 
 try:
@@ -351,7 +346,7 @@ class WebRequest(object):
             debug = self.httprequest.environ.get('HTTP_X_DEBUG_MODE')
 
         if not debug and self.httprequest.referrer:
-            debug = bool(parse_qs(urlparse(self.httprequest.referrer).query, keep_blank_values=True).get('debug'))
+            debug = 'debug' in urls.url_parse(self.httprequest.referrer).decode_query()
         return debug
 
     @contextlib.contextmanager
@@ -1254,7 +1249,7 @@ class DisableCacheMiddleware(object):
     def __call__(self, environ, start_response):
         def start_wrapped(status, headers):
             referer = environ.get('HTTP_REFERER', '')
-            parsed = urlparse(referer)
+            parsed = urls.url_parse(referer)
             debug = parsed.query.count('debug') >= 1
 
             new_headers = []
@@ -1619,7 +1614,7 @@ def send_file(filepath_or_fp, mimetype=None, as_attachment=False, filename=None,
 
 def content_disposition(filename):
     filename = odoo.tools.ustr(filename)
-    escaped = quote(filename.encode('utf8'))
+    escaped = urls.url_quote(filename.encode('utf8'))
     browser = request.httprequest.user_agent.browser
     version = int((request.httprequest.user_agent.version or '0').split('.')[0])
     if browser == 'msie' and version < 9:

odoo/sql_db.py

@@ -13,17 +13,13 @@ from functools import wraps
 import logging
 import time
 import uuid
-try:
-    from urllib import parse as urlparse
-except ImportError:
-    #pylint: disable=bad-python3-import
-    import urlparse
 
 import psycopg2
 import psycopg2.extras
 import psycopg2.extensions
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT, ISOLATION_LEVEL_READ_COMMITTED, ISOLATION_LEVEL_REPEATABLE_READ
 from psycopg2.pool import PoolError
+from werkzeug import urls
 
 from .tools import pycompat
 
@@ -656,7 +652,7 @@ def connection_info_for(db_or_uri):
     """
     if db_or_uri.startswith(('postgresql://', 'postgres://')):
         # extract db from uri
-        us = urlparse.urlsplit(db_or_uri)
+        us = urls.url_parse(db_or_uri)
         if len(us.path) > 1:
             db_name = us.path[1:]
         elif us.username:

odoo/tests/common.py

@@ -19,16 +19,15 @@ import unittest
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 from pprint import pformat
+
+import requests
+
 try:
-    from urllib import request as urllib2
     from xmlrpc import client as xmlrpclib
 except ImportError:
     # pylint: disable=bad-python3-import
-    import urllib2
     import xmlrpclib
 
-import werkzeug
-
 import odoo
 from odoo import api
 
@@ -215,26 +214,6 @@ class SavepointCase(SingleTransactionCase):
         self.registry.clear_caches()
 
 
-class RedirectHandler(urllib2.HTTPRedirectHandler):
-    """
-    HTTPRedirectHandler is predicated upon HTTPErrorProcessor being used and
-    works by intercepting 3xy "errors".
-
-    Inherit from it to handle 3xy non-error responses instead, as we're not
-    using the error processor
-    """
-
-    def http_response(self, request, response):
-        code, msg, hdrs = response.code, response.msg, response.info()
-
-        if 300 <= code < 400:
-            return self.parent.error(
-                'http', request, response, code, msg, hdrs)
-
-        return response
-
-    https_response = http_response
-
 class HttpCase(TransactionCase):
     """ Transactional HTTP TestCase with url_open and phantomjs helpers.
     """
@@ -259,18 +238,15 @@ class HttpCase(TransactionCase):
         self.session.db = get_db_name()
         odoo.http.root.session_store.save(self.session)
         # setup an url opener helper
-        self.opener = urllib2.OpenerDirector()
-        self.opener.add_handler(urllib2.UnknownHandler())
-        self.opener.add_handler(urllib2.HTTPHandler())
-        self.opener.add_handler(urllib2.HTTPSHandler())
-        self.opener.add_handler(urllib2.HTTPCookieProcessor())
-        self.opener.add_handler(RedirectHandler())
-        self.opener.addheaders.append(('Cookie', 'session_id=%s' % self.session_id))
+        self.opener = requests.Session()
+        self.opener.cookies['session_id'] = self.session_id
 
     def url_open(self, url, data=None, timeout=10):
         if url.startswith('/'):
             url = "http://%s:%s%s" % (HOST, PORT, url)
-        return self.opener.open(url, data, timeout)
+        if data:
+            return self.opener.post(url, data=data, timeout=timeout)
+        return self.opener.get(url, timeout=timeout)
 
     def authenticate(self, user, password):
         # stay non-authenticated

odoo/tools/mail.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
-import cgi
 import logging
 import lxml.html.clean as clean
 import random
@@ -16,7 +15,7 @@ from lxml import etree
 
 import odoo
 from odoo.loglevels import ustr
-from odoo.tools import pycompat
+from odoo.tools import pycompat, misc
 
 _logger = logging.getLogger(__name__)
 
@@ -178,10 +177,10 @@ def html_sanitize(src, silent=True, sanitize_tags=True, sanitize_attributes=Fals
     part = re.compile(r"(<(([^a<>]|a[^<>\s])[^<>]*)@[^<>]+>)", re.IGNORECASE | re.DOTALL)
     # remove results containing cite="mid:email_like@address" (ex: blockquote cite)
     # cite_except = re.compile(r"^((?!cite[\s]*=['\"]).)*$", re.IGNORECASE)
-    src = part.sub(lambda m: ('cite=' not in m.group(1) and 'alt=' not in m.group(1)) and cgi.escape(m.group(1)) or m.group(1), src)
+    src = part.sub(lambda m: ('cite=' not in m.group(1) and 'alt=' not in m.group(1)) and misc.html_escape(m.group(1)) or m.group(1), src)
     # html encode mako tags <% ... %> to decode them later and keep them alive, otherwise they are stripped by the cleaner
-    src = src.replace('<%', cgi.escape('<%'))
-    src = src.replace('%>', cgi.escape('%>'))
+    src = src.replace('<%', misc.html_escape('<%'))
+    src = src.replace('%>', misc.html_escape('%>'))
 
     kwargs = {
         'page_structure': True,
@@ -334,7 +333,7 @@ def html2plaintext(html, body_id=None, encoding='utf-8'):
 
 def plaintext2html(text, container_tag=False):
     """ Convert plaintext into html. Content of the text is escaped to manage
-        html entities, using cgi.escape().
+        html entities, using misc.html_escape().
         - all \n,\r are replaced by <br />
         - enclose content into <p>
         - convert url into clickable link
@@ -343,7 +342,7 @@ def plaintext2html(text, container_tag=False):
         :param string container_tag: container of the html; by default the
             content is embedded into a <div>
     """
-    text = cgi.escape(ustr(text))
+    text = misc.html_escape(ustr(text))
 
     # 1. replace \n and \r
     text = text.replace('\n', '<br/>')

odoo/tools/misc.py

@@ -5,7 +5,6 @@
 """
 Miscellaneous tools used by OpenERP.
 """
-
 from functools import wraps
 import babel
 from contextlib import contextmanager
@@ -14,11 +13,13 @@ import subprocess
 import io
 import os
 import passlib.utils
+import pickle as pickle_
 import re
 import socket
 import sys
 import threading
 import time
+import types
 import werkzeug.utils
 import zipfile
 from collections import defaultdict, Iterable, Mapping, MutableSet, OrderedDict
@@ -26,7 +27,6 @@ from itertools import islice, groupby, repeat
 from lxml import etree
 
 from .which import which
-from threading import local
 import traceback
 import csv
 from operator import itemgetter
@@ -37,16 +37,6 @@ try:
 except ImportError:
     import profile as cProfile
 
-try:
-    # pylint: disable=bad-python3-import
-    import cPickle as pickle_
-except ImportError:
-    import pickle as pickle_
-
-try:
-    from html2text import html2text
-except ImportError:
-    html2text = None
 
 from .config import config
 from .cache import *
@@ -677,29 +667,6 @@ def split_every(n, iterable, piece_maker=tuple):
         yield piece
         piece = piece_maker(islice(iterator, n))
 
-if __name__ == '__main__':
-    import doctest
-    doctest.testmod()
-
-class upload_data_thread(threading.Thread):
-    def __init__(self, email, data, type):
-        self.args = [('email',email),('type',type),('data',data)]
-        super(upload_data_thread,self).__init__()
-    def run(self):
-        try:
-            import urllib
-            args = urllib.urlencode(self.args)
-            fp = urllib.urlopen('http://www.openerp.com/scripts/survey.php', args)
-            fp.read()
-            fp.close()
-        except Exception:
-            pass
-
-def upload_data(email, data, type='SURVEY'):
-    a = upload_data_thread(email, data, type)
-    a.start()
-    return True
-
 def get_and_group_by_field(cr, uid, obj, ids, field, context=None):
     """ Read the values of ``field´´ for the given ``ids´´ and group ids by value.
 
@@ -1128,23 +1095,18 @@ def _consteq(str1, str2):
 
 consteq = getattr(passlib.utils, 'consteq', _consteq)
 
-class Pickle(object):
-    @classmethod
-    def load(cls, stream, errors=False):
-        unpickler = pickle_.Unpickler(stream)
-        # pickle builtins: str/unicode, int/long, float, bool, tuple, list, dict, None
-        unpickler.find_global = None
-        try:
-            return unpickler.load()
-        except Exception:
-            _logger.warning('Failed unpickling data, returning default: %r', errors, exc_info=True)
-            return errors
-
-    @classmethod
-    def loads(cls, text):
-        return cls.load(io.BytesIO(text))
-
-    dumps = pickle_.dumps
-    dump = pickle_.dump
-
-pickle = Pickle
+def _pickle_load(stream, errors=False):
+    unpickler = pickle_.Unpickler(stream)
+    # pickle builtins: str/unicode, int/long, float, bool, tuple, list, dict, None
+    unpickler.find_global = None
+    try:
+        return unpickler.load()
+    except Exception:
+        _logger.warning('Failed unpickling data, returning default: %r',
+                        errors, exc_info=True)
+        return errors
+pickle = types.ModuleType(__name__ + '.pickle')
+pickle.load = _pickle_load
+pickle.loads = lambda text: _pickle_load(io.BytesIO(text))
+pickle.dump = pickle_.dump
+pickle.dumps = pickle_.dumps

requirements.txt

@@ -6,6 +6,7 @@ feedparser==5.2.1
 gevent==1.1.2
 greenlet==0.4.10
 jcconv==0.2.3
+html2text==2016.9.19
 Jinja2==2.8
 lxml==3.5.0
 Mako==1.0.4

setup.py

@@ -140,6 +140,7 @@ setup(
         'docutils',
         'feedparser',
         'gevent',
+        'html2text',
         'Jinja2',
         'lxml',  # windows binary http://www.lfd.uci.edu/~gohlke/pythonlibs/
         'mako',