From 6e9d29df1d6335d41a950633478551c78484a794 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Tue, 16 May 2017 11:48:05 +0200 Subject: [PATCH 1/8] [FIX] P3: bytecode changes * Python 3.4 adds new convenient bytecode-inspection API to dis which is nice, however Python 3.6 also switches from the old bytecode to a new wordcode which breaks _get_opcodes and *requires* that we use the new API * add various opcodes which either were added in Python 3 (CALL_FUNCTION_EX) or were only used for some small corners of Python 2 but have been expanded or despecialised in Python 3 (BUILD_SLICE) --- odoo/tools/safe_eval.py | 78 +++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/odoo/tools/safe_eval.py b/odoo/tools/safe_eval.py index 569df6455e2..b34757df23c 100644 --- a/odoo/tools/safe_eval.py +++ b/odoo/tools/safe_eval.py @@ -14,8 +14,8 @@ condition/math builtins. # - http://code.activestate.com/recipes/286134/ # - safe_eval in lp:~xrg/openobject-server/optimize-5.0 # - safe_eval in tryton http://hg.tryton.org/hgwebdir.cgi/trytond/rev/bbb5f73319ad - -from opcode import HAVE_ARGUMENT, opmap +import dis +from opcode import HAVE_ARGUMENT, opmap, opname import functools from psycopg2 import OperationalError @@ -40,11 +40,34 @@ _ALLOWED_MODULES = ['_strptime', 'math', 'time'] _UNSAFE_ATTRIBUTES = ['f_builtins', 'f_globals', 'f_locals', 'gi_frame', 'co_code', 'func_globals'] +_POSSIBLE_OPCODES_P3 = [ + # comprehensions, actually in P2 + 'MAP_ADD', 'SET_ADD', + # opcodes for `with` statement cleanup process + 'WITH_CLEANUP_START', 'WITH_CLEANUP_FINISH', + # f-strings + 'FORMAT_VALUE', 'BUILD_STRING', + # extended iterable unpacking: LHS has * e.g. `a, *b, c = thing()` + 'UNPACK_EX', + # collection literals with unpacking e.g. [*a, *b] + 'BUILD_LIST_UNPACK', 'BUILD_TUPLE_UNPACK', 'BUILD_SET_UNPACK', 'BUILD_MAP_UNPACK', + # packs args/kwargs for calls with multiple unpacks e.g. foo(*a, *b, *c) + 'BUILD_TUPLE_UNPACK_WITH_CALL', 'BUILD_MAP_UNPACK_WITH_CALL', + # ??? + 'GET_YIELD_FROM_ITER', + # matrix operator + 'BINARY_MATRIX_MULTIPLY', 'INPLACE_MATRIX_MULTIPLY', +] _CONST_OPCODES = set(opmap[x] for x in [ 'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP', 'DUP_TOPX', 'POP_BLOCK','SETUP_LOOP', 'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE', - 'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR', 'STORE_MAP'] if x in opmap) + 'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR', 'STORE_MAP', + # maps with constant keys optimisation https://bugs.python.org/issue27140 + 'BUILD_CONST_KEY_MAP', + 'POP_EXCEPT', # Seems to be a special-case of POP_BLOCK for P3 + 'DUP_TOP_TWO', # replaces DUP_TOPX in P3 +] if x in opmap) _EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [ 'UNARY_POSITIVE', 'UNARY_NEGATIVE', 'UNARY_NOT', @@ -62,36 +85,44 @@ _SAFE_OPCODES = _EXPR_OPCODES.union(set(opmap[x] for x in [ 'LOAD_NAME', 'CALL_FUNCTION', 'COMPARE_OP', 'LOAD_ATTR', 'STORE_NAME', 'GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'DELETE_NAME', 'JUMP_FORWARD', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE', 'JUMP_ABSOLUTE', - 'MAKE_FUNCTION', 'SLICE+0', 'SLICE+1', 'SLICE+2', 'SLICE+3', 'BREAK_LOOP', - 'CONTINUE_LOOP', 'RAISE_VARARGS', 'YIELD_VALUE', + 'MAKE_FUNCTION', 'SLICE+0', 'SLICE+1', 'SLICE+2', 'SLICE+3', 'BUILD_SLICE', + 'BREAK_LOOP', 'CONTINUE_LOOP', 'RAISE_VARARGS', 'YIELD_VALUE', # New in Python 2.7 - http://bugs.python.org/issue4715 : 'JUMP_IF_FALSE_OR_POP', 'JUMP_IF_TRUE_OR_POP', 'POP_JUMP_IF_FALSE', 'POP_JUMP_IF_TRUE', 'SETUP_EXCEPT', 'END_FINALLY', 'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'UNPACK_SEQUENCE', 'LOAD_GLOBAL', # Only allows access to restricted globals + # P3: https://bugs.python.org/issue27213 + 'CALL_FUNCTION_EX', + # Already in P2 but apparently the first one is used more aggressively in P3 + 'CALL_FUNCTION_KW', 'CALL_FUNCTION_VAR', 'CALL_FUNCTION_VAR_KW', ] if x in opmap)) _logger = logging.getLogger(__name__) -def _get_opcodes(codeobj): - """_get_opcodes(codeobj) -> [opcodes] +if hasattr(dis, 'get_instructions'): + def _get_opcodes(codeobj): + """_get_opcodes(codeobj) -> [opcodes] - Extract the actual opcodes as a list from a code object + Extract the actual opcodes as an iterator from a code object - >>> c = compile("[1 + 2, (1,2)]", "", "eval") - >>> _get_opcodes(c) - [100, 100, 23, 100, 100, 102, 103, 83] - """ - i = 0 - byte_codes = codeobj.co_code - while i < len(byte_codes): - code = ord(byte_codes[i]) - yield code + >>> c = compile("[1 + 2, (1,2)]", "", "eval") + >>> list(_get_opcodes(c)) + [100, 100, 23, 100, 100, 102, 103, 83] + """ + return (i.opcode for i in dis.get_instructions(codeobj)) +else: + def _get_opcodes(codeobj): + i = 0 + byte_codes = codeobj.co_code + while i < len(byte_codes): + code = ord(byte_codes[i:i+1]) + yield code - if code >= HAVE_ARGUMENT: - i += 3 - else: - i += 1 + if code >= HAVE_ARGUMENT: + i += 3 + else: + i += 1 def assert_no_dunder_name(code_obj, expr): """ assert_no_dunder_name(code_obj, expr) -> None @@ -138,8 +169,9 @@ def assert_valid_codeobj(allowed_codes, code_obj, expr): # almost twice as fast as a manual iteration + condition when loading # /web according to line_profiler - if set(_get_opcodes(code_obj)) - allowed_codes: - raise ValueError("forbidden opcode(s) in %r" % expr) + codes = set(_get_opcodes(code_obj)) - allowed_codes + if codes: + raise ValueError("forbidden opcode(s) in %r: %s" % (expr, ', '.join(opname[x] for x in codes))) for const in code_obj.co_consts: if isinstance(const, CodeType): From 3c95f7f25c02e97cb3c3c966b3102368b9a6a474 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:04:06 +0200 Subject: [PATCH 2/8] [FIX] P3: ast changes * P3 gets a "proper" arg node * P3's arguments adds two parameters (kwonlyargs and kw_defaults) for keyword-only parameters * P3's ``fn(**bar)`` only works if ``bar`` *only has string keys*, some of the contexts in qweb passed non-string keys and blew up * P3's ast.If requires a non-empty body, add ``[ast.Pass()]`` ballback for non-literal bodies --- odoo/addons/base/ir/ir_qweb/qweb.py | 70 ++++++++++++++++++----------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/odoo/addons/base/ir/ir_qweb/qweb.py b/odoo/addons/base/ir/ir_qweb/qweb.py index 06aba8ccdb4..69a01635f2d 100644 --- a/odoo/addons/base/ir/ir_qweb/qweb.py +++ b/odoo/addons/base/ir/ir_qweb/qweb.py @@ -33,6 +33,18 @@ unsafe_eval = eval _logger = logging.getLogger(__name__) +# in Python 2, arguments (within the ast.arguments structure) are expressions +# (since they can be tuples), generally +# ast.Name(id: identifyer, ctx=ast.Param()), whereas in Python 3 they are +# ast.arg(arg: identifier, annotation: expr?) provide a toplevel arg() +# function which matches ast.arg producing the relevant ast.Name in Python 2. +arg = getattr(ast, 'arg', lambda arg, annotation: ast.Name(id=arg, ctx=ast.Param())) +# also Python 3's arguments has grown *2* new mandatory arguments, kwonlyargs +# and kw_defaults for keyword-only arguments and their default values (if any) +# so add a shim for *that* based on the signature of Python 3 I guess? +arguments = ast.arguments +if pycompat.PY2: + arguments = lambda args, vararg, kwonlyargs, kw_defaults, kwarg, defaults: ast.arguments(args=args, vararg=vararg, kwarg=kwarg, defaults=defaults) #################################### ### qweb tools ### #################################### @@ -76,16 +88,23 @@ class Contextifier(ast.NodeTransformer): def visit_Lambda(self, node): args = node.args # assume we don't have any tuple parameter, just names - names = [arg.id for arg in args.args] + if pycompat.PY2: + names = [arg.id for arg in args.args] + else: + names = [arg.arg for arg in args.args] if args.vararg: names.append(args.vararg) if args.kwarg: names.append(args.kwarg) # remap defaults in case there's any return ast.copy_location(ast.Lambda( - args=ast.arguments( + args=arguments( args=args.args, defaults=[self.visit(default) for default in args.defaults], vararg=args.vararg, kwarg=args.kwarg, + # assume we don't have any, not sure it's even possible to + # handle that cross-version + kwonlyargs=[], + kw_defaults=[], ), body=Contextifier(self._safe_names + tuple(names)).visit(node.body) ), node) @@ -296,7 +315,7 @@ class QWeb(object): except Exception as e: path = _options['last_path_node'] node = element.getroottree().xpath(path) - raise QWebException("Error when compiling AST", e, path, etree.tostring(node[0]), name) + raise QWebException("Error when compiling AST", e, path, etree.tostring(node[0], encoding='unicode'), name) astmod.body.extend(_options['ast_calls']) if 'profile' in options: @@ -316,28 +335,29 @@ class QWeb(object): except Exception as e: path = _options['last_path_node'] node = element.getroottree().xpath(path) - raise QWebException("Error when compiling AST", e, path, node and etree.tostring(node[0]), name) + raise QWebException("Error when compiling AST", e, path, node and etree.tostring(node[0], encoding='unicode'), name) # return the wrapped function def _compiled_fn(self, append, values): log = {'last_path_node': None} - values = dict(self.default_values(), **values) + new = self.default_values() + new.update(values) try: - return compiled(self, append, values, options, log) + return compiled(self, append, new, options, log) except QWebException as e: raise e except Exception as e: path = log['last_path_node'] element, document = self.get_template(template, options) node = element.getroottree().xpath(path) - raise QWebException("Error to render compiling AST", e, path, node and etree.tostring(node[0]), name) + raise QWebException("Error to render compiling AST", e, path, node and etree.tostring(node[0], encoding='unicode'), name) return _compiled_fn def default_values(self): """ Return attributes added to the values for each computed template. """ - return dict(format=self.format) + return {'format': self.format} def get_template(self, template, options): """ Retrieve the given template, and return it as a pair ``(element, @@ -528,13 +548,13 @@ class QWeb(object): # def $name(self, append, values, options, log) fn = ast.FunctionDef( name=name, - args=ast.arguments(args=[ - ast.Name(id='self', ctx=ast.Param()), - ast.Name(id='append', ctx=ast.Param()), - ast.Name(id='values', ctx=ast.Param()), - ast.Name(id='options', ctx=ast.Param()), - ast.Name(id='log', ctx=ast.Param()), - ], defaults=[], vararg=None, kwarg=None), + args=arguments(args=[ + arg(arg='self', annotation=None), + arg(arg='append', annotation=None), + arg(arg='values', annotation=None), + arg(arg='options', annotation=None), + arg(arg='log', annotation=None), + ], defaults=[], vararg=None, kwarg=None, kwonlyargs=[], kw_defaults=[]), body=body or [ast.Return()], decorator_list=[]) if lineno is not None: @@ -601,7 +621,7 @@ class QWeb(object): ] ), # append(escape($content)) - body=body, + body=body or [ast.Pass()], # append(body default value) orelse=orelse, ) @@ -667,7 +687,7 @@ class QWeb(object): # all directives have been compiled, there should be none left if any(att.startswith('t-') for att in el.attrib): - raise NameError("Unknown directive on %s" % etree.tostring(el)) + raise NameError("Unknown directive on %s" % etree.tostring(el, encoding='unicode')) return [] def _values_var(self, varname, ctx): @@ -750,8 +770,8 @@ class QWeb(object): # since `options['nsmap']` is a dict (and therefore mutable) and we do **not** # want changes done in deeper recursion to bevisible in earlier ones, we'll pass # a copy before continuing the recursion and restore the original afterwards. - original_nsmap = options['nsmap'] - options['nsmap'] = dict(options['nsmap'], **el.nsmap) + original_nsmap = dict(options['nsmap']) + options['nsmap'].update(el.nsmap) content = self._compile_directive_content(el, options) options['nsmap'] = original_nsmap @@ -964,9 +984,9 @@ class QWeb(object): # since `options['nsmap']` is a dict (and therefore mutable) and we do **not** # want changes done in deeper recursion to bevisible in earlier ones, we'll pass # a copy before continuing the recursion and restore the original afterwards. + original_nsmap = dict(options['nsmap']) if el.nsmap: - original_nsmap = options['nsmap'] - options['nsmap'] = dict(options['nsmap'], **el.nsmap) + options['nsmap'].update(el.nsmap) content = self._compile_directives(el, options) if el.nsmap: options['nsmap'] = original_nsmap @@ -1069,7 +1089,7 @@ class QWeb(object): # $t-else ast.If( test=self._compile_expr(el.attrib.pop('t-if')), - body=self._compile_directives(el, options), + body=self._compile_directives(el, options) or [ast.Pass()], orelse=orelse ) ] @@ -1088,7 +1108,7 @@ class QWeb(object): args=[ast.Str(el.attrib.pop('t-groups'))], keywords=[], starargs=None, kwargs=None ), - body=self._compile_directives(el, options), + body=self._compile_directives(el, options) or [ast.Pass()], orelse=[] ) ] @@ -1262,7 +1282,7 @@ class QWeb(object): # display the tag without content orelse = [ast.If( test=ast.Name(id='force_display', ctx=ast.Load()), - body=self._compile_tag(el, [], options, True), + body=self._compile_tag(el, [], options, True) or [ast.Pass()], orelse=[], )] @@ -1306,7 +1326,7 @@ class QWeb(object): # display the tag without content ast.If( test=ast.Name(id=default_content, ctx=ast.Load()), - body=self._compile_tag(el, [self._append(ast.Name(id=default_content, ctx=ast.Load()))], options, True), + body=self._compile_tag(el, [self._append(ast.Name(id=default_content, ctx=ast.Load()))], options, True) or [ast.Pass()], orelse=orelse, ) ] From 8f6e5dbdd378dccc6fcdb8e41d0cee3d9afc7941 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 10:10:51 +0200 Subject: [PATCH 3/8] [FIX] P3: assertItemsEqual is called assertCountEqual The python 3 name is more correct as to what the assertion actually does, but still makes for a breaking change. --- odoo/tests/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/odoo/tests/common.py b/odoo/tests/common.py index 15d17dbf611..799863c76eb 100644 --- a/odoo/tests/common.py +++ b/odoo/tests/common.py @@ -23,6 +23,8 @@ from pprint import pformat import requests +from odoo.tools import pycompat + try: from itertools import zip_longest as izip_longest except ImportError: @@ -158,6 +160,10 @@ class BaseCase(TreeCase): doc = self._testMethodDoc return doc and ' '.join(l.strip() for l in doc.splitlines() if not l.isspace()) or None + if not pycompat.PY2: + # turns out this thing may not be quite as useful as we thought... + def assertItemsEqual(self, a, b, msg=None): + self.assertCountEqual(a, b, msg=None) class TransactionCase(BaseCase): """ TestCase in which each test method is run in its own transaction, From d5c80fd92d9f38f998a55c95dd295fb3cd00c362 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Wed, 17 May 2017 17:06:05 +0200 Subject: [PATCH 4/8] [FIX] P3: xmlrpclib -> xmlrpc.client --- addons/mail/models/mail_thread.py | 5 ++++- addons/mail/static/scripts/openerp_mailgate.py | 5 ++++- setup/package.py | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/addons/mail/models/mail_thread.py b/addons/mail/models/mail_thread.py index b07adf60608..ebe11eb09d9 100644 --- a/addons/mail/models/mail_thread.py +++ b/addons/mail/models/mail_thread.py @@ -13,7 +13,10 @@ import pytz import re import socket import time -import xmlrpclib +try: + from xmlrpc import client as xmlrpclib +except ImportError: + import xmlrpclib from collections import namedtuple from email.message import Message diff --git a/addons/mail/static/scripts/openerp_mailgate.py b/addons/mail/static/scripts/openerp_mailgate.py index aed6db0681f..7ad0901ef22 100755 --- a/addons/mail/static/scripts/openerp_mailgate.py +++ b/addons/mail/static/scripts/openerp_mailgate.py @@ -9,7 +9,10 @@ import cgitb import time import optparse import sys -import xmlrpclib +try: + from xmlrpc import client as xmlrpclib +except ImportError: + import xmlrpclib import smtplib from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase diff --git a/setup/package.py b/setup/package.py index fcb174435ac..d7c712aba7e 100755 --- a/setup/package.py +++ b/setup/package.py @@ -10,7 +10,10 @@ import signal import subprocess import tempfile import time -import xmlrpclib +try: + from xmlrpc import client as xmlrpclib +except ImportError: + import xmlrpclib from contextlib import contextmanager from glob import glob from os.path import abspath, dirname, join From 5810ea2a97d63b7302f335cc64a2101d53995297 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 19 May 2017 11:11:42 +0200 Subject: [PATCH 5/8] [FIX] P3: random.sample on a recordset In P3, random.sample requires that its parameter be a Sequence or a Set. This blows up _cart_accessories. So opt BaseModel into both ABC, seems to work fine. --- odoo/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/odoo/models.py b/odoo/models.py index e3e90a12975..8adb0d9c7d8 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -22,6 +22,8 @@ """ import datetime + +import collections import dateutil import functools import itertools @@ -4952,7 +4954,9 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): } return result - +collections.Set.register(BaseModel) +# not exactly true as BaseModel doesn't have __reversed__, index or count +collections.Sequence.register(BaseModel) class RecordCache(MutableMapping): """ Implements a proxy dictionary to read/update the cache of a record. From aaf8f3e37fa3c84c3b0cd0df1b7cb4f56c2ff95c Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 22 May 2017 16:40:31 +0200 Subject: [PATCH 6/8] [IMP] reorganise opcodes in categories --- odoo/addons/base/tests/test_base.py | 7 ++++- odoo/tools/safe_eval.py | 47 ++++++++++++++++++----------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/odoo/addons/base/tests/test_base.py b/odoo/addons/base/tests/test_base.py index 5a45f00e95c..6d6d209d352 100644 --- a/odoo/addons/base/tests/test_base.py +++ b/odoo/addons/base/tests/test_base.py @@ -7,10 +7,15 @@ import unittest from odoo.exceptions import ValidationError from odoo.tests.common import TransactionCase from odoo.tools import mute_logger -from odoo.tools.safe_eval import safe_eval +from odoo.tools.safe_eval import safe_eval, const_eval class TestSafeEval(unittest.TestCase): + def test_const(self): + # NB: True and False are names in Python 2 not consts + expected = (1, {"a": {2.5}}, [None, u"foo"]) + actual = const_eval('(1, {"a": {2.5}}, [None, u"foo"])') + self.assertEqual(actual, expected) def test_01_safe_eval(self): """ Try a few common expressions to verify they work with safe_eval """ diff --git a/odoo/tools/safe_eval.py b/odoo/tools/safe_eval.py index b34757df23c..de767fb7453 100644 --- a/odoo/tools/safe_eval.py +++ b/odoo/tools/safe_eval.py @@ -41,8 +41,6 @@ _ALLOWED_MODULES = ['_strptime', 'math', 'time'] _UNSAFE_ATTRIBUTES = ['f_builtins', 'f_globals', 'f_locals', 'gi_frame', 'co_code', 'func_globals'] _POSSIBLE_OPCODES_P3 = [ - # comprehensions, actually in P2 - 'MAP_ADD', 'SET_ADD', # opcodes for `with` statement cleanup process 'WITH_CLEANUP_START', 'WITH_CLEANUP_FINISH', # f-strings @@ -59,16 +57,23 @@ _POSSIBLE_OPCODES_P3 = [ 'BINARY_MATRIX_MULTIPLY', 'INPLACE_MATRIX_MULTIPLY', ] +# opcodes necessary to build literal values _CONST_OPCODES = set(opmap[x] for x in [ + # stack manipulations 'POP_TOP', 'ROT_TWO', 'ROT_THREE', 'ROT_FOUR', 'DUP_TOP', 'DUP_TOPX', - 'POP_BLOCK','SETUP_LOOP', 'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE', - 'LOAD_CONST', 'RETURN_VALUE', 'STORE_SUBSCR', 'STORE_MAP', - # maps with constant keys optimisation https://bugs.python.org/issue27140 + 'DUP_TOP_TWO', # replaces DUP_TOPX in P3 + 'LOAD_CONST', + 'RETURN_VALUE', # return the result of the literal/expr evaluation + # literal collections + 'BUILD_LIST', 'BUILD_MAP', 'BUILD_TUPLE', 'BUILD_SET', + # 3.6: literal map with constant keys https://bugs.python.org/issue27140 'BUILD_CONST_KEY_MAP', - 'POP_EXCEPT', # Seems to be a special-case of POP_BLOCK for P3 - 'DUP_TOP_TWO', # replaces DUP_TOPX in P3 + # until Python 3.5, literal maps are compiled to creating an empty map + # (pre-sized) then filling it key by key + 'STORE_MAP', ] if x in opmap) +# operations on literal values _EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [ 'UNARY_POSITIVE', 'UNARY_NEGATIVE', 'UNARY_NOT', 'UNARY_INVERT', 'BINARY_POWER', 'BINARY_MULTIPLY', @@ -78,24 +83,30 @@ _EXPR_OPCODES = _CONST_OPCODES.union(set(opmap[x] for x in [ 'BINARY_OR', 'INPLACE_ADD', 'INPLACE_SUBTRACT', 'INPLACE_MULTIPLY', 'INPLACE_DIVIDE', 'INPLACE_REMAINDER', 'INPLACE_POWER', 'INPLACE_LEFTSHIFT', 'INPLACE_RIGHTSHIFT', 'INPLACE_AND', - 'INPLACE_XOR','INPLACE_OR' + 'INPLACE_XOR','INPLACE_OR', 'STORE_SUBSCR', + # slice operations (Python 3 only has BUILD_SLICE) + 'SLICE+0', 'SLICE+1', 'SLICE+2', 'SLICE+3', 'BUILD_SLICE', + # comprehensions + 'LIST_APPEND', 'MAP_ADD', 'SET_ADD', + 'COMPARE_OP', ] if x in opmap)) _SAFE_OPCODES = _EXPR_OPCODES.union(set(opmap[x] for x in [ - 'LOAD_NAME', 'CALL_FUNCTION', 'COMPARE_OP', 'LOAD_ATTR', - 'STORE_NAME', 'GET_ITER', 'FOR_ITER', 'LIST_APPEND', 'DELETE_NAME', - 'JUMP_FORWARD', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE', 'JUMP_ABSOLUTE', - 'MAKE_FUNCTION', 'SLICE+0', 'SLICE+1', 'SLICE+2', 'SLICE+3', 'BUILD_SLICE', - 'BREAK_LOOP', 'CONTINUE_LOOP', 'RAISE_VARARGS', 'YIELD_VALUE', - # New in Python 2.7 - http://bugs.python.org/issue4715 : - 'JUMP_IF_FALSE_OR_POP', 'JUMP_IF_TRUE_OR_POP', 'POP_JUMP_IF_FALSE', - 'POP_JUMP_IF_TRUE', 'SETUP_EXCEPT', 'END_FINALLY', - 'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'UNPACK_SEQUENCE', - 'LOAD_GLOBAL', # Only allows access to restricted globals + 'POP_BLOCK', 'POP_EXCEPT', # Seems to be a special-case of POP_BLOCK for P3 + 'SETUP_LOOP', 'BREAK_LOOP', 'CONTINUE_LOOP', + 'MAKE_FUNCTION', 'CALL_FUNCTION', # P3: https://bugs.python.org/issue27213 'CALL_FUNCTION_EX', # Already in P2 but apparently the first one is used more aggressively in P3 'CALL_FUNCTION_KW', 'CALL_FUNCTION_VAR', 'CALL_FUNCTION_VAR_KW', + 'GET_ITER', 'FOR_ITER', 'YIELD_VALUE', + 'JUMP_FORWARD', 'JUMP_IF_TRUE', 'JUMP_IF_FALSE', 'JUMP_ABSOLUTE', + # New in Python 2.7 - http://bugs.python.org/issue4715 : + 'JUMP_IF_FALSE_OR_POP', 'JUMP_IF_TRUE_OR_POP', 'POP_JUMP_IF_FALSE', + 'POP_JUMP_IF_TRUE', 'SETUP_EXCEPT', 'END_FINALLY', 'RAISE_VARARGS', + 'LOAD_NAME', 'STORE_NAME', 'DELETE_NAME', 'LOAD_ATTR', + 'LOAD_FAST', 'STORE_FAST', 'DELETE_FAST', 'UNPACK_SEQUENCE', + 'LOAD_GLOBAL', # Only allows access to restricted globals ] if x in opmap)) _logger = logging.getLogger(__name__) From c5dcf75a813174ac5b27b4bc1fdeba1243e04347 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Tue, 23 May 2017 11:05:03 +0200 Subject: [PATCH 7/8] [FIX] P3: Unpickler.find_global -> Unpickler.find_class --- odoo/tools/misc.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py index a698bf394e8..03b784dec2e 100644 --- a/odoo/tools/misc.py +++ b/odoo/tools/misc.py @@ -1104,10 +1104,12 @@ def _consteq(str1, str2): consteq = getattr(passlib.utils, 'consteq', _consteq) +# forbid globals entirely: str/unicode, int/long, float, bool, tuple, list, dict, None +class Unpickler(pickle_.Unpickler, object): + find_global = None # Python 2 + find_class = None # Python 3 def _pickle_load(stream, errors=False): - unpickler = pickle_.Unpickler(stream) - # pickle builtins: str/unicode, int/long, float, bool, tuple, list, dict, None - unpickler.find_global = None + unpickler = Unpickler(stream) try: return unpickler.load() except Exception: From 5ee9ec7d6c5608ad75c63852557244ae65547c25 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Tue, 23 May 2017 11:05:37 +0200 Subject: [PATCH 8/8] [FIX] P3: __nonzero__ -> __bool__ Also ~removed bool-testing on cursors --- odoo/models.py | 6 ++++-- odoo/sql_db.py | 12 +++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/odoo/models.py b/odoo/models.py index 8adb0d9c7d8..8a44c40d006 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -169,8 +169,9 @@ class MetaModel(api.Meta): class NewId(object): """ Pseudo-ids for new records. """ - def __nonzero__(self): + def __bool__(self): return False + __nonzero__ = __bool__ IdType = pycompat.integer_types + (str, unicode, NewId) @@ -4501,9 +4502,10 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): # "Dunder" methods # - def __nonzero__(self): + def __bool__(self): """ Test whether ``self`` is nonempty. """ return bool(getattr(self, '_ids', True)) + __nonzero__ = __bool__ def __len__(self): """ Return the size of ``self``. """ diff --git a/odoo/sql_db.py b/odoo/sql_db.py index f2448259e10..81b717ef026 100644 --- a/odoo/sql_db.py +++ b/odoo/sql_db.py @@ -642,15 +642,9 @@ class Connection(object): # serialized_cursor is deprecated - cursors are serialized by default serialized_cursor = cursor - def __nonzero__(self): - """Check if connection is possible""" - try: - _logger.info("__nonzero__() is deprecated. (It is too expensive to test a connection.)") - cr = self.cursor() - cr.close() - return True - except Exception: - return False + def __bool__(self): + raise NotImplementedError() + __nonzero__ = __bool__ def connection_info_for(db_or_uri): """ parse the given `db_or_uri` and return a 2-tuple (dbname, connection_params)