pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ADD] tools: patch C types, e.g. str.format

Browse Source 
Monkey-patching C types is not straightforward.
It relies on changing the attributes or methods in memory
at the right address with the exact right size.
This requires the greatest caution.
A simple mistake can mess up the memory used by the Python interpreter,
and for instance lead to `SegmentationFault` exceptions
or unforeseen behaviors.

However, being able to patch C type is a very powerful tool.

With great power comes great responsibility.

`patch_c_type` is implemented with the greateast caution.
The Python C-API documentation has been thoroughly followed
and understood.
In addition, this patch has been battle tested in real
conditions.

In the end, this allows to patch unwanted behaviors
from types implemented in C.

Co-authored-by: Denis Ledoux <dle@odoo.com>
Co-authored-by: Christophe Simonis <chs@odoo.com>
Co-authored-by: Mathieu Walravens <wama@odoo.com>
This commit is contained in:
Olivier Dony
2024-03-26 16:51:36 +00:00
co-authored by Denis Ledoux Christophe Simonis Mathieu Walravens
parent 515ea2afb5
commit 66832f0ba0
2 changed files with 205 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
odoo/addons/base/tests/test_base.py
+64
View File
@@ -3,6 +3,8 @@
import ast
from markupsafe import Markup
from odoo import Command
from odoo.tests.common import TransactionCase, BaseCase
from odoo.tools import mute_logger
@@ -60,6 +62,68 @@ class TestSafeEval(BaseCase):
with self.assertRaises(NameError):
safe_eval("self.__name__", {'self': self}, mode="exec")
def test_06_safe_eval_format(self):
# string.format
self.assertEqual(safe_eval("'__{0}__'.format('Foo')"), '__Foo__')
self.assertEqual(safe_eval("'{0.__self__}'.format(abs)"), '{0.__self__}')
self.assertEqual(safe_eval("'{0.f_globals}'.format(abs)"), '{0.f_globals}')
# string.format_map
self.assertEqual(safe_eval("'__{foo}__'.format_map({'foo': 'Foo'})"), '__Foo__')
self.assertEqual(safe_eval("'{foo.__self__}'.format_map({'foo': abs})"), '{foo.__self__}')
self.assertEqual(safe_eval("'{foo.f_globals}'.format_map({'foo': abs})"), '{foo.f_globals}')
# Evaluation context for Markup asserts
c = {"Markup": Markup}
# Markup.format
self.assertEqual(safe_eval("Markup('__{0}__').format('Foo')", c), Markup('__Foo__'))
with self.assertRaisesRegex(ValueError, 'Access to forbidden name'):
safe_eval("Markup('{0.__self__}').format(abs)", c)
with self.assertRaisesRegex(ValueError, 'Access to forbidden name'):
safe_eval("Markup('{0.f_globals}').format(abs)", c)
# Markup.format_map
self.assertEqual(safe_eval("Markup('__{foo}__').format_map({'foo': 'Foo'})", c), Markup('__Foo__'))
self.assertEqual(safe_eval("Markup('{foo.__self__}').format_map({'foo': abs})", c), Markup('{foo.__self__}'))
self.assertEqual(safe_eval("Markup('{foo.f_globals}').format_map({'foo': abs})", c), Markup('{foo.f_globals}'))
def test_07_safe_eval_attribute_error_obj(self):
locals_dict = {}
try:
safe_eval("""
try:
dict.foo
except Exception as e:
action = {'args': e.args, 'obj': e.obj, 'name': e.name}
""", locals_dict=locals_dict, mode="exec", nocopy=True)
except ValueError as e:
# AttributeError.name, AttributeError.obj added in Python 3.10
# https://github.com/python/cpython/commit/37494b441aced0362d7edd2956ab3ea7801e60c8
self.assertIn("'AttributeError' object has no attribute 'obj'", e.args[0])
else:
exception = locals_dict.get('action')
self.assertEqual(exception['args'], ("type object 'dict' has no attribute 'foo'",))
self.assertIsNone(exception['name'])
self.assertIsNone(exception['obj'])
attribute_error = None
try:
raise AttributeError('Foo', name='Bar', obj=[])
except TypeError as e:
# AttributeError does not take keyword arguments before Python 3.10
# https://github.com/python/cpython/commit/37494b441aced0362d7edd2956ab3ea7801e60c8
# Error can be either, according to the Python version:
# - AttributeError does not take keyword arguments
# - AttributeError() takes no keyword arguments
self.assertIn("keyword arguments", e.args[0])
except AttributeError as e:
attribute_error = e
if attribute_error:
self.assertEqual(attribute_error.args, ('Foo',))
self.assertEqual(attribute_error.name, 'Bar')
self.assertIsNone(attribute_error.obj)
class TestParentStore(TransactionCase):
""" Verify that parent_store computation is done right """
odoo/tools/_monkeypatches.py
+141
View File
@@ -1,6 +1,8 @@
import ast
import ctypes
import os
import logging
import string
from shutil import copyfileobj
from types import CodeType
@@ -12,6 +14,8 @@ from werkzeug.wrappers import Request, Response
from .json import scriptsafe
from odoo.tools.safe_eval import _UNSAFE_ATTRIBUTES
try:
from xlrd import xlsx
except ImportError:
@@ -65,3 +69,140 @@ def literal_eval(expr):
return orig_literal_eval(expr)
ast.literal_eval = literal_eval
def _is_safe_expr(expr):
return '__' not in expr and not any(att_name in expr for att_name in _UNSAFE_ATTRIBUTES)
origin_formatter_get_field = string.Formatter.get_field
def get_field(self, field_name, args, kwargs):
# Monkey-patch `get_field` to raise in case of access to a forbidden name
# Ref: https://github.com/python/cpython/blob/812245ecce2d8344c3748228047bab456816180a/Lib/string.py#L267
if not _is_safe_expr(field_name):
raise NameError('Access to forbidden name %r' % (field_name))
return origin_formatter_get_field(self, field_name, args, kwargs)
string.Formatter.get_field = get_field
#
# Monkey-Patch C types
#
# PyTypeObject is not in Python ABI, so we map it to a custom ctypes Struct
class PyTypeObject(ctypes.Structure):
# Ref: https://docs.python.org/3/c-api/typeobj.html
_fields_ = [
# cover PyObject variable header https://docs.python.org/3/c-api/typeobj.html#pyobject-slots
# + 15 first PyTypeObject slots: https://docs.python.org/3/c-api/typeobj.html#pytypeobject-slots
('_', 18 * ctypes.c_void_p),
# https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_getattro
('tp_getattro', ctypes.CFUNCTYPE(ctypes.py_object, ctypes.py_object, ctypes.py_object)),
('_', 14 * ctypes.c_void_p), # cover 14 slots, not needed so far
# https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dict
('tp_dict', ctypes.py_object),
('_', 3 * ctypes.c_void_p), # cover 3 slots, not needed so far
# https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_init
# /!\ last param mapped to c_void_p because it's nullable and PyObject doesn't support it
('tp_init', ctypes.CFUNCTYPE(ctypes.c_int, ctypes.py_object, ctypes.py_object, ctypes.c_void_p)),
]
_slot_mapping = {
# python_method: slot_name
'__init__': 'tp_init',
}
def patch_c_type(cls, attr, value):
# obtain the address of the C type - don't assume id(cls) is guaranteed to hold it
cls_pt = ctypes.py_object(cls)
cls_addr = ctypes.POINTER(ctypes.c_void_p)(cls_pt)[0]
# cast into our custom PyTypeObject struct
obj = PyTypeObject.from_address(cls_addr)
# CASE 1. Slot functions: replace slot function pointer
if attr.startswith("__") and attr.endswith("__"):
slot_func = PyTypeObject._slot_mapping[attr]
c_func_type = dict(PyTypeObject._fields_)[slot_func]
c_func = c_func_type(value) # C callback pointer for our patch function
# incref: store a ref in the type's __dict__
cls_dict = getattr(obj, "tp_dict")
cls_dict.setdefault('__patch_refs__', []).append(c_func)
# apply patch function
setattr(obj, slot_func, c_func)
# CASE 2. Other methods: replace the method in the __dict__ of the type
else:
cls_dict = getattr(obj, "tp_dict")
origin = cls_dict.get(attr)
if origin:
value.__name__ = origin.__name__
value.__qualname__ = origin.__qualname__
# apply patch function
cls_dict[attr] = value
# clear internal caches: https://docs.python.org/3/c-api/type.html#c.PyType_Modified
ctypes.pythonapi.PyType_Modified(cls_pt)
# Monkey-patch AttributeError to suppress assignation of `.obj`
if hasattr(AttributeError, 'obj'):
# AttributeError.name, AttributeError.obj added in Python 3.10
# https://github.com/python/cpython/commit/37494b441aced0362d7edd2956ab3ea7801e60c8
def __init__(self, args, kwargs):
if kwargs:
# py_object isn't NULLABLE, so we manually handle the NULL case from a void pointer
kwargs = ctypes.cast(kwargs, ctypes.py_object).value
else:
kwargs = {}
# emulate super __init__, we don't want to call it, it's cheaper
# it's here: https://github.com/python/cpython/blob/d0524caed0f3b77f271640460d0dff1a4c784087/Objects/exceptions.c#L1404
self.name = kwargs.get('name')
# assign .obj immediately so set_attribute_error_context() won't do it
# cfr https://github.com/python/cpython/commit/3b3be05a164da43f201e35b6dafbc840993a4d18
self.obj = None # pretend it was not really assigned (None != NULL pointer)
return 0
patch_c_type(AttributeError, "__init__", __init__)
def _mkpatch_str_format():
# Monkey-patch str.format to forbid usage of dunder attributes in replacement fields,
# keeping all refs in the closure
formatter = string.Formatter() # thread-safe parsing, can be shared
origin_format = str.format
origin_format_map = str.format_map
def _safe_format_fields(s):
# First quick pass with strstr(), matching both `{0.__foo__}` and `__{0}__`
if not _is_safe_expr(s):
# Second pass with a real parsing for "format fields" to avoid blocking `__{0}__`. cfr PEP-3101.
field_exprs = tuple(field_expr for _lit, field_expr, _fmt, _conv in formatter.parse(s) if field_expr)
for expr in field_exprs:
if not _is_safe_expr(expr):
return False
return True
def format(*args, **kwargs):
if not _safe_format_fields(args[0]):
return args[0] # pretend we forgot to format
return origin_format(*args, **kwargs)
def format_map(*args, **kwargs):
if not _safe_format_fields(args[0]):
return args[0] # pretend we forgot to format
return origin_format_map(*args, **kwargs)
patch_c_type(str, "format", format)
patch_c_type(str, "format_map", format_map)
_mkpatch_str_format()