From f61aa39ff1190f66cebb8efb8432723214932161 Mon Sep 17 00:00:00 2001 From: Julien Castiaux Date: Thu, 10 Feb 2022 09:49:23 +0000 Subject: [PATCH] [REF] core: HTTPocalypse (9) ORM initialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit is the 9th commit of a comprehensive refactor of our HTTP framework. See odoo/odoo#78857 for complete historic, discussions and rationnals. See also [REF] core: HTTPocalypse (11) ir.http base model. This is a two-part commit with "(11) ir.http base model". In this commit we focus on the initialization of the various ORM objects, namely: the registry, the cursor, the environment, the user and the context. In the other n°11 commit we focus on the ir.http model and its relation to the current http.py module. One of the objectives of this comprehensive refactor was to ease the cognitive complexity of the http framework, in other words to make it simplier. One of the problem identified quite early during the preparation of this work is the way the various ORM objects are initialized, modified and cleaned during the request lifetime. Before this work, all the ORM internals were lazily initialized via properties. It is the first time one uses `request.cr` that a cursor is opened to `request.db` and stored on `request._cr`. It is the first time one uses `request.env` that an environment is create with the current `request.user` and `request.context`. Upon user or context modification, the current environment is discarded, the next usage of `request.env` will create yet another environment on the fly using the modified user and/or context. Using this model, no ressource is initialized if not necessary. It is possible for nodb-compatible endpoint to be served via the db-compatible router and ir.http without ever opening a cursor to the database. But this model is harder to reason about and ultimately to maintain. In this work we propose to drop the lazy approach for a greedy one. In this work the first steps of `_serve_db`, the db-compatible counter-part of `_serve_nodb`, are dedicated to setup a registry, open a cursor to the database and create an environment using the session's user and context. In this work, when one wants to change the environ's user or context, he must call `request.update_env` or `request.update_context`, both method will recreate the environment *now* with the given values. The downside of this approach is that resources are always allocated even when it is not necessary. We argue that, in general, the controllers that do not use the ORM are rare thus it is rare we allocate unecessary resources. We also argue that the cognitive benefits are more than welcome and that the new APIs will help at writing more robust applications. PR: odoo#78857 Task: 2571224 --- odoo/http.py | 70 +++++++++++- odoo/service/model.py | 250 ++++++++++++++++++++---------------------- 2 files changed, 189 insertions(+), 131 deletions(-) diff --git a/odoo/http.py b/odoo/http.py index 9da9e10d4d7..4c7d933c467 100644 --- a/odoo/http.py +++ b/odoo/http.py @@ -845,7 +845,47 @@ class Request: # ===================================================== # Getters and setters # ===================================================== - ... + def update_env(self, user=None, context=None, su=None): + """ Update the environment of the current request. """ + cr = None # None is a sentinel, it keeps the same cursor + self.env = self.env(cr, user, context, su) + threading.current_thread().uid = self.env.uid + + def update_context(self, **overrides): + """ + Override the environment context of the current request with the + values of ``overrides``. To replace the entire context, please + use :meth:`~update_env`: instead. + """ + self.update_env(context=dict(self.env.context, **overrides)) + + @property + def context(self): + return self.env.context + + @context.setter + def context(self, value): + raise NotImplementedError("Use request.update_context instead.") + + @property + def uid(self): + return self.env.uid + + @uid.setter + def uid(self, value): + raise NotImplementedError("Use request.update_env instead.") + + @property + def cr(self): + return self.env.cr + + @cr.setter + def cr(self, value): + if value is None: + raise NotImplementedError("Close the cursor instead.") + raise ValueError("You cannot replace the cursor attached to the current request.") + + _cr = cr # ===================================================== # Helpers @@ -957,6 +997,34 @@ class Request: return response def _serve_db(self): + """ + Prepare the user session and load the ORM before forwarding the + request to ``_serve_ir_http``. + """ + try: + self.registry = Registry(self.db) + self.registry.check_signaling() + except (AttributeError, psycopg2.OperationalError, psycopg2.ProgrammingError): + # psycopg2 error or attribute error while constructing + # the registry. That means either + # - the database probably does not exists anymore, or + # - the database is corrupted, or + # - the database version doesnt match the server version. + # So remove the database from the cookie + self.db = None + self.session.db = None + root.session_store.save(self.session) + return self.redirect('/web/database/selector') + + with contextlib.closing(self.registry.cursor()) as cr: + self.env = odoo.api.Environment(cr, self.session.uid, self.session.context) + threading.current_thread().uid = self.env.uid + try: + return service_model.retrying(self._serve_ir_http, self.env) + except Exception as exc: + ... + + def _serve_ir_http(self): ... diff --git a/odoo/service/model.py b/odoo/service/model.py index bcc08912ecd..9b26388e001 100644 --- a/odoo/service/model.py +++ b/odoo/service/model.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. -from contextlib import closing -from functools import wraps +from functools import partial +import json import logging from psycopg2 import IntegrityError, OperationalError, errorcodes import random @@ -10,8 +10,10 @@ import time import odoo from odoo.exceptions import UserError, ValidationError +from odoo.http import request from odoo.models import check_method_name -from odoo.tools.translate import translate, translate_sql_constraint +from odoo.tools import DotDict +from odoo.tools.translate import translate_sql_constraint from odoo.tools.translate import _ from . import security @@ -22,142 +24,31 @@ _logger = logging.getLogger(__name__) PG_CONCURRENCY_ERRORS_TO_RETRY = (errorcodes.LOCK_NOT_AVAILABLE, errorcodes.SERIALIZATION_FAILURE, errorcodes.DEADLOCK_DETECTED) MAX_TRIES_ON_CONCURRENCY_FAILURE = 5 + def dispatch(method, params): - (db, uid, passwd ) = params[0], int(params[1]), params[2] + db, uid, passwd = params[0], int(params[1]), params[2] + security.check(db, uid, passwd) - # set uid tracker - cleaned up at the WSGI - # dispatching phase in odoo.service.wsgi_server.application + threading.current_thread().dbname = db threading.current_thread().uid = uid - - params = params[3:] - if method == 'obj_list': - raise NameError("obj_list has been discontinued via RPC as of 6.0, please query ir.model directly!") - if method not in ['execute', 'execute_kw']: - raise NameError("Method not available %s" % method) - security.check(db,uid,passwd) registry = odoo.registry(db).check_signaling() - fn = globals()[method] with registry.manage_changes(): - res = fn(db, uid, *params) + if method == 'execute': + res = execute(db, uid, *params[3:]) + elif method == 'execute_kw': + res = execute_kw(db, uid, *params[3:]) + else: + raise NameError("Method not available %s" % method) return res -def check(f): - @wraps(f) - def wrapper(___dbname, *args, **kwargs): - """ Wraps around OSV functions and normalises a few exceptions - """ - dbname = ___dbname # NOTE: this forbid to use "___dbname" as arguments in http routes - - def tr(src, ttype): - # We try to do the same as the _(), but without the frame - # inspection, since we already are wrapping an osv function - # trans_obj = self.get('ir.translation') cannot work yet :( - ctx = {} - if not kwargs: - if args and isinstance(args[-1], dict): - ctx = args[-1] - elif isinstance(kwargs, dict): - if 'context' in kwargs: - ctx = kwargs['context'] - elif 'kwargs' in kwargs and kwargs['kwargs'].get('context'): - # http entry points such as call_kw() - ctx = kwargs['kwargs'].get('context') - else: - try: - from odoo.http import request - ctx = request.env.context - except Exception: - pass - - lang = ctx and ctx.get('lang') - if not (lang or hasattr(src, '__call__')): - return src - - # We open a *new* cursor here, one reason is that failed SQL - # queries (as in IntegrityError) will invalidate the current one. - with closing(odoo.sql_db.db_connect(dbname).cursor()) as cr: - if ttype == 'sql_constraint': - res = translate_sql_constraint(cr, key=key, lang=lang) - else: - res = translate(cr, name=False, source_type=ttype, - lang=lang, source=src) - return res or src - - def _(src): - return tr(src, 'code') - - tries = 0 - while True: - try: - if odoo.registry(dbname)._init and not odoo.tools.config['test_enable']: - raise odoo.exceptions.Warning('Currently, this database is not fully loaded and can not be used.') - return f(dbname, *args, **kwargs) - except OperationalError as e: - # Automatically retry the typical transaction serialization errors - if e.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: - raise - if tries >= MAX_TRIES_ON_CONCURRENCY_FAILURE: - _logger.info("%s, maximum number of tries reached" % errorcodes.lookup(e.pgcode)) - raise - wait_time = random.uniform(0.0, 2 ** tries) - tries += 1 - _logger.info("%s, retry %d/%d in %.04f sec..." % (errorcodes.lookup(e.pgcode), tries, MAX_TRIES_ON_CONCURRENCY_FAILURE, wait_time)) - time.sleep(wait_time) - except IntegrityError as inst: - registry = odoo.registry(dbname) - key = inst.diag.constraint_name - if key in registry._sql_constraints: - raise ValidationError(tr(key, 'sql_constraint') or inst.pgerror) - if inst.pgcode in (errorcodes.NOT_NULL_VIOLATION, errorcodes.FOREIGN_KEY_VIOLATION, errorcodes.RESTRICT_VIOLATION): - msg = _('The operation cannot be completed:') - _logger.debug("IntegrityError", exc_info=True) - try: - # Get corresponding model and field - model = field = None - for name, rclass in registry.items(): - if inst.diag.table_name == rclass._table: - model = rclass - field = model._fields.get(inst.diag.column_name) - break - if inst.pgcode == errorcodes.NOT_NULL_VIOLATION: - # This is raised when a field is set with `required=True`. 2 cases: - # - Create/update: a mandatory field is not set. - # - Delete: another model has a not nullable using the deleted record. - msg += '\n' - msg += _( - '- Create/update: a mandatory field is not set.\n' - '- Delete: another model requires the record being deleted. If possible, archive it instead.' - ) - if model: - msg += '\n\n{} {} ({}), {} {} ({})'.format( - _('Model:'), model._description, model._name, - _('Field:'), field.string if field else _('Unknown'), field.name if field else _('Unknown'), - ) - elif inst.pgcode == errorcodes.FOREIGN_KEY_VIOLATION: - # This is raised when a field is set with `ondelete='restrict'`, at - # unlink only. - msg += _(' another model requires the record being deleted. If possible, archive it instead.') - constraint = inst.diag.constraint_name - if model or constraint: - msg += '\n\n{} {} ({}), {} {}'.format( - _('Model:'), model._description if model else _('Unknown'), model._name if model else _('Unknown'), - _('Constraint:'), constraint if constraint else _('Unknown'), - ) - except Exception: - pass - raise ValidationError(msg) - else: - raise ValidationError(inst.args[0]) - - return wrapper def execute_cr(cr, uid, obj, method, *args, **kw): # clean cache etc if we retry the same transaction - cr.reset() - recs = odoo.api.Environment(cr, uid, {}).get(obj) + env = odoo.api.Environment(cr, uid, {}) + recs = env.get(obj) if recs is None: raise UserError(_("Object %s doesn't exist", obj)) - result = odoo.api.call_kw(recs, method, args, kw) + result = retrying(partial(odoo.api.call_kw, recs, method, args, kw), env) # force evaluation of lazy values before the cursor is closed, as it would # error afterwards if the lazy isn't already evaluated (and cached) for l in traverse_containers(result, lazy): @@ -168,12 +59,111 @@ def execute_cr(cr, uid, obj, method, *args, **kw): def execute_kw(db, uid, obj, method, args, kw=None): return execute(db, uid, obj, method, *args, **kw or {}) -@check + def execute(db, uid, obj, method, *args, **kw): - threading.currentThread().dbname = db with odoo.registry(db).cursor() as cr: check_method_name(method) res = execute_cr(cr, uid, obj, method, *args, **kw) if res is None: _logger.info('The method %s of the object %s can not return `None` !', method, obj) return res + + +def _as_validation_error(env, exc): + """ Return the IntegrityError encapsuled in a nice ValidationError """ + + unknown = _('Unknown') + for _name, rclass in env.registry.items(): + if exc.diag.table_name == rclass._table: + model = rclass + field = model._fields.get(exc.diag.column_name) + break + else: + model = DotDict({'_name': unknown.lower(), '_description': unknown}) + field = DotDict({'name': unknown.lower(), 'string': unknown}) + + if exc.pgcode == errorcodes.NOT_NULL_VIOLATION: + return ValidationError(_( + "The operation cannot be completed:\n" + "- Create/update: a mandatory field is not set.\n" + "- Delete: another model requires the record being deleted." + " If possible, archive it instead.\n\n" + "Model: %(model_name)s (%(model_tech_name)s)\n" + "Field: %(field_name)s (%(field_tech_name)s)\n", + model_name=model._description, + model_tech_name=model._name, + field_name=field.string, + field_tech_name=field.name, + )) + + if exc.pgcode == errorcodes.FOREIGN_KEY_VIOLATION: + return ValidationError(_( + "The operation cannot be completed: another model requires " + "the record being deleted. If possible, archive it instead.\n\n" + "Model: %(model_name)s (%(model_tech_name)s)\n" + "Constraint: %(constraint)s\n", + model_name=model._description, + model_tech_name=model._name, + constraint=exc.diag.constraint_name, + )) + + if exc.diag.constraint_name in env.registry._sql_constraints: + return ValidationError(_( + "The operation cannot be completed: %s", + translate_sql_constraint(env.cr, exc.diag.constraint_name, env.context['lang']) + )) + + return ValidationError(_("The operation cannot be completed: %s", exc.args[0])) + + +def retrying(func, env): + """ + Call ``func`` until the function returns without serialisation + error. A serialisation error occurs when two requests in independent + cursors perform incompatible changes (such as writing different + values on a same record). By default, it retries up to 5 times. + + :param callable func: The function to call, you can pass arguments + using :func:`functools.partial`:. + :param odoo.api.Environment env: The environment where the registry + and the cursor are taken. + """ + try: + for tryno in range(1, MAX_TRIES_ON_CONCURRENCY_FAILURE + 1): + tryleft = MAX_TRIES_ON_CONCURRENCY_FAILURE - tryno + try: + result = func() + if not env.cr._closed: + env.cr.flush() # submit the changes to the database + break + except (IntegrityError, OperationalError) as exc: + if env.cr._closed: + raise + env.cr.rollback() + env.registry.reset_changes() + if request: + request.session.clear() + request.session.update(json.loads(request.session.json_data)) + if isinstance(exc, IntegrityError): + raise _as_validation_error(env, exc) from exc + if exc.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: + raise + if not tryleft: + _logger.info("%s, maximum number of tries reached!", errorcodes.lookup(exc.pgcode)) + raise + + wait_time = random.uniform(0.0, 2 ** tryno) + _logger.info("%s, %s tries left, try again in %.04f sec...", errorcodes.lookup(exc.pgcode), tryleft, wait_time) + time.sleep(wait_time) + else: + # handled in the "if not tryleft" case + raise RuntimeError("unreachable") + + except Exception: + env.registry.reset_changes() + raise + + if not env.cr._closed: + env.cr.commit() # effectively commits and execute post-commits + env.registry.signal_changes() + return result