From 058cf208a8e7d43ebfc2e761ec1cbfc6e7a56351 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 26 Nov 2019 16:04:02 +0000 Subject: [PATCH] [ADD] sql_db: pre/post-commit/rollback hooks --- odoo/addons/base/models/ir_cron.py | 10 +- odoo/addons/base/tests/test_misc.py | 32 +++++ odoo/sql_db.py | 174 ++++++++++++++++------------ odoo/tests/common.py | 4 + odoo/tools/misc.py | 50 ++++++++ 5 files changed, 189 insertions(+), 81 deletions(-) diff --git a/odoo/addons/base/models/ir_cron.py b/odoo/addons/base/models/ir_cron.py index a9821e782f7..818a8e70d5d 100644 --- a/odoo/addons/base/models/ir_cron.py +++ b/odoo/addons/base/models/ir_cron.py @@ -119,8 +119,8 @@ class ir_cron(models.Model): :param cron_cr: cursor holding lock on the cron job row, to use to update the next exec date, must not be committed/rolled back! """ - try: - with api.Environment.manage(): + with api.Environment.manage(): + try: cron = api.Environment(job_cr, job['user_id'], { 'lastcall': fields.Datetime.from_string(job['lastcall']) })[cls._name] @@ -153,9 +153,9 @@ class ir_cron(models.Model): cron.flush() cron.invalidate_cache() - finally: - job_cr.commit() - cron_cr.commit() + finally: + job_cr.commit() + cron_cr.commit() @classmethod def _process_jobs(cls, db_name): diff --git a/odoo/addons/base/tests/test_misc.py b/odoo/addons/base/tests/test_misc.py index 2f504756deb..b37ee11246a 100644 --- a/odoo/addons/base/tests/test_misc.py +++ b/odoo/addons/base/tests/test_misc.py @@ -270,3 +270,35 @@ class TestFormatLangDate(TransactionCase): # Check given `lang_code` overwites context lang self.assertEqual(misc.format_time(lang.with_context(lang='fr_FR').env, time_part, time_format='short', lang_code='zh_CN'), '\u4e0b\u53484:30') self.assertEqual(misc.format_time(lang.with_context(lang='zh_CN').env, time_part, time_format='medium', lang_code='fr_FR'), '16:30:22') + + +class TestGroupCalls(BaseCase): + def test_callbacks(self): + log = [] + + def foo(): + log.append("foo") + + def bar(items): + log.extend(items) + callbacks.add(baz) + + def baz(): + log.append("baz") + + callbacks = misc.GroupCalls() + callbacks.add(foo) + callbacks.add(bar, list)[0].append(1) + callbacks.add(bar, list)[0].append(2) + self.assertEqual(log, []) + + callbacks() + self.assertEqual(log, ["foo", 1, 2, "baz"]) + + callbacks() + self.assertEqual(log, ["foo", 1, 2, "baz"]) + + callbacks.add(bar, list)[0].append(3) + callbacks.clear() + callbacks() + self.assertEqual(log, ["foo", 1, 2, "baz"]) diff --git a/odoo/sql_db.py b/odoo/sql_db.py index 922540220b7..516eb175633 100644 --- a/odoo/sql_db.py +++ b/odoo/sql_db.py @@ -15,6 +15,7 @@ import logging import time import uuid +from decorator import decorator import psycopg2 import psycopg2.extras import psycopg2.extensions @@ -80,7 +81,72 @@ re_into = re.compile('.* into "?([a-zA-Z_0-9]+)"? .*$') sql_counter = 0 -class Cursor(object): + +@decorator +def check(f, self, *args, **kwargs): + """ Wrap a cursor method that cannot be called when the cursor is closed. """ + if self._closed: + msg = 'Unable to use a closed cursor.' + if self.__closer: + msg += ' It was closed at %s, line %s' % self.__closer + raise psycopg2.OperationalError(msg) + return f(self, *args, **kwargs) + + +class BaseCursor: + """ Base class for cursors that manages pre/post commit/rollback hooks. """ + + def __init__(self): + self.precommit = tools.GroupCalls() + self.postcommit = tools.GroupCalls() + self.prerollback = tools.GroupCalls() + self.postrollback = tools.GroupCalls() + + @contextmanager + @check + def savepoint(self, flush=True): + """context manager entering in a new savepoint""" + name = uuid.uuid1().hex + if flush: + flush_env(self) + self.precommit() + self.prerollback.clear() + self.execute('SAVEPOINT "%s"' % name) + try: + yield + if flush: + flush_env(self) + self.precommit() + self.prerollback.clear() + except Exception: + if flush: + clear_env(self) + self.precommit.clear() + self.prerollback() + self.execute('ROLLBACK TO SAVEPOINT "%s"' % name) + raise + else: + self.execute('RELEASE SAVEPOINT "%s"' % name) + + def __enter__(self): + """ Using the cursor as a contextmanager automatically commits and + closes it:: + + with cr: + cr.execute(...) + + # cr is committed if no failure occurred + # cr is closed in any case + """ + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + self.commit() + self.close() + + +class Cursor(BaseCursor): """Represents an open transaction to the PostgreSQL DB backend, acting as a lightweight wrapper around psycopg2's ``cursor`` objects. @@ -152,18 +218,9 @@ class Cursor(object): """ IN_MAX = 1000 # decent limit on size of IN queries - guideline = Oracle limit - def check(f): - @wraps(f) - def wrapper(self, *args, **kwargs): - if self._closed: - msg = 'Unable to use a closed cursor.' - if self.__closer: - msg += ' It was closed at %s, line %s' % self.__closer - raise psycopg2.OperationalError(msg) - return f(self, *args, **kwargs) - return wrapper - def __init__(self, pool, dbname, dsn, serialized=True): + super().__init__() + self.sql_from_log = {} self.sql_into_log = {} @@ -197,9 +254,6 @@ class Cursor(object): self.cache = {} - # event handlers, see method after() below - self._event_handlers = {'commit': [], 'rollback': []} - def __build_dict(self, row): return {d.name: row[i] for i, d in enumerate(self._obj.description)} def dictfetchone(self): @@ -373,71 +427,33 @@ class Cursor(object): back or committed independently. You may consider the use of a dedicated temporary cursor to do some database operation. """ - self._event_handlers[event].append(func) - - def _pop_event_handlers(self): - # return the current handlers, and reset them on self - result = self._event_handlers - self._event_handlers = {'commit': [], 'rollback': []} - return result + if event == 'commit': + self.postcommit.add(func) + elif event == 'rollback': + self.postrollback.add(func) @check def commit(self): - """ Perform an SQL `COMMIT` - """ + """ Perform an SQL `COMMIT` """ flush_env(self) + self.precommit() result = self._cnx.commit() - for func in self._pop_event_handlers()['commit']: - func() + self.prerollback.clear() + self.postrollback.clear() + self.postcommit() return result @check def rollback(self): - """ Perform an SQL `ROLLBACK` - """ + """ Perform an SQL `ROLLBACK` """ clear_env(self) + self.precommit.clear() + self.postcommit.clear() + self.prerollback() result = self._cnx.rollback() - for func in self._pop_event_handlers()['rollback']: - func() + self.postrollback() return result - def __enter__(self): - """ Using the cursor as a contextmanager automatically commits and - closes it:: - - with cr: - cr.execute(...) - - # cr is committed if no failure occurred - # cr is closed in any case - """ - return self - - def __exit__(self, exc_type, exc_value, traceback): - if exc_type is None: - self.commit() - self.close() - - @contextmanager - @check - def savepoint(self, flush=True): - """context manager entering in a new savepoint""" - name = uuid.uuid1().hex - if flush: - flush_env(self) - self.execute('SAVEPOINT "%s"' % name) - try: - yield - if flush: - flush_env(self) - except Exception: - if flush: - clear_env(self) - self.execute('ROLLBACK TO SAVEPOINT "%s"' % name) - raise - else: - self.execute('RELEASE SAVEPOINT "%s"' % name) - @check def __getattr__(self, name): return getattr(self._obj, name) @@ -447,7 +463,7 @@ class Cursor(object): return self._closed -class TestCursor(object): +class TestCursor(BaseCursor): """ A pseudo-cursor to be used for tests, on top of a real cursor. It keeps the transaction open across requests, and simulates committing, rolling back, and closing: @@ -488,21 +504,27 @@ class TestCursor(object): def autocommit(self, on): _logger.debug("TestCursor.autocommit(%r) does nothing", on) + @check def commit(self): + """ Perform an SQL `COMMIT` """ flush_env(self) + self.precommit() self._cursor.execute('SAVEPOINT "%s"' % self._savepoint) + self.prerollback.clear() + # ignore post-commit/rollback hooks + self.postcommit.clear() + self.postrollback.clear() + @check def rollback(self): + """ Perform an SQL `ROLLBACK` """ clear_env(self) + self.precommit.clear() + self.prerollback() self._cursor.execute('ROLLBACK TO SAVEPOINT "%s"' % self._savepoint) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - if exc_type is None: - self.commit() - self.close() + # ignore post-commit/rollback hooks + self.postcommit.clear() + self.postrollback.clear() def __getattr__(self, name): value = getattr(self._cursor, name) diff --git a/odoo/tests/common.py b/odoo/tests/common.py index 87a7889e4ec..c72832b4ad0 100644 --- a/odoo/tests/common.py +++ b/odoo/tests/common.py @@ -382,10 +382,12 @@ class BaseCase(TreeCase, MetaCase('DummyCase', (object,), {})): expected = counters.get(login, default) if flush: self.env.user.flush() + self.env.cr.precommit() count0 = self.cr.sql_log_count yield if flush: self.env.user.flush() + self.env.cr.precommit() count = self.cr.sql_log_count - count0 if count != expected: # add some info on caller to allow semi-automatic update of query count @@ -404,9 +406,11 @@ class BaseCase(TreeCase, MetaCase('DummyCase', (object,), {})): # same operations, otherwise the caches might not be ready! if flush: self.env.user.flush() + self.env.cr.precommit() yield if flush: self.env.user.flush() + self.env.cr.precommit() def assertRecordValues(self, records, expected_values): ''' Compare a recordset with a list of dictionaries representing the expected results. diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py index d2fe3e5c79f..534493afa95 100644 --- a/odoo/tools/misc.py +++ b/odoo/tools/misc.py @@ -1088,6 +1088,56 @@ class LastOrderedSet(OrderedSet): OrderedSet.add(self, elem) +class GroupCalls: + """ A collection of callbacks with support for aggregated arguments. Upon + call, every registered function is called once with positional arguments. + When registering a function, a tuple of positional arguments is returned, so + that the caller can modify the arguments in place. This allows to + accumulate some data to process once:: + + callbacks = GroupCalls() + + # register print (by default with a list) + [args] = callbacks.register(print, list) + args.append(42) + + # add an element to the list to print + [args] = callbacks.register(print, list) + args.append(43) + + # print "[42, 43]" + callbacks() + """ + def __init__(self): + self._func_args = {} # {func: args} + + def __call__(self): + """ Call all the registered functions (in first addition order) with + their respective arguments. Only recurrent functions remain registered + after the call. + """ + func_args = self._func_args + while func_args: + func = next(iter(func_args)) + args = func_args.pop(func) + func(*args) + + def add(self, func, *types): + """ Register the given function, and return the tuple of positional + arguments to call the function with. If the function is not registered + yet, the list of arguments is made up by invoking the given types. + """ + try: + return self._func_args[func] + except KeyError: + args = self._func_args[func] = [type_() for type_ in types] + return args + + def clear(self): + """ Remove all callbacks from self. """ + self._func_args.clear() + + class IterableGenerator: """ An iterable object based on a generator function, which is called each time the object is iterated over.