[FIX] *: Reset module states on registry init error

Commit 763d714 introduced cron job locking for databases which had
modules with states set to 'to x', however if an
installation/uninstallation/upgrade fails, the state will stay at 'to
x', and it may stay in that state for an indefinite amount of time,
meaning that cron jobs could stay locked forever.

This commit fixes this in part by adding a cleanup function to loading.py that
will be executed whenever load_modules fails, the function will change
every 'to x' module to their original state, effectively unlocking the
execution of cron jobs.

This however only works to prevent "zombie" transient states for
brand new databases, however for existing databases which already
contain some modules in a zombie state it won't do anything unless
a module is installed/uninstalled/upgraded, which may never happen.

This is where the second part comes in (ir_cron.py), when failing to
execute crons, we check if the failure was due to bad module state
and if an arbitrary amount of time (5 hours as of this commit) has passed
since the last time it was supposed to be executed, if it is the case, it means
that the cron execution failed around 5 * 60 times (1 failure per minute for 5h)
in which case we assume that the crons are stuck because the db
has zombie states and we force a call to reset_module_states.

odoo/addons/base/ir/ir_cron.py

@@ -5,7 +5,7 @@ import threading
 import time
 import psycopg2
 import pytz
-from datetime import datetime
+from datetime import datetime, timedelta
 from dateutil.relativedelta import relativedelta
 
 import odoo
@@ -16,6 +16,7 @@ from odoo.tools.safe_eval import safe_eval
 _logger = logging.getLogger(__name__)
 
 BASE_VERSION = odoo.modules.load_information_from_description_file('base')['version']
+MAX_FAIL_TIME = timedelta(hours=5)  # chosen with a fair roll of the dice
 
 
 class BadVersion(Exception):
@@ -198,7 +199,7 @@ class ir_cron(models.Model):
                 (version,) = cr.fetchone()
                 cr.execute("SELECT COUNT(*) FROM ir_module_module WHERE state LIKE %s", ['to %'])
                 (changes,) = cr.fetchone()
-                if not version or changes:
+                if version is None:
                     raise BadModuleState()
                 elif version != BASE_VERSION:
                     raise BadVersion()
@@ -209,6 +210,19 @@ class ir_cron(models.Model):
                               ORDER BY priority""")
                 jobs = cr.dictfetchall()
 
+            if changes:
+                if not jobs:
+                    raise BadModuleState()
+                # nextcall is never updated if the cron is not executed,
+                # it is used as a sentinel value to check whether cron jobs
+                # have been locked for a long time (stuck)
+                parse = fields.Datetime.from_string
+                oldest = min([parse(job['nextcall']) for job in jobs])
+                if datetime.now() - oldest > MAX_FAIL_TIME:
+                    odoo.modules.reset_modules_state(db_name)
+                else:
+                    raise BadModuleState()
+
             for job in jobs:
                 lock_cr = db.cursor()
                 try:

odoo/modules/__init__.py

@@ -7,7 +7,7 @@
 
 from . import db, graph, loading, migration, module, registry
 
-from odoo.modules.loading import load_modules
+from odoo.modules.loading import load_modules, reset_modules_state
 
 from odoo.modules.module import (
     adapt_version,

odoo/modules/loading.py

@@ -431,3 +431,24 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
             _logger.log(25, "All post-tested in %.2fs, %s queries", time.time() - t0, odoo.sql_db.sql_counter - t0_sql)
     finally:
         cr.close()
+
+
+def reset_modules_state(db_name):
+    """
+    Resets modules flagged as "to x" to their original state
+    """
+    # Warning, this function was introduced in response to commit 763d714
+    # which locks cron jobs for dbs which have modules marked as 'to %'.
+    # The goal of this function is to be called ONLY when module
+    # installation/upgrade/uninstallation fails, which is the only known case
+    # for which modules can stay marked as 'to %' for an indefinite amount
+    # of time
+    db = odoo.sql_db.db_connect(db_name)
+    with db.cursor() as cr:
+        cr.execute(
+            "UPDATE ir_module_module SET state='installed' WHERE state IN ('to remove', 'to upgrade')"
+        )
+        cr.execute(
+            "UPDATE ir_module_module SET state='uninstalled' WHERE state='to install'"
+        )
+        _logger.warning("Transient module states were reset")

odoo/modules/registry.py

@@ -79,7 +79,11 @@ class Registry(Mapping):
                 try:
                     registry.setup_signaling()
                     # This should be a method on Registry
-                    odoo.modules.load_modules(registry._db, force_demo, status, update_module)
+                    try:
+                        odoo.modules.load_modules(registry._db, force_demo, status, update_module)
+                    except Exception:
+                        odoo.modules.reset_modules_state(db_name)
+                        raise
                 except Exception:
                     _logger.exception('Failed to load registry')
                     del cls.registries[db_name]