[IMP] core: allow declaration of end migration scripts

These `end` migration script will be executed when all modules have
been updated and models setup.

Also accelerate the search of migration scripts, speeding up registry
loading.

openerp/modules/loading.py

@@ -193,6 +193,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
             # Set new modules and dependencies
             modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver})
 
+            package.load_state = package.state
+            package.load_version = package.installed_version
             package.state = 'installed'
             for kind in ('init', 'demo', 'update'):
                 if hasattr(package, kind):
@@ -339,6 +341,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
 
         registry.setup_models(cr)
 
+        # STEP 3.5: execute migration end-scripts
+        migrations = openerp.modules.migration.MigrationManager(cr, graph)
+        for package in graph:
+            migrations.migrate_module(package, 'end')
+
         # STEP 4: Finish and cleanup installations
         if processed_modules:
             cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""")

openerp/modules/migration.py

@@ -3,12 +3,14 @@
 
 """ Modules migration handling. """
 
+from collections import defaultdict
+import glob
 import imp
 import logging
 import os
 from os.path import join as opj
 
-import openerp
+from openerp.modules.module import get_resource_path
 import openerp.release as release
 import openerp.tools as tools
 from openerp.tools.parse_version import parse_version
@@ -19,11 +21,14 @@ _logger = logging.getLogger(__name__)
 class MigrationManager(object):
     """
         This class manage the migration of modules
-        Migrations files must be python files containing a "migrate(cr, installed_version)" function.
+        Migrations files must be python files containing a `migrate(cr, installed_version)`
-        Theses files must respect a directory tree structure: A 'migrations' folder which containt a
+        function. Theses files must respect a directory tree structure: A 'migrations' folder
-        folder by version. Version can be 'module' version or 'server.module' version (in this case,
+        which containt a folder by version. Version can be 'module' version or 'server.module'
-        the files will only be processed by this version of the server). Python file names must start
+        version (in this case, the files will only be processed by this version of the server).
-        by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation
+        Python file names must start by `pre` or `post` and will be executed, respectively,
+        before and after the module initialisation. `end` scripts are run after all modules have
+        been updated.
+
         Example:
 
             <moduledir>
@@ -31,47 +36,52 @@ class MigrationManager(object):
                 |-- 1.0
                 |   |-- pre-update_table_x.py
                 |   |-- pre-update_table_y.py
-                |   |-- post-clean-data.py
+                |   |-- post-create_plop_records.py
-                |   `-- README.txt              # not processed
+                |   |-- end-cleanup.py
-                |-- 5.0.1.1                     # files in this folder will be executed only on a 5.0 server
+                |   `-- README.txt                      # not processed
+                |-- 9.0.1.1                             # processed only on a 9.0 server
                 |   |-- pre-delete_table_z.py
                 |   `-- post-clean-data.py
-                `-- foo.py                      # not processed
+                `-- foo.py                              # not processed
-
-        This similar structure is generated by the maintenance module with the migrations files get by
-        the maintenance contract
 
     """
+
     def __init__(self, cr, graph):
         self.cr = cr
         self.graph = graph
-        self.migrations = {}
+        self.migrations = defaultdict(dict)
         self._get_files()
 
     def _get_files(self):
-
+        def get_scripts(path):
-        """
+            if not path:
-        import addons.base.maintenance.utils as maintenance_utils
+                return {}
-        maintenance_utils.update_migrations_files(self.cr)
+            return {
-        #"""
+                version: glob.glob1(opj(path, version), '*.py')
+                for version in os.listdir(path)
+                if os.path.isdir(opj(path, version))
+            }
 
         for pkg in self.graph:
-            self.migrations[pkg.name] = {}
+            if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade' or
-            if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'):
+                    getattr(pkg, 'load_state', None) == 'to upgrade'):
                 continue
 
-            get_module_filetree = openerp.modules.module.get_module_filetree
+            self.migrations[pkg.name] = {
-            self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {}
+                'module': get_scripts(get_resource_path(pkg.name, 'migrations')),
-            self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {}
+                'maintenance': get_scripts(get_resource_path('base', 'maintenance', 'migrations', pkg.name)),
+            }
 
     def migrate_module(self, pkg, stage):
-        assert stage in ('pre', 'post')
+        assert stage in ('pre', 'post', 'end')
         stageformat = {
             'pre': '[>%s]',
             'post': '[%s>]',
+            'end': '[$%s]',
         }
+        state = pkg.state if stage in ('pre', 'post') else getattr(pkg, 'load_state', None)
 
-        if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade') or pkg.state == 'to install':
+        if not (hasattr(pkg, 'update') or state == 'to upgrade') or state == 'to install':
             return
 
         def convert_version(version):
@@ -80,18 +90,17 @@ class MigrationManager(object):
             return "%s.%s" % (release.major_version, version)
 
         def _get_migration_versions(pkg):
-            def __get_dir(tree):
-                return [d for d in tree if tree[d] is not None]
-
             versions = list(set(
-                __get_dir(self.migrations[pkg.name]['module']) +
+                ver
-                __get_dir(self.migrations[pkg.name]['maintenance'])
+                for lv in self.migrations[pkg.name].values()
+                for ver, lf in lv.items()
+                if lf
             ))
             versions.sort(key=lambda k: parse_version(convert_version(k)))
             return versions
 
         def _get_migration_files(pkg, version, stage):
-            """ return a list of tuple (module, file)
+            """ return a list of migration script files
             """
             m = self.migrations[pkg.name]
             lst = []
@@ -102,17 +111,15 @@ class MigrationManager(object):
             }
 
             for x in mapping.keys():
-                if version in m[x]:
+                if version in m.get(x):
                     for f in m[x][version]:
-                        if m[x][version][f] is not None:
-                            continue
                         if not f.startswith(stage + '-'):
                             continue
                         lst.append(opj(mapping[x], version, f))
             lst.sort()
             return lst
 
-        parsed_installed_version = parse_version(pkg.installed_version or '')
+        parsed_installed_version = parse_version(getattr(pkg, 'load_version', pkg.installed_version) or '')
         current_version = parse_version(convert_version(pkg.data['version']))
 
         versions = _get_migration_versions(pkg)