From cbff8629fa0d0e7b1f4d2847b01d86893ba91b77 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Tue, 29 Nov 2016 18:43:50 +0100 Subject: [PATCH] [IMP] core: allow declaration of `end` migration scripts These `end` migration script will be executed when all modules have been updated and models setup. Also accelerate the search of migration scripts, speeding up registry loading. --- openerp/modules/loading.py | 7 ++++ openerp/modules/migration.py | 79 ++++++++++++++++++++---------------- 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index 174b9b7c7a9..985ba53cb8f 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -193,6 +193,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= # Set new modules and dependencies modobj.write(cr, SUPERUSER_ID, [module_id], {'state': 'installed', 'latest_version': ver}) + package.load_state = package.state + package.load_version = package.installed_version package.state = 'installed' for kind in ('init', 'demo', 'update'): if hasattr(package, kind): @@ -339,6 +341,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False): registry.setup_models(cr) + # STEP 3.5: execute migration end-scripts + migrations = openerp.modules.migration.MigrationManager(cr, graph) + for package in graph: + migrations.migrate_module(package, 'end') + # STEP 4: Finish and cleanup installations if processed_modules: cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") diff --git a/openerp/modules/migration.py b/openerp/modules/migration.py index 9776a6816ad..78e2c1e55a8 100644 --- a/openerp/modules/migration.py +++ b/openerp/modules/migration.py @@ -3,12 +3,14 @@ """ Modules migration handling. """ +from collections import defaultdict +import glob import imp import logging import os from os.path import join as opj -import openerp +from openerp.modules.module import get_resource_path import openerp.release as release import openerp.tools as tools from openerp.tools.parse_version import parse_version @@ -19,11 +21,14 @@ _logger = logging.getLogger(__name__) class MigrationManager(object): """ This class manage the migration of modules - Migrations files must be python files containing a "migrate(cr, installed_version)" function. - Theses files must respect a directory tree structure: A 'migrations' folder which containt a - folder by version. Version can be 'module' version or 'server.module' version (in this case, - the files will only be processed by this version of the server). Python file names must start - by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation + Migrations files must be python files containing a `migrate(cr, installed_version)` + function. Theses files must respect a directory tree structure: A 'migrations' folder + which containt a folder by version. Version can be 'module' version or 'server.module' + version (in this case, the files will only be processed by this version of the server). + Python file names must start by `pre` or `post` and will be executed, respectively, + before and after the module initialisation. `end` scripts are run after all modules have + been updated. + Example: @@ -31,47 +36,52 @@ class MigrationManager(object): |-- 1.0 | |-- pre-update_table_x.py | |-- pre-update_table_y.py - | |-- post-clean-data.py - | `-- README.txt # not processed - |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server + | |-- post-create_plop_records.py + | |-- end-cleanup.py + | `-- README.txt # not processed + |-- 9.0.1.1 # processed only on a 9.0 server | |-- pre-delete_table_z.py | `-- post-clean-data.py - `-- foo.py # not processed - - This similar structure is generated by the maintenance module with the migrations files get by - the maintenance contract + `-- foo.py # not processed """ + def __init__(self, cr, graph): self.cr = cr self.graph = graph - self.migrations = {} + self.migrations = defaultdict(dict) self._get_files() def _get_files(self): - - """ - import addons.base.maintenance.utils as maintenance_utils - maintenance_utils.update_migrations_files(self.cr) - #""" + def get_scripts(path): + if not path: + return {} + return { + version: glob.glob1(opj(path, version), '*.py') + for version in os.listdir(path) + if os.path.isdir(opj(path, version)) + } for pkg in self.graph: - self.migrations[pkg.name] = {} - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade' or + getattr(pkg, 'load_state', None) == 'to upgrade'): continue - get_module_filetree = openerp.modules.module.get_module_filetree - self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {} - self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {} + self.migrations[pkg.name] = { + 'module': get_scripts(get_resource_path(pkg.name, 'migrations')), + 'maintenance': get_scripts(get_resource_path('base', 'maintenance', 'migrations', pkg.name)), + } def migrate_module(self, pkg, stage): - assert stage in ('pre', 'post') + assert stage in ('pre', 'post', 'end') stageformat = { 'pre': '[>%s]', 'post': '[%s>]', + 'end': '[$%s]', } + state = pkg.state if stage in ('pre', 'post') else getattr(pkg, 'load_state', None) - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade') or pkg.state == 'to install': + if not (hasattr(pkg, 'update') or state == 'to upgrade') or state == 'to install': return def convert_version(version): @@ -80,18 +90,17 @@ class MigrationManager(object): return "%s.%s" % (release.major_version, version) def _get_migration_versions(pkg): - def __get_dir(tree): - return [d for d in tree if tree[d] is not None] - versions = list(set( - __get_dir(self.migrations[pkg.name]['module']) + - __get_dir(self.migrations[pkg.name]['maintenance']) + ver + for lv in self.migrations[pkg.name].values() + for ver, lf in lv.items() + if lf )) versions.sort(key=lambda k: parse_version(convert_version(k))) return versions def _get_migration_files(pkg, version, stage): - """ return a list of tuple (module, file) + """ return a list of migration script files """ m = self.migrations[pkg.name] lst = [] @@ -102,17 +111,15 @@ class MigrationManager(object): } for x in mapping.keys(): - if version in m[x]: + if version in m.get(x): for f in m[x][version]: - if m[x][version][f] is not None: - continue if not f.startswith(stage + '-'): continue lst.append(opj(mapping[x], version, f)) lst.sort() return lst - parsed_installed_version = parse_version(pkg.installed_version or '') + parsed_installed_version = parse_version(getattr(pkg, 'load_version', pkg.installed_version) or '') current_version = parse_version(convert_version(pkg.data['version'])) versions = _get_migration_versions(pkg)