diff --git a/bin/addons/__init__.py b/bin/addons/__init__.py index feb635055f5..a5f6fe5daa6 100644 --- a/bin/addons/__init__.py +++ b/bin/addons/__init__.py @@ -3,7 +3,7 @@ # # OpenERP, Open Source Management Solution # Copyright (C) 2004-2009 Tiny SPRL (). -# Copyright (C) 2010 OpenERP s.a. (). +# Copyright (C) 2010-2011 OpenERP s.a. (). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -47,8 +47,8 @@ import logging logger = netsvc.Logger() -_ad = os.path.abspath(opj(tools.config['root_path'], 'addons')) # default addons path (base) -ad_paths= map(lambda m: os.path.abspath(m.strip()),tools.config['addons_path'].split(',')) +_ad = os.path.abspath(opj(tools.ustr(tools.config['root_path']), u'addons')) # default addons path (base) +ad_paths= map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(',')) sys.path.insert(1, _ad) @@ -225,6 +225,10 @@ def zip_directory(directory, b64enc=True, src=True): archname = StringIO() archive = PyZipFile(archname, "w", ZIP_DEFLATED) + + # for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8) + directory = tools.ustr(directory).encode('utf-8') + archive.writepy(directory) _zippy(archive, directory, src=src) archive.close() @@ -312,7 +316,11 @@ def load_information_from_description_file(module): for filename in ['__openerp__.py', '__terp__.py']: description_file = get_module_resource(module, filename) if description_file : - return eval(tools.file_open(description_file).read()) + desc_f = tools.file_open(description_file) + try: + return eval(desc_f.read()) + finally: + desc_f.close() #TODO: refactor the logger in this file to follow the logging guidelines # for 6.0 @@ -351,11 +359,14 @@ def upgrade_graph(graph, cr, module_list, force=None): continue if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'): + terp_f = tools.file_open(terp_file) try: - info = eval(tools.file_open(terp_file).read()) - except: + info = eval(terp_f.read()) + except Exception: logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: eval file %s' % (module, terp_file)) raise + finally: + terp_f.close() if info.get('installable', True): packages.append((module, info.get('depends', []), info)) else: @@ -599,18 +610,14 @@ class MigrationManager(object): log = logging.getLogger('init') -def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup=False, **kwargs): +def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, **kwargs): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load :param status: status dictionary for keeping track of progress :param perform_checks: whether module descriptors should be checked for validity (prints warnings for same cases, and even raise osv_except if certificate is invalid) - :param skip_cleanup: whether the auto-cleanup of records should be executed (unlinks any object that - appears to be from one of the updated modules, but have not been loaded during - last loading (i.e. records that seem to have been removed from the module). - This is best left disabled when loading stand-alone modules that could contain - records from dependent modules (i.e. other modules have put records in their - namespace) + :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped + :return: list of modules that were installed or updated """ def process_sql_file(cr, fp): queries = fp.read().split(';') @@ -624,29 +631,33 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup= logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, filename)) _, ext = os.path.splitext(filename) fp = tools.file_open(opj(m, filename)) - if ext == '.csv': - noupdate = (kind == 'init') - tools.convert_csv_import(cr, m, os.path.basename(filename), fp.read(), idref, mode=mode, noupdate=noupdate) - elif ext == '.sql': - process_sql_file(cr, fp) - elif ext == '.yml': - tools.convert_yaml_import(cr, m, fp, idref, mode=mode, **kwargs) - else: - tools.convert_xml_import(cr, m, fp, idref, mode=mode, **kwargs) - fp.close() + try: + if ext == '.csv': + noupdate = (kind == 'init') + tools.convert_csv_import(cr, m, os.path.basename(filename), fp.read(), idref, mode=mode, noupdate=noupdate) + elif ext == '.sql': + process_sql_file(cr, fp) + elif ext == '.yml': + tools.convert_yaml_import(cr, m, fp, idref, mode=mode, **kwargs) + else: + tools.convert_xml_import(cr, m, fp, idref, mode=mode, **kwargs) + finally: + fp.close() def load_demo_xml(cr, m, idref, mode): for xml in package.data.get('demo_xml', []): name, ext = os.path.splitext(xml) logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, xml)) fp = tools.file_open(opj(m, xml)) - if ext == '.csv': - tools.convert_csv_import(cr, m, os.path.basename(xml), fp.read(), idref, mode=mode, noupdate=True) - elif ext == '.yml': - tools.convert_yaml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) - else: - tools.convert_xml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) - fp.close() + try: + if ext == '.csv': + tools.convert_csv_import(cr, m, os.path.basename(xml), fp.read(), idref, mode=mode, noupdate=True) + elif ext == '.yml': + tools.convert_yaml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) + else: + tools.convert_xml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) + finally: + fp.close() def load_data(cr, module_name, id_map, mode): _load_data(cr, module_name, id_map, mode, 'data') @@ -660,7 +671,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup= try: _load_data(cr, module_name, id_map, mode, 'test') except Exception, e: - logging.getLogger('test').exception('Tests failed to execute in %s module %s', module_name) + logging.getLogger('test').exception('Tests failed to execute in module %s', module_name) finally: if tools.config.options['test_commit']: cr.commit() @@ -668,41 +679,40 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup= cr.rollback() def _load_data(cr, module_name, id_map, mode, kind): - noupdate = (kind == 'demo') for filename in package.data.get(kind, []): + noupdate = (kind == 'demo') _, ext = os.path.splitext(filename) log.info("module %s: loading %s", module_name, filename) pathname = os.path.join(module_name, filename) file = tools.file_open(pathname) - # TODO manage .csv file with noupdate == (kind == 'init') - if ext == '.sql': - process_sql_file(cr, file) - elif ext == '.csv': - noupdate = (kind == 'init') - tools.convert_csv_import(cr, module_name, pathname, file.read(), id_map, mode, noupdate) - elif ext == '.yml': - tools.convert_yaml_import(cr, module_name, file, id_map, mode, noupdate) - else: - tools.convert_xml_import(cr, module_name, file, id_map, mode, noupdate) - file.close() + try: + if ext == '.sql': + process_sql_file(cr, file) + elif ext == '.csv': + noupdate = (kind == 'init') + tools.convert_csv_import(cr, module_name, pathname, file.read(), id_map, mode, noupdate) + elif ext == '.yml': + tools.convert_yaml_import(cr, module_name, file, id_map, mode, noupdate) + else: + tools.convert_xml_import(cr, module_name, file, id_map, mode, noupdate) + finally: + file.close() # **kwargs is passed directly to convert_xml_import if not status: status = {} status = status.copy() - package_todo = [] + processed_modules = [] statusi = 0 pool = pooler.get_pool(cr.dbname) - migrations = MigrationManager(cr, graph) - - has_updates = False modobj = None - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) for package in graph: + if skip_modules and package.name in skip_modules: + continue logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name) migrations.migrate_module(package, 'pre') register_class(package.name) @@ -716,6 +726,9 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup= m = package.name mid = package.id + if skip_modules and m in skip_modules: + continue + if modobj is None: modobj = pool.get('ir.module.module') @@ -730,7 +743,6 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup= mode = 'init' if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): - has_updates = True for kind in ('init', 'update'): if package.state=='to upgrade': # upgrading the module information @@ -749,7 +761,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup= # as there is no rollback. load_test(cr, m, idref, mode) - package_todo.append(package.name) + processed_modules.append(package.name) migrations.migrate_module(package, 'post') @@ -769,16 +781,9 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_cleanup= statusi += 1 - cr.execute('select model from ir_model where state=%s', ('manual',)) - for model in cr.dictfetchall(): - pool.get('ir.model').instanciate(cr, 1, model['model'], {}) - - if not skip_cleanup: - # Cleanup orphan records - pool.get('ir.model.data')._process_end(cr, 1, package_todo) cr.commit() - return has_updates + return processed_modules def _check_module_names(cr, module_names): mod_names = set(module_names) @@ -810,17 +815,30 @@ def load_modules(db, force_demo=False, status=None, update_module=False): force = [] if force_demo: force.append('demo') + + # This is a brand new pool, just created in pooler.get_db_and_pool() pool = pooler.get_pool(cr.dbname) + try: + processed_modules = [] report = tools.assertion_report() # NOTE: Try to also load the modules that have been marked as uninstallable previously... STATES_TO_LOAD = ['installed', 'to upgrade', 'uninstallable'] + if 'base' in tools.config['update']: + cr.execute("update ir_module_module set state=%s where name=%s", ('to upgrade', 'base')) + + # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) graph = create_graph(cr, ['base'], force) if not graph: logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)') raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)')) - has_updates = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report, skip_cleanup=True) + processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)) + if tools.config['load_language']: + for lang in tools.config['load_language'].split(','): + tools.load_language(cr, lang) + + # STEP 2: Mark other modules to be loaded/updated if update_module: modobj = pool.get('ir.module.module') logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list') @@ -845,6 +863,8 @@ def load_modules(db, force_demo=False, status=None, update_module=False): STATES_TO_LOAD += ['to install'] + + # STEP 3: Load marked modules (skipping base which was done in STEP 1) loop_guardrail = 0 while True: loop_guardrail += 1 @@ -862,10 +882,15 @@ def load_modules(db, force_demo=False, status=None, update_module=False): break logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list))) - r = load_module_graph(cr, graph, status, report=report) - has_updates = has_updates or r + processed_modules.extend(load_module_graph(cr, graph, status, report=report, skip_modules=processed_modules)) + + # STEP 4: Finish and cleanup + if processed_modules: + # load custom models + cr.execute('select model from ir_model where state=%s', ('manual',)) + for model in cr.dictfetchall(): + pool.get('ir.model').instanciate(cr, 1, model['model'], {}) - if has_updates: cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") for (model, name) in cr.fetchall(): model_obj = pool.get(model) @@ -888,6 +913,9 @@ def load_modules(db, force_demo=False, status=None, update_module=False): else: logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is referenced but not present in the orm pool!" % model) + # Cleanup orphan records + pool.get('ir.model.data')._process_end(cr, 1, processed_modules) + if report.get_report(): logger.notifyChannel('init', netsvc.LOG_INFO, report) diff --git a/bin/addons/base/__openerp__.py b/bin/addons/base/__openerp__.py index eefaf9f4442..bb419fc90d3 100644 --- a/bin/addons/base/__openerp__.py +++ b/bin/addons/base/__openerp__.py @@ -89,6 +89,7 @@ 'test/base_test.yml', 'test/test_context.xml', 'test/bug_lp541545.xml', + 'test/test_osv_expression.yml', ], 'installable': True, 'active': True, diff --git a/bin/addons/base/base_data.xml b/bin/addons/base/base_data.xml index 143d2e26b07..db74b9b1b1e 100644 --- a/bin/addons/base/base_data.xml +++ b/bin/addons/base/base_data.xml @@ -28,6 +28,8 @@ True + +