From 923e4da38c576fc2272abc21604321e957f1d8f2 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 1 Aug 2011 09:25:29 +0200 Subject: [PATCH 1/6] [IMP] db-creation thread code: no need for a class with just a __call__ method, just provide a function bzr revid: xmo@openerp.com-20110801072529-92okkttbl5h041q9 --- openerp/service/web_services.py | 87 ++++++++++++++++----------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/openerp/service/web_services.py b/openerp/service/web_services.py index 40e7ccf1d69..8474e38b2d4 100644 --- a/openerp/service/web_services.py +++ b/openerp/service/web_services.py @@ -84,6 +84,48 @@ class db(netsvc.ExportService): finally: cr.close() + def _initialize_db(self, serv, id, db_name, demo, lang, user_password='admin'): + cr = None + try: + serv.actions[id]['progress'] = 0 + cr = sql_db.db_connect(db_name).cursor() + openerp.modules.db.initialize(cr) # TODO this should be removed as it is done by pooler.restart_pool. + tools.config['lang'] = lang + cr.commit() + cr.close() + + pool = pooler.restart_pool(db_name, demo, serv.actions[id], + update_module=True)[1] + + cr = sql_db.db_connect(db_name).cursor() + + if lang: + modobj = pool.get('ir.module.module') + mids = modobj.search(cr, 1, [('state', '=', 'installed')]) + modobj.update_translations(cr, 1, mids, lang) + + cr.execute('UPDATE res_users SET password=%s, context_lang=%s, active=True WHERE login=%s', ( + user_password, lang, 'admin')) + cr.execute('SELECT login, password, name ' \ + ' FROM res_users ' \ + ' ORDER BY login') + serv.actions[id]['users'] = cr.dictfetchall() + serv.actions[id]['clean'] = True + cr.commit() + cr.close() + except Exception, e: + serv.actions[id]['clean'] = False + serv.actions[id]['exception'] = e + import traceback + e_str = StringIO() + traceback.print_exc(file=e_str) + traceback_str = e_str.getvalue() + e_str.close() + netsvc.Logger().notifyChannel('web-services', netsvc.LOG_ERROR, 'CREATE DATABASE\n%s' % (traceback_str)) + serv.actions[id]['traceback'] = traceback_str + if cr: + cr.close() + def exp_create(self, db_name, demo, lang, user_password='admin'): self.id_protect.acquire() self.id += 1 @@ -94,52 +136,9 @@ class db(netsvc.ExportService): self._create_empty_database(db_name) - class DBInitialize(object): - def __call__(self, serv, id, db_name, demo, lang, user_password='admin'): - cr = None - try: - serv.actions[id]['progress'] = 0 - cr = sql_db.db_connect(db_name).cursor() - openerp.modules.db.initialize(cr) # TODO this should be removed as it is done by pooler.restart_pool. - tools.config['lang'] = lang - cr.commit() - cr.close() - - pool = pooler.restart_pool(db_name, demo, serv.actions[id], - update_module=True)[1] - - cr = sql_db.db_connect(db_name).cursor() - - if lang: - modobj = pool.get('ir.module.module') - mids = modobj.search(cr, 1, [('state', '=', 'installed')]) - modobj.update_translations(cr, 1, mids, lang) - - cr.execute('UPDATE res_users SET password=%s, context_lang=%s, active=True WHERE login=%s', ( - user_password, lang, 'admin')) - cr.execute('SELECT login, password, name ' \ - ' FROM res_users ' \ - ' ORDER BY login') - serv.actions[id]['users'] = cr.dictfetchall() - serv.actions[id]['clean'] = True - cr.commit() - cr.close() - except Exception, e: - serv.actions[id]['clean'] = False - serv.actions[id]['exception'] = e - import traceback - e_str = StringIO() - traceback.print_exc(file=e_str) - traceback_str = e_str.getvalue() - e_str.close() - netsvc.Logger().notifyChannel('web-services', netsvc.LOG_ERROR, 'CREATE DATABASE\n%s' % (traceback_str)) - serv.actions[id]['traceback'] = traceback_str - if cr: - cr.close() logger = netsvc.Logger() logger.notifyChannel("web-services", netsvc.LOG_INFO, 'CREATE DATABASE: %s' % (db_name.lower())) - dbi = DBInitialize() - create_thread = threading.Thread(target=dbi, + create_thread = threading.Thread(target=self._initialize_db, args=(self, id, db_name, demo, lang, user_password)) create_thread.start() self.actions[id]['thread'] = create_thread From 92a9836bbb5c905ed0ce3d41df7f22ed86f347e6 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 1 Aug 2011 09:30:50 +0200 Subject: [PATCH 2/6] [IMP] use dict.update instead of __setitem__ sequences bzr revid: xmo@openerp.com-20110801073050-xr37h37sc8w8xhi5 --- openerp/service/web_services.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openerp/service/web_services.py b/openerp/service/web_services.py index 8474e38b2d4..0491b7ec206 100644 --- a/openerp/service/web_services.py +++ b/openerp/service/web_services.py @@ -109,13 +109,15 @@ class db(netsvc.ExportService): cr.execute('SELECT login, password, name ' \ ' FROM res_users ' \ ' ORDER BY login') - serv.actions[id]['users'] = cr.dictfetchall() - serv.actions[id]['clean'] = True + serv.actions[id].update( + users=cr.dictfetchall(), + clean=True) cr.commit() cr.close() except Exception, e: - serv.actions[id]['clean'] = False - serv.actions[id]['exception'] = e + serv.actions[id].update( + clean=False, + exception=e) import traceback e_str = StringIO() traceback.print_exc(file=e_str) From 1e904fe6adc4527a08c46c1e9caf8f57f7ca674f Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 1 Aug 2011 10:54:46 +0200 Subject: [PATCH 3/6] [IMP] db creation: cleanup logging and exception traceback reification traceback.format_exc was introduced in Python 2.5, no need to muck around with StringIO and print_exc to get a traceback as a string bzr revid: xmo@openerp.com-20110801085446-8t4oiiccinf9frvs --- openerp/service/web_services.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/openerp/service/web_services.py b/openerp/service/web_services.py index 0491b7ec206..37e99df6888 100644 --- a/openerp/service/web_services.py +++ b/openerp/service/web_services.py @@ -20,13 +20,17 @@ ############################################################################## import base64 +import locale +import logging import os +import platform import security +import sys import thread import threading import time -import sys -import platform +import traceback +from cStringIO import StringIO from openerp.tools.translate import _ import openerp.netsvc as netsvc import openerp.pooler as pooler @@ -34,9 +38,6 @@ import openerp.release as release import openerp.sql_db as sql_db import openerp.tools as tools import openerp.modules -import locale -import logging -from cStringIO import StringIO #.apidoc title: Exported Service methods #.apidoc module-mods: member-order: bysource @@ -109,22 +110,13 @@ class db(netsvc.ExportService): cr.execute('SELECT login, password, name ' \ ' FROM res_users ' \ ' ORDER BY login') - serv.actions[id].update( - users=cr.dictfetchall(), - clean=True) + serv.actions[id].update(users=cr.dictfetchall(), clean=True) cr.commit() cr.close() except Exception, e: - serv.actions[id].update( - clean=False, - exception=e) - import traceback - e_str = StringIO() - traceback.print_exc(file=e_str) - traceback_str = e_str.getvalue() - e_str.close() - netsvc.Logger().notifyChannel('web-services', netsvc.LOG_ERROR, 'CREATE DATABASE\n%s' % (traceback_str)) - serv.actions[id]['traceback'] = traceback_str + serv.actions[id].update(clean=False, exception=e) + logging.getLogger('db.create').exception('CREATE DATABASE failed:') + serv.actions[id]['traceback'] = traceback.format_exc() if cr: cr.close() @@ -138,8 +130,7 @@ class db(netsvc.ExportService): self._create_empty_database(db_name) - logger = netsvc.Logger() - logger.notifyChannel("web-services", netsvc.LOG_INFO, 'CREATE DATABASE: %s' % (db_name.lower())) + logging.getLogger('db.create').info('CREATE DATABASE %s', db_name.lower()) create_thread = threading.Thread(target=self._initialize_db, args=(self, id, db_name, demo, lang, user_password)) create_thread.start() From f4fef55e2b04ce44bb9519667a6aa82dc1418969 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 1 Aug 2011 10:54:57 +0200 Subject: [PATCH 4/6] [IMP] don't manually manage extracted indexes when enumerate() can do it for us bzr revid: xmo@openerp.com-20110801085457-bw0x6af0xwqmf7e5 --- openerp/modules/loading.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index 6bb45e77598..b24f4888424 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -153,13 +153,12 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= processed_modules = [] loaded_modules = [] - statusi = 0 pool = pooler.get_pool(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) # register, instantiate and initialize models for each modules - for package in graph: + for index, package in enumerate(graph): module_name = package.name module_id = package.id @@ -174,7 +173,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) - status['progress'] = float(statusi) / len(graph) + status['progress'] = float(index) / len(graph) # Can't put this line out of the loop: ir.module.module will be # registered by init_module_models() above. @@ -197,7 +196,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= load_update_xml(cr, module_name, idref, mode) load_data(cr, module_name, idref, mode) if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): - status['progress'] = (float(statusi)+0.75) / len(graph) + status['progress'] = (index + 0.75) / len(graph) load_demo_xml(cr, module_name, idref, mode) load_demo(cr, module_name, idref, mode) cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) @@ -224,7 +223,6 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= delattr(package, kind) cr.commit() - statusi += 1 cr.commit() From 4f6bc445a99dde634c1b1ad51c32add4649f6e5b Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 1 Aug 2011 10:55:06 +0200 Subject: [PATCH 5/6] [IMP] use logging in db initialization bzr revid: xmo@openerp.com-20110801085506-5qjuq36mx3ql7b93 --- openerp/modules/loading.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index b24f4888424..7726960bd47 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -81,6 +81,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped :return: list of modules that were installed or updated """ + logger = logging.getLogger('init.load') def process_sql_file(cr, fp): queries = fp.read().split(';') for query in queries: @@ -109,7 +110,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= try: _load_data(cr, module_name, idref, mode, 'test') except Exception, e: - logging.getLogger('test').exception('Tests failed to execute in module %s', module_name) + logging.getLogger('init.test').exception( + 'Tests failed to execute in module %s', module_name) finally: if tools.config.options['test_commit']: cr.commit() @@ -126,8 +128,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= """ for filename in package.data[kind]: - log = logging.getLogger('init') - log.info("module %s: loading %s", module_name, filename) + logger.info("module %s: loading %s", module_name, filename) _, ext = os.path.splitext(filename) pathname = os.path.join(module_name, filename) fp = tools.file_open(pathname) @@ -155,7 +156,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= loaded_modules = [] pool = pooler.get_pool(cr.dbname) migrations = openerp.modules.migration.MigrationManager(cr, graph) - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) + logger.debug('loading %d packages...', len(graph)) # register, instantiate and initialize models for each modules for index, package in enumerate(graph): @@ -165,7 +166,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= if skip_modules and module_name in skip_modules: continue - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name) + logger.info('module %s: loading objects', package.name) migrations.migrate_module(package, 'pre') register_module_classes(package.name) models = pool.instanciate(package.name, cr) From 4ae3738376ae9071b504a3c00fa031a0cf17ce1c Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 1 Aug 2011 10:57:28 +0200 Subject: [PATCH 6/6] [IMP] sort imports, improve functional shortcuts to _load_data (include cr partial application) bzr revid: xmo@openerp.com-20110801085728-hg81gop09evfzuzu --- openerp/modules/loading.py | 71 ++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index 7726960bd47..45a466e5462 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -24,36 +24,34 @@ """ -import os, sys, imp -from os.path import join as opj +import base64 +import imp import itertools +import logging +import os +import re +import sys +import zipfile import zipimport -import openerp - -import openerp.osv as osv -import openerp.tools as tools -import openerp.tools.osutil as osutil -from openerp.tools.safe_eval import safe_eval as eval -import openerp.pooler as pooler -from openerp.tools.translate import _ - -import openerp.netsvc as netsvc - -import zipfile -import openerp.release as release - -import re -import base64 -from zipfile import PyZipFile, ZIP_DEFLATED from cStringIO import StringIO +from os.path import join as opj +from zipfile import PyZipFile, ZIP_DEFLATED -import logging +import openerp import openerp.modules.db import openerp.modules.graph import openerp.modules.migration +import openerp.netsvc as netsvc +import openerp.osv as osv +import openerp.pooler as pooler +import openerp.release as release +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +from openerp.tools.translate import _ from openerp.modules.module import \ get_modules, get_modules_with_version, \ load_information_from_description_file, \ @@ -89,22 +87,13 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= if new_query: cr.execute(new_query) - def load_init_xml(cr, module_name, idref, mode): - _load_data(cr, module_name, idref, mode, 'init_xml') + load_init_xml = lambda *args: _load_data(cr, *args, kind='init_xml') + load_update_xml = lambda *args: _load_data(cr, *args, kind='update_xml') + load_demo_xml = lambda *args: _load_data(cr, *args, kind='demo_xml') + load_data = lambda *args: _load_data(cr, *args, kind='data') + load_demo = lambda *args: _load_data(cr, *args, kind='demo') - def load_update_xml(cr, module_name, idref, mode): - _load_data(cr, module_name, idref, mode, 'update_xml') - - def load_demo_xml(cr, module_name, idref, mode): - _load_data(cr, module_name, idref, mode, 'demo_xml') - - def load_data(cr, module_name, idref, mode): - _load_data(cr, module_name, idref, mode, 'data') - - def load_demo(cr, module_name, idref, mode): - _load_data(cr, module_name, idref, mode, 'demo') - - def load_test(cr, module_name, idref, mode): + def load_test(module_name, idref, mode): cr.commit() if not tools.config.options['test_disable']: try: @@ -193,20 +182,20 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= if package.state=='to upgrade': # upgrading the module information modobj.write(cr, 1, [module_id], modobj.get_values_from_terp(package.data)) - load_init_xml(cr, module_name, idref, mode) - load_update_xml(cr, module_name, idref, mode) - load_data(cr, module_name, idref, mode) + load_init_xml(module_name, idref, mode) + load_update_xml(module_name, idref, mode) + load_data(module_name, idref, mode) if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): status['progress'] = (index + 0.75) / len(graph) - load_demo_xml(cr, module_name, idref, mode) - load_demo(cr, module_name, idref, mode) + load_demo_xml(module_name, idref, mode) + load_demo(module_name, idref, mode) cr.execute('update ir_module_module set demo=%s where id=%s', (True, module_id)) # launch tests only in demo mode, as most tests will depend # on demo data. Other tests can be added into the regular # 'data' section, but should probably not alter the data, # as there is no rollback. - load_test(cr, module_name, idref, mode) + load_test(module_name, idref, mode) processed_modules.append(package.name)