From 12580e690aea8b9ea3a4862e547cf30e323c683a Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Sun, 8 Sep 2013 19:35:35 +0200 Subject: [PATCH 01/10] [IMP] openerp threaded, gevent, prefork service cleanup - unify signal handling - unify start and stop no new feature yet, it paves the way for - developement mode auto reload - graceful restart on HUP - multiprocessing and gevent on windows bzr revid: al@openerp.com-20130908173535-xomt5w7xmqtwkmyy --- openerp-long-polling | 11 -- openerp/__init__.py | 11 +- openerp/cli/server.py | 120 +------------ openerp/service/__init__.py | 116 +------------ openerp/service/cron.py | 76 -------- openerp/service/workers.py | 309 +++++++++++++++++++++++++++++++-- openerp/service/wsgi_server.py | 66 ------- 7 files changed, 317 insertions(+), 392 deletions(-) delete mode 100755 openerp-long-polling delete mode 100644 openerp/service/cron.py diff --git a/openerp-long-polling b/openerp-long-polling deleted file mode 100755 index 304cd01c1ee..00000000000 --- a/openerp-long-polling +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import gevent.monkey -gevent.monkey.patch_all() -import gevent_psycopg2 -gevent_psycopg2.monkey_patch() - -import openerp - -if __name__ == "__main__": - openerp.cli.main() diff --git a/openerp/__init__.py b/openerp/__init__.py index 22c8c98256a..161b06b0990 100644 --- a/openerp/__init__.py +++ b/openerp/__init__.py @@ -27,9 +27,16 @@ import sys # Is the server running with gevent. evented = False -if sys.modules.get("gevent") is not None: - evented = True +for i in sys.argv: + if i.startswith('--gevent'): + evented = True + break +if evented: + import gevent.monkey + gevent.monkey.patch_all() + import gevent_psycopg2 + gevent_psycopg2.monkey_patch() # Make sure the OpenERP server runs in UTC. This is especially necessary # under Windows as under Linux it seems the real import of time is diff --git a/openerp/cli/server.py b/openerp/cli/server.py index fc1bf61b617..0c081b6156b 100644 --- a/openerp/cli/server.py +++ b/openerp/cli/server.py @@ -147,112 +147,15 @@ def import_translation(): cr.commit() cr.close() -# Variable keeping track of the number of calls to the signal handler defined -# below. This variable is monitored by ``quit_on_signals()``. -quit_signals_received = 0 - -def signal_handler(sig, frame): - """ Signal handler: exit ungracefully on the second handled signal. - - :param sig: the signal number - :param frame: the interrupted stack frame or None - """ - global quit_signals_received - quit_signals_received += 1 - if quit_signals_received > 1: - # logging.shutdown was already called at this point. - sys.stderr.write("Forced shutdown.\n") - os._exit(0) - -def dumpstacks(sig, frame): - """ Signal handler: dump a stack trace for each existing thread.""" - # code from http://stackoverflow.com/questions/132058/getting-stack-trace-from-a-running-python-application#answer-2569696 - # modified for python 2.5 compatibility - threads_info = dict([(th.ident, {'name': th.name, - 'uid': getattr(th,'uid','n/a')}) - for th in threading.enumerate()]) - code = [] - for threadId, stack in sys._current_frames().items(): - thread_info = threads_info.get(threadId) - code.append("\n# Thread: %s (id:%s) (uid:%s)" % \ - (thread_info and thread_info['name'] or 'n/a', - threadId, - thread_info and thread_info['uid'] or 'n/a')) - for filename, lineno, name, line in traceback.extract_stack(stack): - code.append('File: "%s", line %d, in %s' % (filename, lineno, name)) - if line: - code.append(" %s" % (line.strip())) - _logger.info("\n".join(code)) - -def setup_signal_handlers(signal_handler): - """ Register the given signal handler. """ - SIGNALS = (signal.SIGINT, signal.SIGTERM) - if os.name == 'posix': - map(lambda sig: signal.signal(sig, signal_handler), SIGNALS) - signal.signal(signal.SIGQUIT, dumpstacks) - elif os.name == 'nt': - import win32api - win32api.SetConsoleCtrlHandler(lambda sig: signal_handler(sig, None), 1) - -def quit_on_signals(): - """ Wait for one or two signals then shutdown the server. - - The first SIGINT or SIGTERM signal will initiate a graceful shutdown while - a second one if any will force an immediate exit. - - """ - # Wait for a first signal to be handled. (time.sleep will be interrupted - # by the signal handler.) The try/except is for the win32 case. - try: - while quit_signals_received == 0: - time.sleep(60) - except KeyboardInterrupt: - pass - - config = openerp.tools.config - openerp.service.stop_services() - - if getattr(openerp, 'phoenix', False): - # like the phoenix, reborn from ashes... - openerp.service._reexec() - return - - if config['pidfile']: - os.unlink(config['pidfile']) - sys.exit(0) - -def watch_parent(beat=4): - import gevent - ppid = os.getppid() - while True: - if ppid != os.getppid(): - pid = os.getpid() - _logger.info("LongPolling (%s) Parent changed", pid) - # suicide !! - os.kill(pid, signal.SIGTERM) - return - gevent.sleep(beat) - def main(args): check_root_user() openerp.tools.config.parse_config(args) - - if openerp.tools.config.options["gevent"]: - openerp.evented = True - _logger.info('Using gevent mode') - import gevent.monkey - gevent.monkey.patch_all() - import gevent_psycopg2 - gevent_psycopg2.monkey_patch() - check_postgres_user() openerp.netsvc.init_logger() report_configuration() config = openerp.tools.config - setup_signal_handlers(signal_handler) - if config["test_file"]: run_test_file(config['db_name'], config['test_file']) sys.exit(0) @@ -265,28 +168,19 @@ def main(args): import_translation() sys.exit(0) - if not config["stop_after_init"]: - setup_pid_file() - # Some module register themselves when they are loaded so we need the - # services to be running before loading any registry. - if not openerp.evented: - if config['workers']: - openerp.service.start_services_workers() - else: - openerp.service.start_services() - else: - config['xmlrpc_port'] = config['longpolling_port'] - import gevent - gevent.spawn(watch_parent) - openerp.service.start_services() - + # preload registryies, needed for -u --stop_after_init rc = 0 if config['db_name']: for dbname in config['db_name'].split(','): if not preload_registry(dbname): rc += 1 - if config["stop_after_init"]: + if not config["stop_after_init"]: + setup_pid_file() + openerp.service.workers.start() + if config['pidfile']: + os.unlink(config['pidfile']) + else: sys.exit(rc) _logger.info('OpenERP server is running, waiting for connections...') diff --git a/openerp/service/__init__.py b/openerp/service/__init__.py index 2c6c80eb80c..02401d455e1 100644 --- a/openerp/service/__init__.py +++ b/openerp/service/__init__.py @@ -20,29 +20,12 @@ # ############################################################################## -import logging -import os -import signal -import subprocess -import sys -import threading -import time - -import cron -import wsgi_server - -import openerp -import openerp.modules -import openerp.netsvc -import openerp.osv -from openerp.release import nt_service_name -import openerp.tools -from openerp.tools.misc import stripped_sys_argv - import common import db import model import report +import wsgi_server +import workers #.apidoc title: RPC Services @@ -55,100 +38,5 @@ import report low-level behavior of the wire. """ -_logger = logging.getLogger(__name__) - -def load_server_wide_modules(): - for m in openerp.conf.server_wide_modules: - try: - openerp.modules.module.load_openerp_module(m) - except Exception: - msg = '' - if m == 'web': - msg = """ -The `web` module is provided by the addons found in the `openerp-web` project. -Maybe you forgot to add those addons in your addons_path configuration.""" - _logger.exception('Failed to load server-wide module `%s`.%s', m, msg) - -start_internal_done = False -main_thread_id = threading.currentThread().ident - -def start_internal(): - global start_internal_done - if start_internal_done: - return - openerp.netsvc.init_logger() - - load_server_wide_modules() - start_internal_done = True - -def start_services(): - """ Start all services including http, and cron """ - start_internal() - # Start the WSGI server. - wsgi_server.start_service() - # Start the main cron thread. - if not openerp.evented: - cron.start_service() - -def stop_services(): - """ Stop all services. """ - # stop services - if not openerp.evented: - cron.stop_service() - wsgi_server.stop_service() - - _logger.info("Initiating shutdown") - _logger.info("Hit CTRL-C again or send a second signal to force the shutdown.") - - # Manually join() all threads before calling sys.exit() to allow a second signal - # to trigger _force_quit() in case some non-daemon threads won't exit cleanly. - # threading.Thread.join() should not mask signals (at least in python 2.5). - me = threading.currentThread() - _logger.debug('current thread: %r', me) - for thread in threading.enumerate(): - _logger.debug('process %r (%r)', thread, thread.isDaemon()) - if thread != me and not thread.isDaemon() and thread.ident != main_thread_id: - while thread.isAlive(): - _logger.debug('join and sleep') - # Need a busyloop here as thread.join() masks signals - # and would prevent the forced shutdown. - thread.join(0.05) - time.sleep(0.05) - - _logger.debug('--') - openerp.modules.registry.RegistryManager.delete_all() - logging.shutdown() - -def start_services_workers(): - import openerp.service.workers - openerp.multi_process = True - openerp.service.workers.Multicorn(openerp.service.wsgi_server.application).run() - -def _reexec(): - """reexecute openerp-server process with (nearly) the same arguments""" - if openerp.tools.osutil.is_running_as_nt_service(): - subprocess.call('net stop {0} && net start {0}'.format(nt_service_name), shell=True) - exe = os.path.basename(sys.executable) - args = stripped_sys_argv() - if not args or args[0] != exe: - args.insert(0, exe) - os.execv(sys.executable, args) - -def restart_server(): - if openerp.multi_process: - raise NotImplementedError("Multicorn is not supported (but gunicorn was)") - pid = openerp.wsgi.core.arbiter_pid - os.kill(pid, signal.SIGHUP) - else: - if os.name == 'nt': - def reborn(): - stop_services() - _reexec() - - # run in a thread to let the current thread return response to the caller. - threading.Thread(target=reborn).start() - else: - openerp.phoenix = True - os.kill(os.getpid(), signal.SIGINT) # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/service/cron.py b/openerp/service/cron.py deleted file mode 100644 index f21b27f7fea..00000000000 --- a/openerp/service/cron.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# Copyright (C) 2004-2011 OpenERP SA () -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -""" Cron jobs scheduling - -Cron jobs are defined in the ir_cron table/model. This module deals with all -cron jobs, for all databases of a single OpenERP server instance. - -""" - -import logging -import threading -import time -from datetime import datetime - -import openerp - -_logger = logging.getLogger(__name__) - -SLEEP_INTERVAL = 60 # 1 min - -def cron_runner(number): - while True: - time.sleep(SLEEP_INTERVAL + number) # Steve Reich timing style - registries = openerp.modules.registry.RegistryManager.registries - _logger.debug('cron%d polling for jobs', number) - for db_name, registry in registries.items(): - while True and registry.ready: - acquired = openerp.addons.base.ir.ir_cron.ir_cron._acquire_job(db_name) - if not acquired: - break - -def start_service(): - """ Start the above runner function in a daemon thread. - - The thread is a typical daemon thread: it will never quit and must be - terminated when the main process exits - with no consequence (the processing - threads it spawns are not marked daemon). - - """ - - # Force call to strptime just before starting the cron thread - # to prevent time.strptime AttributeError within the thread. - # See: http://bugs.python.org/issue7980 - datetime.strptime('2012-01-01', '%Y-%m-%d') - - for i in range(openerp.tools.config['max_cron_threads']): - def target(): - cron_runner(i) - t = threading.Thread(target=target, name="openerp.service.cron.cron%d" % i) - t.setDaemon(True) - t.start() - _logger.debug("cron%d started!" % i) - -def stop_service(): - pass - -# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/service/workers.py b/openerp/service/workers.py index c8b8b84bca2..382d581720f 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -1,7 +1,7 @@ #----------------------------------------------------------- -# Multicorn, multiprocessing inspired by gunicorn -# TODO rename class: Multicorn -> Arbiter ? +# Threaded, Gevent and Prefork Servers #----------------------------------------------------------- +import datetime import errno import fcntl import logging @@ -13,10 +13,14 @@ import select import signal import socket import sys +import threading import time +import traceback import subprocess import os.path +import wsgi_server + import werkzeug.serving try: from setproctitle import setproctitle @@ -25,11 +29,238 @@ except ImportError: import openerp import openerp.tools.config as config +from openerp.release import nt_service_name from openerp.tools.misc import stripped_sys_argv _logger = logging.getLogger(__name__) -class Multicorn(object): +SLEEP_INTERVAL = 60 # 1 min + +#---------------------------------------------------------- +# Common +#---------------------------------------------------------- + +class CommonServer(object): + def __init__(self, app): + # TODO Change the xmlrpc_* options to http_* + self.app = app + # config + self.interface = config['xmlrpc_interface'] or '0.0.0.0' + self.port = config['xmlrpc_port'] + # runtime + self.pid = os.getpid() + + def dumpstacks(self): + """ Signal handler: dump a stack trace for each existing thread.""" + # code from http://stackoverflow.com/questions/132058/getting-stack-trace-from-a-running-python-application#answer-2569696 + # modified for python 2.5 compatibility + threads_info = dict([(th.ident, {'name': th.name, + 'uid': getattr(th,'uid','n/a')}) + for th in threading.enumerate()]) + code = [] + for threadId, stack in sys._current_frames().items(): + thread_info = threads_info.get(threadId) + code.append("\n# Thread: %s (id:%s) (uid:%s)" % \ + (thread_info and thread_info['name'] or 'n/a', + threadId, + thread_info and thread_info['uid'] or 'n/a')) + for filename, lineno, name, line in traceback.extract_stack(stack): + code.append('File: "%s", line %d, in %s' % (filename, lineno, name)) + if line: + code.append(" %s" % (line.strip())) + _logger.info("\n".join(code)) + + def close_socket(self, sock): + """ Closes a socket instance cleanly + :param sock: the network socket to close + :type sock: socket.socket + """ + try: + sock.shutdown(socket.SHUT_RDWR) + except socket.error, e: + # On OSX, socket shutdowns both sides if any side closes it + # causing an error 57 'Socket is not connected' on shutdown + # of the other side (or something), see + # http://bugs.python.org/issue4397 + # note: stdlib fixed test, not behavior + if e.errno != errno.ENOTCONN or platform.system() not in ['Darwin', 'Windows']: + raise + sock.close() + +#---------------------------------------------------------- +# Threaded +#---------------------------------------------------------- + +class ThreadedServer(CommonServer): + def __init__(self, app): + super(ThreadedServer, self).__init__(app) + self.main_thread_id = threading.currentThread().ident + # Variable keeping track of the number of calls to the signal handler defined + # below. This variable is monitored by ``quit_on_signals()``. + self.quit_signals_received = 0 + + #self.socket = None + #self.queue = [] + self.httpd = None + + def signal_handler(self, sig, frame): + if sig in [signal.SIGINT,signal.SIGTERM]: + # shutdown on kill -INT or -TERM + self.quit_signals_received += 1 + if self.quit_signals_received > 1: + # logging.shutdown was already called at this point. + sys.stderr.write("Forced shutdown.\n") + os._exit(0) + elif sig == signal.SIGHUP: + # restart on kill -HUP + openerp.phoenix = True + self.quit_signals_received += 1 + elif sig == signal.SIGQUIT: + # dump stacks on kill -3 + self.dumpstacks() + + def cron_thread(self, number): + while True: + time.sleep(SLEEP_INTERVAL + number) # Steve Reich timing style + registries = openerp.modules.registry.RegistryManager.registries + _logger.debug('cron%d polling for jobs', number) + for db_name, registry in registries.items(): + while True and registry.ready: + acquired = openerp.addons.base.ir.ir_cron.ir_cron._acquire_job(db_name) + if not acquired: + break + + def cron_spawn(self): + """ Start the above runner function in a daemon thread. + + The thread is a typical daemon thread: it will never quit and must be + terminated when the main process exits - with no consequence (the processing + threads it spawns are not marked daemon). + + """ + # Force call to strptime just before starting the cron thread + # to prevent time.strptime AttributeError within the thread. + # See: http://bugs.python.org/issue7980 + datetime.datetime.strptime('2012-01-01', '%Y-%m-%d') + for i in range(openerp.tools.config['max_cron_threads']): + def target(): + self.cron_thread(i) + t = threading.Thread(target=target, name="openerp.service.cron.cron%d" % i) + t.setDaemon(True) + t.start() + _logger.debug("cron%d started!" % i) + + def http_thread(self): + self.httpd = werkzeug.serving.make_server(self.interface, self.port, self.app, threaded=True) + self.httpd.serve_forever() + + def http_spawn(self): + threading.Thread(target=self.http_thread).start() + _logger.info('HTTP service (werkzeug) running on %s:%s', self.interface, self.port) + + def start(self): + _logger.debug("Setting signal handlers") + if os.name == 'posix': + signal.signal(signal.SIGINT, self.signal_handler) + signal.signal(signal.SIGTERM, self.signal_handler) + signal.signal(signal.SIGCHLD, self.signal_handler) + signal.signal(signal.SIGHUP, self.signal_handler) + signal.signal(signal.SIGQUIT, self.signal_handler) + elif os.name == 'nt': + import win32api + win32api.SetConsoleCtrlHandler(lambda sig: signal_handler(sig, None), 1) + self.cron_spawn() + self.http_spawn() + + def stop(self): + """ Shutdown the WSGI server. Wait for non deamon threads. + """ + _logger.info("Initiating shutdown") + _logger.info("Hit CTRL-C again or send a second signal to force the shutdown.") + + self.httpd.shutdown() + self.close_socket(self.httpd.socket) + + # Manually join() all threads before calling sys.exit() to allow a second signal + # to trigger _force_quit() in case some non-daemon threads won't exit cleanly. + # threading.Thread.join() should not mask signals (at least in python 2.5). + me = threading.currentThread() + _logger.debug('current thread: %r', me) + for thread in threading.enumerate(): + _logger.debug('process %r (%r)', thread, thread.isDaemon()) + if thread != me and not thread.isDaemon() and thread.ident != main_thread_id: + while thread.isAlive(): + _logger.debug('join and sleep') + # Need a busyloop here as thread.join() masks signals + # and would prevent the forced shutdown. + thread.join(0.05) + time.sleep(0.05) + + _logger.debug('--') + openerp.modules.registry.RegistryManager.delete_all() + logging.shutdown() + + def run(self): + """ Start the http server and the cron thread then wait for a signal. + + The first SIGINT or SIGTERM signal will initiate a graceful shutdown while + a second one if any will force an immediate exit. + """ + self.start() + + # Wait for a first signal to be handled. (time.sleep will be interrupted + # by the signal handler.) The try/except is for the win32 case. + try: + while self.quit_signals_received == 0: + time.sleep(60) + except KeyboardInterrupt: + pass + + self.stop() + +#---------------------------------------------------------- +# Gevent +#---------------------------------------------------------- + +class GeventServer(CommonServer): + def __init__(self, app): + super(GeventServer, self).__init__(app) + self.port = config['longpolling_port'] + self.httpd = None + + def watch_parent(self, beat=4): + import gevent + ppid = os.getppid() + while True: + if ppid != os.getppid(): + pid = os.getpid() + _logger.info("LongPolling (%s) Parent changed", pid) + # suicide !! + os.kill(pid, signal.SIGTERM) + return + gevent.sleep(beat) + + def start(self): + import gevent + from gevent.wsgi import WSGIServer + gevent.spawn(self.watch_parent) + self.httpd = WSGIServer((self.interface, self.port), self.app) + self.httpd.serve_forever() + + def stop(self): + import gevent + self.httpd.stop() + gevent.shutdown() + + def run(self): + self.start() + self.stop() + +#---------------------------------------------------------- +# Prefork +#---------------------------------------------------------- + +class Multicorn(CommonServer): """ Multiprocessing inspired by (g)unicorn. Multicorn currently uses accept(2) as dispatching method between workers but we plan to replace it by a more intelligent dispatcher to will parse @@ -92,10 +323,8 @@ class Multicorn(object): sys.exit(0) def long_polling_spawn(self): - nargs = stripped_sys_argv('--pidfile') - cmd = nargs[0] - cmd = os.path.join(os.path.dirname(cmd), "openerp-long-polling") - nargs[0] = cmd + nargs = stripped_sys_argv('--pidfile','--workers') + nargs += ['--gevent'] popen = subprocess.Popen(nargs) self.long_polling_pid = popen.pid @@ -122,6 +351,13 @@ class Multicorn(object): sig = self.queue.pop(0) if sig in [signal.SIGINT,signal.SIGTERM]: raise KeyboardInterrupt + elif sig == signal.SIGHUP: + # restart on kill -HUP + openerp.phoenix = True + raise KeyboardInterrupt + elif sig == signal.SIGQUIT: + # dump stacks on kill -3 + self.dumpstacks() def process_zombie(self): # reap dead workers @@ -211,7 +447,6 @@ class Multicorn(object): for pid in self.workers.keys(): self.worker_kill(pid, signal.SIGTERM) self.socket.close() - openerp.cli.server.quit_signals_received = 1 def run(self): self.start() @@ -389,7 +624,7 @@ class WorkerCron(Worker): def sleep(self): # Really sleep once all the databases have been processed. if self.db_index == 0: - interval = 60 + self.pid % 10 # chorus effect + interval = SLEEP_INTERVAL + self.pid % 10 # chorus effect time.sleep(interval) def _db_list(self): @@ -437,12 +672,66 @@ class WorkerCron(Worker): os.nice(10) # mommy always told me to be nice with others... Worker.start(self) self.multi.socket.close() - openerp.service.start_internal() # chorus effect: make cron workers do not all start at first database mct = config['max_cron_threads'] p = float(self.pid % mct) / mct self.db_index = int(len(self._db_list()) * p) +#---------------------------------------------------------- +# start/stop public api +#---------------------------------------------------------- + +server = None + +def load_server_wide_modules(): + for m in openerp.conf.server_wide_modules: + try: + openerp.modules.module.load_openerp_module(m) + except Exception: + msg = '' + if m == 'web': + msg = """ +The `web` module is provided by the addons found in the `openerp-web` project. +Maybe you forgot to add those addons in your addons_path configuration.""" + _logger.exception('Failed to load server-wide module `%s`.%s', m, msg) + +def _reexec(): + """reexecute openerp-server process with (nearly) the same arguments""" + if openerp.tools.osutil.is_running_as_nt_service(): + subprocess.call('net stop {0} && net start {0}'.format(nt_service_name), shell=True) + exe = os.path.basename(sys.executable) + args = stripped_sys_argv() + if not args or args[0] != exe: + args.insert(0, exe) + os.execv(sys.executable, args) + +def start(): + """ Start the openerp http server and cron processor. + """ + load_server_wide_modules() + if config['workers']: + openerp.multi_process = True + server = Multicorn(openerp.service.wsgi_server.application) + elif openerp.evented: + server = GeventServer(openerp.service.wsgi_server.application) + else: + server = ThreadedServer(openerp.service.wsgi_server.application) + server.run() + + # like the legend of the phoenix, all ends with beginnings + if getattr(openerp, 'phoenix', False): + _reexec() + sys.exit(0) + +def restart_server(): + """ Restart the server + """ + if os.name == 'nt': + # run in a thread to let the current thread return response to the caller. + threading.Thread(target=_reexec).start() + else: + os.kill(server.pid, signal.SIGHUP) + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/service/wsgi_server.py b/openerp/service/wsgi_server.py index acbfc67dd0b..002f8c248b9 100644 --- a/openerp/service/wsgi_server.py +++ b/openerp/service/wsgi_server.py @@ -400,8 +400,6 @@ def application_unproxied(environ, start_response): if hasattr(threading.current_thread(), 'dbname'): del threading.current_thread().dbname - openerp.service.start_internal() - # Try all handlers until one returns some result (i.e. not None). wsgi_handlers = [wsgi_xmlrpc_1, wsgi_xmlrpc, wsgi_xmlrpc_legacy, wsgi_webdav] wsgi_handlers += module_handlers @@ -422,69 +420,5 @@ def application(environ, start_response): else: return application_unproxied(environ, start_response) -# The WSGI server, started by start_server(), stopped by stop_server(). -httpd = None - -def serve(interface, port, threaded): - """ Serve HTTP requests via werkzeug development server. - - Calling this function is blocking, you might want to call it in its own - thread. - """ - - global httpd - if not openerp.evented: - httpd = werkzeug.serving.make_server(interface, port, application, threaded=threaded) - else: - from gevent.wsgi import WSGIServer - httpd = WSGIServer((interface, port), application) - httpd.serve_forever() - -def start_service(): - """ Call serve() in its own thread. - - The WSGI server can be shutdown with stop_server() below. - """ - # TODO Change the xmlrpc_* options to http_* - interface = config['xmlrpc_interface'] or '0.0.0.0' - port = config['xmlrpc_port'] - _logger.info('HTTP service (werkzeug) running on %s:%s', interface, port) - if not openerp.evented: - threading.Thread(target=serve, args=(interface, port, True)).start() - else: - serve(interface, port, True) - -def stop_service(): - """ Initiate the shutdown of the WSGI server. - - The server is supposed to have been started by start_server() above. - """ - if httpd: - if not openerp.evented: - httpd.shutdown() - close_socket(httpd.socket) - else: - import gevent - httpd.stop() - gevent.shutdown() - -def close_socket(sock): - """ Closes a socket instance cleanly - - :param sock: the network socket to close - :type sock: socket.socket - """ - try: - sock.shutdown(socket.SHUT_RDWR) - except socket.error, e: - # On OSX, socket shutdowns both sides if any side closes it - # causing an error 57 'Socket is not connected' on shutdown - # of the other side (or something), see - # http://bugs.python.org/issue4397 - # note: stdlib fixed test, not behavior - if e.errno != errno.ENOTCONN or platform.system() not in ['Darwin', 'Windows']: - raise - sock.close() - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: From 093dec2ab01cd5cd8e3f1962cd707f4afdaea17f Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Mon, 9 Sep 2013 15:10:24 +0200 Subject: [PATCH 02/10] [FIX] Add the platform lib [IMP] Log when the Long Polling Service is running bzr revid: stw@openerp.com-20130909131024-23an5cvt1qaa90su --- openerp/service/workers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openerp/service/workers.py b/openerp/service/workers.py index 382d581720f..73df8cc5aaf 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -18,6 +18,7 @@ import time import traceback import subprocess import os.path +import platform import wsgi_server @@ -245,6 +246,7 @@ class GeventServer(CommonServer): from gevent.wsgi import WSGIServer gevent.spawn(self.watch_parent) self.httpd = WSGIServer((self.interface, self.port), self.app) + _logger.info('Evented Service (longpolling) running on %s:%s', self.interface, self.port) self.httpd.serve_forever() def stop(self): From f8c0310d8ea8364b13a1a6b7c51862f1760a4ca9 Mon Sep 17 00:00:00 2001 From: Stephane Wirtel Date: Mon, 9 Sep 2013 15:11:23 +0200 Subject: [PATCH 03/10] [FIX] Use psycogreen instead of gevent-psycopg2 green-psycopg2 has been deprecated by the developer, he asks to use an alternative, in this case, the alternative is psycogreen https://bitbucket.org/dvarrazzo/psycogreen bzr revid: stw@openerp.com-20130909131123-u0gqhvxqwv13yqke --- openerp/__init__.py | 4 ++-- openerpcommand/web.py | 4 ++-- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openerp/__init__.py b/openerp/__init__.py index 161b06b0990..2cb3bce21d6 100644 --- a/openerp/__init__.py +++ b/openerp/__init__.py @@ -35,8 +35,8 @@ for i in sys.argv: if evented: import gevent.monkey gevent.monkey.patch_all() - import gevent_psycopg2 - gevent_psycopg2.monkey_patch() + import psycogreen.gevent + psycogreen.gevent.patch_psycopg() # Make sure the OpenERP server runs in UTC. This is especially necessary # under Windows as under Linux it seems the real import of time is diff --git a/openerpcommand/web.py b/openerpcommand/web.py index 5875978ad32..5d9d429244b 100644 --- a/openerpcommand/web.py +++ b/openerpcommand/web.py @@ -30,9 +30,9 @@ def run(args): import gevent import gevent.monkey import gevent.wsgi - import gevent_psycopg2 + import psycogreen.gevent gevent.monkey.patch_all() - gevent_psycopg2.monkey_patch() + psycogreen.gevent.patch_psycopg() import threading import openerp import openerp.cli.server diff --git a/setup.py b/setup.py index ff801a97933..4f4ffe6e505 100644 --- a/setup.py +++ b/setup.py @@ -120,7 +120,7 @@ setuptools.setup( 'feedparser', 'gdata', 'gevent', - 'gevent-psycopg2', + 'psycogreen', 'Jinja2', 'lxml', # windows binary http://www.lfd.uci.edu/~gohlke/pythonlibs/ 'mako', From 2a6d0299d97251d93e1a10976096b2354f8f625b Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Mon, 9 Sep 2013 22:08:25 +0200 Subject: [PATCH 04/10] revert to use a separate executable for gevent, add --dev option bzr revid: al@openerp.com-20130909200825-sm5c3invmg7tt70j --- openerp-gevent | 13 +++++++++++++ openerp/__init__.py | 10 ---------- openerp/service/workers.py | 4 +++- openerp/tools/config.py | 4 ++-- 4 files changed, 18 insertions(+), 13 deletions(-) create mode 100755 openerp-gevent diff --git a/openerp-gevent b/openerp-gevent new file mode 100755 index 00000000000..e1c9fa797e1 --- /dev/null +++ b/openerp-gevent @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import gevent.monkey +gevent.monkey.patch_all() +import psycogreen.gevent +psycogreen.gevent.patch_psycopg() + +import openerp + +openerp.evented = True + +if __name__ == "__main__": + openerp.cli.main() diff --git a/openerp/__init__.py b/openerp/__init__.py index 2cb3bce21d6..fe439838497 100644 --- a/openerp/__init__.py +++ b/openerp/__init__.py @@ -27,16 +27,6 @@ import sys # Is the server running with gevent. evented = False -for i in sys.argv: - if i.startswith('--gevent'): - evented = True - break - -if evented: - import gevent.monkey - gevent.monkey.patch_all() - import psycogreen.gevent - psycogreen.gevent.patch_psycopg() # Make sure the OpenERP server runs in UTC. This is especially necessary # under Windows as under Linux it seems the real import of time is diff --git a/openerp/service/workers.py b/openerp/service/workers.py index 73df8cc5aaf..50aa8f245d0 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -326,7 +326,9 @@ class Multicorn(CommonServer): def long_polling_spawn(self): nargs = stripped_sys_argv('--pidfile','--workers') - nargs += ['--gevent'] + cmd = nargs[0] + cmd = os.path.join(os.path.dirname(cmd), "openerp-gevent") + nargs[0] = cmd popen = subprocess.Popen(nargs) self.long_polling_pid = popen.pid diff --git a/openerp/tools/config.py b/openerp/tools/config.py index cc3c187c775..a399f6d133c 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -106,7 +106,7 @@ class configmanager(object): help="specify additional addons paths (separated by commas).", action="callback", callback=self._check_addons_path, nargs=1, type="string") group.add_option("--load", dest="server_wide_modules", help="Comma-separated list of server-wide modules default=web") - group.add_option("--gevent", dest="gevent", action="store_true", my_default=False, help="Activate the GEvent mode, this also desactivate the cron.") + group.add_option("--dev", dest="dev", action="store_true", my_default=False, help="Activate the developer mode. (code and views auto-reload).") parser.add_option_group(group) # XML-RPC / HTTP @@ -399,7 +399,7 @@ class configmanager(object): 'list_db', 'xmlrpcs', 'proxy_mode', 'test_file', 'test_enable', 'test_commit', 'test_report_directory', 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', 'unaccent', - 'workers', 'limit_memory_hard', 'limit_memory_soft', 'limit_time_cpu', 'limit_time_real', 'limit_request', 'gevent' + 'workers', 'limit_memory_hard', 'limit_memory_soft', 'limit_time_cpu', 'limit_time_real', 'limit_request', 'dev' ] for arg in keys: From 3bc9a499d4e872176db809c80222edd5d6cd6f11 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Mon, 9 Sep 2013 22:17:06 +0200 Subject: [PATCH 05/10] add signals to increase/decrease number of workers bzr revid: al@openerp.com-20130909201706-sllijtol7p6mi5nr --- openerp/service/workers.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openerp/service/workers.py b/openerp/service/workers.py index 50aa8f245d0..0e7e15fcc39 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -362,6 +362,12 @@ class Multicorn(CommonServer): elif sig == signal.SIGQUIT: # dump stacks on kill -3 self.dumpstacks() + elif sig == signal.SIGTTIN: + # increase number of workers + self.population += 1 + elif sig == signal.SIGTTOUT: + # decrease number of workers + self.population -= 1 def process_zombie(self): # reap dead workers @@ -423,10 +429,14 @@ class Multicorn(CommonServer): # by a signal simulating a pseudo SA_RESTART. We write to a pipe in the # signal handler to overcome this behaviour self.pipe = self.pipe_new() - # set signal + # set signal handlers signal.signal(signal.SIGINT, self.signal_handler) signal.signal(signal.SIGTERM, self.signal_handler) + signal.signal(signal.SIGHUP, self.signal_handler) signal.signal(signal.SIGCHLD, self.signal_handler) + signal.signal(signal.SIGQUIT, self.signal_handler) + signal.signal(signal.SIGTTIN, self.signal_handler) + signal.signal(signal.SIGTTOUT, self.signal_handler) # listen to socket self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) From 65c0538a8359c5836a3e42ac6c2c0e739d4d6bff Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Tue, 10 Sep 2013 01:05:53 +0200 Subject: [PATCH 06/10] autoreload, first working version for both python and data, still wip bzr revid: al@openerp.com-20130909230553-jn26ue5qenv0sd3p --- openerp/service/workers.py | 117 +++++++++++++++++++++++++++++++++---- 1 file changed, 107 insertions(+), 10 deletions(-) diff --git a/openerp/service/workers.py b/openerp/service/workers.py index 0e7e15fcc39..f2f4c60d5db 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -41,6 +41,97 @@ SLEEP_INTERVAL = 60 # 1 min # Common #---------------------------------------------------------- +class AutoReload(object): + def __init__(self): + self.files = {} + import pyinotify + class EventHandler(pyinotify.ProcessEvent): + def __init__(self, autoreload): + self.autoreload = autoreload + + def process_IN_CREATE(self, event): + _logger.debug('File created: %s', event.pathname) + self.autoreload.files[event.pathname] = 1 + + def process_IN_MODIFY(self, event): + _logger.debug('File modified: %s', event.pathname) + self.autoreload.files[event.pathname] = 1 + + self.wm = pyinotify.WatchManager() + self.handler = EventHandler(self) + self.notifier = pyinotify.Notifier(self.wm, self.handler, timeout=0) + mask = pyinotify.IN_MODIFY | pyinotify.IN_CREATE # IN_MOVED_FROM, IN_MOVED_TO ? + for path in openerp.tools.config.options["addons_path"].split(','): + _logger.info('Watching addons folder %s', path) + self.wm.add_watch(path, mask, rec=True) + + def process_data(self, touched_files): + # pyinotify notifier + fs modiciation tracker + from openerp.modules.module import load_information_from_description_file as load_manifest + addons_path = openerp.tools.config.options["addons_path"].split(',') + registries = openerp.modules.registry.RegistryManager.registries + keys = ['data', 'demo', 'test', 'init_xml', 'update_xml', 'demo_xml'] + # This will only work for loaded registies, so we have to use -d + # al: proposed to move this code in the registry manager so it can be lazy + for db_name, registry in registries.items(): + cr = registry.db.cursor() + try: + for tfile in touched_files: + for path in addons_path: + if tfile.startswith(path): + # find out wich addons path the file belongs to + # and extract it's module name + right = tfile[len(path) + 1:].split('/') + if len(right) < 2: + continue + module = right[0] + relname = "/".join(right[1:]) + domain = [('name', '=', module), ('state', 'in', ['installed', 'to upgrade'])] + if registry.get('ir.module.module').search(cr, openerp.SUPERUSER_ID, domain): + manifest = load_manifest(module) + kind = [key for key in keys if relname in manifest[key]] + if kind: + _logger.info('Updating changed xml file: %s', tfile) + idref = {} + openerp.tools.convert_file(cr, module, relname, idref, mode='update', kind=kind[0]) + cr.commit() + except Exception,e: + _logger.exception(e) + finally: + cr.close() + + def process_python(self, files): + # process python changes + py_files = [i for i in files if i.endswith('.py')] + py_errors = [] + # TODO keep python errors until they are ok + if py_files: + for i in py_files: + try: + source = open(i, 'rb').read() + '\n' + compile(source, i, 'exec') + except SyntaxError: + py_errors.append(i) + if py_errors: + _logger.info('autoreload: python code change detected, errors found') + for i in py_errors: + _logger.info('autoreload: SyntaxError %s',i) + else: + _logger.info('autoreload: python code updated, autoreload activated') + restart_server() + + def check(self): + # Check if some files have been touched in the addons path. + # If true, check if the touched file belongs to an installed module + # in any of the database used in the registry manager. + while self.notifier.check_events(0): + self.notifier.read_events() + self.notifier.process_events() + l = self.files.keys() + self.files.clear() + self.process_data(l) + self.process_python(l) + class CommonServer(object): def __init__(self, app): # TODO Change the xmlrpc_* options to http_* @@ -101,8 +192,8 @@ class ThreadedServer(CommonServer): self.quit_signals_received = 0 #self.socket = None - #self.queue = [] self.httpd = None + self.autoreload = None def signal_handler(self, sig, frame): if sig in [signal.SIGINT,signal.SIGTERM]: @@ -152,7 +243,11 @@ class ThreadedServer(CommonServer): _logger.debug("cron%d started!" % i) def http_thread(self): - self.httpd = werkzeug.serving.make_server(self.interface, self.port, self.app, threaded=True) + def app(e,s): + if self.autoreload: + self.autoreload.check() + return self.app(e,s) + self.httpd = werkzeug.serving.make_server(self.interface, self.port, app, threaded=True) self.httpd.serve_forever() def http_spawn(self): @@ -172,6 +267,7 @@ class ThreadedServer(CommonServer): win32api.SetConsoleCtrlHandler(lambda sig: signal_handler(sig, None), 1) self.cron_spawn() self.http_spawn() + self.autoreload = AutoReload() def stop(self): """ Shutdown the WSGI server. Wait for non deamon threads. @@ -189,7 +285,7 @@ class ThreadedServer(CommonServer): _logger.debug('current thread: %r', me) for thread in threading.enumerate(): _logger.debug('process %r (%r)', thread, thread.isDaemon()) - if thread != me and not thread.isDaemon() and thread.ident != main_thread_id: + if thread != me and not thread.isDaemon() and thread.ident != self.main_thread_id: while thread.isAlive(): _logger.debug('join and sleep') # Need a busyloop here as thread.join() masks signals @@ -262,11 +358,11 @@ class GeventServer(CommonServer): # Prefork #---------------------------------------------------------- -class Multicorn(CommonServer): +class PreforkServer(CommonServer): """ Multiprocessing inspired by (g)unicorn. - Multicorn currently uses accept(2) as dispatching method between workers - but we plan to replace it by a more intelligent dispatcher to will parse - the first HTTP request line. + PreforkServer (aka Multicorn) currently uses accept(2) as dispatching + method between workers but we plan to replace it by a more intelligent + dispatcher to will parse the first HTTP request line. """ def __init__(self, app): # config @@ -617,12 +713,12 @@ class WorkerBaseWSGIServer(werkzeug.serving.BaseWSGIServer): def __init__(self, app): werkzeug.serving.BaseWSGIServer.__init__(self, "1", "1", app) def server_bind(self): - # we dont bind beause we use the listen socket of Multicorn#socket + # we dont bind beause we use the listen socket of PreforkServer#socket # instead we close the socket if self.socket: self.socket.close() def server_activate(self): - # dont listen as we use Multicorn#socket + # dont listen as we use PreforkServer#socket pass class WorkerCron(Worker): @@ -723,10 +819,11 @@ def _reexec(): def start(): """ Start the openerp http server and cron processor. """ + global server load_server_wide_modules() if config['workers']: openerp.multi_process = True - server = Multicorn(openerp.service.wsgi_server.application) + server = PreforkServer(openerp.service.wsgi_server.application) elif openerp.evented: server = GeventServer(openerp.service.wsgi_server.application) else: From 082b1dc9fc91200d4eeb9a78c05930dbfac510c6 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Tue, 10 Sep 2013 01:19:46 +0200 Subject: [PATCH 07/10] restore the original gevent detection bzr revid: al@openerp.com-20130909231946-aglho23enxhfwm0i --- openerp-gevent | 2 -- openerp/__init__.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openerp-gevent b/openerp-gevent index e1c9fa797e1..55044976492 100755 --- a/openerp-gevent +++ b/openerp-gevent @@ -7,7 +7,5 @@ psycogreen.gevent.patch_psycopg() import openerp -openerp.evented = True - if __name__ == "__main__": openerp.cli.main() diff --git a/openerp/__init__.py b/openerp/__init__.py index fe439838497..a06c85d0c21 100644 --- a/openerp/__init__.py +++ b/openerp/__init__.py @@ -27,6 +27,8 @@ import sys # Is the server running with gevent. evented = False +if sys.modules.get("gevent") is not None: + evented = True # Make sure the OpenERP server runs in UTC. This is especially necessary # under Windows as under Linux it seems the real import of time is From 0a75a6b5ff175feef9348cdbbc4ab2d90f686304 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Sat, 5 Oct 2013 23:22:40 +0200 Subject: [PATCH 08/10] simplier autoreload working bzr revid: al@openerp.com-20131005212240-5lp8tgwukeg5wwdq --- openerp/addons/base/module/module.py | 2 +- openerp/service/workers.py | 185 +++++++++++++++------------ openerp/tools/config.py | 3 +- 3 files changed, 103 insertions(+), 87 deletions(-) diff --git a/openerp/addons/base/module/module.py b/openerp/addons/base/module/module.py index 39d59a4ab2c..32306fbd0ff 100644 --- a/openerp/addons/base/module/module.py +++ b/openerp/addons/base/module/module.py @@ -726,7 +726,7 @@ class module(osv.osv): if already_installed: # in this case, force server restart to reload python code... cr.commit() - openerp.service.restart_server() + openerp.service.restart() return { 'type': 'ir.actions.client', 'tag': 'home', diff --git a/openerp/service/workers.py b/openerp/service/workers.py index f2f4c60d5db..f7d6f97b85b 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -6,21 +6,19 @@ import errno import fcntl import logging import os +import os.path +import platform import psutil import random import resource import select import signal import socket +import subprocess import sys import threading import time import traceback -import subprocess -import os.path -import platform - -import wsgi_server import werkzeug.serving try: @@ -33,17 +31,60 @@ import openerp.tools.config as config from openerp.release import nt_service_name from openerp.tools.misc import stripped_sys_argv +import wsgi_server + _logger = logging.getLogger(__name__) SLEEP_INTERVAL = 60 # 1 min #---------------------------------------------------------- -# Common +# Werkzeug WSGI servers patched +#---------------------------------------------------------- + +class BaseWSGIServerNoBind(werkzeug.serving.BaseWSGIServer): + """ werkzeug Base WSGI Server patched to skip socket binding. PreforkServer + use this class, sets the socket and calls the process_request() manually + """ + def __init__(self, app): + werkzeug.serving.BaseWSGIServer.__init__(self, "1", "1", app) + def server_bind(self): + # we dont bind beause we use the listen socket of PreforkServer#socket + # instead we close the socket + if self.socket: + self.socket.close() + def server_activate(self): + # dont listen as we use PreforkServer#socket + pass + +# MAybe NOT useful BECAUSE of SOCKET_REUSE, need to test + +class ThreadedWSGIServerReloadable(werkzeug.serving.ThreadedWSGIServer): + """ werkzeug Threaded WSGI Server patched to allow reusing a listen socket + given by the environement, this is used by autoreload to keep the listen + socket open when a reload happens. + """ + def server_bind(self): + envfd = os.environ.get('OPENERP_AUTO_RELOAD_FD') + if envfd: + self.reload_socket = socket.fromfd + # close os.close()fd if fd has been diplucated ?! + else: + self.reload_socket = False + super(ThreadedWSGIServerReloadable, self).server_bind() + + def server_activate(self): + if not self.reload_socket: + super(ThreadedWSGIServerReloadable, self).server_activate() + +#---------------------------------------------------------- +# AutoReload watcher #---------------------------------------------------------- class AutoReload(object): - def __init__(self): + def __init__(self, server): + self.server = server self.files = {} + self.modules = {} import pyinotify class EventHandler(pyinotify.ProcessEvent): def __init__(self, autoreload): @@ -65,40 +106,22 @@ class AutoReload(object): _logger.info('Watching addons folder %s', path) self.wm.add_watch(path, mask, rec=True) - def process_data(self, touched_files): - # pyinotify notifier + fs modiciation tracker - from openerp.modules.module import load_information_from_description_file as load_manifest + def process_data(self, files): + xml_files = [i for i in files if i.endswith('.xml')] addons_path = openerp.tools.config.options["addons_path"].split(',') - registries = openerp.modules.registry.RegistryManager.registries - keys = ['data', 'demo', 'test', 'init_xml', 'update_xml', 'demo_xml'] - # This will only work for loaded registies, so we have to use -d - # al: proposed to move this code in the registry manager so it can be lazy - for db_name, registry in registries.items(): - cr = registry.db.cursor() - try: - for tfile in touched_files: - for path in addons_path: - if tfile.startswith(path): - # find out wich addons path the file belongs to - # and extract it's module name - right = tfile[len(path) + 1:].split('/') - if len(right) < 2: - continue - module = right[0] - relname = "/".join(right[1:]) - domain = [('name', '=', module), ('state', 'in', ['installed', 'to upgrade'])] - if registry.get('ir.module.module').search(cr, openerp.SUPERUSER_ID, domain): - manifest = load_manifest(module) - kind = [key for key in keys if relname in manifest[key]] - if kind: - _logger.info('Updating changed xml file: %s', tfile) - idref = {} - openerp.tools.convert_file(cr, module, relname, idref, mode='update', kind=kind[0]) - cr.commit() - except Exception,e: - _logger.exception(e) - finally: - cr.close() + for i in xml_files: + for path in addons_path: + if i.startswith(path): + # find out wich addons path the file belongs to + # and extract it's module name + right = i[len(path) + 1:].split('/') + if len(right) < 2: + continue + module = right[0] + self.modules[module]=1 + if self.modules: + _logger.info('autoreload: xml change detected, autoreload activated') + restart() def process_python(self, files): # process python changes @@ -118,19 +141,30 @@ class AutoReload(object): _logger.info('autoreload: SyntaxError %s',i) else: _logger.info('autoreload: python code updated, autoreload activated') - restart_server() + restart() - def check(self): + def check_thread(self): # Check if some files have been touched in the addons path. # If true, check if the touched file belongs to an installed module # in any of the database used in the registry manager. - while self.notifier.check_events(0): - self.notifier.read_events() - self.notifier.process_events() - l = self.files.keys() - self.files.clear() - self.process_data(l) - self.process_python(l) + while 1: + while self.notifier.check_events(1000): + self.notifier.read_events() + self.notifier.process_events() + l = self.files.keys() + self.files.clear() + self.process_data(l) + self.process_python(l) + + def run(self): + t = threading.Thread(target=self.check_thread) + t.setDaemon(True) + t.start() + _logger.info('AutoReload watcher running') + +#---------------------------------------------------------- +# Servers: Threaded, Gevented and Prefork +#---------------------------------------------------------- class CommonServer(object): def __init__(self, app): @@ -179,10 +213,6 @@ class CommonServer(object): raise sock.close() -#---------------------------------------------------------- -# Threaded -#---------------------------------------------------------- - class ThreadedServer(CommonServer): def __init__(self, app): super(ThreadedServer, self).__init__(app) @@ -193,7 +223,6 @@ class ThreadedServer(CommonServer): #self.socket = None self.httpd = None - self.autoreload = None def signal_handler(self, sig, frame): if sig in [signal.SIGINT,signal.SIGTERM]: @@ -244,10 +273,8 @@ class ThreadedServer(CommonServer): def http_thread(self): def app(e,s): - if self.autoreload: - self.autoreload.check() return self.app(e,s) - self.httpd = werkzeug.serving.make_server(self.interface, self.port, app, threaded=True) + self.httpd = ThreadedWSGIServerReloadable(self.interface, self.port, app) self.httpd.serve_forever() def http_spawn(self): @@ -267,7 +294,6 @@ class ThreadedServer(CommonServer): win32api.SetConsoleCtrlHandler(lambda sig: signal_handler(sig, None), 1) self.cron_spawn() self.http_spawn() - self.autoreload = AutoReload() def stop(self): """ Shutdown the WSGI server. Wait for non deamon threads. @@ -315,9 +341,8 @@ class ThreadedServer(CommonServer): self.stop() -#---------------------------------------------------------- -# Gevent -#---------------------------------------------------------- + def reload(self): + os.kill(self.pid, signal.SIGHUP) class GeventServer(CommonServer): def __init__(self, app): @@ -354,10 +379,6 @@ class GeventServer(CommonServer): self.start() self.stop() -#---------------------------------------------------------- -# Prefork -#---------------------------------------------------------- - class PreforkServer(CommonServer): """ Multiprocessing inspired by (g)unicorn. PreforkServer (aka Multicorn) currently uses accept(2) as dispatching @@ -684,7 +705,7 @@ class WorkerHTTP(Worker): # Prevent fd inherientence close_on_exec flags = fcntl.fcntl(client, fcntl.F_GETFD) | fcntl.FD_CLOEXEC fcntl.fcntl(client, fcntl.F_SETFD, flags) - # do request using WorkerBaseWSGIServer monkey patched with socket + # do request using BaseWSGIServerNoBind monkey patched with socket self.server.socket = client # tolerate broken pipe when the http client closes the socket before # receiving the full reply @@ -705,21 +726,7 @@ class WorkerHTTP(Worker): def start(self): Worker.start(self) - self.server = WorkerBaseWSGIServer(self.multi.app) - -class WorkerBaseWSGIServer(werkzeug.serving.BaseWSGIServer): - """ werkzeug WSGI Server patched to allow using an external listen socket - """ - def __init__(self, app): - werkzeug.serving.BaseWSGIServer.__init__(self, "1", "1", app) - def server_bind(self): - # we dont bind beause we use the listen socket of PreforkServer#socket - # instead we close the socket - if self.socket: - self.socket.close() - def server_activate(self): - # dont listen as we use PreforkServer#socket - pass + self.server = BaseWSGIServerNoBind(self.multi.app) class WorkerCron(Worker): """ Cron workers """ @@ -806,12 +813,13 @@ The `web` module is provided by the addons found in the `openerp-web` project. Maybe you forgot to add those addons in your addons_path configuration.""" _logger.exception('Failed to load server-wide module `%s`.%s', m, msg) -def _reexec(): +def _reexec(updated_modules=None): """reexecute openerp-server process with (nearly) the same arguments""" if openerp.tools.osutil.is_running_as_nt_service(): subprocess.call('net stop {0} && net start {0}'.format(nt_service_name), shell=True) exe = os.path.basename(sys.executable) args = stripped_sys_argv() + args += ["-u", ','.join(updated_modules)] if not args or args[0] != exe: args.insert(0, exe) os.execv(sys.executable, args) @@ -828,14 +836,22 @@ def start(): server = GeventServer(openerp.service.wsgi_server.application) else: server = ThreadedServer(openerp.service.wsgi_server.application) + + if config['auto_reload']: + autoreload = AutoReload(server) + autoreload.run() + server.run() # like the legend of the phoenix, all ends with beginnings if getattr(openerp, 'phoenix', False): - _reexec() + modules = [] + if config['auto_reload']: + modules = autoreload.modules.keys() + _reexec(modules) sys.exit(0) -def restart_server(): +def restart(): """ Restart the server """ if os.name == 'nt': @@ -844,5 +860,4 @@ def restart_server(): else: os.kill(server.pid, signal.SIGHUP) - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tools/config.py b/openerp/tools/config.py index a399f6d133c..05242b538c2 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -246,6 +246,7 @@ class configmanager(object): # Advanced options group = optparse.OptionGroup(parser, "Advanced options") + group.add_option('--auto-reload', dest='auto_reload', action='store_true', my_default=False, help='enable auto reload') group.add_option('--debug', dest='debug_mode', action='store_true', my_default=False, help='enable debug mode') group.add_option("--stop-after-init", action="store_true", dest="stop_after_init", my_default=False, help="stop the server after its initialization") @@ -399,7 +400,7 @@ class configmanager(object): 'list_db', 'xmlrpcs', 'proxy_mode', 'test_file', 'test_enable', 'test_commit', 'test_report_directory', 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', 'unaccent', - 'workers', 'limit_memory_hard', 'limit_memory_soft', 'limit_time_cpu', 'limit_time_real', 'limit_request', 'dev' + 'workers', 'limit_memory_hard', 'limit_memory_soft', 'limit_time_cpu', 'limit_time_real', 'limit_request', 'dev', 'auto_reload' ] for arg in keys: From c52df72706844589ca8bc4c3fac3fe8550612c23 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Sat, 5 Oct 2013 23:58:21 +0200 Subject: [PATCH 09/10] fix signal name bzr revid: al@openerp.com-20131005215821-r60pgkgsrp1lo2ug --- openerp/service/workers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openerp/service/workers.py b/openerp/service/workers.py index f7d6f97b85b..e9b0615e70c 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -482,7 +482,7 @@ class PreforkServer(CommonServer): elif sig == signal.SIGTTIN: # increase number of workers self.population += 1 - elif sig == signal.SIGTTOUT: + elif sig == signal.SIGTTOU: # decrease number of workers self.population -= 1 @@ -553,7 +553,7 @@ class PreforkServer(CommonServer): signal.signal(signal.SIGCHLD, self.signal_handler) signal.signal(signal.SIGQUIT, self.signal_handler) signal.signal(signal.SIGTTIN, self.signal_handler) - signal.signal(signal.SIGTTOUT, self.signal_handler) + signal.signal(signal.SIGTTOU, self.signal_handler) # listen to socket self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) From e6b18fc7aac067d6e905a0f1a68d89670e959fc3 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Sun, 6 Oct 2013 00:18:29 +0200 Subject: [PATCH 10/10] rename server bzr revid: al@openerp.com-20131005221829-a18vx5ag6ef4ufee --- openerp/addons/base/module/module.py | 2 +- openerp/cli/server.py | 2 +- openerp/service/__init__.py | 2 +- openerp/service/{workers.py => server.py} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename openerp/service/{workers.py => server.py} (100%) diff --git a/openerp/addons/base/module/module.py b/openerp/addons/base/module/module.py index 6f986ac01b3..6279a5fe6ae 100644 --- a/openerp/addons/base/module/module.py +++ b/openerp/addons/base/module/module.py @@ -716,7 +716,7 @@ class module(osv.osv): if already_installed: # in this case, force server restart to reload python code... cr.commit() - openerp.service.restart() + openerp.service.server.restart() return { 'type': 'ir.actions.client', 'tag': 'home', diff --git a/openerp/cli/server.py b/openerp/cli/server.py index 0c081b6156b..24987c098e4 100644 --- a/openerp/cli/server.py +++ b/openerp/cli/server.py @@ -177,7 +177,7 @@ def main(args): if not config["stop_after_init"]: setup_pid_file() - openerp.service.workers.start() + openerp.service.server.start() if config['pidfile']: os.unlink(config['pidfile']) else: diff --git a/openerp/service/__init__.py b/openerp/service/__init__.py index 02401d455e1..6e3ae985fab 100644 --- a/openerp/service/__init__.py +++ b/openerp/service/__init__.py @@ -25,7 +25,7 @@ import db import model import report import wsgi_server -import workers +import server #.apidoc title: RPC Services diff --git a/openerp/service/workers.py b/openerp/service/server.py similarity index 100% rename from openerp/service/workers.py rename to openerp/service/server.py