diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 9962493d549..7f23553a1de 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -22,11 +22,15 @@ workers = 4 # Some application-wide initialization is needed. on_starting = openerp.wsgi.on_starting when_ready = openerp.wsgi.when_ready +pre_request = openerp.wsgi.pre_request +post_request = openerp.wsgi.post_request # openerp request-response cycle can be quite long for # big reports for example timeout = 240 +max_requests = 2000 + # Equivalent of --load command-line option openerp.conf.server_wide_modules = ['web'] @@ -52,5 +56,4 @@ conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons # If --static-http-enable is used, path for the static web directory #conf['static_http_document_root'] = '/var/www' - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/__init__.py b/openerp/tests/addons/test_limits/__init__.py new file mode 100644 index 00000000000..fe4487156b1 --- /dev/null +++ b/openerp/tests/addons/test_limits/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +import models +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/__openerp__.py b/openerp/tests/addons/test_limits/__openerp__.py new file mode 100644 index 00000000000..333f1fe4372 --- /dev/null +++ b/openerp/tests/addons/test_limits/__openerp__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'test-limits', + 'version': '0.1', + 'category': 'Tests', + 'description': """A module with dummy methods.""", + 'author': 'OpenERP SA', + 'maintainer': 'OpenERP SA', + 'website': 'http://www.openerp.com', + 'depends': ['base'], + 'data': [], + 'installable': True, + 'active': False, +} +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/models.py b/openerp/tests/addons/test_limits/models.py new file mode 100644 index 00000000000..5240acd23ab --- /dev/null +++ b/openerp/tests/addons/test_limits/models.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +import time + +import openerp + +class m(openerp.osv.osv.Model): + """ This model exposes a few methods that will consume between 'almost no + resource' and 'a lot of resource'. + """ + _name = 'test.limits.model' + + def consume_nothing(self, cr, uid, context=None): + return True + + def consume_memory(self, cr, uid, size, context=None): + l = [0] * size + return True + + def leak_memory(self, cr, uid, size, context=None): + if not hasattr(self, 'l'): + self.l = [] + self.l.append([0] * size) + return True + + def consume_time(self, cr, uid, seconds, context=None): + time.sleep(seconds) + return True + + def consume_cpu_time(self, cr, uid, seconds, context=None): + import os + t0 = time.clock() + t1 = time.clock() + while t1 - t0 < seconds: + for i in xrange(10000000): + x = i * i + t1 = time.clock() + return True +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tools/config.py b/openerp/tools/config.py index 8107e10bf85..4f33c3b79f1 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -266,6 +266,20 @@ class configmanager(object): group.add_option("--max-cron-threads", dest="max_cron_threads", my_default=4, help="Maximum number of threads processing concurrently cron jobs.", type="int") + # TODO sensible default for the three following limits. + group.add_option("--virtual-memory-limit", dest="virtual_memory_limit", my_default=768 * 1024 * 1024, + help="Maximum allowed virtual memory per Gunicorn process. " + "When the limit is reached, any memory allocation will fail.", + type="int") + group.add_option("--virtual-memory-reset", dest="virtual_memory_reset", my_default=640 * 1024 * 1024, + help="Maximum allowed virtual memory per Gunicorn process. " + "When the limit is reached, the worker will be reset after " + "the current request.", + type="int") + group.add_option("--cpu-time-limit", dest="cpu_time_limit", my_default=60, + help="Maximum allowed CPU time per Gunicorn process. " + "When the limit is reached, an exception is raised.", + type="int") group.add_option("--unaccent", dest="unaccent", my_default=False, action="store_true", help="Use the unaccent function provided by the database when available.") @@ -371,7 +385,8 @@ class configmanager(object): 'stop_after_init', 'logrotate', 'without_demo', 'netrpc', 'xmlrpc', 'syslog', 'list_db', 'xmlrpcs', 'test_file', 'test_disable', 'test_commit', 'test_report_directory', - 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', 'unaccent', + 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', + 'virtual_memory_limit', 'virtual_memory_reset', 'cpu_time_limit', 'unaccent', ] for arg in keys: diff --git a/openerp/wsgi.py b/openerp/wsgi.py index aeba38dcfbc..2b721648196 100644 --- a/openerp/wsgi.py +++ b/openerp/wsgi.py @@ -480,12 +480,44 @@ def when_ready(server): # Hijack gunicorn's SIGWINCH handling; we can choose another one. signal.signal(signal.SIGWINCH, make_winch_handler(server)) +# Install limits on virtual memory and CPU time consumption. +def pre_request(worker, req): + import os + import psutil + import resource + import signal + # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space + rss, vms = psutil.Process(os.getpid()).get_memory_info() + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + resource.setrlimit(resource.RLIMIT_AS, (config['virtual_memory_limit'], hard)) + + r = resource.getrusage(resource.RUSAGE_SELF) + cpu_time = r.ru_utime + r.ru_stime + signal.signal(signal.SIGXCPU, time_expired) + soft, hard = resource.getrlimit(resource.RLIMIT_CPU) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['cpu_time_limit'], hard)) + +# Reset the worker if it consumes too much memory (e.g. caused by a memory leak). +def post_request(worker, req, environ): + import os + import psutil + rss, vms = psutil.Process(os.getpid()).get_memory_info() + if vms > config['virtual_memory_reset']: + logging.getLogger('wsgi.worker').info('Virtual memory consumption ' + 'too high, rebooting the worker.') + worker.alive = False # Commit suicide after the request. + # Our signal handler will signal a SGIQUIT to all workers. def make_winch_handler(server): def handle_winch(sig, fram): server.kill_workers(signal.SIGQUIT) # This is gunicorn specific. return handle_winch +# SIGXCPU (exceeded CPU time) signal handler will raise an exception. +def time_expired(n, stack): + logging.getLogger('wsgi.worker').info('CPU time limit exceeded.') + raise Exception('CPU time limit exceeded.') # TODO one of openerp.exception + # Kill gracefuly the workers (e.g. because we want to clear their cache). # This is done by signaling a SIGWINCH to the master process, so it can be # called by the workers themselves.