diff --git a/bin/netsvc.py b/bin/netsvc.py index 624623081fc..3b77f3b1cf2 100644 --- a/bin/netsvc.py +++ b/bin/netsvc.py @@ -323,6 +323,7 @@ class Server: """ __is_started = False __servers = [] + __starter_threads = [] # we don't want blocking server calls (think select()) to # wait forever and possibly prevent exiting the process, @@ -335,12 +336,26 @@ class Server: __logger = logging.getLogger('server') def __init__(self): - if Server.__is_started: - raise Exception('All instances of servers must be inited before the startAll()') Server.__servers.append(self) + if Server.__is_started: + # raise Exception('All instances of servers must be inited before the startAll()') + # Since the startAll() won't be called again, allow this server to + # init and then start it after 1sec (hopefully). Register that + # timer thread in a list, so that we can abort the start if quitAll + # is called in the meantime + t = threading.Timer(1.0, self._late_start) + t.name = 'Late start timer for %s' % str(self.__class__) + Server.__starter_threads.append(t) + t.start() def start(self): self.__logger.debug("called stub Server.start") + + def _late_start(self): + self.start() + for thr in Server.__starter_threads: + if thr.finished.is_set(): + Server.__starter_threads.remove(thr) def stop(self): self.__logger.debug("called stub Server.stop") @@ -363,6 +378,11 @@ class Server: if not cls.__is_started: return cls.__logger.info("Stopping %d services" % len(cls.__servers)) + for thr in cls.__starter_threads: + if not thr.finished.is_set(): + thr.cancel() + cls.__starter_threads.remove(thr) + for srv in cls.__servers: srv.stop() cls.__is_started = False diff --git a/bin/openerp-server.py b/bin/openerp-server.py index c56977fe1f0..5a68440af27 100755 --- a/bin/openerp-server.py +++ b/bin/openerp-server.py @@ -108,6 +108,7 @@ if not ( tools.config["stop_after_init"] or \ tools.config["translate_out"] ): service.http_server.init_servers() service.http_server.init_xmlrpc() + service.http_server.init_static_http() import service.netrpc_server service.netrpc_server.init_servers() diff --git a/bin/service/http_server.py b/bin/service/http_server.py index bab38b06609..519b8fca675 100644 --- a/bin/service/http_server.py +++ b/bin/service/http_server.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # -# Copyright P. Christeas 2008,2009 +# Copyright P. Christeas 2008-2010 +# Copyright 2010 OpenERP SA. (http://www.openerp.com) # # # WARNING: This program as such is intended to be used by professional @@ -34,10 +35,13 @@ import netsvc import errno import threading import tools +import posixpath +import urllib import os import select import socket import xmlrpclib +import logging from SimpleXMLRPCServer import SimpleXMLRPCDispatcher @@ -69,6 +73,9 @@ class ThreadedHTTPServer(ConnThreadingMixIn, SimpleXMLRPCDispatcher, HTTPServer) SimpleXMLRPCDispatcher.__init__(self, allow_none, encoding) HTTPServer.__init__(self, addr, requestHandler) + + self.numThreads = 0 + self.__threadno = 0 # [Bug #1222790] If possible, set close-on-exec flag; if a # method spawns a subprocess, the subprocess shouldn't have @@ -81,21 +88,44 @@ class ThreadedHTTPServer(ConnThreadingMixIn, SimpleXMLRPCDispatcher, HTTPServer) def handle_error(self, request, client_address): """ Override the error handler """ - import traceback - netsvc.Logger().notifyChannel("init", netsvc.LOG_ERROR,"Server error in request from %s:\n%s" % - (client_address,traceback.format_exc())) + + logging.getLogger("init").exception("Server error in request from %s:" % (client_address,)) -class MultiHandler2(MultiHTTPHandler): + def _mark_start(self, thread): + self.numThreads += 1 + + def _mark_end(self, thread): + self.numThreads -= 1 + + + def _get_next_name(self): + self.__threadno += 1 + return 'http-client-%d' % self.__threadno +class HttpLogHandler: + """ helper class for uniform log handling + Please define self._logger at each class that is derived from this + """ + _logger = None + def log_message(self, format, *args): - netsvc.Logger().notifyChannel('http',netsvc.LOG_DEBUG,format % args) + self._logger.debug(format % args) # todo: perhaps other level def log_error(self, format, *args): - netsvc.Logger().notifyChannel('http',netsvc.LOG_ERROR,format % args) + self._logger.error(format % args) + + def log_exception(self, format, *args): + self._logger.exception(format, *args) + + def log_request(self, code='-', size='-'): + self._logger.log(netsvc.logging.DEBUG_RPC, '"%s" %s %s', + self.requestline, str(code), str(size)) + +class MultiHandler2(HttpLogHandler, MultiHTTPHandler): + _logger = logging.getLogger('http') -class SecureMultiHandler2(SecureMultiHTTPHandler): - def log_message(self, format, *args): - netsvc.Logger().notifyChannel('https',netsvc.LOG_DEBUG,format % args) +class SecureMultiHandler2(HttpLogHandler, SecureMultiHTTPHandler): + _logger = logging.getLogger('https') def getcert_fnames(self): tc = tools.config @@ -103,13 +133,9 @@ class SecureMultiHandler2(SecureMultiHTTPHandler): fkey = tc.get_misc('httpsd','sslkey', 'ssl/server.key') return (fcert,fkey) - def log_message(self, format, *args): - netsvc.Logger().notifyChannel('http',netsvc.LOG_DEBUG,format % args) - - def log_error(self, format, *args): - netsvc.Logger().notifyChannel('http',netsvc.LOG_ERROR,format % args) - class BaseHttpDaemon(threading.Thread, netsvc.Server): + _RealProto = '??' + def __init__(self, interface, port, handler): threading.Thread.__init__(self) netsvc.Server.__init__(self) @@ -121,10 +147,11 @@ class BaseHttpDaemon(threading.Thread, netsvc.Server): self.server.vdirs = [] self.server.logRequests = True self.server.timeout = self._busywait_timeout + logging.getLogger("web-services").info( + "starting %s service at %s port %d" % + (self._RealProto, interface or '0.0.0.0', port,)) except Exception, e: - netsvc.Logger().notifyChannel( - 'httpd', netsvc.LOG_CRITICAL, - "Error occur when starting the server daemon: %s" % (e,)) + logging.getLogger("httpd").exception("Error occured when starting the server daemon.") raise @property @@ -148,29 +175,51 @@ class BaseHttpDaemon(threading.Thread, netsvc.Server): raise return True + def stats(self): + res = "%sd: " % self._RealProto + ((self.running and "running") or "stopped") + if self.server: + res += ", %d threads" % (self.server.numThreads,) + return res + + def append_svc(self, service): + if not isinstance(service, HTTPDir): + raise Exception("Wrong class for http service") + + pos = len(self.server.vdirs) + lastpos = pos + while pos > 0: + pos -= 1 + if self.server.vdirs[pos].matches(service.path): + lastpos = pos + # we won't break here, but search all way to the top, to + # ensure there is no lesser entry that will shadow the one + # we are inserting. + self.server.vdirs.insert(lastpos, service) + + def list_services(self): + ret = [] + for svc in self.server.vdirs: + ret.append( ( svc.path, str(svc.handler)) ) + + return ret + + class HttpDaemon(BaseHttpDaemon): + _RealProto = 'HTTP' def __init__(self, interface, port): super(HttpDaemon, self).__init__(interface, port, handler=MultiHandler2) - netsvc.Logger().notifyChannel( - "web-services", netsvc.LOG_INFO, - "starting HTTP service at %s port %d" % - (interface or '0.0.0.0', port,)) class HttpSDaemon(BaseHttpDaemon): + _RealProto = 'HTTPS' def __init__(self, interface, port): try: super(HttpSDaemon, self).__init__(interface, port, handler=SecureMultiHandler2) except SSLError, e: - netsvc.Logger().notifyChannel( - 'httpd-ssl', netsvc.LOG_CRITICAL, - "Can not load the certificate and/or the private key files") + logging.getLogger('httpsd').exception( \ + "Can not load the certificate and/or the private key files") raise - netsvc.Logger().notifyChannel( - "web-services", netsvc.LOG_INFO, - "starting HTTPS service at %s port %d" % - (interface or '0.0.0.0', port,)) httpd = None httpsd = None @@ -190,23 +239,32 @@ def reg_http_service(hts, secure_only = False): hts must be an HTTPDir """ global httpd, httpsd - if not isinstance(hts, HTTPDir): - raise Exception("Wrong class for http service") if httpd and not secure_only: - httpd.server.vdirs.append(hts) + httpd.append_svc(hts) if httpsd: - httpsd.server.vdirs.append(hts) + httpsd.append_svc(hts) if (not httpd) and (not httpsd): - netsvc.Logger().notifyChannel('httpd',netsvc.LOG_WARNING,"No httpd available to register service %s" % hts.path) + logging.getLogger('httpd').warning("No httpd available to register service %s" % hts.path) return +def list_http_services(protocol=None): + global httpd, httpsd + if httpd and (protocol == 'http' or protocol == None): + return httpd.list_services() + elif httpsd and (protocol == 'https' or protocol == None): + return httpsd.list_services() + else: + raise Exception("Incorrect protocol or no http services") + import SimpleXMLRPCServer -class XMLRPCRequestHandler(netsvc.OpenERPDispatcher,FixSendError,SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): +class XMLRPCRequestHandler(netsvc.OpenERPDispatcher,FixSendError,HttpLogHandler,SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): rpc_paths = [] protocol_version = 'HTTP/1.1' + _logger = logging.getLogger('xmlrpc') + def _dispatch(self, method, params): try: service_name = self.path.split("/")[-1] @@ -214,9 +272,6 @@ class XMLRPCRequestHandler(netsvc.OpenERPDispatcher,FixSendError,SimpleXMLRPCSer except netsvc.OpenERPDispatcherException, e: raise xmlrpclib.Fault(tools.exception_to_unicode(e.exception), e.traceback) - def log_message(self, format, *args): - netsvc.Logger().notifyChannel('xmlrpc',netsvc.LOG_DEBUG_RPC,format % args) - def handle(self): pass @@ -235,13 +290,57 @@ def init_xmlrpc(): # Example of http file serving: # reg_http_service(HTTPDir('/test/',HTTPHandler)) reg_http_service(HTTPDir('/xmlrpc/', XMLRPCRequestHandler)) - netsvc.Logger().notifyChannel("web-services", netsvc.LOG_INFO, - "Registered XML-RPC over HTTP") + logging.getLogger("web-services").info("Registered XML-RPC over HTTP") - if tools.config.get('xmlrpcs', False): - reg_http_service(HTTPDir('/xmlrpc/', XMLRPCRequestHandler, True)) - netsvc.Logger().notifyChannel('web-services', netsvc.LOG_INFO, - "Registered XML-RPC over HTTPS") + if tools.config.get('xmlrpcs', False) \ + and not tools.config.get('xmlrpc', False): + # only register at the secure server + reg_http_service(HTTPDir('/xmlrpc/', XMLRPCRequestHandler), True) + logging.getLogger("web-services").info("Registered XML-RPC over HTTPS only") + +class StaticHTTPHandler(HttpLogHandler, FixSendError, HttpOptions, HTTPHandler): + _logger = logging.getLogger('httpd') + _HTTP_OPTIONS = { 'Allow': ['OPTIONS', 'GET', 'HEAD'] } + + def __init__(self,request, client_address, server): + HTTPHandler.__init__(self,request,client_address,server) + dir_path = tools.config.get_misc('static-http', 'dir_path', False) + assert dir_path, "Please specify static-http/dir_path in config, or disable static-httpd!" + self.__basepath = dir_path + + def translate_path(self, path): + """Translate a /-separated PATH to the local filename syntax. + + Components that mean special things to the local file system + (e.g. drive or directory names) are ignored. (XXX They should + probably be diagnosed.) + + """ + # abandon query parameters + path = path.split('?',1)[0] + path = path.split('#',1)[0] + path = posixpath.normpath(urllib.unquote(path)) + words = path.split('/') + words = filter(None, words) + path = self.__basepath + for word in words: + if word in (os.curdir, os.pardir): continue + path = os.path.join(path, word) + return path + +def init_static_http(): + if not tools.config.get_misc('static-http','enable', False): + return + + dir_path = tools.config.get_misc('static-http', 'dir_path', False) + assert dir_path + + base_path = tools.config.get_misc('static-http', 'base_path', '/') + + reg_http_service(HTTPDir(base_path,StaticHTTPHandler)) + + logging.getLogger("web-services").info("Registered HTTP dir %s for %s" % \ + (dir_path, base_path)) class OerpAuthProxy(AuthProxy): """ Require basic authentication.. @@ -260,7 +359,6 @@ class OerpAuthProxy(AuthProxy): try: if not db: db = handler.get_db_from_path(path) - print "Got db:",db except Exception: if path.startswith('/'): path = path[1:] @@ -305,10 +403,10 @@ class OpenERPAuthProvider(AuthProvider): return False return (user, passwd, db, uid) except Exception,e: - netsvc.Logger().notifyChannel("auth",netsvc.LOG_DEBUG,"Fail auth:"+ str(e)) + logging.getLogger("auth").debug("Fail auth: %s" % e ) return False - def log(self, msg): - netsvc.Logger().notifyChannel("auth",netsvc.LOG_INFO,msg) + def log(self, msg, lvl=logging.INFO): + logging.getLogger("auth").log(lvl,msg) #eof diff --git a/bin/service/netrpc_server.py b/bin/service/netrpc_server.py index 7f204751576..80cb968c8e9 100644 --- a/bin/service/netrpc_server.py +++ b/bin/service/netrpc_server.py @@ -36,7 +36,9 @@ import tools class TinySocketClientThread(threading.Thread, netsvc.OpenERPDispatcher): def __init__(self, sock, threads): - threading.Thread.__init__(self) + spn = sock and sock.getpeername() + spn = 'netrpc-client-%s:%s' % spn[0:2] + threading.Thread.__init__(self, name=spn) self.sock = sock # Only at the server side, use a big timeout: close the # clients connection when they're idle for 20min. @@ -48,7 +50,8 @@ class TinySocketClientThread(threading.Thread, netsvc.OpenERPDispatcher): try: self.socket.shutdown( getattr(socket, 'SHUT_RDWR', 2)) - except Exception: pass + except Exception: + pass # That should garbage-collect and close it, too self.sock = None diff --git a/bin/service/web_services.py b/bin/service/web_services.py index 31075ec015d..8f1fc910739 100644 --- a/bin/service/web_services.py +++ b/bin/service/web_services.py @@ -305,7 +305,7 @@ class db(netsvc.ExportService): else: cr.execute("select decode(datname, 'escape') from pg_database where datname not in('template0', 'template1','postgres') order by datname") res = [str(name) for (name,) in cr.fetchall()] - except: + except Exception: res = [] finally: cr.close() @@ -385,7 +385,8 @@ class common(_ObjectService): logger.notifyChannel("web-service", netsvc.LOG_INFO,'Logout %s from database %s'%(login,db)) return True elif method in ['about', 'timezone_get', 'get_server_environment', - 'login_message','get_stats', 'check_connectivity']: + 'login_message','get_stats', 'check_connectivity', + 'list_http_services']: pass elif method in ['get_available_updates', 'get_migration_scripts', 'set_loglevel']: passwd = params[0] @@ -492,7 +493,7 @@ GNU Public Licence. try: try: base64_decoded = base64.decodestring(zips[module]) - except: + except Exception: l.notifyChannel('migration', netsvc.LOG_ERROR, 'unable to read the module %s' % (module,)) raise @@ -501,13 +502,13 @@ GNU Public Licence. try: try: tools.extract_zip_file(zip_contents, tools.config['addons_path'] ) - except: + except Exception: l.notifyChannel('migration', netsvc.LOG_ERROR, 'unable to extract the module %s' % (module, )) rmtree(module) raise finally: zip_contents.close() - except: + except Exception: l.notifyChannel('migration', netsvc.LOG_ERROR, 'restore the previous version of the module %s' % (module, )) nmp = os.path.join(backup_directory, module) if os.path.isdir(nmp): @@ -563,6 +564,10 @@ GNU Public Licence. res += netsvc.Server.allStats() return res + def exp_list_http_services(self): + from service import http_server + return http_server.list_http_services() + def exp_check_connectivity(self): return bool(sql_db.db_connect('template1')) diff --git a/bin/service/websrv_lib.py b/bin/service/websrv_lib.py index 80f24cf4b2b..8565ab6ccaa 100644 --- a/bin/service/websrv_lib.py +++ b/bin/service/websrv_lib.py @@ -142,6 +142,9 @@ class noconnection: def makefile(self, mode, bufsize): return None + def close(self): + pass + class dummyconn: def shutdown(self, tru): pass @@ -169,10 +172,45 @@ class FixSendError: self.send_header('Connection', 'close') self.send_header('Content-Length', len(content) or 0) self.end_headers() + if hasattr(self, '_flush'): + self._flush() + if self.command != 'HEAD' and code >= 200 and code not in (204, 304): self.wfile.write(content) -class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler): +class HttpOptions: + _HTTP_OPTIONS = {'Allow': ['OPTIONS' ] } + + def do_OPTIONS(self): + """return the list of capabilities """ + + opts = self._HTTP_OPTIONS + nopts = self._prep_OPTIONS(opts) + if nopts: + opts = nopts + + self.send_response(200) + self.send_header("Content-Length", 0) + + for key, value in opts.items(): + if isinstance(value, basestring): + self.send_header(key, value) + elif isinstance(value, (tuple, list)): + self.send_header(key, ', '.join(value)) + self.end_headers() + + def _prep_OPTIONS(self, opts): + """Prepare the OPTIONS response, if needed + + Sometimes, like in special DAV folders, the OPTIONS may contain + extra keywords, perhaps also dependant on the request url. + @param the options already. MUST be copied before being altered + @return the updated options. + + """ + return opts + +class MultiHTTPHandler(FixSendError, HttpOptions, BaseHTTPRequestHandler): """ this is a multiple handler, that will dispatch each request to a nested handler, iff it matches @@ -226,11 +264,30 @@ class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler): return mname = 'do_' + fore.command if not hasattr(fore, mname): - fore.send_error(501, "Unsupported method (%r)" % fore.command) + if fore.command == 'OPTIONS': + self.do_OPTIONS() + return + self.send_error(501, "Unsupported method (%r)" % fore.command) return fore.close_connection = 0 method = getattr(fore, mname) - method() + try: + method() + except (AuthRejectedExc, AuthRequiredExc): + raise + except Exception, e: + if hasattr(self, 'log_exception'): + self.log_exception("Could not run %s", mname) + else: + self.log_error("Could not run %s: %s", mname, e) + self.send_error(500, "Internal error") + # may not work if method has already sent data + fore.close_connection = 1 + self.close_connection = 1 + if hasattr(fore, '_flush'): + fore._flush() + return + if fore.close_connection: # print "Closing connection because of handler" self.close_connection = fore.close_connection @@ -289,6 +346,7 @@ class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler): [command, path] = words self.close_connection = 1 if command != 'GET': + self.log_error("Junk http request: %s", self.raw_requestline) self.send_error(400, "Bad HTTP/0.9 request type (%r)" % command) return False @@ -315,6 +373,14 @@ class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler): self.log_message("Could not parse rawline.") return # self.parse_request(): # Do NOT parse here. the first line should be the only + + if self.path == '*' and self.command == 'OPTIONS': + # special handling of path='*', must not use any vdir at all. + if not self.parse_request(): + return + self.do_OPTIONS() + return + for vdir in self.server.vdirs: p = vdir.matches(self.path) if p == False: @@ -397,13 +463,26 @@ class ConnThreadingMixIn: # main process daemon_threads = False + def _get_next_name(self): + return None + def _handle_request_noblock(self): """Start a new thread to process the request.""" - t = threading.Thread(target = self._handle_request2) + if not threading: # happens while quitting python + return + t = threading.Thread(name=self._get_next_name(), target=self._handle_request2) if self.daemon_threads: t.setDaemon (1) t.start() + def _mark_start(self, thread): + """ Mark the start of a request thread """ + pass + + def _mark_end(self, thread): + """ Mark the end of a request thread """ + pass + def _handle_request2(self): """Handle one request, without blocking. @@ -412,14 +491,17 @@ class ConnThreadingMixIn: no risk of blocking in get_request(). """ try: + self._mark_start(threading.currentThread()) request, client_address = self.get_request() + if self.verify_request(request, client_address): + try: + self.process_request(request, client_address) + except Exception: + self.handle_error(request, client_address) + self.close_request(request) except socket.error: return - if self.verify_request(request, client_address): - try: - self.process_request(request, client_address) - except Exception: - self.handle_error(request, client_address) - self.close_request(request) + finally: + self._mark_end(threading.currentThread()) #eof diff --git a/doc/openerp-server.conf b/doc/openerp-server.conf index 3b58a008329..ac15f2ef255 100644 --- a/doc/openerp-server.conf +++ b/doc/openerp-server.conf @@ -25,3 +25,7 @@ translate_modules = ['all'] demo = {} addons_path = None reportgz = False + +[static-http] +enable = False +dir_path = /var/www/html \ No newline at end of file diff --git a/list-services.sh b/list-services.sh new file mode 100755 index 00000000000..a90e58479d5 --- /dev/null +++ b/list-services.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# ADMIN_PASSWD='admin' +method_1() { + cat '-' << EOF + + + list_http_services + + + +EOF +} + +method_1 | POST -c 'text/xml' http://localhost:8069/xmlrpc/common +#eof