pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] server.py: --dev=reload with inotify

Browse Source 
Add the alternative of inotify instead of watchdog to watch the addons
paths the server was started with.

Reason: watchdog spawns 2 threads per path to watch. When there are a
lot of addons paths, this can become too costly. With inotify we watch
all the repositories in a single thread.

https://github.com/dsoprea/PyInotify

installation:

    pip install inotify
This commit is contained in:
Denis Vermylen
2019-03-15 14:18:09 +00:00
parent 83b2d2e1b5
commit 7b6cfc412e
1 changed files with 91 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
odoo/service/server.py
+91 -24
View File
@@ -25,9 +25,25 @@ if os.name == 'posix':
import fcntl
import resource
import psutil
try:
import inotify
from inotify.adapters import InotifyTrees
from inotify.constants import IN_MODIFY, IN_CREATE, IN_MOVED_TO
INOTIFY_LISTEN_EVENTS = IN_MODIFY | IN_CREATE | IN_MOVED_TO
except ImportError:
inotify = None
else:
# Windows shim
signal.SIGHUP = -1
inotify = None
if not inotify:
try:
import watchdog
from watchdog.observers import Observer
from watchdog.events import FileCreatedEvent, FileModifiedEvent, FileMovedEvent
except ImportError:
watchdog = None
# Optional process names for workers
try:
@@ -43,13 +59,6 @@ from odoo.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats
_logger = logging.getLogger(__name__)
try:
import watchdog
from watchdog.observers import Observer
from watchdog.events import FileCreatedEvent, FileModifiedEvent, FileMovedEvent
except ImportError:
watchdog = None
SLEEP_INTERVAL = 60 # 1 min
def memory_info(process):
@@ -119,7 +128,25 @@ class ThreadedWSGIServerReloadable(LoggingBaseWSGIServerMixIn, werkzeug.serving.
#----------------------------------------------------------
# FileSystem Watcher for autoreload and cache invalidation
#----------------------------------------------------------
class FSWatcher(object):
class FSWatcherBase(object):
def handle_file(self, path):
if path.endswith('.py') and not os.path.basename(path).startswith('.~'):
try:
# Forward-ports: watch out PY3 compatibility!
source = open(path, 'rb').read() + '\n'
compile(source, path, 'exec')
except IOError:
_logger.error('autoreload: python code change detected, IOError for %s', path)
except SyntaxError:
_logger.error('autoreload: python code change detected, SyntaxError in %s', path)
else:
if not getattr(odoo, 'phoenix', False):
_logger.info('autoreload: python code updated, autoreload activated')
restart()
return True
class FSWatcherWatchdog(FSWatcherBase):
def __init__(self):
self.observer = Observer()
for path in odoo.modules.module.ad_paths:
@@ -130,27 +157,60 @@ class FSWatcher(object):
if isinstance(event, (FileCreatedEvent, FileModifiedEvent, FileMovedEvent)):
if not event.is_directory:
path = getattr(event, 'dest_path', event.src_path)
if path.endswith('.py') and not os.path.basename(path).startswith('.~'):
try:
source = open(path, 'rb').read() + '\n'
compile(source, path, 'exec')
except IOError:
_logger.error('autoreload: python code change detected, IOError for %s', path)
except SyntaxError:
_logger.error('autoreload: python code change detected, SyntaxError in %s', path)
else:
if not getattr(odoo, 'phoenix', False):
_logger.info('autoreload: python code updated, autoreload activated')
restart()
self.handle_file(path)
def start(self):
self.observer.start()
_logger.info('AutoReload watcher running')
_logger.info('AutoReload watcher running with watchdog')
def stop(self):
self.observer.stop()
self.observer.join()
class FSWatcherInotify(FSWatcherBase):
def __init__(self):
self.started = False
# ignore warnings from inotify in case we have duplicate addons paths.
inotify.adapters._LOGGER.setLevel(logging.ERROR)
# recreate a list as InotifyTrees' __init__ deletes the list's items
paths_to_watch = []
for path in odoo.modules.module.ad_paths:
paths_to_watch.append(path)
_logger.info('Watching addons folder %s', path)
self.watcher = InotifyTrees(paths_to_watch, mask=INOTIFY_LISTEN_EVENTS, block_duration_s=.5)
def run(self):
_logger.info('AutoReload watcher running with inotify')
dir_creation_events = set(('IN_MOVED_TO', 'IN_CREATE'))
while self.started:
for event in self.watcher.event_gen(timeout_s=0, yield_nones=False):
(_, type_names, path, filename) = event
if 'IN_ISDIR' not in type_names:
# despite not having IN_DELETE in the watcher's mask, the
# watcher sends these events when a directory is deleted.
if 'IN_DELETE' not in type_names:
full_path = os.path.join(path, filename)
if self.handle_file(full_path):
return
elif dir_creation_events.intersection(type_names):
full_path = os.path.join(path, filename)
for root, _, files in os.walk(full_path):
for file in files:
if self.handle_file(os.path.join(root, file)):
return
def start(self):
self.started = True
self.thread = threading.Thread(target=self.run, name="odoo.service.autoreload.watcher")
self.thread.setDaemon(True)
self.thread.start()
def stop(self):
self.started = False
self.thread.join()
#----------------------------------------------------------
# Servers: Threaded, Gevented and Prefork
#----------------------------------------------------------
@@ -942,11 +1002,18 @@ def start(preload=None, stop=False):
watcher = None
if 'reload' in config['dev_mode'] and not odoo.evented:
if watchdog:
watcher = FSWatcher()
if inotify:
watcher = FSWatcherInotify()
watcher.start()
elif watchdog:
watcher = FSWatcherWatchdog()
watcher.start()
else:
_logger.warning("'watchdog' module not installed. Code autoreload feature is disabled")
if os.name == 'posix' and platform.system() != 'Darwin':
module = 'inotify'
else:
module = 'watchdog'
_logger.warning("'%s' module not installed. Code autoreload feature is disabled", module)
if 'werkzeug' in config['dev_mode']:
server.app = DebuggedApplication(server.app, evalex=True)