From a1fa9faa313da8fc64e3eb7c9efd293d7c6373ef Mon Sep 17 00:00:00 2001 From: Denis Vermylen Date: Thu, 14 Mar 2019 10:06:04 +0000 Subject: [PATCH 1/5] [FIX] server.py: FileNotFoundError doesn't exist in P2 Python2 uses IOError instead. /!\ DO NOT FORWARD PORT AFTER 11.* /!\ --- odoo/service/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/odoo/service/server.py b/odoo/service/server.py index d9beb9f6378..60b009e1f28 100644 --- a/odoo/service/server.py +++ b/odoo/service/server.py @@ -134,8 +134,8 @@ class FSWatcher(object): try: source = open(path, 'rb').read() + '\n' compile(source, path, 'exec') - except FileNotFoundError: - _logger.error('autoreload: python code change detected, FileNotFound for %s', path) + except IOError: + _logger.error('autoreload: python code change detected, IOError for %s', path) except SyntaxError: _logger.error('autoreload: python code change detected, SyntaxError in %s', path) else: From 83b2d2e1b5fb5343e036545c7ea7f7335eea1a14 Mon Sep 17 00:00:00 2001 From: Denis Vermylen Date: Thu, 14 Mar 2019 14:37:57 +0000 Subject: [PATCH 2/5] [FIX] server.py: don't start multiple FSWatcher in multiworker mode Before this commit every worker would start his own FSWatcher. --- odoo/service/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odoo/service/server.py b/odoo/service/server.py index 60b009e1f28..516b97652a9 100644 --- a/odoo/service/server.py +++ b/odoo/service/server.py @@ -941,7 +941,7 @@ def start(preload=None, stop=False): server = ThreadedServer(odoo.service.wsgi_server.application) watcher = None - if 'reload' in config['dev_mode']: + if 'reload' in config['dev_mode'] and not odoo.evented: if watchdog: watcher = FSWatcher() watcher.start() From 7b6cfc412e1af3f8887a02e7c013561126bf1d7c Mon Sep 17 00:00:00 2001 From: Denis Vermylen Date: Thu, 14 Mar 2019 15:02:54 +0000 Subject: [PATCH 3/5] [IMP] server.py: --dev=reload with inotify Add the alternative of inotify instead of watchdog to watch the addons paths the server was started with. Reason: watchdog spawns 2 threads per path to watch. When there are a lot of addons paths, this can become too costly. With inotify we watch all the repositories in a single thread. https://github.com/dsoprea/PyInotify installation: pip install inotify --- odoo/service/server.py | 115 ++++++++++++++++++++++++++++++++--------- 1 file changed, 91 insertions(+), 24 deletions(-) diff --git a/odoo/service/server.py b/odoo/service/server.py index 516b97652a9..fd1566c4431 100644 --- a/odoo/service/server.py +++ b/odoo/service/server.py @@ -25,9 +25,25 @@ if os.name == 'posix': import fcntl import resource import psutil + try: + import inotify + from inotify.adapters import InotifyTrees + from inotify.constants import IN_MODIFY, IN_CREATE, IN_MOVED_TO + INOTIFY_LISTEN_EVENTS = IN_MODIFY | IN_CREATE | IN_MOVED_TO + except ImportError: + inotify = None else: # Windows shim signal.SIGHUP = -1 + inotify = None + +if not inotify: + try: + import watchdog + from watchdog.observers import Observer + from watchdog.events import FileCreatedEvent, FileModifiedEvent, FileMovedEvent + except ImportError: + watchdog = None # Optional process names for workers try: @@ -43,13 +59,6 @@ from odoo.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats _logger = logging.getLogger(__name__) -try: - import watchdog - from watchdog.observers import Observer - from watchdog.events import FileCreatedEvent, FileModifiedEvent, FileMovedEvent -except ImportError: - watchdog = None - SLEEP_INTERVAL = 60 # 1 min def memory_info(process): @@ -119,7 +128,25 @@ class ThreadedWSGIServerReloadable(LoggingBaseWSGIServerMixIn, werkzeug.serving. #---------------------------------------------------------- # FileSystem Watcher for autoreload and cache invalidation #---------------------------------------------------------- -class FSWatcher(object): +class FSWatcherBase(object): + def handle_file(self, path): + if path.endswith('.py') and not os.path.basename(path).startswith('.~'): + try: + # Forward-ports: watch out PY3 compatibility! + source = open(path, 'rb').read() + '\n' + compile(source, path, 'exec') + except IOError: + _logger.error('autoreload: python code change detected, IOError for %s', path) + except SyntaxError: + _logger.error('autoreload: python code change detected, SyntaxError in %s', path) + else: + if not getattr(odoo, 'phoenix', False): + _logger.info('autoreload: python code updated, autoreload activated') + restart() + return True + + +class FSWatcherWatchdog(FSWatcherBase): def __init__(self): self.observer = Observer() for path in odoo.modules.module.ad_paths: @@ -130,27 +157,60 @@ class FSWatcher(object): if isinstance(event, (FileCreatedEvent, FileModifiedEvent, FileMovedEvent)): if not event.is_directory: path = getattr(event, 'dest_path', event.src_path) - if path.endswith('.py') and not os.path.basename(path).startswith('.~'): - try: - source = open(path, 'rb').read() + '\n' - compile(source, path, 'exec') - except IOError: - _logger.error('autoreload: python code change detected, IOError for %s', path) - except SyntaxError: - _logger.error('autoreload: python code change detected, SyntaxError in %s', path) - else: - if not getattr(odoo, 'phoenix', False): - _logger.info('autoreload: python code updated, autoreload activated') - restart() + self.handle_file(path) def start(self): self.observer.start() - _logger.info('AutoReload watcher running') + _logger.info('AutoReload watcher running with watchdog') def stop(self): self.observer.stop() self.observer.join() + +class FSWatcherInotify(FSWatcherBase): + def __init__(self): + self.started = False + # ignore warnings from inotify in case we have duplicate addons paths. + inotify.adapters._LOGGER.setLevel(logging.ERROR) + # recreate a list as InotifyTrees' __init__ deletes the list's items + paths_to_watch = [] + for path in odoo.modules.module.ad_paths: + paths_to_watch.append(path) + _logger.info('Watching addons folder %s', path) + self.watcher = InotifyTrees(paths_to_watch, mask=INOTIFY_LISTEN_EVENTS, block_duration_s=.5) + + def run(self): + _logger.info('AutoReload watcher running with inotify') + dir_creation_events = set(('IN_MOVED_TO', 'IN_CREATE')) + while self.started: + for event in self.watcher.event_gen(timeout_s=0, yield_nones=False): + (_, type_names, path, filename) = event + if 'IN_ISDIR' not in type_names: + # despite not having IN_DELETE in the watcher's mask, the + # watcher sends these events when a directory is deleted. + if 'IN_DELETE' not in type_names: + full_path = os.path.join(path, filename) + if self.handle_file(full_path): + return + elif dir_creation_events.intersection(type_names): + full_path = os.path.join(path, filename) + for root, _, files in os.walk(full_path): + for file in files: + if self.handle_file(os.path.join(root, file)): + return + + def start(self): + self.started = True + self.thread = threading.Thread(target=self.run, name="odoo.service.autoreload.watcher") + self.thread.setDaemon(True) + self.thread.start() + + def stop(self): + self.started = False + self.thread.join() + + #---------------------------------------------------------- # Servers: Threaded, Gevented and Prefork #---------------------------------------------------------- @@ -942,11 +1002,18 @@ def start(preload=None, stop=False): watcher = None if 'reload' in config['dev_mode'] and not odoo.evented: - if watchdog: - watcher = FSWatcher() + if inotify: + watcher = FSWatcherInotify() + watcher.start() + elif watchdog: + watcher = FSWatcherWatchdog() watcher.start() else: - _logger.warning("'watchdog' module not installed. Code autoreload feature is disabled") + if os.name == 'posix' and platform.system() != 'Darwin': + module = 'inotify' + else: + module = 'watchdog' + _logger.warning("'%s' module not installed. Code autoreload feature is disabled", module) if 'werkzeug' in config['dev_mode']: server.app = DebuggedApplication(server.app, evalex=True) From 744bdd275eff08fe7593dd8ca7aa9162b54ab275 Mon Sep 17 00:00:00 2001 From: Denis Vermylen Date: Thu, 14 Mar 2019 15:08:52 +0000 Subject: [PATCH 4/5] [FIX] server.py: stop the FSWatcher when stopping the server Avoid potential tracebacks from the FSWatcher's thread being killed. Drawback: Server shut down can have an extra small delay (only applies when the --dev=reload option is given) closes odoo/odoo#31855 Signed-off-by: Christophe Simonis --- odoo/service/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/odoo/service/server.py b/odoo/service/server.py index fd1566c4431..240d816ebd3 100644 --- a/odoo/service/server.py +++ b/odoo/service/server.py @@ -1019,10 +1019,10 @@ def start(preload=None, stop=False): rc = server.run(preload, stop) + if watcher: + watcher.stop() # like the legend of the phoenix, all ends with beginnings if getattr(odoo, 'phoenix', False): - if watcher: - watcher.stop() _reexec() return rc if rc else 0 From f1c72ee8ee44c35707d5a18167eb8efeca3c4a6f Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Thu, 7 Mar 2019 17:32:04 +0000 Subject: [PATCH 5/5] [FIX] base: fix ir.logging database locking when using --log-db Before this patch, the ir.logging's write_uid field was a many2one which could cause a module install/update to hang when the module is changing the res.users model schema and when this module causes to orm to warn through the logger. (eg: declaring two res.users fields with the same string attribute) In such situation the transaction cursor that is processing the res_users table alteration will be granted an exclusive postgresql lock hence causing the ir_logging insertion to block because of the write_uid foreign key to res_users. This issue has never been raised by runbot as it is using a remote database with --log-db Note: the write_uid conversion from m2o to int was left over in commit e6a5d82 closes odoo/odoo#32015 Signed-off-by: Christophe Simonis --- odoo/addons/base/ir/ir_logging.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/odoo/addons/base/ir/ir_logging.py b/odoo/addons/base/ir/ir_logging.py index 0397bff7425..4e3caef0794 100644 --- a/odoo/addons/base/ir/ir_logging.py +++ b/odoo/addons/base/ir/ir_logging.py @@ -1,13 +1,27 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. -from odoo import fields, models +from odoo import api, fields, models class IrLogging(models.Model): _name = 'ir.logging' _order = 'id DESC' - create_date = fields.Datetime(readonly=True) - create_uid = fields.Integer(string='Uid', readonly=True) # Integer not m2o is intentionnal + # The _log_access fields are defined manually for the following reasons: + # + # - The entries in ir_logging are filled in with sql queries bypassing the orm. As the --log-db + # cli option allows to insert ir_logging entries into a remote database, the one2many *_uid + # fields make no sense in the first place but we will keep it for backward compatibility. + # + # - Also, when an ir_logging entry is triggered by the orm (when using --log-db) at the moment + # it is making changes to the res.users model, the ALTER TABLE will aquire an exclusive lock + # on res_users, preventing the ir_logging INSERT to be processed, hence the ongoing module + # install/update will hang forever as the orm is blocked by the ir_logging query that will + # never occur. + create_uid = fields.Integer(string='Created by', readonly=True) + create_date = fields.Datetime(string='Created on', readonly=True) + write_uid = fields.Integer(string='Last Updated by', readonly=True) + write_date = fields.Datetime(string='Last Updated on', readonly=True) + name = fields.Char(required=True) type = fields.Selection([('client', 'Client'), ('server', 'Server')], required=True, index=True) dbname = fields.Char(string='Database Name', index=True) @@ -16,3 +30,8 @@ class IrLogging(models.Model): path = fields.Char(required=True) func = fields.Char(string='Function', required=True) line = fields.Char(required=True) + + @api.model_cr + def init(self): + super(IrLogging, self).init() + self._cr.execute("ALTER TABLE ir_logging DROP CONSTRAINT IF EXISTS ir_logging_write_uid_fkey")