From c8d596e9849aa19edc4cf89c6ded798efc241213 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Wed, 15 Jan 2014 18:12:05 +0100 Subject: [PATCH 01/51] [IMP] release.py: add "series" variable bzr revid: chs@openerp.com-20140115171205-458hinraek04wfr2 --- openerp/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openerp/release.py b/openerp/release.py index 1ad35340496..1a9c863660c 100644 --- a/openerp/release.py +++ b/openerp/release.py @@ -32,7 +32,7 @@ RELEASE_LEVELS_DISPLAY = {ALPHA: ALPHA, # (6,1,0,'candidate',2) < (6,1,0,'final',0) < (6,1,2,'final',0) version_info = (8, 0, 0, ALPHA, 1) version = '.'.join(map(str, version_info[:2])) + RELEASE_LEVELS_DISPLAY[version_info[3]] + str(version_info[4] or '') -serie = major_version = '.'.join(map(str, version_info[:2])) +series = serie = major_version = '.'.join(map(str, version_info[:2])) description = 'OpenERP Server' long_desc = '''OpenERP is a complete ERP and CRM. The main features are accounting (analytic @@ -50,6 +50,6 @@ author = 'OpenERP S.A.' author_email = 'info@openerp.com' license = 'AGPL-3' -nt_service_name = "openerp-server-" + serie +nt_service_name = "openerp-server-" + series # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: From 5d87753ef2094faea8796e4f5ce199eae52b1682 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Wed, 15 Jan 2014 18:13:23 +0100 Subject: [PATCH 02/51] [ADD] new cli argument: -D/--data-dir bzr revid: chs@openerp.com-20140115171323-eiy2h61qi098n0q3 --- openerp/tools/config.py | 23 +- openerp/vendors/__init__.py | 0 openerp/vendors/appdirs.py | 477 ++++++++++++++++++++++++++++++++++++ 3 files changed, 496 insertions(+), 4 deletions(-) create mode 100644 openerp/vendors/__init__.py create mode 100644 openerp/vendors/appdirs.py diff --git a/openerp/tools/config.py b/openerp/tools/config.py index ebfd8cea20d..101724f4159 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -3,7 +3,7 @@ # # OpenERP, Open Source Management Solution # Copyright (C) 2004-2009 Tiny SPRL (). -# Copyright (C) 2010-2012 OpenERP s.a. (). +# Copyright (C) 2010-2014 OpenERP s.a. (). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -29,6 +29,7 @@ import openerp.conf import openerp.loglevels as loglevels import logging import openerp.release as release +from openerp.vendors import appdirs class MyOption (optparse.Option, object): """ optparse Option with two additional attributes. @@ -59,6 +60,9 @@ def check_ssl(): DEFAULT_LOG_HANDLER = [':INFO'] +def _get_default_datadir(): + return appdirs.user_data_dir(appname='OpenERP', appauthor=release.author) + class configmanager(object): def __init__(self, fname=None): # Options not exposed on the command line. Command line options will be added @@ -106,6 +110,9 @@ class configmanager(object): help="specify additional addons paths (separated by commas).", action="callback", callback=self._check_addons_path, nargs=1, type="string") group.add_option("--load", dest="server_wide_modules", help="Comma-separated list of server-wide modules default=web") + + group.add_option("-D", "--data-dir", dest="data_dir", my_default=_get_default_datadir(), + help="Directory where to store OpenERP data") parser.add_option_group(group) # XML-RPC / HTTP @@ -348,6 +355,7 @@ class configmanager(object): # (../etc from the server) # if the server is run by an unprivileged user, he has to specify location of a config file where he has the rights to write, # else he won't be able to save the configurations, or even to start the server... + # TODO use appdirs if os.name == 'nt': rcfilepath = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'openerp-server.conf') else: @@ -358,7 +366,6 @@ class configmanager(object): or os.environ.get('OPENERP_SERVER') or rcfilepath) self.load() - # Verify that we want to log or not, if not the output will go to stdout if self.options['logfile'] in ('None', 'False'): self.options['logfile'] = False @@ -387,7 +394,6 @@ class configmanager(object): elif isinstance(self.options[arg], basestring) and self.casts[arg].type in optparse.Option.TYPE_CHECKER: self.options[arg] = optparse.Option.TYPE_CHECKER[self.casts[arg].type](self.casts[arg], arg, self.options[arg]) - if isinstance(self.options['log_handler'], basestring): self.options['log_handler'] = self.options['log_handler'].split(',') @@ -399,7 +405,8 @@ class configmanager(object): 'list_db', 'xmlrpcs', 'proxy_mode', 'test_file', 'test_enable', 'test_commit', 'test_report_directory', 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', 'unaccent', - 'workers', 'limit_memory_hard', 'limit_memory_soft', 'limit_time_cpu', 'limit_time_real', 'limit_request', 'auto_reload' + 'workers', 'limit_memory_hard', 'limit_memory_soft', 'limit_time_cpu', 'limit_time_real', 'limit_request', + 'auto_reload', 'data_dir', ] for arg in keys: @@ -617,6 +624,14 @@ class configmanager(object): def __getitem__(self, key): return self.options[key] + @property + def addons_data_dir(self): + return os.path.join(self['data_dir'], 'addons', release.series) + + @property + def session_dir(self): + return os.path.join(self['data_dir'], 'sessions') + config = configmanager() diff --git a/openerp/vendors/__init__.py b/openerp/vendors/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/openerp/vendors/appdirs.py b/openerp/vendors/appdirs.py new file mode 100644 index 00000000000..1d24dd8586a --- /dev/null +++ b/openerp/vendors/appdirs.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2005-2010 ActiveState Software Inc. +# Copyright (c) 2013 Eddy Petrișor + +"""Utilities for determining application-specific dirs. + +See for details and usage. +""" +# Dev Notes: +# - MSDN on where to store app data files: +# http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120 +# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html +# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html + +__version_info__ = (1, 3, 0) +__version__ = '.'.join(map(str, __version_info__)) + + +import sys +import os + +PY3 = sys.version_info[0] == 3 + +if PY3: + unicode = str + + + +def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + + for a discussion of issues. + + Typical user data directories are: + Mac OS X: ~/Library/Application Support/ + Unix: ~/.local/share/ # or in $XDG_DATA_HOME, if defined + Win XP (not roaming): C:\Documents and Settings\\Application Data\\ + Win XP (roaming): C:\Documents and Settings\\Local Settings\Application Data\\ + Win 7 (not roaming): C:\Users\\AppData\Local\\ + Win 7 (roaming): C:\Users\\AppData\Roaming\\ + + For Unix, we follow the XDG spec and support $XDG_DATA_HOME. + That means, by deafult "~/.local/share/". + """ + if sys.platform == "win32": + if appauthor is None: + appauthor = appname + const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" + path = os.path.normpath(_get_win_folder(const)) + if appname: + path = os.path.join(path, appauthor, appname) + elif sys.platform == 'darwin': + path = os.path.expanduser('~/Library/Application Support/') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of data dirs should be + returned. By default, the first item from XDG_DATA_DIRS is + returned, or '/usr/local/share/', + if XDG_DATA_DIRS is not set + + Typical user data directories are: + Mac OS X: /Library/Application Support/ + Unix: /usr/local/share/ or /usr/share/ + Win XP: C:\Documents and Settings\All Users\Application Data\\ + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + Win 7: C:\ProgramData\\ # Hidden, but writeable on Win 7. + + For Unix, this is using the $XDG_DATA_DIRS[0] default. + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if sys.platform == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) + if appname: + path = os.path.join(path, appauthor, appname) + elif sys.platform == 'darwin': + path = os.path.expanduser('/Library/Application Support') + if appname: + path = os.path.join(path, appname) + else: + # XDG default for $XDG_DATA_DIRS + # only first, if multipath is False + path = os.getenv('XDG_DATA_DIRS', + os.pathsep.join(['/usr/local/share', '/usr/share'])) + pathlist = [ os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) ] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [ os.sep.join([x, appname]) for x in pathlist ] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + if appname and version: + path = os.path.join(path, version) + return path + + +def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific config dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + + for a discussion of issues. + + Typical user data directories are: + Mac OS X: same as user_data_dir + Unix: ~/.config/ # or in $XDG_CONFIG_HOME, if defined + Win *: same as user_data_dir + + For Unix, we follow the XDG spec and support $XDG_DATA_HOME. + That means, by deafult "~/.local/share/". + """ + if sys.platform in [ "win32", "darwin" ]: + path = user_data_dir(appname, appauthor, None, roaming) + else: + path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): + """Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of config dirs should be + returned. By default, the first item from XDG_CONFIG_DIRS is + returned, or '/etc/xdg/', if XDG_CONFIG_DIRS is not set + + Typical user data directories are: + Mac OS X: same as site_data_dir + Unix: /etc/xdg/ or $XDG_CONFIG_DIRS[i]/ for each value in + $XDG_CONFIG_DIRS + Win *: same as site_data_dir + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + + For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if sys.platform in [ "win32", "darwin" ]: + path = site_data_dir(appname, appauthor) + if appname and version: + path = os.path.join(path, version) + else: + # XDG default for $XDG_CONFIG_DIRS + # only first, if multipath is False + path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') + pathlist = [ os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) ] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [ os.sep.join([x, appname]) for x in pathlist ] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + +def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific cache dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Cache" to the base app data dir for Windows. See + discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Caches/ + Unix: ~/.cache/ (XDG default) + Win XP: C:\Documents and Settings\\Local Settings\Application Data\\\Cache + Vista: C:\Users\\AppData\Local\\\Cache + + On Windows the only suggestion in the MSDN docs is that local settings go in + the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming + app data dir (the default returned by `user_data_dir` above). Apps typically + put cache data somewhere *under* the given dir here. Some examples: + ...\Mozilla\Firefox\Profiles\\Cache + ...\Acme\SuperApp\Cache\1.0 + OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. + This can be disabled with the `opinion=False` option. + """ + if sys.platform == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) + if appname: + path = os.path.join(path, appauthor, appname) + if opinion: + path = os.path.join(path, "Cache") + elif sys.platform == 'darwin': + path = os.path.expanduser('~/Library/Caches') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + +def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific log dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only required and used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Logs" to the base app data dir for Windows, and "log" to the + base cache dir for Unix. See discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Logs/ + Unix: ~/.cache//log # or under $XDG_CACHE_HOME if defined + Win XP: C:\Documents and Settings\\Local Settings\Application Data\\\Logs + Vista: C:\Users\\AppData\Local\\\Logs + + On Windows the only suggestion in the MSDN docs is that local settings + go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in + examples of what some windows apps use for a logs dir.) + + OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA` + value for Windows and appends "log" to the user cache dir for Unix. + This can be disabled with the `opinion=False` option. + """ + if sys.platform == "darwin": + path = os.path.join( + os.path.expanduser('~/Library/Logs'), + appname) + elif sys.platform == "win32": + path = user_data_dir(appname, appauthor, version); version=False + if opinion: + path = os.path.join(path, "Logs") + else: + path = user_cache_dir(appname, appauthor, version); version=False + if opinion: + path = os.path.join(path, "log") + if appname and version: + path = os.path.join(path, version) + return path + + +class AppDirs(object): + """Convenience wrapper for getting application dirs.""" + def __init__(self, appname, appauthor=None, version=None, + roaming=False, multipath=False): + self.appname = appname + self.appauthor = appauthor + self.version = version + self.roaming = roaming + self.multipath = multipath + @property + def user_data_dir(self): + return user_data_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + @property + def site_data_dir(self): + return site_data_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + @property + def user_config_dir(self): + return user_config_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + @property + def site_config_dir(self): + return site_data_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + @property + def user_cache_dir(self): + return user_cache_dir(self.appname, self.appauthor, + version=self.version) + @property + def user_log_dir(self): + return user_log_dir(self.appname, self.appauthor, + version=self.version) + + + + +#---- internal support stuff + +def _get_win_folder_from_registry(csidl_name): + """This is a fallback technique at best. I'm not sure if using the + registry for this guarantees us the correct answer for all CSIDL_* + names. + """ + import _winreg + + shell_folder_name = { + "CSIDL_APPDATA": "AppData", + "CSIDL_COMMON_APPDATA": "Common AppData", + "CSIDL_LOCAL_APPDATA": "Local AppData", + }[csidl_name] + + key = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders") + dir, type = _winreg.QueryValueEx(key, shell_folder_name) + return dir + +def _get_win_folder_with_pywin32(csidl_name): + from win32com.shell import shellcon, shell + dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) + # Try to make this a unicode path because SHGetFolderPath does + # not return unicode strings when there is unicode data in the + # path. + try: + dir = unicode(dir) + + # Downgrade to short path name if have highbit chars. See + # . + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + try: + import win32api + dir = win32api.GetShortPathName(dir) + except ImportError: + pass + except UnicodeError: + pass + return dir + +def _get_win_folder_with_ctypes(csidl_name): + import ctypes + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + }[csidl_name] + + buf = ctypes.create_unicode_buffer(1024) + ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if have highbit chars. See + # . + has_high_char = False + for c in buf: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf2 = ctypes.create_unicode_buffer(1024) + if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + return buf.value + +if sys.platform == "win32": + try: + import win32com.shell + _get_win_folder = _get_win_folder_with_pywin32 + except ImportError: + try: + import ctypes + _get_win_folder = _get_win_folder_with_ctypes + except ImportError: + _get_win_folder = _get_win_folder_from_registry + + + +#---- self test code + +if __name__ == "__main__": + appname = "MyApp" + appauthor = "MyCompany" + + props = ("user_data_dir", "site_data_dir", + "user_config_dir", "site_config_dir", + "user_cache_dir", "user_log_dir") + + print("-- app dirs (with optional 'version')") + dirs = AppDirs(appname, appauthor, version="1.0") + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'version')") + dirs = AppDirs(appname, appauthor) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'appauthor')") + dirs = AppDirs(appname) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + From 5269664102a0b8a607d7f29f6102ebfce023922b Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Wed, 15 Jan 2014 19:03:13 +0100 Subject: [PATCH 03/51] [FIX] correct uses of addons_path bzr revid: chs@openerp.com-20140115180313-pqcrfmstci2w21y8 --- .../base/module/wizard/base_module_import.py | 2 - openerp/cli/server.py | 2 +- openerp/modules/module.py | 14 ++++--- openerp/service/server.py | 5 +-- openerp/tests/common.py | 1 - openerp/tools/translate.py | 37 +++++-------------- 6 files changed, 22 insertions(+), 39 deletions(-) diff --git a/openerp/addons/base/module/wizard/base_module_import.py b/openerp/addons/base/module/wizard/base_module_import.py index 6d6f0e1ef8d..2ba789f3332 100644 --- a/openerp/addons/base/module/wizard/base_module_import.py +++ b/openerp/addons/base/module/wizard/base_module_import.py @@ -28,8 +28,6 @@ from openerp import tools from openerp.osv import osv, fields from openerp.tools.translate import _ -ADDONS_PATH = tools.config['addons_path'].split(",")[-1] - class base_module_import(osv.osv_memory): """ Import Module """ diff --git a/openerp/cli/server.py b/openerp/cli/server.py index ee2e90bccbd..7041d7a4c3c 100644 --- a/openerp/cli/server.py +++ b/openerp/cli/server.py @@ -72,7 +72,7 @@ def report_configuration(): """ config = openerp.tools.config _logger.info("OpenERP version %s", __version__) - for name, value in [('addons paths', config['addons_path']), + for name, value in [('addons paths', openerp.modules.module.ad_paths), ('database hostname', config['db_host'] or 'localhost'), ('database port', config['db_port'] or '5432'), ('database user', config['db_user'])]: diff --git a/openerp/modules/module.py b/openerp/modules/module.py index cf3ff61805e..98511234b75 100644 --- a/openerp/modules/module.py +++ b/openerp/modules/module.py @@ -3,7 +3,7 @@ # # OpenERP, Open Source Management Solution # Copyright (C) 2004-2009 Tiny SPRL (). -# Copyright (C) 2010-2012 OpenERP s.a. (). +# Copyright (C) 2010-2014 OpenERP s.a. (). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -45,7 +45,6 @@ import logging _logger = logging.getLogger(__name__) _test_logger = logging.getLogger('openerp.tests') -_ad = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'addons') # default addons path (base) ad_paths = [] # Modules already loaded @@ -97,8 +96,13 @@ def initialize_sys_path(): if ad_paths: return - ad_paths = map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(',')) - ad_paths.append(os.path.abspath(_ad)) # for get_module_path + ad_paths = [tools.config.addons_data_dir] + ad_paths += map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(',')) + + # add base module path + base_path = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'addons')) + ad_paths += [base_path] + sys.meta_path.append(AddonsImportHook()) def get_module_path(module, downloaded=False, display_warning=True): @@ -115,7 +119,7 @@ def get_module_path(module, downloaded=False, display_warning=True): return opj(adp, module) if downloaded: - return opj(_ad, module) + return opj(tools.config.addons_data_dir, module) if display_warning: _logger.warning('module %s: module not found', module) return False diff --git a/openerp/service/server.py b/openerp/service/server.py index 87f10010645..e7fcb610152 100644 --- a/openerp/service/server.py +++ b/openerp/service/server.py @@ -108,15 +108,14 @@ class AutoReload(object): self.handler = EventHandler(self) self.notifier = pyinotify.Notifier(self.wm, self.handler, timeout=0) mask = pyinotify.IN_MODIFY | pyinotify.IN_CREATE # IN_MOVED_FROM, IN_MOVED_TO ? - for path in openerp.tools.config.options["addons_path"].split(','): + for path in openerp.modules.modules.ad_paths: _logger.info('Watching addons folder %s', path) self.wm.add_watch(path, mask, rec=True) def process_data(self, files): xml_files = [i for i in files if i.endswith('.xml')] - addons_path = openerp.tools.config.options["addons_path"].split(',') for i in xml_files: - for path in addons_path: + for path in openerp.modules.modules.ad_paths: if i.startswith(path): # find out wich addons path the file belongs to # and extract it's module name diff --git a/openerp/tests/common.py b/openerp/tests/common.py index df00e61f140..7bd8bf76cc9 100644 --- a/openerp/tests/common.py +++ b/openerp/tests/common.py @@ -11,7 +11,6 @@ import xmlrpclib import openerp # The openerp library is supposed already configured. -ADDONS_PATH = openerp.tools.config['addons_path'] PORT = openerp.tools.config['xmlrpc_port'] DB = openerp.tools.config['db_name'] diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index f82fa4b87ad..72cf4ec1d0b 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -778,49 +778,32 @@ def trans_generate(lang, modules, cr): if model_obj._sql_constraints: push_local_constraints(module, model_obj, 'sql_constraints') - def get_module_from_path(path, mod_paths=None): - if not mod_paths: - # First, construct a list of possible paths - def_path = os.path.abspath(os.path.join(config.config['root_path'], 'addons')) # default addons path (base) - ad_paths= map(lambda m: os.path.abspath(m.strip()),config.config['addons_path'].split(',')) - mod_paths=[def_path] - for adp in ad_paths: - mod_paths.append(adp) - if not os.path.isabs(adp): - mod_paths.append(adp) - elif adp.startswith(def_path): - mod_paths.append(adp[len(def_path)+1:]) - for mp in mod_paths: - if path.startswith(mp) and (os.path.dirname(path) != mp): - path = path[len(mp)+1:] - return path.split(os.path.sep)[0] - return 'base' # files that are not in a module are considered as being in 'base' module modobj = registry['ir.module.module'] installed_modids = modobj.search(cr, uid, [('state', '=', 'installed')]) installed_modules = map(lambda m: m['name'], modobj.read(cr, uid, installed_modids, ['name'])) - root_path = os.path.join(config.config['root_path'], 'addons') - - apaths = map(os.path.abspath, map(str.strip, config.config['addons_path'].split(','))) - if root_path in apaths: - path_list = apaths - else : - path_list = [root_path,] + apaths - + path_list = list(openerp.modules.module.ad_paths) # Also scan these non-addon paths for bin_path in ['osv', 'report' ]: path_list.append(os.path.join(config.config['root_path'], bin_path)) _logger.debug("Scanning modules at paths: ", path_list) - mod_paths = [] + mod_paths = list(path_list) + + def get_module_from_path(path): + for mp in mod_paths: + if path.startswith(mp) and (os.path.dirname(path) != mp): + path = path[len(mp)+1:] + return path.split(os.path.sep)[0] + return 'base' # files that are not in a module are considered as being in 'base' module def verified_module_filepaths(fname, path, root): fabsolutepath = join(root, fname) frelativepath = fabsolutepath[len(path):] display_path = "addons%s" % frelativepath - module = get_module_from_path(fabsolutepath, mod_paths=mod_paths) + module = get_module_from_path(fabsolutepath) if ('all' in modules or module in modules) and module in installed_modules: return module, fabsolutepath, frelativepath, display_path return None, None, None, None From a73c02075afad47da4edae53659c14d71e1a303f Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 11:02:43 +0100 Subject: [PATCH 04/51] [FIX] ensure data-dir (and subdirs) exists bzr revid: chs@openerp.com-20140116100243-ltp6yunvm1bnvn5q --- openerp/tools/config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/openerp/tools/config.py b/openerp/tools/config.py index 101724f4159..bd87a0d4b44 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -626,11 +626,17 @@ class configmanager(object): @property def addons_data_dir(self): - return os.path.join(self['data_dir'], 'addons', release.series) + d = os.path.join(self['data_dir'], 'addons', release.series) + if not os.path.exists(d): + os.makedirs(d) + return d @property def session_dir(self): - return os.path.join(self['data_dir'], 'sessions') + d = os.path.join(self['data_dir'], 'sessions') + if not os.path.exists(d): + os.makedirs(d) + return d config = configmanager() From db10e1d3a51cd0e315ee8e4fcc80ffe9b6499fbf Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 17:41:42 +0100 Subject: [PATCH 05/51] [FIX] http: store sessions in data-dir bzr revid: chs@openerp.com-20140116164142-ikcw82lyfopoj46k --- openerp/http.py | 23 +---------------------- openerp/tools/config.py | 2 +- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/openerp/http.py b/openerp/http.py index 7069cbdeefe..9b632348280 100644 --- a/openerp/http.py +++ b/openerp/http.py @@ -830,33 +830,12 @@ class DisableCacheMiddleware(object): start_response(status, new_headers) return self.app(environ, start_wrapped) -def session_path(): - try: - import pwd - username = pwd.getpwuid(os.geteuid()).pw_name - except ImportError: - try: - username = getpass.getuser() - except Exception: - username = "unknown" - path = os.path.join(tempfile.gettempdir(), "oe-sessions-" + username) - try: - os.mkdir(path, 0700) - except OSError as exc: - if exc.errno == errno.EEXIST: - # directory exists: ensure it has the correct permissions - # this will fail if the directory is not owned by the current user - os.chmod(path, 0700) - else: - raise - return path - class Root(object): """Root WSGI application for the OpenERP Web Client. """ def __init__(self): # Setup http sessions - path = session_path() + path = os.path.join(openerp.tools.config.session_dir _logger.debug('HTTP sessions stored in: %s', path) self.session_store = werkzeug.contrib.sessions.FilesystemSessionStore(path, session_class=OpenERPSession) diff --git a/openerp/tools/config.py b/openerp/tools/config.py index bd87a0d4b44..a9e4a8a3276 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -633,7 +633,7 @@ class configmanager(object): @property def session_dir(self): - d = os.path.join(self['data_dir'], 'sessions') + d = os.path.join(self['data_dir'], 'sessions', release.series) if not os.path.exists(d): os.makedirs(d) return d From c48f11041bf46ccbc880aba6c3c90f79e932c68f Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 17:44:06 +0100 Subject: [PATCH 06/51] [IMP] chmod data-dir bzr revid: chs@openerp.com-20140116164406-p2gtb2uziwhchp68 --- openerp/tools/config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openerp/tools/config.py b/openerp/tools/config.py index a9e4a8a3276..90013cfedd0 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -628,14 +628,18 @@ class configmanager(object): def addons_data_dir(self): d = os.path.join(self['data_dir'], 'addons', release.series) if not os.path.exists(d): - os.makedirs(d) + os.makedirs(d, 0700) + else: + os.chmod(d, 0700) return d @property def session_dir(self): d = os.path.join(self['data_dir'], 'sessions', release.series) if not os.path.exists(d): - os.makedirs(d) + os.makedirs(d, 0700) + else: + os.chmod(d, 0700) return d config = configmanager() From 3275de59815c2846d6385637c920d0724934c326 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 19:54:15 +0100 Subject: [PATCH 07/51] [IMP] ir.attachment: filestore is now in data-dir and by default bzr revid: chs@openerp.com-20140116185415-ajia02bsty9joox7 --- openerp/addons/base/ir/ir_attachment.py | 65 ++++++----- .../addons/base/tests/test_ir_attachment.py | 108 +++++++++--------- openerp/service/db.py | 7 +- 3 files changed, 95 insertions(+), 85 deletions(-) diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index f6bb00fed5f..616b4168c60 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -63,19 +63,38 @@ class ir_attachment(osv.osv): data[attachment.id] = False return data + def _storage(self, cr, uid, context=None): + return self.pool['ir.config_parameter'].get_param(cr, SUPERUSER_ID, 'ir_attachment.location', 'file') + + @tools.ormcache() + def _filestore(self, cr, uid, context=None): + dbuuid = self.pool['ir.config_paramater'].get_param(cr, SUPERUSER_ID, 'database.uuid') + return os.path.join(tools.config['data_dir'], 'filestore', dbuuid) + # 'data' field implementation def _full_path(self, cr, uid, location, path): - # location = 'file:filestore' - assert location.startswith('file:'), "Unhandled filestore location %s" % location - location = location[5:] - - # sanitize location name and path - location = re.sub('[.]','',location) - location = location.strip('/\\') - - path = re.sub('[.]','',path) + # sanitize ath + path = re.sub('[.]', '', path) path = path.strip('/\\') - return os.path.join(tools.config['root_path'], location, cr.dbname, path) + return os.path.join(self._filestore(cr, uid), path) + + def _get_path(self, cr, uid, location, bin_data): + sha = hashlib.sha1(bin_data).hexdigest() + + # retro compatibility + fname = sha[:3] + '/' + sha + full_path = self._full_path(cr, uid, location, fname) + if os.path.isfile(full_path): + return fname, full_path # keep existing path + + # scatter files across 256 dirs + # we use '/' in the db (even on windows) + fname = sha[:2] + '/' + sha + full_path = self._full_path(cr, uid, location, fname) + dirname = os.path.dirname(full_path) + if not os.path.isdir(dirname): + os.makedirs(dirname) + return fname, full_path def _file_read(self, cr, uid, location, fname, bin_size=False): full_path = self._full_path(cr, uid, location, fname) @@ -91,18 +110,13 @@ class ir_attachment(osv.osv): def _file_write(self, cr, uid, location, value): bin_value = value.decode('base64') - fname = hashlib.sha1(bin_value).hexdigest() - # scatter files across 1024 dirs - # we use '/' in the db (even on windows) - fname = fname[:3] + '/' + fname - full_path = self._full_path(cr, uid, location, fname) - try: - dirname = os.path.dirname(full_path) - if not os.path.isdir(dirname): - os.makedirs(dirname) - open(full_path,'wb').write(bin_value) - except IOError: - _logger.error("_file_write writing %s",full_path) + fname, full_path = self._get_path(cr, uid, location, bin_value) + if not os.path.exists(full_path): + try: + with open(full_path, 'wb') as fp: + fp.write(bin_value) + except IOError: + _logger.error("_file_write writing %s", full_path) return fname def _file_delete(self, cr, uid, location, fname): @@ -121,7 +135,7 @@ class ir_attachment(osv.osv): if context is None: context = {} result = {} - location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location') + location = self._storage(cr, uid, context) bin_size = context.get('bin_size') for attach in self.browse(cr, uid, ids, context=context): if location and attach.store_fname: @@ -136,7 +150,7 @@ class ir_attachment(osv.osv): return True if context is None: context = {} - location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location') + location = self._storage(cr, uid, context) file_size = len(value.decode('base64')) if location: attach = self.browse(cr, uid, id, context=context) @@ -284,7 +298,7 @@ class ir_attachment(osv.osv): if isinstance(ids, (int, long)): ids = [ids] self.check(cr, uid, ids, 'unlink', context=context) - location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location') + location = self._storage(cr, uid, context) if location: for attach in self.browse(cr, uid, ids, context=context): if attach.store_fname: @@ -302,4 +316,3 @@ class ir_attachment(osv.osv): cr, uid, 'base', 'action_attachment', context=context) # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: - diff --git a/openerp/addons/base/tests/test_ir_attachment.py b/openerp/addons/base/tests/test_ir_attachment.py index 4d96ec42b28..fd2b4304eae 100644 --- a/openerp/addons/base/tests/test_ir_attachment.py +++ b/openerp/addons/base/tests/test_ir_attachment.py @@ -1,89 +1,91 @@ import hashlib import os -import unittest2 - import openerp import openerp.tests.common -class test_ir_attachment(openerp.tests.common.TransactionCase): +HASH_SPLIT = 2 # FIXME: testing implementations detail is not a good idea - def test_00_attachment_flow(self): +class test_ir_attachment(openerp.tests.common.TransactionCase): + def setUp(self): registry, cr, uid = self.registry, self.cr, self.uid - root_path = openerp.tools.config['root_path'] - ira = registry('ir.attachment') + self.ira = registry('ir.attachment') + self.filestore = self.ira._filestore(cr, uid) # Blob1 - blob1 = 'blob1' - blob1_b64 = blob1.encode('base64') - blob1_hash = hashlib.sha1(blob1).hexdigest() - blob1_fname = blob1_hash[:3] + '/' + blob1_hash + self.blob1 = 'blob1' + self.blob1_b64 = self.blob1.encode('base64') + blob1_hash = hashlib.sha1(self.blob1).hexdigest() + self.blob1_fname = blob1_hash[:HASH_SPLIT] + '/' + blob1_hash # Blob2 blob2 = 'blob2' - blob2_b64 = blob2.encode('base64') - blob2_hash = hashlib.sha1(blob2).hexdigest() - blob2_fname = blob2_hash[:3] + '/' + blob2_hash + self.blob2_b64 = blob2.encode('base64') + + def test_01_store_in_db(self): + registry, cr, uid = self.registry, self.cr, self.uid + + # force storing in database + registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', '') # 'ir_attachment.location' is undefined test database storage - a1 = ira.create(cr, uid, {'name': 'a1', 'datas': blob1_b64}) - a1_read = ira.read(cr, uid, [a1], ['datas']) - self.assertEqual(a1_read[0]['datas'], blob1_b64) + a1 = self.ira.create(cr, uid, {'name': 'a1', 'datas': self.blob1_b64}) + a1_read = self.ira.read(cr, uid, [a1], ['datas']) + self.assertEqual(a1_read[0]['datas'], self.blob1_b64) - cr.execute("select id,db_datas from ir_attachment where id = %s", (a1,) ) - a1_db_datas = str(cr.fetchall()[0][1]) - self.assertEqual(a1_db_datas, blob1_b64) + a1_db_datas = self.ira.browse(cr, uid, a1).db_datas + self.assertEqual(a1_db_datas, self.blob1_b64) - # define a location for filestore - registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', 'file:///filestore') + def test_02_store_on_disk(self): + registry, cr, uid = self.registry, self.cr, self.uid - # Test file storage - a2 = ira.create(cr, uid, {'name': 'a2', 'datas': blob1_b64}) - a2_read = ira.read(cr, uid, [a2], ['datas']) - self.assertEqual(a2_read[0]['datas'], blob1_b64) + a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64}) + a2_store_fname = self.ira.browse(cr, uid, a2).store_fname - cr.execute("select id,store_fname from ir_attachment where id = %s", (a2,) ) - a2_store_fname = cr.fetchall()[0][1] - self.assertEqual(a2_store_fname, blob1_fname) + self.assertEqual(a2_store_fname, self.blob1_fname) + self.assertTrue(os.path.isdir(os.path.join(self.filestore, a2_store_fname))) - a2_fn = os.path.join(root_path, 'filestore', cr.dbname, blob1_hash[:3], blob1_hash) - fc = file(a2_fn).read() - self.assertEqual(fc, blob1) + def test_03_no_duplication(self): + registry, cr, uid = self.registry, self.cr, self.uid - # create a3 with same blob - a3 = ira.create(cr, uid, {'name': 'a3', 'datas': blob1_b64}) - a3_read = ira.read(cr, uid, [a3], ['datas']) - self.assertEqual(a3_read[0]['datas'], blob1_b64) + a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64}) + a2_store_fname = self.ira.browse(cr, uid, a2).store_fname + + a3 = self.ira.create(cr, uid, {'name': 'a3', 'datas': self.blob1_b64}) + a3_store_fname = self.ira.browse(cr, uid, a3).store_fname - cr.execute("select id,store_fname from ir_attachment where id = %s", (a3,) ) - a3_store_fname = cr.fetchall()[0][1] self.assertEqual(a3_store_fname, a2_store_fname) - # create a4 blob2 - a4 = ira.create(cr, uid, {'name': 'a4', 'datas': blob2_b64}) - a4_read = ira.read(cr, uid, [a4], ['datas']) - self.assertEqual(a4_read[0]['datas'], blob2_b64) + def test_04_keep_file(self): + registry, cr, uid = self.registry, self.cr, self.uid - a4_fn = os.path.join(root_path, 'filestore', cr.dbname, blob2_hash[:3], blob2_hash) - self.assertTrue(os.path.isfile(a4_fn)) + a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64}) + a3 = self.ira.create(cr, uid, {'name': 'a3', 'datas': self.blob1_b64}) - # delete a3 but file stays - ira.unlink(cr, uid, [a3]) + a2_store_fname = self.ira.browse(cr, uid, a2).store_fname + a2_fn = os.path.join(self.filestore, a2_store_fname) + + self.ira.unlink(cr, uid, [a3]) self.assertTrue(os.path.isfile(a2_fn)) # delete a2 it is unlinked - ira.unlink(cr, uid, [a2]) + self.ira.unlink(cr, uid, [a2]) self.assertFalse(os.path.isfile(a2_fn)) - # update a4 blob2 by blob1 - ira.write(cr, uid, [a4], {'datas': blob1_b64}) - a4_read = ira.read(cr, uid, [a4], ['datas']) - self.assertEqual(a4_read[0]['datas'], blob1_b64) + def test_05_change_data_change_file(self): + registry, cr, uid = self.registry, self.cr, self.uid + + a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64}) + a2_store_fname = self.ira.browse(cr, uid, a2).store_fname + a2_fn = os.path.join(self.filestore, a2_store_fname) - # file of a4 disapear and a2 reappear - self.assertFalse(os.path.isfile(a4_fn)) self.assertTrue(os.path.isfile(a2_fn)) - # everybody applause + self.ira.write(cr, uid, [a2], {'datas': self.blob2_b64}) + self.assertFalse(os.path.isfile(a2_fn)) + new_a2_store_fname = self.ira.browse(cr, uid, a2).store_fname + self.assertNotEqual(a2_store_fname, new_a2_store_fname) + new_a2_fn = os.path.join(self.filestore, new_a2_store_fname) + self.assertTrue(os.path.isfile(new_a2_fn)) diff --git a/openerp/service/db.py b/openerp/service/db.py index 1381b5b1e04..acdb8fe2ed9 100644 --- a/openerp/service/db.py +++ b/openerp/service/db.py @@ -276,15 +276,10 @@ def exp_rename(old_name, new_name): cr.autocommit(True) # avoid transaction block try: cr.execute('ALTER DATABASE "%s" RENAME TO "%s"' % (old_name, new_name)) + _logger.info('RENAME DB: %s -> %s', old_name, new_name) except Exception, e: _logger.error('RENAME DB: %s -> %s failed:\n%s', old_name, new_name, e) raise Exception("Couldn't rename database %s to %s: %s" % (old_name, new_name, e)) - else: - fs = os.path.join(openerp.tools.config['root_path'], 'filestore') - if os.path.exists(os.path.join(fs, old_name)): - os.rename(os.path.join(fs, old_name), os.path.join(fs, new_name)) - - _logger.info('RENAME DB: %s -> %s', old_name, new_name) return True def exp_db_exist(db_name): From 080cbb88a32ba603e33a6ef428d65d32cb4305e5 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 21:26:01 +0100 Subject: [PATCH 08/51] [FIX] works better without SyntaxError bzr revid: chs@openerp.com-20140116202601-ckn20h6yhiufzonq --- openerp/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/http.py b/openerp/http.py index 9b632348280..2bd0f48565f 100644 --- a/openerp/http.py +++ b/openerp/http.py @@ -835,7 +835,7 @@ class Root(object): """ def __init__(self): # Setup http sessions - path = os.path.join(openerp.tools.config.session_dir + path = openerp.tools.config.session_dir _logger.debug('HTTP sessions stored in: %s', path) self.session_store = werkzeug.contrib.sessions.FilesystemSessionStore(path, session_class=OpenERPSession) From 45d24fd92b66a06e36598e682310435b322d8d01 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 22:47:22 +0100 Subject: [PATCH 09/51] [FIX] ir.attachement: typo s/config_paramater/config_parameter/ bzr revid: chs@openerp.com-20140116214722-4804mskx7c21ikk5 --- openerp/addons/base/ir/ir_attachment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index 616b4168c60..85770669aac 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -68,7 +68,7 @@ class ir_attachment(osv.osv): @tools.ormcache() def _filestore(self, cr, uid, context=None): - dbuuid = self.pool['ir.config_paramater'].get_param(cr, SUPERUSER_ID, 'database.uuid') + dbuuid = self.pool['ir.config_parameter'].get_param(cr, SUPERUSER_ID, 'database.uuid') return os.path.join(tools.config['data_dir'], 'filestore', dbuuid) # 'data' field implementation From 895b3c96f68d74d99d5fa311670a6e75554f5865 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 23:14:03 +0100 Subject: [PATCH 10/51] [FIX] test ir_attachment: call super() bzr revid: chs@openerp.com-20140116221403-h6h9cvt9rn8y2yj1 --- openerp/addons/base/tests/test_ir_attachment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openerp/addons/base/tests/test_ir_attachment.py b/openerp/addons/base/tests/test_ir_attachment.py index fd2b4304eae..9c02ce6265f 100644 --- a/openerp/addons/base/tests/test_ir_attachment.py +++ b/openerp/addons/base/tests/test_ir_attachment.py @@ -8,6 +8,7 @@ HASH_SPLIT = 2 # FIXME: testing implementations detail is not a good idea class test_ir_attachment(openerp.tests.common.TransactionCase): def setUp(self): + super(test_ir_attachment, self).setUp() registry, cr, uid = self.registry, self.cr, self.uid self.ira = registry('ir.attachment') self.filestore = self.ira._filestore(cr, uid) From c55079f1121b215194d6c0d827d068da7b0ea335 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 23:40:58 +0100 Subject: [PATCH 11/51] [IMP] ir.attachment: active db storage by setting setting to "db" bzr revid: chs@openerp.com-20140116224058-v53fozipvi0obmq1 --- openerp/addons/base/ir/ir_attachment.py | 6 +++--- openerp/addons/base/tests/test_ir_attachment.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index 85770669aac..ca61fb46fc7 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -138,7 +138,7 @@ class ir_attachment(osv.osv): location = self._storage(cr, uid, context) bin_size = context.get('bin_size') for attach in self.browse(cr, uid, ids, context=context): - if location and attach.store_fname: + if location != 'db' and attach.store_fname: result[attach.id] = self._file_read(cr, uid, location, attach.store_fname, bin_size) else: result[attach.id] = attach.db_datas @@ -152,7 +152,7 @@ class ir_attachment(osv.osv): context = {} location = self._storage(cr, uid, context) file_size = len(value.decode('base64')) - if location: + if location != 'db': attach = self.browse(cr, uid, id, context=context) if attach.store_fname: self._file_delete(cr, uid, location, attach.store_fname) @@ -299,7 +299,7 @@ class ir_attachment(osv.osv): ids = [ids] self.check(cr, uid, ids, 'unlink', context=context) location = self._storage(cr, uid, context) - if location: + if location != 'db': for attach in self.browse(cr, uid, ids, context=context): if attach.store_fname: self._file_delete(cr, uid, location, attach.store_fname) diff --git a/openerp/addons/base/tests/test_ir_attachment.py b/openerp/addons/base/tests/test_ir_attachment.py index 9c02ce6265f..a5e546a90c4 100644 --- a/openerp/addons/base/tests/test_ir_attachment.py +++ b/openerp/addons/base/tests/test_ir_attachment.py @@ -27,7 +27,7 @@ class test_ir_attachment(openerp.tests.common.TransactionCase): registry, cr, uid = self.registry, self.cr, self.uid # force storing in database - registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', '') + registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', 'db') # 'ir_attachment.location' is undefined test database storage a1 = self.ira.create(cr, uid, {'name': 'a1', 'datas': self.blob1_b64}) From 143f52039500a420a42ee53d1ef5a411930c9d00 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Thu, 16 Jan 2014 23:42:07 +0100 Subject: [PATCH 12/51] [FIX] test ir_attachment: files are not directories bzr revid: chs@openerp.com-20140116224207-m2s5ljkb1t7q54tk --- openerp/addons/base/tests/test_ir_attachment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/addons/base/tests/test_ir_attachment.py b/openerp/addons/base/tests/test_ir_attachment.py index a5e546a90c4..891b38b7dc3 100644 --- a/openerp/addons/base/tests/test_ir_attachment.py +++ b/openerp/addons/base/tests/test_ir_attachment.py @@ -44,7 +44,7 @@ class test_ir_attachment(openerp.tests.common.TransactionCase): a2_store_fname = self.ira.browse(cr, uid, a2).store_fname self.assertEqual(a2_store_fname, self.blob1_fname) - self.assertTrue(os.path.isdir(os.path.join(self.filestore, a2_store_fname))) + self.assertTrue(os.path.isfile(os.path.join(self.filestore, a2_store_fname))) def test_03_no_duplication(self): registry, cr, uid = self.registry, self.cr, self.uid From 14815d669ff5ae428a74c7b81327667cdbcf44f9 Mon Sep 17 00:00:00 2001 From: Christophe Simonis Date: Fri, 17 Jan 2014 17:14:24 +0100 Subject: [PATCH 13/51] [FIX] attachments: filestore use dbname instead of dbuuid bzr revid: chs@openerp.com-20140117161424-i1ggvzawkjrabbwc --- openerp/addons/base/ir/ir_attachment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index 4fb5f6f16e5..d9089a25f96 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -68,8 +68,7 @@ class ir_attachment(osv.osv): @tools.ormcache() def _filestore(self, cr, uid, context=None): - dbuuid = self.pool['ir.config_parameter'].get_param(cr, SUPERUSER_ID, 'database.uuid') - return os.path.join(tools.config['data_dir'], 'filestore', dbuuid) + return os.path.join(tools.config['data_dir'], 'filestore', cr.dbname) # 'data' field implementation def _full_path(self, cr, uid, location, path): From 476a0b620dcc563420191449daabbecbcc179150 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (Acsone)" Date: Fri, 31 Jan 2014 17:32:10 +0100 Subject: [PATCH 14/51] [FIX] ir_cron: use the same search criteria as when listing job to excecyte when acquiring the lock on the job before its execution to prevent running already executed job bzr revid: laurent.mignon@acsone.eu-20140131163210-2sw8f2e5n4my5o5l --- openerp/addons/base/ir/ir_cron.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 24a522f6309..77269c61550 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -216,12 +216,21 @@ class ir_cron(osv.osv): lock_cr = db.cursor() try: # Try to grab an exclusive lock on the job row from within the task transaction + # Restrict to the same conditions as for the search since the job may have already + # been run by an other thread when cron is running in multi thread lock_cr.execute("""SELECT * FROM ir_cron - WHERE id=%s + WHERE numbercall != 0 + AND active + AND nextcall <= (now() at time zone 'UTC') + AND id=%s FOR UPDATE NOWAIT""", (job['id'],), log_exceptions=False) + locked_job = lock_cr.fetchone() + if not locked_job: + _logger.debug("Job already %s executed by another process/thread. skipping it", job['name']) + continue # Got the lock on the job row, run its code _logger.debug('Starting job `%s`.', job['name']) job_cr = db.cursor() From ef974c7fe5d221c3b1de68f2b6eee2a30b2c01d7 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (Acsone)" Date: Wed, 5 Feb 2014 10:14:52 +0100 Subject: [PATCH 15/51] Fix log message bzr revid: laurent.mignon@acsone.eu-20140205091452-6mamu14lpevyjy28 --- openerp/addons/base/ir/ir_cron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 77269c61550..77c9835d9be 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -229,7 +229,7 @@ class ir_cron(osv.osv): locked_job = lock_cr.fetchone() if not locked_job: - _logger.debug("Job already %s executed by another process/thread. skipping it", job['name']) + _logger.debug("Job `%s` already executed by another process/thread. skipping it", job['name']) continue # Got the lock on the job row, run its code _logger.debug('Starting job `%s`.', job['name']) From 763496be49813da85aecb518116f7ccc7f30eac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Thu, 6 Feb 2014 10:48:31 +0100 Subject: [PATCH 16/51] [IMP] mail: added message_last_update field. This field is a datetime field, updated everytime message_post in called. This allows to build filters, search or actions based on the last message date, for reporting for example. bzr revid: tde@openerp.com-20140206094831-nvyds6nqz0v25t59 --- addons/mail/mail_thread.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/addons/mail/mail_thread.py b/addons/mail/mail_thread.py index ee4dcac5414..c1006182000 100644 --- a/addons/mail/mail_thread.py +++ b/addons/mail/mail_thread.py @@ -296,6 +296,9 @@ class mail_thread(osv.AbstractModel): auto_join=True, string='Messages', help="Messages and communication history"), + 'message_last_update': fields.datetime('Last Message Date', + help='Date of the last message posted on the record. Only messages going' + 'through the message_post API are taken into account for better performances.'), 'message_unread': fields.function(_get_message_data, fnct_search=_search_message_unread, multi="_get_message_data", type='boolean', string='Unread Messages', @@ -1522,8 +1525,13 @@ class mail_thread(osv.AbstractModel): for x in ('from', 'to', 'cc'): values.pop(x, None) - # Create and auto subscribe the author + # Post the message msg_id = mail_message.create(cr, uid, values, context=context) + + # Post-process: subscribe author, update message_last_update + if model and model != 'mail.thread' and thread_id: + # done with SUPERUSER_ID, because on some models users can post only with read access, not necessarily write access + self.write(cr, SUPERUSER_ID, [thread_id], {'message_last_update': fields.datetime.now()}, context=context) message = mail_message.browse(cr, uid, msg_id, context=context) if message.author_id and thread_id and type != 'notification' and not context.get('mail_create_nosubscribe'): self.message_subscribe(cr, uid, [thread_id], [message.author_id.id], context=context) From 7d656e961b69939a1b68abae10741a4f156c60f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me?= Date: Thu, 13 Feb 2014 15:30:44 +0100 Subject: [PATCH 17/51] [ADD] crm_project_issue: link module to create lead from issues and issues from lead bzr revid: jem@openerp.com-20140213143044-p06ef9scr32ihef2 --- addons/crm_project_issue/__init__.py | 3 +++ addons/crm_project_issue/__openerp__.py | 21 +++++++++++++++ addons/crm_project_issue/crm_lead.py | 10 +++++++ addons/crm_project_issue/crm_lead_view.xml | 27 +++++++++++++++++++ addons/crm_project_issue/project_issue.py | 14 ++++++++++ .../crm_project_issue/project_issue_view.xml | 17 ++++++++++++ 6 files changed, 92 insertions(+) create mode 100644 addons/crm_project_issue/__init__.py create mode 100644 addons/crm_project_issue/__openerp__.py create mode 100644 addons/crm_project_issue/crm_lead.py create mode 100644 addons/crm_project_issue/crm_lead_view.xml create mode 100644 addons/crm_project_issue/project_issue.py create mode 100644 addons/crm_project_issue/project_issue_view.xml diff --git a/addons/crm_project_issue/__init__.py b/addons/crm_project_issue/__init__.py new file mode 100644 index 00000000000..40302ae183c --- /dev/null +++ b/addons/crm_project_issue/__init__.py @@ -0,0 +1,3 @@ + +import project_issue +import crm_lead \ No newline at end of file diff --git a/addons/crm_project_issue/__openerp__.py b/addons/crm_project_issue/__openerp__.py new file mode 100644 index 00000000000..89ef20a655e --- /dev/null +++ b/addons/crm_project_issue/__openerp__.py @@ -0,0 +1,21 @@ +{ + 'name' : 'CRM Project Issues', + 'version': '1.0', + 'summary': 'Module linker between Leads and Issues', + 'sequence': '19', + 'category': 'Project Management', + 'complexity': 'easy', + 'description': + """ +CRM Project Issues +================== + +Link module to map leads and issues + """, + 'data': [ + 'project_issue_view.xml', + 'crm_lead_view.xml', + ], + 'depends' : ['crm', 'project_issue'], + 'installable': True, +} diff --git a/addons/crm_project_issue/crm_lead.py b/addons/crm_project_issue/crm_lead.py new file mode 100644 index 00000000000..ec83af18a39 --- /dev/null +++ b/addons/crm_project_issue/crm_lead.py @@ -0,0 +1,10 @@ + +from openerp.osv import osv, fields + + +class crm_lead(osv.Model): + _inherit = 'crm.lead' + + _columns = { + 'project_issue_ids': fields.one2many('project.issue', 'lead_id', "Project Issues"), + } diff --git a/addons/crm_project_issue/crm_lead_view.xml b/addons/crm_project_issue/crm_lead_view.xml new file mode 100644 index 00000000000..0e81f16fa9f --- /dev/null +++ b/addons/crm_project_issue/crm_lead_view.xml @@ -0,0 +1,27 @@ + + + + + + lead.form + crm.lead + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/addons/crm_project_issue/project_issue.py b/addons/crm_project_issue/project_issue.py new file mode 100644 index 00000000000..9409aeb51ae --- /dev/null +++ b/addons/crm_project_issue/project_issue.py @@ -0,0 +1,14 @@ + +from openerp.osv import osv, fields + + +class project_issue(osv.Model): + _inherit = 'project.issue' + + _columns = { + 'lead_id': fields.many2one('crm.lead', ondelete='set null', string="Related lead"), + } + + _defaults = { + + } \ No newline at end of file diff --git a/addons/crm_project_issue/project_issue_view.xml b/addons/crm_project_issue/project_issue_view.xml new file mode 100644 index 00000000000..18a03b5a2c1 --- /dev/null +++ b/addons/crm_project_issue/project_issue_view.xml @@ -0,0 +1,17 @@ + + + + + + project.issue.form + project.issue + + + + + + + + + + \ No newline at end of file From 0ff108c00c72742693c20cb22d251bee7d463978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me?= Date: Mon, 17 Feb 2014 14:57:31 +0100 Subject: [PATCH 18/51] [ADD] change the link between lead and issue. Now : transfrom a lead into a issue and modify the mail thread bzr revid: jem@openerp.com-20140217135731-538vluw7i1gverjf --- addons/crm_project_issue/__init__.py | 3 +- addons/crm_project_issue/__openerp__.py | 3 +- addons/crm_project_issue/project_issue.py | 36 +++++++++--- .../crm_project_issue/project_issue_view.xml | 55 ++++++++++++++----- addons/mail/mail_thread.py | 31 +++++++++++ 5 files changed, 103 insertions(+), 25 deletions(-) diff --git a/addons/crm_project_issue/__init__.py b/addons/crm_project_issue/__init__.py index 40302ae183c..1d0a3f683ea 100644 --- a/addons/crm_project_issue/__init__.py +++ b/addons/crm_project_issue/__init__.py @@ -1,3 +1,2 @@ -import project_issue -import crm_lead \ No newline at end of file +import project_issue \ No newline at end of file diff --git a/addons/crm_project_issue/__openerp__.py b/addons/crm_project_issue/__openerp__.py index 89ef20a655e..e50a6dbfa2a 100644 --- a/addons/crm_project_issue/__openerp__.py +++ b/addons/crm_project_issue/__openerp__.py @@ -13,8 +13,7 @@ CRM Project Issues Link module to map leads and issues """, 'data': [ - 'project_issue_view.xml', - 'crm_lead_view.xml', + 'project_issue_view.xml' ], 'depends' : ['crm', 'project_issue'], 'installable': True, diff --git a/addons/crm_project_issue/project_issue.py b/addons/crm_project_issue/project_issue.py index 9409aeb51ae..8e7a0733932 100644 --- a/addons/crm_project_issue/project_issue.py +++ b/addons/crm_project_issue/project_issue.py @@ -1,14 +1,36 @@ from openerp.osv import osv, fields +class crm_lead_to_project_issue_wizard(osv.TransientModel): + """ wizard to convert a Lead into a Project Issue and move the Mail Thread """ -class project_issue(osv.Model): - _inherit = 'project.issue' + def action_lead_to_project_issue(self, cr, uid, ids, context=None): + # get the wizards + wizards = self.browse(cr, uid, ids, context=context) + lead_model = self.pool.get("crm.lead") + issue_model = self.pool.get("project.issue") + + for wizard in wizards: + # get the lead to transform + lead = lead_model.browse(cr, uid, wizard.lead_id.id, context=context) + # create new project.issue + vals = {} + vals["name"] = lead.name + vals["description"] = lead.description + vals["email_from"] = lead.email_from + vals["partner_id"] = lead.partner_id.id + vals["project_id"] = wizard.project_id.id + issue_id = issue_model.create(cr, uid, vals, context=None) + # move the mail thread + lead_model.transform_model_messages(cr, uid, wizard.lead_id.id, issue_id, "project.issue", context=context) + # delete the lead + lead_model.unlink(cr, uid, [wizard.lead_id.id], context=None) + return False + + + _name = "crm.lead2projectissue.wizard" _columns = { - 'lead_id': fields.many2one('crm.lead', ondelete='set null', string="Related lead"), - } - - _defaults = { - + "lead_id" : fields.many2one("crm.lead","Lead", domain=[("type","=","lead")]), + "project_id" : fields.many2one("project.project", "Project", domain=[("use_issues","=",True)]) } \ No newline at end of file diff --git a/addons/crm_project_issue/project_issue_view.xml b/addons/crm_project_issue/project_issue_view.xml index 18a03b5a2c1..57cf256bdd3 100644 --- a/addons/crm_project_issue/project_issue_view.xml +++ b/addons/crm_project_issue/project_issue_view.xml @@ -1,17 +1,44 @@ - + - - - - project.issue.form - project.issue - - - - - - - + - + + + CRM - Leads Form + crm.lead + + + +