From ca6fada2564ce40efd2d660755bf7c2f1c7cf006 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 26 Nov 2010 12:33:44 +0200 Subject: [PATCH 1/7] convert.py: use the get_object_reference() API for ir.model.data Note that the "return False" would have never worked, since the _get_id() would have raised an exception for missing records. bzr revid: p_christ@hol.gr-20101126103344-ugr20dnti4gezdoe --- bin/tools/convert.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bin/tools/convert.py b/bin/tools/convert.py index 1d050a832a4..3190335f4c2 100644 --- a/bin/tools/convert.py +++ b/bin/tools/convert.py @@ -830,11 +830,7 @@ form: module.record_id""" % (xml_id,) mod = self.module if '.' in id_str: mod,id_str = id_str.split('.') - result = model_data_obj._get_id(cr, self.uid, mod, id_str) - res = model_data_obj.read(cr, self.uid, [result], ['model', 'res_id']) - if res and res[0] and res[0]['res_id']: - return res[0]['model'], int(res[0]['res_id']) - return False + return model_data_obj.get_object_reference(cr, self.uid, mod, id_str) def parse(self, de): if not de.tag in ['terp', 'openerp']: From 7c55e521a3b6aa0e45599c67158cf392e57173ea Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 26 Nov 2010 12:34:59 +0200 Subject: [PATCH 2/7] customfonts: search path for fonts, override the broken of reportlab Several Linux distros still ship with a broken reportlab config, which looks at "c:\winnt\fonts" for fonts! Since they have not fixed that for months, we have to do their job and have a custom search path of: - paths from the config file (ttfonts.search_path key) - sensible defaults for distros, considering os.name and os.uname - default reportlab path (opt-out with ttfonts.use_default_path=False) The result is that we will have more chance of locating TTF fonts and use them (as now required) in the PDF reports. bzr revid: p_christ@hol.gr-20101126103459-5wv7crw3i3b56adr --- bin/report/render/rml2pdf/customfonts.py | 44 +++++++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/bin/report/render/rml2pdf/customfonts.py b/bin/report/render/rml2pdf/customfonts.py index e4f15aa8d8b..5fbfe3b4e12 100644 --- a/bin/report/render/rml2pdf/customfonts.py +++ b/bin/report/render/rml2pdf/customfonts.py @@ -3,6 +3,7 @@ # # OpenERP, Open Source Management Solution # Copyright (C) 2004-2009 P. Christeas, Tiny SPRL (). +# Copyright (C) 2010 OpenERP SA. (http://www.openerp.com) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -20,6 +21,8 @@ ############################################################################## from reportlab import rl_config +from tools import config +import glob import os import logging @@ -28,6 +31,9 @@ the reportlab engine. This file could be customized per distro (although most Linux/Unix ones) should have the same filenames, only need the code below). + +Due to an awful configuration that ships with reportlab at many Linux +and Ubuntu distros, we have to override the search path, too. """ CustomTTFonts = [ ('Helvetica',"DejaVu Sans", "DejaVuSans.ttf", 'normal'), @@ -51,6 +57,16 @@ CustomTTFonts = [ ('Helvetica',"DejaVu Sans", "DejaVuSans.ttf", 'normal'), ('Courier',"FreeMono Oblique", "FreeMonoOblique.ttf", 'italic'), ('Courier',"FreeMono BoldOblique", "FreeMonoBoldOblique.ttf", 'bolditalic'),] + +TTFSearchPath_Linux = ( + '/usr/share/fonts/truetype', # SuSE + '/usr/share/fonts/dejavu', '/usr/share/fonts/liberation', # Fedora, RHEL + '/usr/share/fonts/truetype/*', # Ubuntu, + '/usr/share/fonts/TTF/*', # at Mandriva/Mageia + ) + + +# ----- The code below is less distro-specific, please avoid editing! ------- __foundFonts = [] def FindCustomFonts(): @@ -64,10 +80,28 @@ def FindCustomFonts(): dirpath = [] log = logging.getLogger('report.fonts') global __foundFonts - for dirname in rl_config.TTFSearchPath: - abp = os.path.abspath(dirname) - if os.path.isdir(abp): - dirpath.append(abp) + searchpath = [] + + if config.get_misc('ttfonts', 'search_path', False): + searchpath += map(str.strip, config.get_misc('ttfonts', 'search_path').split(',')) + + if os.name == 'nt': + pass # TODO + elif os.uname()[0] == 'Linux': + searchpath += TTFSearchPath_Linux + else: + pass # TODO for MacOSX, Unix etc. + + if config.get_misc('ttfonts', 'use_default_path', True): + # Append the original search path of reportlab (at the end) + searchpath += rl_config.TTFSearchPath + + for dirglob in searchpath: + dirglob = os.path.expanduser(dirglob) + for dirname in glob.iglob(dirglob): + abp = os.path.abspath(dirname) + if os.path.isdir(abp): + dirpath.append(abp) for k, (name, font, fname, mode) in enumerate(CustomTTFonts): if fname in __foundFonts: @@ -77,7 +111,7 @@ def FindCustomFonts(): log.debug("Found font %s in %s as %s", fname, d, name) __foundFonts.append(fname) break - + # print "Found fonts:", __foundFonts From 0228f8e8891fb4d532e8ef57fec29adf159b3155 Mon Sep 17 00:00:00 2001 From: Olivier Dony Date: Fri, 26 Nov 2010 14:02:25 +0100 Subject: [PATCH 3/7] [IMP] netsvc: order debug levels declaration according to priority bzr revid: odo@openerp.com-20101126130225-o2stqws2mmg98bon --- bin/netsvc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/netsvc.py b/bin/netsvc.py index 7cb7e73c9df..716fe0f5d20 100644 --- a/bin/netsvc.py +++ b/bin/netsvc.py @@ -132,8 +132,8 @@ class ExportService(object): LOG_NOTSET = 'notset' LOG_DEBUG_SQL = 'debug_sql' -LOG_DEBUG_RPC = 'debug_rpc' LOG_DEBUG_RPC_ANSWER = 'debug_rpc_answer' +LOG_DEBUG_RPC = 'debug_rpc' LOG_DEBUG = 'debug' LOG_TEST = 'test' LOG_INFO = 'info' From 38c5f460258f3bf15ae18fbc451459b5f78ccf0b Mon Sep 17 00:00:00 2001 From: Olivier Dony Date: Fri, 26 Nov 2010 18:07:05 +0100 Subject: [PATCH 4/7] [IMP] report.customfonts: more robust fonts detection and loading bzr revid: odo@openerp.com-20101126170705-en0wbjb3rna8r96b --- bin/report/render/rml2pdf/customfonts.py | 79 +++++++++++++++--------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/bin/report/render/rml2pdf/customfonts.py b/bin/report/render/rml2pdf/customfonts.py index 097c28df524..30ff97caa90 100644 --- a/bin/report/render/rml2pdf/customfonts.py +++ b/bin/report/render/rml2pdf/customfonts.py @@ -20,11 +20,13 @@ # ############################################################################## -from reportlab import rl_config -from tools import config import glob -import os import logging +import os +import platform +from reportlab import rl_config + +from tools import config """This module allows the mapping of some system-available TTF fonts to the reportlab engine. @@ -65,8 +67,28 @@ TTFSearchPath_Linux = ( '/usr/share/fonts/TTF/*', # at Mandriva/Mageia ) +TTFSearchPath_Windows = ( + 'c:/winnt/fonts', + 'c:/windows/fonts' + ) + +TTFSearchPath_Darwin = ( + #mac os X - from + #http://developer.apple.com/technotes/tn/tn2024.html + '~/Library/Fonts', + '/Library/Fonts', + '/Network/Library/Fonts', + '/System/Library/Fonts', + ) + +TTFSearchPathMap = { + 'Darwin': TTFSearchPath_Darwin, + 'Windows': TTFSearchPath_Windows, + 'Linux': TTFSearchPath_Linux, +} + # ----- The code below is less distro-specific, please avoid editing! ------- -__foundFonts = [] +__foundFonts = None def FindCustomFonts(): """Fill the __foundFonts list with those filenames, whose fonts @@ -79,41 +101,38 @@ def FindCustomFonts(): dirpath = [] log = logging.getLogger('report.fonts') global __foundFonts + __foundFonts = {} searchpath = [] - if config.get_misc('ttfonts', 'search_path', False): - searchpath += map(str.strip, config.get_misc('ttfonts', 'search_path').split(',')) + if config.get('fonts_search_path'): + searchpath += map(str.strip, config.get('fonts_search_path').split(',')) - if os.name == 'nt': - pass # TODO - elif os.uname()[0] == 'Linux': - searchpath += TTFSearchPath_Linux - else: - pass # TODO for MacOSX, Unix etc. - - if config.get_misc('ttfonts', 'use_default_path', True): - # Append the original search path of reportlab (at the end) - searchpath += rl_config.TTFSearchPath + local_platform = platform.system() + if local_platform in TTFSearchPathMap: + searchpath += TTFSearchPathMap[local_platform] + # Append the original search path of reportlab (at the end) + searchpath += rl_config.TTFSearchPath + + # Perform the search for font files ourselves, as reportlab's + # TTFOpenFile is not very good at it. for dirglob in searchpath: dirglob = os.path.expanduser(dirglob) for dirname in glob.iglob(dirglob): abp = os.path.abspath(dirname) if os.path.isdir(abp): dirpath.append(abp) - - for k, (name, font, fname, mode) in enumerate(CustomTTFonts): - if fname in __foundFonts: + + for k, (name, font, filename, mode) in enumerate(CustomTTFonts): + if filename in __foundFonts: continue for d in dirpath: - if os.path.exists(os.path.join(d, fname)): - log.debug("Found font %s in %s as %s", fname, d, name) - __foundFonts.append(fname) + abs_filename = os.path.join(d, filename) + if os.path.exists(abs_filename): + log.debug("Found font %s at %s", filename, abs_filename) + __foundFonts[filename] = abs_filename break - # print "Found fonts:", __foundFonts - - def SetCustomFonts(rmldoc): """ Map some font names to the corresponding TTF fonts @@ -123,11 +142,13 @@ def SetCustomFonts(rmldoc): avoid system-wide processing (cache it, instead). """ global __foundFonts - if not len(__foundFonts): + if __foundFonts is None: FindCustomFonts() - for name, font, fname, mode in CustomTTFonts: - if os.path.isabs(fname) or fname in __foundFonts: - rmldoc.setTTFontMapping(name, font, fname, mode) + for name, font, filename, mode in CustomTTFonts: + if os.path.isabs(filename): + rmldoc.setTTFontMapping(name, font, filename, mode) + elif filename in __foundFonts: + rmldoc.setTTFontMapping(name, font, __foundFonts[filename], mode) return True #eof From 959d4f962eb9bd0e36a36419a754cb903c0d9b49 Mon Sep 17 00:00:00 2001 From: Olivier Dony Date: Fri, 26 Nov 2010 19:30:43 +0100 Subject: [PATCH 5/7] [IMP] report,fonts: fallback to default Adobe fonts when font is missing - pending improvement We should maybe find a way to let users use their own fonts. At the moment only the 14 default Adobe fonts (cfr reportlab.reportbase.pdfmetrics.standardFonts) + the ones from customfonts.py will work... Also got rid of the mapping for ZapfDingBast, as it is a symbol font that cannot be mapped to a regular one. bzr revid: odo@openerp.com-20101126183043-mrzipe7mdw1zs6g8 --- bin/report/render/rml2pdf/customfonts.py | 6 +----- bin/report/render/rml2pdf/trml2pdf.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/bin/report/render/rml2pdf/customfonts.py b/bin/report/render/rml2pdf/customfonts.py index 30ff97caa90..a60c2d267f5 100644 --- a/bin/report/render/rml2pdf/customfonts.py +++ b/bin/report/render/rml2pdf/customfonts.py @@ -50,10 +50,6 @@ CustomTTFonts = [ ('Helvetica',"DejaVu Sans", "DejaVuSans.ttf", 'normal'), ('Times-Roman',"Liberation Serif Bold", "LiberationSerif-Bold.ttf", 'bold'), ('Times-Roman',"Liberation Serif Italic", "LiberationSerif-Italic.ttf", 'italic'), ('Times-Roman',"Liberation Serif BoldItalic", "LiberationSerif-BoldItalic.ttf", 'bolditalic'), - ('ZapfDingbats',"DejaVu Serif", "DejaVuSerif.ttf", 'normal'), - ('ZapfDingbats',"DejaVu Serif Bold", "DejaVuSerif-Bold.ttf", 'bold'), - ('ZapfDingbats',"DejaVu Serif Italic", "DejaVuSerif-Italic.ttf", 'italic'), - ('ZapfDingbats',"DejaVu Serif BoldItalic", "DejaVuSerif-BoldItalic.ttf", 'bolditalic'), ('Courier',"FreeMono", "FreeMono.ttf", 'normal'), ('Courier',"FreeMono Bold", "FreeMonoBold.ttf", 'bold'), ('Courier',"FreeMono Oblique", "FreeMonoOblique.ttf", 'italic'), @@ -145,7 +141,7 @@ def SetCustomFonts(rmldoc): if __foundFonts is None: FindCustomFonts() for name, font, filename, mode in CustomTTFonts: - if os.path.isabs(filename): + if os.path.isabs(filename) and os.path.exists(filename): rmldoc.setTTFontMapping(name, font, filename, mode) elif filename in __foundFonts: rmldoc.setTTFontMapping(name, font, __foundFonts[filename], mode) diff --git a/bin/report/render/rml2pdf/trml2pdf.py b/bin/report/render/rml2pdf/trml2pdf.py index c8b64f4075b..cbe847eea1c 100644 --- a/bin/report/render/rml2pdf/trml2pdf.py +++ b/bin/report/render/rml2pdf/trml2pdf.py @@ -35,6 +35,7 @@ import base64 from reportlab.platypus.doctemplate import ActionFlowable from tools.safe_eval import safe_eval as eval from reportlab.lib.units import inch,cm,mm +from reportlab.pdfbase import pdfmetrics try: from cStringIO import StringIO @@ -485,11 +486,18 @@ class _rml_canvas(object): self.canvas.drawPath(self.path, **utils.attr_get(node, [], {'fill':'bool','stroke':'bool'})) def setFont(self, node): - fname = node.get('name') - try: - return self.canvas.setFont(fname, utils.unit_get(node.get('size'))) - except KeyError, e: - raise KeyError('Font "%s" is not registered in the engine' % fname) + fontname = node.get('name') + if fontname not in pdfmetrics.getRegisteredFontNames()\ + or fontname not in pdfmetrics.standardFonts: + # let reportlab attempt to find it + try: + pdfmetrics.getFont(fontname) + except Exception: + logging.getLogger('report.fonts').debug('Could not locate font %s, substituting default: %s', + fontname, + self.canvas._fontname) + fontname = self.canvas._fontname + return self.canvas.setFont(fontname, utils.unit_get(node.get('size'))) def render(self, node): tags = { From 4bc20ff99f4251e4705e5836bf656f172bf6fd92 Mon Sep 17 00:00:00 2001 From: Olivier Dony Date: Sat, 27 Nov 2010 04:03:04 +0100 Subject: [PATCH 6/7] [FIX] convert,rng: should support unambiguous view_id attribute + cleanup of useless id_get param bzr revid: odo@openerp.com-20101127030304-mvrb92rmgb0wblvc --- bin/import_xml.rng | 1 - bin/tools/convert.py | 56 ++++++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/bin/import_xml.rng b/bin/import_xml.rng index 0495b334f78..75c4e8b05e7 100644 --- a/bin/import_xml.rng +++ b/bin/import_xml.rng @@ -196,7 +196,6 @@ - diff --git a/bin/tools/convert.py b/bin/tools/convert.py index 61fdb7a82a9..49d56b15b1e 100644 --- a/bin/tools/convert.py +++ b/bin/tools/convert.py @@ -62,7 +62,7 @@ class ConvertError(Exception): return 'Exception:\n\t%s\nUsing file:\n%s' % (self.orig, self.d) def _ref(self, cr): - return lambda x: self.id_get(cr, False, x) + return lambda x: self.id_get(cr, x) def _obj(pool, cr, uid, model_str, context=None): model = pool.get(model_str) @@ -139,7 +139,7 @@ def _eval_xml(self, node, pool, cr, uid, idref, context=None): m = re.findall('[^%]%\((.*?)\)[ds]', s) for id in m: if not id in idref: - idref[id]=self.id_get(cr, False, id) + idref[id]=self.id_get(cr, id) return s % idref _fix_multiple_roots(node) return '\n'\ @@ -177,7 +177,7 @@ def _eval_xml(self, node, pool, cr, uid, idref, context=None): args = [] a_eval = node.get('eval','') if a_eval: - idref['ref'] = lambda x: self.id_get(cr, False, x) + idref['ref'] = lambda x: self.id_get(cr, x) args = unsafe_eval(a_eval, idref) for n in node: return_val = _eval_xml(self,n, pool, cr, uid, idref, context) @@ -262,7 +262,7 @@ class xml_import(object): def get_uid(self, cr, uid, data_node, node): node_uid = node.get('uid','') or (len(data_node) and data_node.get('uid','')) if node_uid: - return self.id_get(cr, None, node_uid) + return self.id_get(cr, node_uid) return uid def _test_xml_id(self, xml_id): @@ -290,7 +290,7 @@ form: module.record_id""" % (xml_id,) ids = self.pool.get(d_model).search(cr, self.uid, unsafe_eval(d_search, idref)) if d_id: try: - ids.append(self.id_get(cr, d_model, d_id)) + ids.append(self.id_get(cr, d_id)) except: # d_id cannot be found. doesn't matter in this case pass @@ -334,10 +334,10 @@ form: module.record_id""" % (xml_id,) groups_value = [] for group in g_names: if group.startswith('-'): - group_id = self.id_get(cr, 'res.groups', group[1:]) + group_id = self.id_get(cr, group[1:]) groups_value.append((3, group_id)) else: - group_id = self.id_get(cr, 'res.groups', group) + group_id = self.id_get(cr, group) groups_value.append((4, group_id)) res['groups_id'] = groups_value @@ -377,10 +377,10 @@ form: module.record_id""" % (xml_id,) groups_value = [] for group in g_names: if group.startswith('-'): - group_id = self.id_get(cr, 'res.groups', group[1:]) + group_id = self.id_get(cr, group[1:]) groups_value.append((3, group_id)) else: - group_id = self.id_get(cr, 'res.groups', group) + group_id = self.id_get(cr, group) groups_value.append((4, group_id)) res['groups_id'] = groups_value @@ -425,8 +425,8 @@ form: module.record_id""" % (xml_id,) self._test_xml_id(xml_id) type = rec.get('type','').encode('utf-8') or 'ir.actions.act_window' view_id = False - if rec.get('view'): - view_id = self.id_get(cr, 'ir.actions.act_window', rec.get('view','').encode('utf-8')) + if rec.get('view_id'): + view_id = self.id_get(cr, rec.get('view_id','').encode('utf-8')) domain = rec.get('domain','').encode('utf-8') or '{}' res_model = rec.get('res_model','').encode('utf-8') src_model = rec.get('src_model','').encode('utf-8') @@ -438,7 +438,7 @@ form: module.record_id""" % (xml_id,) uid = self.uid active_id = str("active_id") # for further reference in client/bin/tools/__init__.py def ref(str_id): - return self.id_get(cr, None, str_id) + return self.id_get(cr, str_id) # Include all locals() in eval_context, for backwards compatibility eval_context = { @@ -489,10 +489,10 @@ form: module.record_id""" % (xml_id,) groups_value = [] for group in g_names: if group.startswith('-'): - group_id = self.id_get(cr, 'res.groups', group[1:]) + group_id = self.id_get(cr, group[1:]) groups_value.append((3, group_id)) else: - group_id = self.id_get(cr, 'res.groups', group) + group_id = self.id_get(cr, group) groups_value.append((4, group_id)) res['groups_id'] = groups_value @@ -527,7 +527,7 @@ form: module.record_id""" % (xml_id,) model = str(rec.get('model','')) w_ref = rec.get('ref','') if w_ref: - id = self.id_get(cr, model, w_ref) + id = self.id_get(cr, w_ref) else: number_children = len(rec) assert number_children > 0,\ @@ -579,7 +579,7 @@ form: module.record_id""" % (xml_id,) # The parent attribute was specified, if non-empty determine its ID, otherwise # explicitly make a top-level menu if rec.get('parent'): - menu_parent_id = self.id_get(cr, 'ir.ui.menu', rec.get('parent','')) + menu_parent_id = self.id_get(cr, rec.get('parent','')) else: # we get here with , explicit clear of parent, or # if no parent attribute at all but menu name is not a menu path @@ -588,7 +588,7 @@ form: module.record_id""" % (xml_id,) if rec.get('name'): values['name'] = rec.get('name') try: - res = [ self.id_get(cr, 'ir.ui.menu', rec.get('id','')) ] + res = [ self.id_get(cr, rec.get('id','')) ] except: res = None @@ -603,7 +603,7 @@ form: module.record_id""" % (xml_id,) } values['icon'] = icons.get(a_type,'STOCK_NEW') if a_type=='act_window': - a_id = self.id_get(cr, 'ir.actions.%s'% a_type, a_action) + a_id = self.id_get(cr, a_action) cr.execute('select view_type,view_mode,name,view_id,target from ir_act_window where id=%s', (int(a_id),)) rrres = cr.fetchone() assert rrres, "No window action defined for this id %s !\n" \ @@ -628,7 +628,7 @@ form: module.record_id""" % (xml_id,) if not values.get('name', False): values['name'] = action_name elif a_type=='wizard': - a_id = self.id_get(cr, 'ir.actions.%s'% a_type, a_action) + a_id = self.id_get(cr, a_action) cr.execute('select name from ir_act_wizard where id=%s', (int(a_id),)) resw = cr.fetchone() if (not values.get('name', False)) and resw: @@ -647,10 +647,10 @@ form: module.record_id""" % (xml_id,) groups_value = [] for group in g_names: if group.startswith('-'): - group_id = self.id_get(cr, 'res.groups', group[1:]) + group_id = self.id_get(cr, group[1:]) groups_value.append((3, group_id)) else: - group_id = self.id_get(cr, 'res.groups', group) + group_id = self.id_get(cr, group) groups_value.append((4, group_id)) values['groups_id'] = groups_value @@ -664,7 +664,7 @@ form: module.record_id""" % (xml_id,) if rec.get('action') and pid: a_action = rec.get('action').encode('utf8') a_type = rec.get('type','').encode('utf8') or 'act_window' - a_id = self.id_get(cr, 'ir.actions.%s' % a_type, a_action) + a_id = self.id_get(cr, a_action) action = "ir.actions.%s,%d" % (a_type, a_id) self.pool.get('ir.model.data').ir_set(cr, self.uid, 'action', 'tree_but_open', 'Menuitem', [('ir.ui.menu', int(pid))], action, True, True, xml_id=rec_id) return ('ir.ui.menu', pid) @@ -692,7 +692,7 @@ form: module.record_id""" % (xml_id,) context = self.get_context(data_node, rec, eval_dict) uid = self.get_uid(cr, self.uid, data_node, rec) if rec_id: - ids = [self.id_get(cr, rec_model, rec_id)] + ids = [self.id_get(cr, rec_id)] elif rec_src: q = unsafe_eval(rec_src, eval_dict) ids = self.pool.get(rec_model).search(cr, uid, q, context=context) @@ -814,10 +814,10 @@ form: module.record_id""" % (xml_id,) else: if f_name in model._columns \ and model._columns[f_name]._type == 'reference': - val = self.model_id_get(cr, f_model, f_ref) + val = self.model_id_get(cr, f_ref) f_val = val[0] + ',' + str(val[1]) else: - f_val = self.id_get(cr, f_model, f_ref) + f_val = self.id_get(cr, f_ref) else: f_val = _eval_xml(self,field, self.pool, cr, self.uid, self.idref) if model._columns.has_key(f_name): @@ -832,14 +832,14 @@ form: module.record_id""" % (xml_id,) cr.commit() return rec_model, id - def id_get(self, cr, model, id_str): + def id_get(self, cr, id_str): if id_str in self.idref: return self.idref[id_str] - res = self.model_id_get(cr, model, id_str) + res = self.model_id_get(cr, id_str) if res and len(res)>1: res = res[1] return res - def model_id_get(self, cr, model, id_str): + def model_id_get(self, cr, id_str): model_data_obj = self.pool.get('ir.model.data') mod = self.module if '.' in id_str: From ec9d236589c383c4ce1e608facf266b7ca7c546a Mon Sep 17 00:00:00 2001 From: Olivier Dony Date: Sat, 27 Nov 2010 04:04:50 +0100 Subject: [PATCH 7/7] [IMP] ir_model,ir_module,tools: tweaks for improving module install/loading time bzr revid: odo@openerp.com-20101127030450-m44krmwbnpd57omf --- bin/addons/base/ir/ir_model.py | 21 +++-- bin/addons/base/ir/ir_values.py | 18 ++-- bin/addons/base/ir/workflow/workflow.py | 2 +- bin/addons/base/module/module.py | 108 +++++++++++++++--------- bin/tools/misc.py | 43 +++++----- 5 files changed, 111 insertions(+), 81 deletions(-) diff --git a/bin/addons/base/ir/ir_model.py b/bin/addons/base/ir/ir_model.py index a0040a836df..583985b5930 100644 --- a/bin/addons/base/ir/ir_model.py +++ b/bin/addons/base/ir/ir_model.py @@ -401,6 +401,12 @@ class ir_model_data(osv.osv): self.doinit = True self.unlink_mark = {} + def _auto_init(self, cr, context=None): + super(ir_model_data, self)._auto_init(cr, context) + cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = \'ir_model_data_module_name_index\'') + if not cr.fetchone(): + cr.execute('CREATE INDEX ir_model_data_module_name_index ON ir_model_data (module, name)') + @tools.cache() def _get_id(self, cr, uid, module, xml_id): """Returns the id of the ir.model.data record corresponding to a given module and xml_id (cached) or raise a ValueError if not found""" @@ -448,18 +454,19 @@ class ir_model_data(osv.osv): action_id = False if xml_id: - cr.execute('select id,res_id from ir_model_data where module=%s and name=%s', (module,xml_id)) + cr.execute('''SELECT imd.id, imd.res_id, md.id + FROM ir_model_data imd LEFT JOIN %s md ON (imd.res_id = md.id) + WHERE imd.module=%%s AND imd.name=%%s''' % model_obj._table, + (module, xml_id)) results = cr.fetchall() - for action_id2,res_id2 in results: - cr.execute('select id from '+model_obj._table+' where id=%s', (res_id2,)) - result3 = cr.fetchone() - if not result3: + for imd_id2,res_id2,real_id2 in results: + if not real_id2: self._get_id.clear_cache(cr.dbname, uid, module, xml_id) self.get_object_reference.clear_cache(cr.dbname, uid, module, xml_id) - cr.execute('delete from ir_model_data where id=%s', (action_id2,)) + cr.execute('delete from ir_model_data where id=%s', (imd_id2,)) res_id = False else: - res_id,action_id = res_id2,action_id2 + res_id,action_id = res_id2,imd_id2 if action_id and res_id: model_obj.write(cr, uid, [res_id], values, context=context) diff --git a/bin/addons/base/ir/ir_values.py b/bin/addons/base/ir/ir_values.py index 21ee1a66f93..602e2a10a80 100644 --- a/bin/addons/base/ir/ir_values.py +++ b/bin/addons/base/ir/ir_values.py @@ -72,21 +72,21 @@ class ir_values(osv.osv): 'name': fields.char('Name', size=128), 'model_id': fields.many2one('ir.model', 'Object', size=128, help="This field is not used, it only helps you to select a good model."), - 'model': fields.char('Object Name', size=128), + 'model': fields.char('Object Name', size=128, select=True), 'action_id': fields.many2one('ir.actions.actions', 'Action', help="This field is not used, it only helps you to select the right action."), 'value': fields.text('Value'), 'value_unpickle': fields.function(_value_unpickle, fnct_inv=_value_pickle, method=True, type='text', string='Value'), 'object': fields.boolean('Is Object'), - 'key': fields.selection([('action','Action'),('default','Default')], 'Type', size=128), - 'key2' : fields.char('Event Type',help="The kind of action or button in the client side that will trigger the action.", size=128), + 'key': fields.selection([('action','Action'),('default','Default')], 'Type', size=128, select=True), + 'key2' : fields.char('Event Type',help="The kind of action or button in the client side that will trigger the action.", size=128, select=True), 'meta': fields.text('Meta Datas'), 'meta_unpickle': fields.function(_value_unpickle, fnct_inv=_value_pickle, method=True, type='text', string='Metadata'), - 'res_id': fields.integer('Object ID', help="Keep 0 if the action must appear on all resources."), - 'user_id': fields.many2one('res.users', 'User', ondelete='cascade'), - 'company_id': fields.many2one('res.company', 'Company') + 'res_id': fields.integer('Object ID', help="Keep 0 if the action must appear on all resources.", select=True), + 'user_id': fields.many2one('res.users', 'User', ondelete='cascade', select=True), + 'company_id': fields.many2one('res.company', 'Company', select=True) } _defaults = { 'key': lambda *a: 'action', @@ -94,11 +94,11 @@ class ir_values(osv.osv): 'company_id': lambda *a: False } - def _auto_init(self, cr, context={}): + def _auto_init(self, cr, context=None): super(ir_values, self)._auto_init(cr, context) - cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = \'ir_values_key_model_key2_index\'') + cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = \'ir_values_key_model_key2_res_id_user_id_idx\'') if not cr.fetchone(): - cr.execute('CREATE INDEX ir_values_key_model_key2_index ON ir_values (key, model, key2)') + cr.execute('CREATE INDEX ir_values_key_model_key2_res_id_user_id_idx ON ir_values (key, model, key2, res_id, user_id)') def set(self, cr, uid, key, key2, name, models, value, replace=True, isobject=False, meta=False, preserve_user=False, company=False): if isinstance(value, unicode): diff --git a/bin/addons/base/ir/workflow/workflow.py b/bin/addons/base/ir/workflow/workflow.py index a117688399d..dda5dd46276 100644 --- a/bin/addons/base/ir/workflow/workflow.py +++ b/bin/addons/base/ir/workflow/workflow.py @@ -175,7 +175,7 @@ class wkf_instance(osv.osv): 'res_type': fields.char('Resource Object', size=64, select=True), 'state': fields.char('State', size=32, select=True), } - def _auto_init(self, cr, context={}): + def _auto_init(self, cr, context=None): super(wkf_instance, self)._auto_init(cr, context) cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = \'wkf_instance_res_id_res_type_state_index\'') if not cr.fetchone(): diff --git a/bin/addons/base/module/module.py b/bin/addons/base/module/module.py index bdd76d75e0c..27979bfb2cc 100644 --- a/bin/addons/base/module/module.py +++ b/bin/addons/base/module/module.py @@ -57,7 +57,7 @@ class module_category(osv.osv): return result _columns = { - 'name': fields.char("Name", size=128, required=True), + 'name': fields.char("Name", size=128, required=True, select=True), 'parent_id': fields.many2one('ir.module.category', 'Parent Category', select=True), 'child_ids': fields.one2many('ir.module.category', 'parent_id', 'Child Categories'), 'module_nr': fields.function(_module_nbr, method=True, string='Number of Modules', type='integer') @@ -96,26 +96,34 @@ class module(osv.osv): mlist = self.browse(cr, uid, ids, context=context) mnames = {} for m in mlist: - mnames[m.name] = m.id + # skip uninstalled modules below, + # no data to find anyway + if m.state in ('installed', 'to upgrade', 'to remove'): + mnames[m.name] = m.id res[m.id] = { 'menus_by_module':[], 'reports_by_module':[], 'views_by_module': [] } + + if not mnames: + return res + view_id = model_data_obj.search(cr,uid,[('module','in', mnames.keys()), ('model','in',('ir.ui.view','ir.actions.report.xml','ir.ui.menu'))]) for data_id in model_data_obj.browse(cr,uid,view_id,context): # We use try except, because views or menus may not exist try: key = data_id.model + res_mod_dic = res[mnames[data_id.module]] if key=='ir.ui.view': v = view_obj.browse(cr,uid,data_id.res_id) aa = v.inherit_id and '* INHERIT ' or '' - res[mnames[data_id.module]]['views_by_module'].append(aa + v.name + '('+v.type+')') + res_mod_dic['views_by_module'].append(aa + v.name + '('+v.type+')') elif key=='ir.actions.report.xml': - res[mnames[data_id.module]]['reports_by_module'].append(report_obj.browse(cr,uid,data_id.res_id).name) + res_mod_dic['reports_by_module'].append(report_obj.browse(cr,uid,data_id.res_id).name) elif key=='ir.ui.menu': - res[mnames[data_id.module]]['menus_by_module'].append(menu_obj.browse(cr,uid,data_id.res_id).complete_name) + res_mod_dic['menus_by_module'].append(menu_obj.browse(cr,uid,data_id.res_id).complete_name) except KeyError, e: self.__logger.warning( 'Data not found for reference %s[%s:%s.%s]', data_id.model, @@ -344,8 +352,8 @@ class module(osv.osv): 'maintainer': terp.get('maintainer', False), 'contributors': ', '.join(terp.get('contributors', [])) or False, 'website': terp.get('website', ''), - 'license': terp.get('license', 'GPL-2'), - 'certificate': terp.get('certificate') or None, + 'license': terp.get('license', 'AGPL-3'), + 'certificate': terp.get('certificate') or False, 'web': terp.get('web') or False, } @@ -353,34 +361,40 @@ class module(osv.osv): def update_list(self, cr, uid, context={}): res = [0, 0] # [update, add] - # iterate through installed modules and mark them as being so + known_mods = self.browse(cr, uid, self.search(cr, uid, [])) + known_mods_names = dict([(m.name, m) for m in known_mods]) + + # iterate through detected modules and update/create them in db for mod_name in addons.get_modules(): - ids = self.search(cr, uid, [('name','=',mod_name)]) + mod = known_mods_names.get(mod_name) terp = self.get_module_info(mod_name) values = self.get_values_from_terp(terp) - if ids: - id = ids[0] - mod = self.browse(cr, uid, id) + if mod: + updated_values = {} + for key in values: + old = getattr(mod, key) + updated = isinstance(values[key], basestring) and tools.ustr(values[key]) or values[key] + if not old == updated: + updated_values[key] = values[key] if terp.get('installable', True) and mod.state == 'uninstallable': - self.write(cr, uid, id, {'state': 'uninstalled'}) + updated_values['state'] = 'uninstalled' if parse_version(terp.get('version', '')) > parse_version(mod.latest_version or ''): - self.write(cr, uid, id, {'url': ''}) res[0] += 1 - self.write(cr, uid, id, values) - cr.execute('DELETE FROM ir_module_module_dependency WHERE module_id = %s', (id,)) + if updated_values: + self.write(cr, uid, mod.id, updated_values) else: mod_path = addons.get_module_path(mod_name) if not mod_path: continue if not terp or not terp.get('installable', True): continue - - ids = self.search(cr, uid, [('name','=',mod_name)]) id = self.create(cr, uid, dict(name=mod_name, state='uninstalled', **values)) + mod = self.browse(cr, uid, id) res[1] += 1 - self._update_dependencies(cr, uid, id, terp.get('depends', [])) - self._update_category(cr, uid, id, terp.get('category', 'Uncategorized')) + + self._update_dependencies(cr, uid, mod, terp.get('depends', [])) + self._update_category(cr, uid, mod, terp.get('category', 'Uncategorized')) return res @@ -412,39 +426,49 @@ class module(osv.osv): self.write(cr, uid, mod.id, self.get_values_from_terp(terp)) cr.execute('DELETE FROM ir_module_module_dependency ' \ 'WHERE module_id = %s', (mod.id,)) - self._update_dependencies(cr, uid, mod.id, terp.get('depends', + self._update_dependencies(cr, uid, mod, terp.get('depends', [])) - self._update_category(cr, uid, mod.id, terp.get('category', + self._update_category(cr, uid, mod, terp.get('category', 'Uncategorized')) # Import module zimp = zipimport.zipimporter(fname) zimp.load_module(mod.name) return res - def _update_dependencies(self, cr, uid, id, depends=None): + def _update_dependencies(self, cr, uid, mod_browse, depends=None): if depends is None: depends = [] - for d in depends: - cr.execute('INSERT INTO ir_module_module_dependency (module_id, name) values (%s, %s)', (id, d)) + existing = set(x.name for x in mod_browse.dependencies_id) + needed = set(depends) + for dep in (needed - existing): + cr.execute('INSERT INTO ir_module_module_dependency (module_id, name) values (%s, %s)', (mod_browse.id, dep)) + for dep in (existing - needed): + cr.execute('DELETE FROM ir_module_module_dependency WHERE module_id = %s and name = %s', (mod_browse.id, dep)) + + def _update_category(self, cr, uid, mod_browse, category='Uncategorized'): + current_category = mod_browse.category_id + current_category_path = [] + while current_category: + current_category_path.insert(0, current_category.name) + current_category = current_category.parent_id - def _update_category(self, cr, uid, id, category='Uncategorized'): categs = category.split('/') - p_id = None - while categs: - if p_id is not None: - cr.execute('select id from ir_module_category where name=%s and parent_id=%s', (categs[0], p_id)) - else: - cr.execute('select id from ir_module_category where name=%s and parent_id is NULL', (categs[0],)) - c_id = cr.fetchone() - if not c_id: - cr.execute('select nextval(\'ir_module_category_id_seq\')') - c_id = cr.fetchone()[0] - cr.execute('insert into ir_module_category (id, name, parent_id) values (%s, %s, %s)', (c_id, categs[0], p_id)) - else: - c_id = c_id[0] - p_id = c_id - categs = categs[1:] - self.write(cr, uid, [id], {'category_id': p_id}) + if categs != current_category_path: + p_id = None + while categs: + if p_id is not None: + cr.execute('SELECT id FROM ir_module_category WHERE name=%s AND parent_id=%s', (categs[0], p_id)) + else: + cr.execute('SELECT id FROM ir_module_category WHERE name=%s AND parent_id is NULL', (categs[0],)) + c_id = cr.fetchone() + if not c_id: + cr.execute('INSERT INTO ir_module_category (name, parent_id) VALUES (%s, %s) RETURNING id', (categs[0], p_id)) + c_id = cr.fetchone()[0] + else: + c_id = c_id[0] + p_id = c_id + categs = categs[1:] + self.write(cr, uid, [mod_browse.id], {'category_id': p_id}) def update_translations(self, cr, uid, ids, filter_lang=None, context=None): logger = logging.getLogger('i18n') diff --git a/bin/tools/misc.py b/bin/tools/misc.py index 7a416052d21..3d7dbd8a4f4 100644 --- a/bin/tools/misc.py +++ b/bin/tools/misc.py @@ -76,20 +76,19 @@ def init_db(cr): p_id = None while categs: if p_id is not None: - cr.execute('select id \ - from ir_module_category \ - where name=%s and parent_id=%s', (categs[0], p_id)) + cr.execute('SELECT id \ + FROM ir_module_category \ + WHERE name=%s AND parent_id=%s', (categs[0], p_id)) else: - cr.execute('select id \ - from ir_module_category \ - where name=%s and parent_id is NULL', (categs[0],)) + cr.execute('SELECT id \ + FROM ir_module_category \ + WHERE name=%s AND parent_id IS NULL', (categs[0],)) c_id = cr.fetchone() if not c_id: - cr.execute('select nextval(\'ir_module_category_id_seq\')') - c_id = cr.fetchone()[0] - cr.execute('insert into ir_module_category \ + cr.execute('INSERT INTO ir_module_category \ (id, name, parent_id) \ - values (%s, %s, %s)', (c_id, categs[0], p_id)) + VALUES (%s, %s) RETURNING id', (categs[0], p_id)) + c_id = cr.fetchone()[0] else: c_id = c_id[0] p_id = c_id @@ -104,23 +103,23 @@ def init_db(cr): state = 'uninstalled' else: state = 'uninstallable' - cr.execute('select nextval(\'ir_module_module_id_seq\')') - id = cr.fetchone()[0] - cr.execute('insert into ir_module_module \ - (id, author, website, name, shortdesc, description, \ - category_id, state, certificate, web) \ - values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)', ( - id, info.get('author', ''), + cr.execute('INSERT INTO ir_module_module \ + (author, website, name, shortdesc, description, \ + category_id, state, certificate, web, license) \ + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id', ( + info.get('author', ''), info.get('website', ''), i, info.get('name', False), info.get('description', ''), p_id, state, info.get('certificate') or None, - info.get('web') or False)) - cr.execute('insert into ir_model_data \ - (name,model,module, res_id, noupdate) values (%s,%s,%s,%s,%s)', ( + info.get('web') or False, + info.get('license') or 'AGPL-3')) + id = cr.fetchone()[0] + cr.execute('INSERT INTO ir_model_data \ + (name,model,module, res_id, noupdate) VALUES (%s,%s,%s,%s,%s)', ( 'module_meta_information', 'ir.module.module', i, id, True)) dependencies = info.get('depends', []) for d in dependencies: - cr.execute('insert into ir_module_module_dependency \ - (module_id,name) values (%s, %s)', (id, d)) + cr.execute('INSERT INTO ir_module_module_dependency \ + (module_id,name) VALUES (%s, %s)', (id, d)) cr.commit() def find_in_path(name):