From 26bb219a74ca586de0f935e544991a9202da311f Mon Sep 17 00:00:00 2001 From: Numerigraphe - Lionel Sausin Date: Wed, 24 Mar 2010 17:12:16 +0100 Subject: [PATCH 01/59] [IMP] wording: "customize" intead of "customise" lp bug: https://launchpad.net/bugs/546138 fixed bzr revid: ls@numerigraphe.fr-20100324161216-uvzz3vrr3s3vth5e --- bin/osv/orm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/osv/orm.py b/bin/osv/orm.py index 165b9a21137..bcd05438a7b 100644 --- a/bin/osv/orm.py +++ b/bin/osv/orm.py @@ -1222,7 +1222,7 @@ class orm_template(object): for field in fields_def: if field == 'id': - # sometime, the view may containt the (invisible) field 'id' needed for a domain (when 2 objects have cross references) + # sometime, the view may contain the (invisible) field 'id' needed for a domain (when 2 objects have cross references) fields['id'] = {'readonly': True, 'type': 'integer', 'string': 'ID'} elif field in fields: fields[field].update(fields_def[field]) @@ -1232,7 +1232,7 @@ class orm_template(object): model = res[0][1] res.insert(0, ("Can't find field '%s' in the following view parts composing the view of object model '%s':" % (field, model), None)) msg = "\n * ".join([r[0] for r in res]) - msg += "\n\nEither you wrongly customised this view, or some modules bringing those views are not compatible with your current data model" + msg += "\n\nEither you wrongly customized this view, or some modules bringing those views are not compatible with your current data model" netsvc.Logger().notifyChannel('orm', netsvc.LOG_ERROR, msg) raise except_orm('View error', msg) return arch, fields From 25a424a89d32e4485ae1e120c1d8b2ada216800e Mon Sep 17 00:00:00 2001 From: Numerigraphe - Lionel Sausin Date: Wed, 24 Mar 2010 17:32:22 +0100 Subject: [PATCH 02/59] [REF] 'children' instead of 'childs' lp bug: https://launchpad.net/bugs/327259 fixed bzr revid: ls@numerigraphe.fr-20100324163222-dj03gxme8sawtdmz --- bin/addons/base/module/module.py | 4 ++-- bin/osv/orm.py | 10 +++++----- bin/report/print_xml.py | 2 +- bin/report/render/rml2pdf/trml2pdf.py | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bin/addons/base/module/module.py b/bin/addons/base/module/module.py index 01ea8cc0745..223a9ac1e42 100644 --- a/bin/addons/base/module/module.py +++ b/bin/addons/base/module/module.py @@ -46,8 +46,8 @@ class module_category(osv.osv): result = dict(cr.fetchall()) for id in ids: cr.execute('select id from ir_module_category where parent_id=%s', (id,)) - childs = [c for c, in cr.fetchall()] - result[id] = reduce(lambda x,y:x+y, [result.get(c, 0) for c in childs], result.get(id, 0)) + children = [c for c, in cr.fetchall()] + result[id] = reduce(lambda x,y:x+y, [result.get(c, 0) for c in children], result.get(id, 0)) return result _columns = { diff --git a/bin/osv/orm.py b/bin/osv/orm.py index bcd05438a7b..3d2e3d7a748 100644 --- a/bin/osv/orm.py +++ b/bin/osv/orm.py @@ -1085,7 +1085,7 @@ class orm_template(object): context = {} result = False fields = {} - childs = True + children = True if node.tag == 'field': if node.get('name'): @@ -1100,7 +1100,7 @@ class orm_template(object): if column: relation = column._obj - childs = False + children = False views = {} for f in node: if f.tag in ('form', 'tree', 'graph'): @@ -1159,7 +1159,7 @@ class orm_template(object): if trans: node.set('sum', trans) - if childs: + if children: for f in node: fields.update(self.__view_look_dom(cr, user, f, view_id, context)) @@ -1966,8 +1966,8 @@ class orm(orm_template): where += ' order by '+self._parent_order cr.execute('SELECT id FROM '+self._table+' WHERE '+where) pos2 = pos + 1 - childs = cr.fetchall() - for id in childs: + children = cr.fetchall() + for id in children: pos2 = browse_rec(id[0], pos2) cr.execute('update '+self._table+' set parent_left=%s, parent_right=%s where id=%s', (pos,pos2,root)) return pos2+1 diff --git a/bin/report/print_xml.py b/bin/report/print_xml.py index 2c24d3766f3..a45270447e1 100644 --- a/bin/report/print_xml.py +++ b/bin/report/print_xml.py @@ -253,7 +253,7 @@ class document(object): for el_cld in node: self.parse_node(el_cld, el, v) else: - # if there is no "type" attribute in the node, copy it to the xml data and parse its childs + # if there is no "type" attribute in the node, copy it to the xml data and parse its children if not node.tag == etree.Comment: if node.tag == parent.tag: el = parent diff --git a/bin/report/render/rml2pdf/trml2pdf.py b/bin/report/render/rml2pdf/trml2pdf.py index 0574f563c90..02352b44cc7 100644 --- a/bin/report/render/rml2pdf/trml2pdf.py +++ b/bin/report/render/rml2pdf/trml2pdf.py @@ -489,8 +489,8 @@ class _rml_flowable(object): return rc1 def _table(self, node): - childs = utils._child_get(node,self,'tr') - if not childs: + children = utils._child_get(node,self,'tr') + if not children: return None length = 0 colwidths = None @@ -498,7 +498,7 @@ class _rml_flowable(object): data = [] styles = [] posy = 0 - for tr in childs: + for tr in children: paraStyle = None if tr.get('style'): st = copy.deepcopy(self.styles.table_styles[tr.get('style')]) From 5905b3ecf05702494c71992beca0ab1505afcf0d Mon Sep 17 00:00:00 2001 From: Numerigraphe - Lionel Sausin Date: Mon, 25 Oct 2010 10:52:22 +0200 Subject: [PATCH 03/59] [IMP] wording: children intead of childs (again) bzr revid: ls@numerigraphe.fr-20101025085222-pm6ovutgvb0jg3af --- bin/osv/orm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/osv/orm.py b/bin/osv/orm.py index bffe82dc913..337b14152fd 100644 --- a/bin/osv/orm.py +++ b/bin/osv/orm.py @@ -4151,9 +4151,9 @@ class orm(orm_template): old_record, new_record = self.read(cr, uid, [old_id, new_id], [field_name], context=context) # here we rely on the order of the ids to match the translations # as foreseen in copy_data() - old_childs = sorted(old_record[field_name]) - new_childs = sorted(new_record[field_name]) - for (old_child, new_child) in zip(old_childs, new_childs): + old_children = sorted(old_record[field_name]) + new_children = sorted(new_record[field_name]) + for (old_child, new_child) in zip(old_children, new_children): # recursive copy of translations here target_obj.copy_translations(cr, uid, old_child, new_child, context=context) # and for translatable fields we keep them for copy From 7a41dd6d9f385ab7c86d4c3f78cff893101ccb5c Mon Sep 17 00:00:00 2001 From: Ferdinand Gassauer Date: Wed, 27 Oct 2010 13:23:22 +0300 Subject: [PATCH 04/59] caldav: Add tree missing fields [Bug 664773] Some fields like * Collection * webdav * order * color should be displayed in tree view (to make the demo - lines different) bzr revid: p_christ@hol.gr-20101027102322-fd32ztdzsg35cjw9 --- addons/caldav/caldav_view.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/addons/caldav/caldav_view.xml b/addons/caldav/caldav_view.xml index 97656a954ad..90b489ac106 100644 --- a/addons/caldav/caldav_view.xml +++ b/addons/caldav/caldav_view.xml @@ -21,6 +21,7 @@ + @@ -127,8 +128,10 @@ - - + + + + From cbac74899019a2aa6283b8054891d3058e5d169c Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:23 +0300 Subject: [PATCH 05/59] Doc: more help on Model-Parented folders. Their usage is a little cryptic, so help the user understand how they work. bzr revid: p_christ@hol.gr-20101027102323-q0i1a5u6ausoj4uv --- addons/document/document_directory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index 9b7ba18b16f..c4df8a7cf18 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -58,8 +58,10 @@ class document_directory(osv.osv): " be located. If false, only ones that have this as parent." ), 'ressource_parent_type_id': fields.many2one('ir.model', 'Parent Model', help="If you put an object here, this directory template will appear bellow all of these objects. " \ + "Such directories are \"attached\" to the specific model or record, just like attachments. " \ "Don't put a parent directory if you select a parent model."), - 'ressource_id': fields.integer('Resource ID'), + 'ressource_id': fields.integer('Resource ID', + help="Along with Parent Model, this ID attaches this folder to a specific record of Parent Model."), 'ressource_tree': fields.boolean('Tree Structure', help="Check this if you want to use the same tree structure as the object selected in the system."), 'dctx_ids': fields.one2many('document.directory.dctx', 'dir_id', 'Context fields'), From 7628e1e4330443ef15b48106c1e2c1b84ce6e4e3 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:23 +0300 Subject: [PATCH 06/59] document: Hide resource attachments/folders from static dirs. If some attachment or folder references a res_model/res_id, it should NOT be visible in the root directory (where it's parent_id points to). This solves functional problems for the root folder, as well as the privacy of those attachments. bzr revid: p_christ@hol.gr-20101027102323-x4h20skrfjger9z7 --- addons/document/nodes.py | 62 +++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 68220885dff..089cde3e41a 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -433,12 +433,12 @@ class node_database(node_class): return res[0] return None - def _child_get(self, cr, name=False, parent_id=False, domain=None): + def _child_get(self, cr, name=False, domain=None): dirobj = self.context._dirobj uid = self.context.uid ctx = self.context.context.copy() ctx.update(self.dctx) - where = [('parent_id','=',parent_id)] + where = [('parent_id','=', False), ('ressource_parent_type_id','=',False)] if name: where.append(('name','=',name)) is_allowed = self.check_perms(1) @@ -448,23 +448,14 @@ class node_database(node_class): if not is_allowed: raise IOError(errno.EPERM, "Permission into directory denied") - if not domain: - domain = [] - - where2 = where + domain + ['|', ('type', '=', 'directory'), \ - '&', ('type', '=', 'ressource'), ('ressource_parent_type_id','=',False)] - ids = dirobj.search(cr, uid, where2, context=ctx) + if domain: + where = where + domain + ids = dirobj.search(cr, uid, where, context=ctx) res = [] for dirr in dirobj.browse(cr, uid, ids, context=ctx): klass = dirr.get_node_class(dirr, context=ctx) res.append(klass(dirr.name, self, self.context,dirr)) - fil_obj = dirobj.pool.get('ir.attachment') - ids = fil_obj.search(cr, uid, where, context=ctx) - if ids: - for fil in fil_obj.browse(cr, uid, ids, context=ctx): - klass = self.context.node_file_class - res.append(klass(fil.name, self, self.context, fil)) return res def _file_get(self,cr, nodename=False): @@ -559,7 +550,46 @@ class node_dir(node_database): return res def _child_get(self, cr, name=None, domain=None): - return super(node_dir,self)._child_get(cr, name, self.dir_id, domain=domain) + dirobj = self.context._dirobj + uid = self.context.uid + ctx = self.context.context.copy() + ctx.update(self.dctx) + where = [('parent_id','=',self.dir_id)] + if name: + where.append(('name','=',name)) + is_allowed = self.check_perms(1) + else: + is_allowed = self.check_perms(5) + + if not is_allowed: + raise IOError(errno.EPERM, "Permission into directory denied") + + if not domain: + domain = [] + + where2 = where + domain + [('ressource_parent_type_id','=',False)] + ids = dirobj.search(cr, uid, where2, context=ctx) + res = [] + for dirr in dirobj.browse(cr, uid, ids, context=ctx): + klass = dirr.get_node_class(dirr, context=ctx) + res.append(klass(dirr.name, self, self.context,dirr)) + + # Static directories should never return files with res_model/res_id + # because static dirs are /never/ related to a record. + # In fact, files related to some model and parented by the root dir + # (the default), will NOT be accessible in the node system unless + # a resource folder for that model exists (with resource_find_all=True). + # Having resource attachments in a common folder is bad practice, + # because they would be visible to all users, and their names may be + # the same, conflicting. + where += [('res_model', '=', False)] + fil_obj = dirobj.pool.get('ir.attachment') + ids = fil_obj.search(cr, uid, where, context=ctx) + if ids: + for fil in fil_obj.browse(cr, uid, ids, context=ctx): + klass = self.context.node_file_class + res.append(klass(fil.name, self, self.context, fil)) + return res def rmcol(self, cr): uid = self.context.uid @@ -963,7 +993,7 @@ class node_res_obj(node_class): continue # TODO Revise klass = directory.get_node_class(directory, dynamic=True, context=ctx) - res.append(klass(res_name, self.dir_id, self, self.context, self.res_model, res_bo = bo)) + res.append(klass(res_name, dir_id=self.dir_id, parent=self, context=self.context, res_model=self.res_model, res_bo=bo)) where2 = where + [('parent_id','=',self.dir_id) ] From c3b4dba6713b7dc987f5e4a1bc4780392bd4903a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:23 +0300 Subject: [PATCH 07/59] Document: by default N/I functions raise IO error. Attempting to create a file or folder at nodes that don't support it, should be reported as a permission error. bzr revid: p_christ@hol.gr-20101027102323-3l15biqitaryqelb --- addons/document/nodes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 089cde3e41a..d32ccf49440 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -366,12 +366,14 @@ class node_class(object): def create_child(self, cr, path, data=None): """ Create a regular file under this node """ - raise NotImplementedError(repr(self)) + logger.warning("Attempted to create a file under %r, not possible.", self) + raise IOError(errno.EPERM, "Not allowed to create files here") def create_child_collection(self, cr, objname): """ Create a child collection (directory) under self """ - raise NotImplementedError(repr(self)) + logger.warning("Attempted to create a collection under %r, not possible.", self) + raise IOError(errno.EPERM, "Not allowed to create folders here") def rm(self, cr): raise NotImplementedError(repr(self)) From d1ee286bcb7f73daa856ef96ae31836d6e8fcd9d Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:23 +0300 Subject: [PATCH 08/59] doc ftp: yaml tests for files/folders in wrong dirs. bzr revid: p_christ@hol.gr-20101027102323-jtiwjjr52lht0dvr --- .../document_ftp/test/document_ftp_test4.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/addons/document_ftp/test/document_ftp_test4.yml b/addons/document_ftp/test/document_ftp_test4.yml index 2bb616dc1bd..2441be3bf92 100644 --- a/addons/document_ftp/test/document_ftp_test4.yml +++ b/addons/document_ftp/test/document_ftp_test4.yml @@ -113,6 +113,35 @@ - parent_id.name == 'Documents' - res_model == 'res.partner' - res_id != False +- + I try to create a file directly under the Partners Testing folder +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + import ftplib + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Partners Testing') + fdata = StringIO('abcd') + try: + ftp.storbinary('STOR stray.txt', fdata) + assert False, "We should't be able to create files here" + except ftplib.error_perm: + # That's what should happen + pass +- + I try to create a folder directly under the Partners Testing folder +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + import ftplib + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Partners Testing') + try: + ftp.mkd('Weird folder') + assert False, "We should't be able to create folders here" + except ftplib.error_perm: + # That's what should happen + pass - I check that all/Partner1 also has the file - | From da5aaac52c3e8e397b33b85a23f5454172ca3c43 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:23 +0300 Subject: [PATCH 09/59] document: minor improvement at nodes. bzr revid: p_christ@hol.gr-20101027102323-dmy92wbks216awe6 --- addons/document/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index d32ccf49440..3a3e6fab54b 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -985,9 +985,9 @@ class node_res_obj(node_class): where1 = [] if obj._parent_name in obj.fields_get(cr, uid): where1 = where + [(obj._parent_name, '=', self.res_id)] + namefield = directory.resource_field.name or 'name' resids = obj.search(cr, uid, where1, context=ctx) for bo in obj.browse(cr, uid, resids, context=ctx): - namefield = directory.resource_field.name or 'name' if not bo: continue res_name = getattr(bo, namefield) From 7539d4051c3236bb33cae36917ab4d39716db120 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:23 +0300 Subject: [PATCH 10/59] document: fix recursive (tree) resource directories The 'node_res_obj' has been the right class to use for the intermediate nodes, ever since. The assertion that it should only apply to static dirs was wrong. bzr revid: p_christ@hol.gr-20101027102323-qewjjzzudj0zz643 --- addons/document/document_directory.py | 1 - addons/document_webdav/document_webdav.py | 1 - 2 files changed, 2 deletions(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index c4df8a7cf18..907a3aaccad 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -190,7 +190,6 @@ class document_directory(osv.osv): dbro = self.browse(cr, uid, ids, context=context) if dynamic: - assert dbro.type == 'directory' return nodes.node_res_obj elif dbro.type == 'directory': return nodes.node_dir diff --git a/addons/document_webdav/document_webdav.py b/addons/document_webdav/document_webdav.py index 28b70c8d7e4..2e1eb8cabd2 100644 --- a/addons/document_webdav/document_webdav.py +++ b/addons/document_webdav/document_webdav.py @@ -37,7 +37,6 @@ class document_davdir(osv.osv): dbro = self.browse(cr, uid, ids, context=context) if dynamic: - assert dbro.type == 'directory' return nodes.node_res_obj elif dbro.type == 'directory': return nodes.node_dir From b4a630ec007ffd394c1f20262015d118d2b7ec9c Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:24 +0300 Subject: [PATCH 11/59] caldav: Prevent the collections views from replacing doc.dir views. bzr revid: p_christ@hol.gr-20101027102324-ki28alynxaw3m8t4 --- addons/caldav/caldav_view.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/addons/caldav/caldav_view.xml b/addons/caldav/caldav_view.xml index 90b489ac106..1b83ce105f0 100644 --- a/addons/caldav/caldav_view.xml +++ b/addons/caldav/caldav_view.xml @@ -4,6 +4,7 @@ Calendar Collections : Form document.directory + 40 form
@@ -18,6 +19,7 @@ Calendar Collections : Tree document.directory tree + 40 From 0f5a276365e9fd06306466b82938bfd3e5a5998f Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:24 +0300 Subject: [PATCH 12/59] document: use 'change_default' on key fields, cleanup defaults As from a later commit, we will need to adjust default values to different directories (such as permissions etc.). bzr revid: p_christ@hol.gr-20101027102324-duffxdogcczqo67r --- addons/document/document.py | 4 ++-- addons/document/document_directory.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/addons/document/document.py b/addons/document/document.py index 470d67c582b..604d18f861c 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -74,7 +74,7 @@ class document_file(osv.osv): 'create_uid': fields.many2one('res.users', 'Creator', readonly=True), 'write_date': fields.datetime('Date Modified', readonly=True), 'write_uid': fields.many2one('res.users', 'Last Modification User', readonly=True), - 'res_model': fields.char('Attached Model', size=64, readonly=True), + 'res_model': fields.char('Attached Model', size=64, readonly=True, change_default=True), 'res_id': fields.integer('Attached ID', readonly=True), # If ir.attachment contained any data before document is installed, preserve @@ -86,7 +86,7 @@ class document_file(osv.osv): 'user_id': fields.many2one('res.users', 'Owner', select=1), # 'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'), # the directory id now is mandatory. It can still be computed automatically. - 'parent_id': fields.many2one('document.directory', 'Directory', select=1, required=True), + 'parent_id': fields.many2one('document.directory', 'Directory', select=1, required=True, change_default=True), 'index_content': fields.text('Indexed Content'), 'partner_id':fields.many2one('res.partner', 'Partner', select=1), 'file_size': fields.integer('File Size', required=True), diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index 907a3aaccad..e75d1861201 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -38,9 +38,9 @@ class document_directory(osv.osv): 'create_uid': fields.many2one('res.users', 'Creator', readonly=True), 'domain': fields.char('Domain', size=128, help="Use a domain if you want to apply an automatic filter on visible resources."), 'user_id': fields.many2one('res.users', 'Owner'), - 'storage_id': fields.many2one('document.storage', 'Storage'), + 'storage_id': fields.many2one('document.storage', 'Storage', change_default=True), 'group_ids': fields.many2many('res.groups', 'document_directory_group_rel', 'item_id', 'group_id', 'Groups'), - 'parent_id': fields.many2one('document.directory', 'Parent Directory', select=1), + 'parent_id': fields.many2one('document.directory', 'Parent Directory', select=1, change_default=True), 'child_ids': fields.one2many('document.directory', 'parent_id', 'Children'), 'file_ids': fields.one2many('ir.attachment', 'parent_id', 'Files'), 'content_ids': fields.one2many('document.directory.content', 'directory_id', 'Virtual Files'), @@ -48,15 +48,15 @@ class document_directory(osv.osv): ('directory','Static Directory'), ('ressource','Folders per resource'), ], - 'Type', required=True, select=1, + 'Type', required=True, select=1, change_default=True, help="Each directory can either have the type Static or be linked to another resource. A static directory, as with Operating Systems, is the classic directory that can contain a set of files. The directories linked to systems resources automatically possess sub-directories for each of resource types defined in the parent directory."), - 'ressource_type_id': fields.many2one('ir.model', 'Resource model', + 'ressource_type_id': fields.many2one('ir.model', 'Resource model', change_default=True, help="Select an object here and there will be one folder per record of that resource."), 'resource_field': fields.many2one('ir.model.fields', 'Name field', help='Field to be used as name on resource directories. If empty, the "name" will be used.'), 'resource_find_all': fields.boolean('Find all resources', required=True, help="If true, all attachments that match this resource will " \ " be located. If false, only ones that have this as parent." ), - 'ressource_parent_type_id': fields.many2one('ir.model', 'Parent Model', + 'ressource_parent_type_id': fields.many2one('ir.model', 'Parent Model', change_default=True, help="If you put an object here, this directory template will appear bellow all of these objects. " \ "Such directories are \"attached\" to the specific model or record, just like attachments. " \ "Don't put a parent directory if you select a parent model."), @@ -65,7 +65,7 @@ class document_directory(osv.osv): 'ressource_tree': fields.boolean('Tree Structure', help="Check this if you want to use the same tree structure as the object selected in the system."), 'dctx_ids': fields.one2many('document.directory.dctx', 'dir_id', 'Context fields'), - 'company_id': fields.many2one('res.company', 'Company'), + 'company_id': fields.many2one('res.company', 'Company', change_default=True), } @@ -100,10 +100,10 @@ class document_directory(osv.osv): _defaults = { 'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'document.directory', context=c), 'user_id': lambda self,cr,uid,ctx: uid, - 'domain': lambda self,cr,uid,ctx: '[]', - 'type': lambda *args: 'directory', - 'ressource_id': lambda *a: 0, - 'storage_id': _get_def_storage, + 'domain': '[]', + 'type': 'directory', + 'ressource_id': 0, + 'storage_id': _get_def_storage, # Still, it is bad practice to set it everywhere. 'resource_find_all': True, } _sql_constraints = [ From 24d70f724b5f2cb01e03661957dfbd5d9cbced3a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:24 +0300 Subject: [PATCH 13/59] document: access permissions documentation, draft. bzr revid: p_christ@hol.gr-20101027102324-zcvyhz13iw75tb0k --- addons/document/doc/access_permissions.rst | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 addons/document/doc/access_permissions.rst diff --git a/addons/document/doc/access_permissions.rst b/addons/document/doc/access_permissions.rst new file mode 100644 index 00000000000..2cdc45dadab --- /dev/null +++ b/addons/document/doc/access_permissions.rst @@ -0,0 +1,65 @@ +Access control in the Document Management system +================================================ + +The purpose is to let the DMS act as a real-life management system for +the file handling of some small business. +The key concept, there, is the separation of access according to users +and groups. + +Fact 1: Users, in general, must NOT see each other's documents, not even +their names (because they usually imply sensitive data, like eg. a doc: +"Acme Company's Patent 012356 about fooobars.odf" ) +Fact 2: Users, sometimes, fail to comprehend complex ACL schemes, so we +owe to keep things simple, a main principle applied all over the place. +Fact 3: our system has both statically placed files and directories, as +well as dynamic (aka "resources" in our terminology) nodes. + +We allow/limit the access based on 3 factors (fields): + - The "owner" field, which holds the user that created or "owns" the + file or directory. + - The "group_ids" field, on directories, which specifieds group-wise + access + - The "company_id" field, for multi-company access rules [1] + +[1] at multi-company setups, we may want the same file hierarchy to apply +to different companies, and also nodes to be company-specific in it. + +Principle of "owner" +---------------------- +Files or directories that have an empty "owner" field are public. All users +will be able to _read_ them. Only the OpenERP Administrator or specified +groups, however, will be able to modify them! +Files or directories that have an "owner" are private. Only their users will +be able to read or modify (including delete) them. +By default, all user's files are created with "owner" field set, thus private. + +Principle of "group_ids" +------------------------- +Directories that have any group ids set will only (apart from their owner) +allow members of these groups to read them. +Directories that are created into the above directories will initially inherit +(that is, copy) the group_ids of their parents, so that they also allow +access to the same users. + +Implementation note +--------------------- +Most of the principles are applied through record rules (see ir.rule object), +so an administrator can actually readjust them. +In order to have logical "areas" of folders, where different policies apply +(like group folders, personal areas), default values for directories' owners +and group_ids can be tuned (through the 'set default' functionality of +fields). + +Summary +-------- + +Table of permissions and behavior + +|| Type | Owner set | Groups set | Description || +|| Public | - | - | Public-readable folders, admin can write || +|| Group | - | X | Group can read, write, delete in them || +|| Group-read | X | X | Group can read[2], owner can write/delete || +|| Private | X | - | Only owner can read, write, delete in. || + +[2] hint: using a wide group, like "Internal users" at this setup creates the + effect of public-readable folders, with write permission to a non-admin user. From d0b6bf11812fd80812aef2ad79e297a7dc8d537a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:23:24 +0300 Subject: [PATCH 14/59] document: richer eval() context for res_dirs. We need to provide 'ctx' to the eval(), so that "uid" is defined in the expression. Covers the case of user directories. bzr revid: p_christ@hol.gr-20101027102324-ujy2293kpdzouna2 --- addons/document/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 3a3e6fab54b..eed424ccb6a 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -791,7 +791,7 @@ class node_res_dir(node_class): ctx.update(self.dctx) where = [] if self.domain: - app = safe_eval(self.domain, self.dctx) + app = safe_eval(self.domain, ctx) if not app: pass elif isinstance(app, list): From 4dae4d0dbfc0d9740e36475b248dab181c8f1aae Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:24:28 +0300 Subject: [PATCH 15/59] document: base the dir permissions on ir.rule, not on sql call. bzr revid: p_christ@hol.gr-20101027102428-6n2huvvv2o1tiyh6 --- addons/document/document_directory.py | 21 ++++++++++----------- addons/document/nodes.py | 16 +++++++++++++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index e75d1861201..82f53b6120b 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -21,6 +21,7 @@ from osv import osv, fields +from osv.orm import except_orm import os import nodes @@ -212,22 +213,20 @@ class document_directory(osv.osv): """ return - def get_dir_permissions(self, cr, uid, ids ): + def get_dir_permissions(self, cr, uid, ids, context=None ): """Check what permission user 'uid' has on directory 'id' """ assert len(ids) == 1 id = ids[0] - cr.execute( "SELECT count(dg.item_id) AS needs, count(ug.uid) AS has " \ - " FROM document_directory_group_rel dg " \ - " LEFT OUTER JOIN res_groups_users_rel ug " \ - " ON (dg.group_id = ug.gid AND ug.uid = %s) " \ - " WHERE dg.item_id = %s ", (uid, id)) - needs, has = cr.fetchone() - if needs and not has: - return 1 # still allow to descend into. - else: - return 15 + res = 0 + for pperms in [('read', 5), ('write', 2), ('unlink', 8)]: + try: + self.check_access_rule(cr, uid, ids, pperms[0], context=context) + res |= pperms[1] + except except_orm: + pass + return res def _locate_child(self, cr, uid, root_id, uri,nparent, ncontext): """ try to locate the node in uri, diff --git a/addons/document/nodes.py b/addons/document/nodes.py index eed424ccb6a..190e0d9c3ce 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -478,6 +478,16 @@ def mkdosname(company_name, default='noname'): return n +def _uid2unixperms(perms, has_owner): + """ Convert the uidperms and the owner flag to full unix bits + """ + res = 0 + res |= (perms & 0x07) << 6 + res |= (perms & 0x05) << 3 + if not has_owner: + res |= 0x05 + return res + class node_dir(node_database): our_type = 'collection' def __init__(self, path, parent, context, dirr, dctx=None): @@ -492,13 +502,13 @@ class node_dir(node_database): # TODO: the write date should be MAX(file.write).. self.write_date = dirr and (dirr.write_date or dirr.create_date) or False self.content_length = 0 - self.unixperms = 040750 try: self.uuser = (dirr.user_id and dirr.user_id.login) or 'nobody' except Exception: self.uuser = 'nobody' self.ugroup = mkdosname(dirr.company_id and dirr.company_id.name, default='nogroup') self.uidperms = dirr.get_dir_permissions() + self.unixperms = 040000 | _uid2unixperms(self.uidperms, dirr and dirr.user_id) if dctx: self.dctx.update(dctx) dc2 = self.context.context @@ -731,13 +741,13 @@ class node_res_dir(node_class): # TODO: the write date should be MAX(file.write).. self.write_date = dirr.write_date or dirr.create_date self.content_length = 0 - self.unixperms = 040750 try: self.uuser = (dirr.user_id and dirr.user_id.login) or 'nobody' except Exception: self.uuser = 'nobody' self.ugroup = mkdosname(dirr.company_id and dirr.company_id.name, default='nogroup') self.uidperms = dirr.get_dir_permissions() + self.unixperms = 040000 | _uid2unixperms(self.uidperms, dirr and dirr.user_id) self.res_model = dirr.ressource_type_id and dirr.ressource_type_id.model or False self.resm_id = dirr.ressource_id self.res_find_all = dirr.resource_find_all @@ -855,8 +865,8 @@ class node_res_obj(node_class): # TODO: the write date should be MAX(file.write).. self.write_date = parent.write_date self.content_length = 0 - self.unixperms = 040750 self.uidperms = parent.uidperms & 15 + self.unixperms = 040000 | _uid2unixperms(self.uidperms, True) self.uuser = parent.uuser self.ugroup = parent.ugroup self.res_model = res_model From 9624ef2907b34464383f857b351144eb45afb346 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:24:28 +0300 Subject: [PATCH 16/59] document: ACL setup, based on ir.rules bzr revid: p_christ@hol.gr-20101027102428-pyanpwxzhta3zqy3 --- addons/document/document_data.xml | 20 ++++++------- .../document/security/document_security.xml | 24 ++++++++++++++- addons/document/security/ir.model.access.csv | 1 + .../document_ftp/test/document_ftp_test5.yml | 30 +++++++++++++++++++ 4 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 addons/document_ftp/test/document_ftp_test5.yml diff --git a/addons/document/document_data.xml b/addons/document/document_data.xml index ef829b6e258..a4f546cbf83 100644 --- a/addons/document/document_data.xml +++ b/addons/document/document_data.xml @@ -16,13 +16,13 @@ Documents - + 0 - My Folder + Admin Folder 0 @@ -37,7 +37,7 @@ 0 - + @@ -46,7 +46,7 @@ [('category_id','in',[active_id])] - + 0 @@ -55,14 +55,14 @@ Personal Folders ressource + 0 - Products - + 0 @@ -70,7 +70,7 @@ Sales Order - + 0 @@ -78,7 +78,7 @@ All Sales Order - + 0 @@ -86,7 +86,7 @@ Quotations - + 0 @@ -94,7 +94,7 @@ Projects - + 0 diff --git a/addons/document/security/document_security.xml b/addons/document/security/document_security.xml index 2bf1a2adb57..e1a60165150 100644 --- a/addons/document/security/document_security.xml +++ b/addons/document/security/document_security.xml @@ -14,6 +14,28 @@ - + + + ['|',('user_id', '=', False), ('user_id', '=', user.id), '|', ('group_ids','=',False), ('group_ids','in',user.groups_id), '|', ('company_id','=',False), ('company_id','child_of',[user.company_id.id])] + Read public directories + + + + + + + + + + + [('user_id', '=', user.id), '|', ('group_ids','=',False), ('group_ids','in',user.groups_id), '|', ('company_id','=',False), ('company_id','child_of',[user.company_id.id]) ] + Document modify own directories + + + + + + + diff --git a/addons/document/security/ir.model.access.csv b/addons/document/security/ir.model.access.csv index 132daa1b7a6..d7aba7cbde7 100644 --- a/addons/document/security/ir.model.access.csv +++ b/addons/document/security/ir.model.access.csv @@ -1,6 +1,7 @@ "id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" "access_document_directory_all","document.directory all","model_document_directory",,1,0,0,0 "access_document_directory_group_document_manager","document.directory document manager","model_document_directory","base.group_system",1,1,1,1 +"access_document_directory_group_knowledge","document.directory modify","model_document_directory","base.group_document_user",1,1,1,1 "access_document_directory_group_system","document.directory group system","model_document_directory","base.group_system",1,1,1,1 "access_document_directory_content_all","document.directory.content all","model_document_directory_content",,1,0,0,0 "access_document_directory_content_group_document_manager","document.directory.content document manager","model_document_directory_content","base.group_system",1,1,1,1 diff --git a/addons/document_ftp/test/document_ftp_test5.yml b/addons/document_ftp/test/document_ftp_test5.yml new file mode 100644 index 00000000000..a66f409905b --- /dev/null +++ b/addons/document_ftp/test/document_ftp_test5.yml @@ -0,0 +1,30 @@ +- + In order to check the permissions setup and functionality of the + document module: +- + I create a testing user for the documents +- + I assign some ... group to the testing user +- + I create a "group testing" user, which also belongs to the same ... group +- + I create a "blocked" user. +- + I create (as root) a testing folder in the document hierarchy, and + assign ownership to the testing user, groups to the ... group. +- + I create a "private" folder inside the testing folder. +- + I try to read the testing folder as the testing user +- + I try to read the folder as the group user, it should fail. +- + I try to read the folder as the blocked user. +- + I create a "group" folder, with the ... group. +- + I try to read the "group" folder as the testing user +- + I try to read the "group" folder as the group user +- + I try to read the "group" folder as the blocked user From cccdc06d06d1c81f0f91b4d4f7f64f5aaa6ae245 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 27 Oct 2010 13:24:29 +0300 Subject: [PATCH 17/59] webdav, caldav: Let folders be public, restrict users' folders. bzr revid: p_christ@hol.gr-20101027102429-yzwaf3qh3e97k7po --- addons/caldav/caldav_setup.xml | 12 +++++++----- addons/document_webdav/webdav_setup.xml | 11 ++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/addons/caldav/caldav_setup.xml b/addons/caldav/caldav_setup.xml index c33edbc75f7..e2e9d7eae5a 100644 --- a/addons/caldav/caldav_setup.xml +++ b/addons/caldav/caldav_setup.xml @@ -6,7 +6,7 @@ [] - + directory @@ -17,6 +17,7 @@ [] + directory @@ -24,10 +25,10 @@ resources - [] + [('id','=',uid)] - + ressource @@ -36,9 +37,10 @@ __uids__ - [] + [('id','=',uid)] + @@ -56,7 +58,7 @@ [] - + diff --git a/addons/document_webdav/webdav_setup.xml b/addons/document_webdav/webdav_setup.xml index 2bd41ddb69c..6ea894b6a22 100644 --- a/addons/document_webdav/webdav_setup.xml +++ b/addons/document_webdav/webdav_setup.xml @@ -12,7 +12,7 @@ [] - + directory @@ -24,7 +24,7 @@ [] - + @@ -45,9 +45,10 @@ resources - [] + [('id','=',uid)] + @@ -58,10 +59,10 @@ __uids__ - [] + [('id','=',uid)] - + From 4cc8404d364911c3cf754c335c7cacceea39abbb Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:34 +0200 Subject: [PATCH 18/59] doc webdav: copy code from http-client.py, for testing bzr revid: p_christ@hol.gr-20101103112534-albm51p5cfidisop --- addons/document_webdav/test_davclient.py | 381 +++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 addons/document_webdav/test_davclient.py diff --git a/addons/document_webdav/test_davclient.py b/addons/document_webdav/test_davclient.py new file mode 100644 index 00000000000..c06eb0cec68 --- /dev/null +++ b/addons/document_webdav/test_davclient.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# Copyright P. Christeas 2008,2009 +# Copyright OpenERP SA. (http://www.openerp.com) 2010 +# +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +############################################################################### + +""" A trivial HTTP/WebDAV client, used for testing the server +""" +# code taken from the 'http-client.py' script: +# http://git.hellug.gr/?p=xrg/openerp;a=history;f=tests/http-client.py;hb=refs/heads/xrg-60 + +import imp +import sys +import os +import glob +import subprocess +import re +import gzip +import logging +import xml.dom.minidom + +import httplib + +# from xmlrpclib import Transport + +log = logging.getLogger('http-client') + +class HTTP11(httplib.HTTP): + _http_vsn = 11 + _http_vsn_str = 'HTTP/1.1' + +class PersistentTransport(Transport): + """Handles an HTTP transaction to an XML-RPC server, persistently.""" + + def __init__(self, use_datetime=0): + self._use_datetime = use_datetime + self._http = {} + log.debug("Using persistent transport") + + def make_connection(self, host): + # create a HTTP connection object from a host descriptor + if not self._http.has_key(host): + host, extra_headers, x509 = self.get_host_info(host) + self._http[host] = HTTP11(host) + log.debug("New connection to %s", host) + return self._http[host] + + def get_host_info(self, host): + host, extra_headers, x509 = Transport.get_host_info(self,host) + if extra_headers == None: + extra_headers = [] + + extra_headers.append( ( 'Connection', 'keep-alive' )) + + return host, extra_headers, x509 + + def _parse_response(self, file, sock, response): + """ read response from input file/socket, and parse it + We are persistent, so it is important to only parse + the right amount of input + """ + + p, u = self.getparser() + + if response.msg.get('content-encoding') == 'gzip': + gzdata = StringIO.StringIO() + while not response.isclosed(): + rdata = response.read(1024) + if not rdata: + break + gzdata.write(rdata) + gzdata.seek(0) + rbuffer = gzip.GzipFile(mode='rb', fileobj=gzdata) + while True: + respdata = rbuffer.read() + if not respdata: + break + p.feed(respdata) + else: + while not response.isclosed(): + rdata = response.read(1024) + if not rdata: + break + p.feed(rdata) + if len(rdata)<1024: + break + + p.close() + return u.close() + + def request(self, host, handler, request_body, verbose=0): + # issue XML-RPC request + + h = self.make_connection(host) + if verbose: + h.set_debuglevel(1) + + self.send_request(h, handler, request_body) + self.send_host(h, host) + self.send_user_agent(h) + self.send_content(h, request_body) + + resp = h._conn.getresponse() + # TODO: except BadStatusLine, e: + + errcode, errmsg, headers = resp.status, resp.reason, resp.msg + + + if errcode != 200: + raise ProtocolError( + host + handler, + errcode, errmsg, + headers + ) + + self.verbose = verbose + + try: + sock = h._conn.sock + except AttributeError: + sock = None + + return self._parse_response(h.getfile(), sock, resp) + +class CompressedTransport(PersistentTransport): + def send_content(self, connection, request_body): + connection.putheader("Content-Type", "text/xml") + + if len(request_body) > 512 or True: + buffer = StringIO.StringIO() + output = gzip.GzipFile(mode='wb', fileobj=buffer) + output.write(request_body) + output.close() + buffer.seek(0) + request_body = buffer.getvalue() + connection.putheader('Content-Encoding', 'gzip') + + connection.putheader("Content-Length", str(len(request_body))) + connection.putheader("Accept-Encoding",'gzip') + connection.endheaders() + if request_body: + connection.send(request_body) + + def send_request(self, connection, handler, request_body): + connection.putrequest("POST", handler, skip_accept_encoding=1) + +class SafePersistentTransport(PersistentTransport): + def make_connection(self, host): + # create a HTTP connection object from a host descriptor + if not self._http.has_key(host): + host, extra_headers, x509 = self.get_host_info(host) + self._http[host] = httplib.HTTPS(host, None, **(x509 or {})) + log.debug("New connection to %s", host) + return self._http[host] + +class AuthClient(object): + def getAuth(self, atype, realm): + raise NotImplementedError("Cannot authenticate for %s" % atype) + + def resolveFailedRealm(self, realm): + """ Called when, using a known auth type, the realm is not in cache + """ + raise NotImplementedError("Cannot authenticate for realm %s" % realm) + +class BasicAuthClient(AuthClient): + def __init__(self): + self._realm_dict = {} + + def getAuth(self, atype, realm): + if atype != 'Basic' : + return super(BasicAuthClient,self).getAuth(atype, realm) + + if not self._realm_dict.has_key(realm): + log.debug("realm dict: %r", self._realm_dict) + log.debug("missing key: \"%s\"" % realm) + self.resolveFailedRealm(realm) + return 'Basic '+ self._realm_dict[realm] + + def addLogin(self, realm, username, passwd): + """ Add some known username/password for a specific login. + This function should be called once, for each realm + that we want to authenticate against + """ + assert realm + auths = base64.encodestring(username + ':' + passwd) + if auths[-1] == "\n": + auths = auths[:-1] + self._realm_dict[realm] = auths + +class addAuthTransport: + """ Intermediate class that authentication algorithm to http transport + """ + + def setAuthClient(self, authobj): + """ Set the authentication client object. + This method must be called before any request is issued, that + would require http authentication + """ + assert isinstance(authobj, AuthClient) + self._auth_client = authobj + + + def request(self, host, handler, request_body, verbose=0): + # issue XML-RPC request + + h = self.make_connection(host) + if verbose: + h.set_debuglevel(1) + + tries = 0 + atype = None + realm = None + + while(tries < 3): + self.send_request(h, handler, request_body) + self.send_host(h, host) + self.send_user_agent(h) + if atype: + # This line will bork if self.setAuthClient has not + # been issued. That is a programming error, fix your code! + auths = self._auth_client.getAuth(atype, realm) + log.debug("sending authorization: %s", auths) + h.putheader('Authorization', auths) + self.send_content(h, request_body) + + resp = h._conn.getresponse() + # except BadStatusLine, e: + tries += 1 + + if resp.status == 401: + if 'www-authenticate' in resp.msg: + (atype,realm) = resp.msg.getheader('www-authenticate').split(' ',1) + data1 = resp.read() + if realm.startswith('realm="') and realm.endswith('"'): + realm = realm[7:-1] + log.debug("Resp: %r %r", resp.version,resp.isclosed(), resp.will_close) + log.debug("Want to do auth %s for realm %s", atype, realm) + if atype != 'Basic': + raise ProtocolError(host+handler, 403, + "Unknown authentication method: %s" % atype, resp.msg) + continue # with the outer while loop + else: + raise ProtocolError(host+handler, 403, + 'Server-incomplete authentication', resp.msg) + + if resp.status != 200: + raise ProtocolError( host + handler, + resp.status, resp.reason, resp.msg ) + + self.verbose = verbose + + try: + sock = h._conn.sock + except AttributeError: + sock = None + + return self._parse_response(h.getfile(), sock, resp) + + raise ProtocolError(host+handler, 403, "No authentication",'') + +class PersistentAuthTransport(addAuthTransport,PersistentTransport): + pass + +class PersistentAuthCTransport(addAuthTransport,CompressedTransport): + pass + +class HTTPSConnection(httplib.HTTPSConnection): + certs_file = None + def connect(self): + "Connect to a host on a given (SSL) port. check the certificate" + import socket, ssl + + if HTTPSConnection.certs_file: + ca_certs = HTTPSConnection.certs_file + cert_reqs = ssl.CERT_REQUIRED + else: + ca_certs = None + cert_reqs = ssl.CERT_NONE + sock = socket.create_connection((self.host, self.port), self.timeout) + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, + ca_certs=ca_certs, + cert_reqs=cert_reqs) + + + def getpeercert(self): + import ssl + cert = None + if self.sock: + cert = self.sock.getpeercert() + else: + cert = ssl.get_server_certificate((self.host,self.port), + ssl_version=ssl.PROTOCOL_SSLv23 ) + lf = (len(ssl.PEM_FOOTER)+1) + if cert[0-lf] != '\n': + cert = cert[:0-lf]+'\n'+cert[0-lf:] + log.debug("len-footer: %s cert: %r", lf, cert[0-lf]) + + return cert + +def http_request(host, path, user=None, method='GET', hdrs=None, body=None, dbg=2): + if not hdrs: + hdrs = {} + passwd=None + if user: + import getpass + passwd = getpass.getpass("Password for %s@%s: " %(user,host)) + import base64 + log.debug("Getting %s http://%s/%s", method, host , path) + conn = httplib.HTTPConnection(host) + conn.set_debuglevel(dbg) + if not path: + path = "/index.html" + if not hdrs.has_key('Connection'): + hdrs['Connection']= 'keep-alive' + conn.request(method, path, body, hdrs ) + try: + r1 = conn.getresponse() + except httplib.BadStatusLine, bsl: + log.warning("Bad status line: %s", bsl.line) + return + if r1.status == 401: # and r1.headers: + if 'www-authenticate' in r1.msg: + (atype,realm) = r1.msg.getheader('www-authenticate').split(' ',1) + data1 = r1.read() + if not user: + raise Exception('Must auth, have no user/pass!') + log.debug("Ver: %s, closed: %s, will close: %s", r1.version,r1.isclosed(), r1.will_close) + log.debug("Want to do auth %s for realm %s", atype, realm) + if atype == 'Basic' : + auths = base64.encodestring(user + ':' + passwd) + if auths[-1] == "\n": + auths = auths[:-1] + hdrs['Authorization']= 'Basic '+ auths + #sleep(1) + conn.request(method, path, body, hdrs ) + r1 = conn.getresponse() + else: + raise Exception("Unknown auth type %s" %atype) + else: + log.warning("Got 401, cannot auth") + raise Exception('No auth') + + log.debug("Reponse: %s %s",r1.status, r1.reason) + data1 = r1.read() + did_print = False + log.debug("Body:\n%s\nEnd of body\n", data1) + try: + ctype = r1.msg.getheader('content-type') + if ctype and ';' in ctype: + ctype, encoding = ctype.split(';',1) + if ctype == 'text/xml': + doc = xml.dom.minidom.parseString(data1) + log.debug("XML Body:\n %s", doc.toprettyxml(indent="\t")) + did_print = True + except Exception, e: + log.warning("could not print xml", exc_info=True) + pass + conn.close() From 97a1f09deb2b97b80dbd3237492b54384d122145 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:34 +0200 Subject: [PATCH 19/59] doc webdav: form the testing class, implement options test. bzr revid: p_christ@hol.gr-20101103112534-868gjh5eqseb7voo --- addons/document_webdav/test/webdav_test1.yml | 12 +++ addons/document_webdav/test_davclient.py | 105 +++++++++++++++---- 2 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 addons/document_webdav/test/webdav_test1.yml diff --git a/addons/document_webdav/test/webdav_test1.yml b/addons/document_webdav/test/webdav_test1.yml new file mode 100644 index 00000000000..80abb838823 --- /dev/null +++ b/addons/document_webdav/test/webdav_test1.yml @@ -0,0 +1,12 @@ +- + In order to test the document_ftp functionality +- + I open the HTTP port and perform an OPTIONS request to the server +- + !python {model: ir.attachment}: | + from document_webdav import test_davclient as te + reload(te) # reload.. + dc = te.DAVClient() + dc.gd_options() + dc.gd_options(path=cr.dbname, expect={'DAV': ['1',]}) +- diff --git a/addons/document_webdav/test_davclient.py b/addons/document_webdav/test_davclient.py index c06eb0cec68..cafd6e36e2f 100644 --- a/addons/document_webdav/test_davclient.py +++ b/addons/document_webdav/test_davclient.py @@ -32,11 +32,6 @@ # code taken from the 'http-client.py' script: # http://git.hellug.gr/?p=xrg/openerp;a=history;f=tests/http-client.py;hb=refs/heads/xrg-60 -import imp -import sys -import os -import glob -import subprocess import re import gzip import logging @@ -44,7 +39,10 @@ import xml.dom.minidom import httplib -# from xmlrpclib import Transport +from tools import config +from xmlrpclib import Transport, ProtocolError +import StringIO +import base64 log = logging.getLogger('http-client') @@ -320,16 +318,41 @@ class HTTPSConnection(httplib.HTTPSConnection): return cert -def http_request(host, path, user=None, method='GET', hdrs=None, body=None, dbg=2): + +class DAVClient(object): + """An instance of a WebDAV client, connected to the OpenERP server + """ + + def __init__(self, user=None, passwd=None, dbg=0, use_ssl=False): + if use_ssl: + self.host = config.get_misc('httpsd', 'interface', False) + self.port = config.get_misc('httpsd', 'port', 8071) + if not self.host: + self.host = config.get('xmlrpcs_interface') + self.port = config.get('xmlrpcs_port') + else: + self.host = config.get_misc('httpd', 'interface') + self.port = config.get_misc('httpd', 'port', 8069) + if not self.host: + self.host = config.get('xmlrpc_interface') + self.port = config.get('xmlrpc_port') or self.port + if self.host == '0.0.0.0' or not self.host: + self.host = '127.0.0.1' + self.port = int(self.port) + if not config.get_misc('webdav','enable',True): + raise Exception("WebDAV is disabled, cannot continue") + self.davpath = '/' + config.get_misc('webdav','vdir','webdav') + self.user = user + self.passwd = passwd + self.dbg = dbg + + def _http_request(self, path, method='GET', hdrs=None, body=None): if not hdrs: hdrs = {} - passwd=None - if user: - import getpass - passwd = getpass.getpass("Password for %s@%s: " %(user,host)) import base64 - log.debug("Getting %s http://%s/%s", method, host , path) - conn = httplib.HTTPConnection(host) + dbg = self.dbg + log.debug("Getting %s http://%s:%d/%s", method, self.host, self.port, path) + conn = httplib.HTTPConnection(self.host, port=self.port) conn.set_debuglevel(dbg) if not path: path = "/index.html" @@ -340,17 +363,17 @@ def http_request(host, path, user=None, method='GET', hdrs=None, body=None, dbg= r1 = conn.getresponse() except httplib.BadStatusLine, bsl: log.warning("Bad status line: %s", bsl.line) - return + raise Exception('Bad status line') if r1.status == 401: # and r1.headers: if 'www-authenticate' in r1.msg: (atype,realm) = r1.msg.getheader('www-authenticate').split(' ',1) data1 = r1.read() - if not user: + if not self.user: raise Exception('Must auth, have no user/pass!') log.debug("Ver: %s, closed: %s, will close: %s", r1.version,r1.isclosed(), r1.will_close) log.debug("Want to do auth %s for realm %s", atype, realm) if atype == 'Basic' : - auths = base64.encodestring(user + ':' + passwd) + auths = base64.encodestring(self.user + ':' + self.passwd) if auths[-1] == "\n": auths = auths[:-1] hdrs['Authorization']= 'Basic '+ auths @@ -365,8 +388,7 @@ def http_request(host, path, user=None, method='GET', hdrs=None, body=None, dbg= log.debug("Reponse: %s %s",r1.status, r1.reason) data1 = r1.read() - did_print = False - log.debug("Body:\n%s\nEnd of body\n", data1) + log.debug("Body:\n%s\nEnd of body", data1) try: ctype = r1.msg.getheader('content-type') if ctype and ';' in ctype: @@ -374,8 +396,51 @@ def http_request(host, path, user=None, method='GET', hdrs=None, body=None, dbg= if ctype == 'text/xml': doc = xml.dom.minidom.parseString(data1) log.debug("XML Body:\n %s", doc.toprettyxml(indent="\t")) - did_print = True - except Exception, e: + except Exception: log.warning("could not print xml", exc_info=True) pass conn.close() + return r1.status, r1.msg, data1 + + def _assert_headers(self, expect, msg): + """ Assert that the headers in msg contain the expect values + """ + for k, v in expect.items(): + hval = msg.getheader(k) + if not hval: + raise AssertionError("Header %s not defined in http response" % k) + if isinstance(v, (list, tuple)): + delim = ',' + hits = map(str.strip, hval.split(delim)) + mvits= [] + for vit in v: + if vit not in hits: + mvits.append(vit) + if mvits: + raise AssertionError("HTTP header \"%s\" is missing: %s" %(k, ', '.join(mvits))) + else: + if hval.strip() != v.strip(): + raise AssertionError("HTTP header \"%s: %s\"" % (k, hval)) + + + + def gd_options(self, path='*', expect=None): + """ Test the http options functionality + If a dictionary is defined in expect, those options are + asserted. + """ + if path != '*': + path = self.davpath + path + hdrs = { 'Content-Length': 0 + } + s, m, d = self._http_request(path, method='OPTIONS', hdrs=hdrs) + assert s == 200, "Status: %r" % s + assert 'OPTIONS' in m.getheader('Allow') + log.debug('Options: %r', m.getheader('Allow')) + + if expect: + self._assert_headers(expect, m) + + + +#eof \ No newline at end of file From 23103d89cf40e39e5e966ad8353044b8e7182601 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:34 +0200 Subject: [PATCH 20/59] doc webdav: use credentials at tests bzr revid: p_christ@hol.gr-20101103112534-ig2s9ijq0qlrw4ug --- addons/document_webdav/test/webdav_test1.yml | 1 + addons/document_webdav/test_davclient.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/addons/document_webdav/test/webdav_test1.yml b/addons/document_webdav/test/webdav_test1.yml index 80abb838823..30d4ad3f8ed 100644 --- a/addons/document_webdav/test/webdav_test1.yml +++ b/addons/document_webdav/test/webdav_test1.yml @@ -8,5 +8,6 @@ reload(te) # reload.. dc = te.DAVClient() dc.gd_options() + dc.get_creds(self, cr, uid) dc.gd_options(path=cr.dbname, expect={'DAV': ['1',]}) - diff --git a/addons/document_webdav/test_davclient.py b/addons/document_webdav/test_davclient.py index cafd6e36e2f..c961faca18b 100644 --- a/addons/document_webdav/test_davclient.py +++ b/addons/document_webdav/test_davclient.py @@ -346,6 +346,22 @@ class DAVClient(object): self.passwd = passwd self.dbg = dbg + def get_creds(self, obj, cr, uid): + """Read back the user credentials from cr, uid + + @param obj is any orm object, in order to use its pool + @param uid is the numeric id, which we will try to reverse resolve + + note: this is a hackish way to get the credentials. It is expected + to break if "base_crypt" is used. + """ + ruob = obj.pool.get('res.users') + res = ruob.read(cr, 1, [uid,], ['login', 'password']) + assert res, "uid %s not found" % uid + self.user = res[0]['login'] + self.passwd = res[0]['password'] + return True + def _http_request(self, path, method='GET', hdrs=None, body=None): if not hdrs: hdrs = {} @@ -422,8 +438,6 @@ class DAVClient(object): if hval.strip() != v.strip(): raise AssertionError("HTTP header \"%s: %s\"" % (k, hval)) - - def gd_options(self, path='*', expect=None): """ Test the http options functionality If a dictionary is defined in expect, those options are From 872f6be3408fb4b7f86dd443f0a28194d83554fb Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:34 +0200 Subject: [PATCH 21/59] doc webdav: more tests in lib, yaml bzr revid: p_christ@hol.gr-20101103112534-7vdnxjxr9wb3736t --- addons/document_webdav/test/webdav_test1.yml | 26 +++ addons/document_webdav/test_davclient.py | 181 ++++++++++++++++++- 2 files changed, 206 insertions(+), 1 deletion(-) diff --git a/addons/document_webdav/test/webdav_test1.yml b/addons/document_webdav/test/webdav_test1.yml index 30d4ad3f8ed..c9384a5c324 100644 --- a/addons/document_webdav/test/webdav_test1.yml +++ b/addons/document_webdav/test/webdav_test1.yml @@ -11,3 +11,29 @@ dc.get_creds(self, cr, uid) dc.gd_options(path=cr.dbname, expect={'DAV': ['1',]}) - + I will test the propnames at the document root +- + !python {model: ir.attachment}: | + from document_webdav import test_davclient as te + dc = te.DAVClient() + dc.get_creds(self, cr, uid) + dc.gd_propname(path=cr.dbname+'/Documents/') +- + I will test the ETags of the document root +- + !python {model: ir.attachment}: | + from document_webdav import test_davclient as te + dc = te.DAVClient() + dc.get_creds(self, cr, uid) + dc.gd_getetag(path=cr.dbname+'/Documents/') + +- + I will now ls -l the document root. +- + !python {model: ir.attachment}: | + from document_webdav import test_davclient as te + dc = te.DAVClient() + dc.get_creds(self, cr, uid) + res = dc.gd_lsl(path=cr.dbname+'/Documents/') + for lin in res: + print "%(type)s\t%(uid)s\t%(gid)s\t%(size)s\t%(mtime)s\t%(name)s" % lin diff --git a/addons/document_webdav/test_davclient.py b/addons/document_webdav/test_davclient.py index c961faca18b..0af83f48f0e 100644 --- a/addons/document_webdav/test_davclient.py +++ b/addons/document_webdav/test_davclient.py @@ -323,7 +323,7 @@ class DAVClient(object): """An instance of a WebDAV client, connected to the OpenERP server """ - def __init__(self, user=None, passwd=None, dbg=0, use_ssl=False): + def __init__(self, user=None, passwd=None, dbg=0, use_ssl=False, useragent=False): if use_ssl: self.host = config.get_misc('httpsd', 'interface', False) self.port = config.get_misc('httpsd', 'port', 8071) @@ -345,6 +345,9 @@ class DAVClient(object): self.user = user self.passwd = passwd self.dbg = dbg + self.hdrs = {} + if useragent: + self.set_useragent(useragent) def get_creds(self, obj, cr, uid): """Read back the user credentials from cr, uid @@ -362,11 +365,25 @@ class DAVClient(object): self.passwd = res[0]['password'] return True + def set_useragent(self, uastr): + """ Set the user-agent header to something meaningful. + Some shorthand names will be replaced by stock strings. + """ + if uastr in ('KDE4', 'Korganizer'): + self.hdrs['User-Agent'] = "Mozilla/5.0 (compatible; Konqueror/4.4; Linux) KHTML/4.4.3 (like Gecko)" + elif uastr == 'iPhone3': + self.hdrs['User-Agent'] = "DAVKit/5.0 (765); iCalendar/5.0 (79); iPhone/4.1 8B117" + elif uastr == "MacOS": + self.hdrs['User-Agent'] = "WebDAVFS/1.8 (01808000) Darwin/9.8.0 (i386)" + else: + self.hdrs['User-Agent'] = uastr + def _http_request(self, path, method='GET', hdrs=None, body=None): if not hdrs: hdrs = {} import base64 dbg = self.dbg + hdrs.update(self.hdrs) log.debug("Getting %s http://%s:%d/%s", method, self.host, self.port, path) conn = httplib.HTTPConnection(self.host, port=self.port) conn.set_debuglevel(dbg) @@ -454,7 +471,169 @@ class DAVClient(object): if expect: self._assert_headers(expect, m) + + def _parse_prop_response(self, data): + """ Parse a propfind/propname response + """ + def getText(node): + rc = [] + for node in node.childNodes: + if node.nodeType == node.TEXT_NODE: + rc.append(node.data) + return ''.join(rc) + + def getElements(node, namespaces=None, strict=False): + for cnod in node.childNodes: + if cnod.nodeType != node.ELEMENT_NODE: + if strict: + log.debug("Found %r inside <%s>", cnod, node.tagName) + continue + if namespaces and (cnod.namespaceURI not in namespaces): + log.debug("Ignoring <%s> in <%s>", cnod.tagName, node.localName) + continue + yield cnod + + nod = xml.dom.minidom.parseString(data) + nod_r = nod.documentElement + res = {} + assert nod_r.localName == 'multistatus', nod_r.tagName + for resp in nod_r.getElementsByTagNameNS('DAV:', 'response'): + href = None + status = 200 + res_nss = {} + for cno in getElements(resp, namespaces=['DAV:',]): + if cno.localName == 'href': + assert href is None, "Second href in same response" + href = getText(cno) + elif cno.localName == 'propstat': + for pno in getElements(cno, namespaces=['DAV:',]): + rstatus = None + if pno.localName == 'prop': + for prop in getElements(pno): + key = prop.localName + tval = getText(prop).strip() + val = tval or (True, rstatus or status) + if prop.namespaceURI == 'DAV:' and prop.localName == 'resourcetype': + val = 'plain' + for rte in getElements(prop, namespaces=['DAV:',]): + # Note: we only look at DAV:... elements, we + # actually expect only one DAV:collection child + val = rte.localName + res_nss.setdefault(prop.namespaceURI,{})[key] = val + elif pno.localName == 'status': + rstr = getText(pno) + htver, sta, msg = rstr.split(' ', 3) + assert htver == 'HTTP/1.1' + rstatus = int(sta) + else: + log.debug("What is <%s> inside a ?", pno.tagName) + + else: + log.debug("Unknown node: %s", cno.tagName) + + res.setdefault(href,[]).append((status, res_nss)) + + return res + + def gd_propfind(self, path, props=None, depth=0): + if not props: + propstr = '' + else: + propstr = '' + nscount = 0 + for p in props: + ns = None + if isinstance(p, tuple): + p, ns = p + if ns is None or ns == 'DAV:': + propstr += '<%s/>' % p + else: + propstr += '' %(nscount, p, nscount, ns) + nscount += 1 + propstr += '' + + body=""" + %s""" % propstr + hdrs = { 'Content-Type': 'text/xml; charset=utf-8', + 'Accept': 'text/xml', + 'Depth': depth, + } + + s, m, d = self._http_request(self.davpath + path, method='PROPFIND', + hdrs=hdrs, body=body) + assert s == 207, "Bad status: %s" % s + ctype = m.getheader('Content-Type').split(';',1)[0] + assert ctype == 'text/xml', m.getheader('Content-Type') + res = self._parse_prop_response(d) + if depth == 0: + assert len(res) == 1 + res = res.values()[0] + else: + assert len(res) >= 1 + return res + def gd_propname(self, path, depth=0): + body=""" + """ + hdrs = { 'Content-Type': 'text/xml; charset=utf-8', + 'Accept': 'text/xml', + 'Depth': depth + } + s, m, d = self._http_request(self.davpath + path, method='PROPFIND', + hdrs=hdrs, body=body) + assert s == 207, "Bad status: %s" % s + ctype = m.getheader('Content-Type').split(';',1)[0] + assert ctype == 'text/xml', m.getheader('Content-Type') + res = self._parse_prop_response(d) + if depth == 0: + assert len(res) == 1 + res = res.values()[0] + else: + assert len(res) >= 1 + return res + + def gd_getetag(self, path, depth=0): + return self.gd_propfind(path, props=['getetag',], depth=depth) + + def gd_lsl(self, path): + """ Return a list of 'ls -l' kind of data for a folder + + This is based on propfind. + """ + + lspairs = [ ('name', 'displayname', 'n/a'), ('size', 'getcontentlength', '0'), + ('type', 'resourcetype', '----------'), ('uid', 'owner', 'nobody'), + ('gid', 'group', 'nogroup'), ('mtime', 'getlastmodified', 'n/a'), + ('mime', 'getcontenttype', 'application/data'), ] + + propnames = [ l[1] for l in lspairs] + propres = self.gd_propfind(path, props=propnames, depth=1) + + res = [] + for href, pr in propres.items(): + lsline = {} + for st, nsdic in pr: + davprops = nsdic['DAV:'] + if st == 200: + for lsp in lspairs: + if lsp[1] in davprops: + if lsp[1] == 'resourcetype': + if davprops[lsp[1]] == 'collection': + lsline[lsp[0]] = 'dr-xr-x---' + else: + lsline[lsp[0]] = '-r-xr-x---' + else: + lsline[lsp[0]] = davprops[lsp[1]] + elif st in (404, 403): + for lsp in lspairs: + if lsp[1] in davprops: + lsline[lsp[0]] = lsp[2] + else: + log.debug("Strange status: %s", st) + + res.append(lsline) + + return res #eof \ No newline at end of file From deffd9703d2a42b079db87dfff3633a63d268a9b Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:34 +0200 Subject: [PATCH 22/59] doc webdav: support for GET and PUT at yaml tests bzr revid: p_christ@hol.gr-20101103112534-9xyjp4xup0pjoxwm --- addons/document_webdav/test/webdav_test1.yml | 20 +++++ addons/document_webdav/test_davclient.py | 79 +++++++++++++++++--- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/addons/document_webdav/test/webdav_test1.yml b/addons/document_webdav/test/webdav_test1.yml index c9384a5c324..a64c2efae4c 100644 --- a/addons/document_webdav/test/webdav_test1.yml +++ b/addons/document_webdav/test/webdav_test1.yml @@ -37,3 +37,23 @@ res = dc.gd_lsl(path=cr.dbname+'/Documents/') for lin in res: print "%(type)s\t%(uid)s\t%(gid)s\t%(size)s\t%(mtime)s\t%(name)s" % lin +- + I will put a file to the server +- + !python {model: ir.attachment}: | + from document_webdav import test_davclient as te + import addons + dc = te.DAVClient() + dc.get_creds(self, cr, uid) + tdp = addons.get_module_resource('document_webdav', 'test_davclient.py') + res = dc.gd_put(path=cr.dbname+'/Documents/test_davclient.py', srcpath=tdp) +- + I will try to get the file from the root +- + !python {model: ir.attachment}: | + from document_webdav import test_davclient as te + import addons + dc = te.DAVClient() + dc.get_creds(self, cr, uid) + tdp = addons.get_module_resource('document_webdav', 'test_davclient.py') + res = dc.gd_get(path=cr.dbname+'/Documents/test_davclient.py', compare=tdp) diff --git a/addons/document_webdav/test_davclient.py b/addons/document_webdav/test_davclient.py index 0af83f48f0e..35c870838ec 100644 --- a/addons/document_webdav/test_davclient.py +++ b/addons/document_webdav/test_davclient.py @@ -252,6 +252,8 @@ class addAuthTransport: if 'www-authenticate' in resp.msg: (atype,realm) = resp.msg.getheader('www-authenticate').split(' ',1) data1 = resp.read() + if data1: + log.warning("Why have data on a 401 auth. message?") if realm.startswith('realm="') and realm.endswith('"'): realm = realm[7:-1] log.debug("Resp: %r %r", resp.version,resp.isclosed(), resp.will_close) @@ -421,17 +423,18 @@ class DAVClient(object): log.debug("Reponse: %s %s",r1.status, r1.reason) data1 = r1.read() - log.debug("Body:\n%s\nEnd of body", data1) - try: - ctype = r1.msg.getheader('content-type') - if ctype and ';' in ctype: - ctype, encoding = ctype.split(';',1) - if ctype == 'text/xml': - doc = xml.dom.minidom.parseString(data1) - log.debug("XML Body:\n %s", doc.toprettyxml(indent="\t")) - except Exception: - log.warning("could not print xml", exc_info=True) - pass + if method != 'GET': + log.debug("Body:\n%s\nEnd of body", data1) + try: + ctype = r1.msg.getheader('content-type') + if ctype and ';' in ctype: + ctype, encoding = ctype.split(';',1) + if ctype == 'text/xml': + doc = xml.dom.minidom.parseString(data1) + log.debug("XML Body:\n %s", doc.toprettyxml(indent="\t")) + except Exception: + log.warning("could not print xml", exc_info=True) + pass conn.close() return r1.status, r1.msg, data1 @@ -636,4 +639,58 @@ class DAVClient(object): return res + def gd_get(self, path, crange=None, mime=None, compare=None): + """ HTTP GET for path, supporting Partial ranges + """ + hdrs = { 'Accept': mime or '*/*', } + if crange: + if isinstance(crange, tuple): + crange = [crange,] + if not isinstance(crange, list): + raise TypeError("Range must be a tuple or list of tuples") + rs = [] + for r in crange: + rs.append('%d-%d' % r) + hdrs['Range'] = 'bytes='+ (','.join(rs)) + s, m, d = self._http_request(self.davpath + path, method='GET', hdrs=hdrs) + assert s in (200, 206), "Bad status: %s" % s + ctype = m.getheader('Content-Type').split(';',1)[0] + if mime: + assert ctype == mime, m.getheader('Content-Type') + rrange = None + rrh = m.getheader('Content-Range') + if rrh: + assert rrh.startswith('bytes '), rrh + rrh=rrh[6:].split('/',1)[0] + rrange = map(int, rrh.split('-',1)) + if compare: + # we need to compare the returned data with that of compare + fd = open(compare, 'rb') + d2 = fd.read() + fd.close() + assert d2 == d, "Data does not match" + return ctype, rrange, d + + def gd_put(self, path, body=None, srcpath=None, mime=None, noclobber=False, ): + """ HTTP PUT + @param noclobber will prevent overwritting a resource (If-None-Match) + @param mime will set the content-type + """ + hdrs = { } + if not (body or srcpath): + raise ValueError("PUT must have something to send") + if (not body) and srcpath: + fd = open(srcpath, 'rb') + body = fd.read() + fd.close() + if mime: + hdrs['Content-Type'] = mime + if noclobber: + hdrs['If-None-Match'] = '*' + s, m, d = self._http_request(self.davpath + path, method='PUT', + hdrs=hdrs, body=body) + assert s == (201), "Bad status: %s" % s + etag = m.getheader('ETag') + return etag or True + #eof \ No newline at end of file From 7f882e14cf2948a4fe4b2b2b34fa085e6dba4278 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:34 +0200 Subject: [PATCH 23/59] document: enhance API for iter() on node_descriptors Starting from WebDAV, the node_descriptors will need to act as stream objects, iterable. This way, we hope to minimize copies of the huge chunks of data each descriptor may contain (still, pywebdav will use a single large buffer). bzr revid: p_christ@hol.gr-20101103112534-go57aeaw946utfc5 --- addons/document/document_storage.py | 24 +++++++++++++++++++++--- addons/document/nodes.py | 14 ++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 0ecc01941b4..dd0247e52fb 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -48,7 +48,7 @@ DMS_ROOT_PATH = tools.config.get('document_path', os.path.join(tools.config.get( We have to consider 3 cases of data /retrieval/: Given (context,path) we need to access the file (aka. node). given (directory, context), we need one of its children (for listings, views) - given (ir.attachment, context), we needs its data and metadata (node). + given (ir.attachment, context), we need its data and metadata (node). For data /storage/ we have the cases: Have (ir.attachment, context), we modify the file (save, update, rename etc). @@ -100,10 +100,17 @@ class nodefd_file(nodes.node_descriptor): if mode.endswith('b'): mode = mode[:-1] self.mode = mode + self._size = os.stat(path).st_size - for attr in ('closed', 'read', 'write', 'seek', 'tell'): + for attr in ('closed', 'read', 'write', 'seek', 'tell', 'next'): setattr(self,attr, getattr(self.__file, attr)) + def size(self): + return self._size + + def __iter__(self): + return self + def close(self): # TODO: locking in init, close() fname = self.__file.name @@ -165,6 +172,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): """ def __init__(self, parent, ira_browse, mode): nodes.node_descriptor.__init__(self, parent) + self._size = 0L if mode.endswith('b'): mode = mode[:-1] @@ -172,6 +180,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): cr = ira_browse._cr # reuse the cursor of the browse object, just now cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,)) data = cr.fetchone()[0] + self._size = len(data) StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) @@ -184,6 +193,9 @@ class nodefd_db(StringIO, nodes.node_descriptor): raise IOError(errno.EINVAL, "Invalid file mode") self.mode = mode + def size(self): + return self._size + def close(self): # we now open a *separate* cursor, to update the data. # FIXME: this may be improved, for concurrency handling @@ -241,11 +253,14 @@ class nodefd_db64(StringIO, nodes.node_descriptor): """ def __init__(self, parent, ira_browse, mode): nodes.node_descriptor.__init__(self, parent) + self._size = 0L if mode.endswith('b'): mode = mode[:-1] if mode in ('r', 'r+'): - StringIO.__init__(self, base64.decodestring(ira_browse.db_datas)) + data = base64.decodestring(ira_browse.db_datas) + self._size = len(data) + StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) # at write, we start at 0 (= overwrite), but have the original @@ -257,6 +272,9 @@ class nodefd_db64(StringIO, nodes.node_descriptor): raise IOError(errno.EINVAL, "Invalid file mode") self.mode = mode + def size(self): + return self._size + def close(self): # we now open a *separate* cursor, to update the data. # FIXME: this may be improved, for concurrency handling diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 190e0d9c3ce..1a903614028 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -175,6 +175,15 @@ class node_descriptor(object): def write(self, str): raise NotImplementedError + def size(self): + raise NotImplementedError + + def __len__(self): + return self.size() + + def next(self, str): + raise NotImplementedError + class node_class(object): """ this is a superclass for our inodes It is an API for all code that wants to access the document files. @@ -1403,11 +1412,13 @@ class nodefd_content(StringIO, node_descriptor): def __init__(self, parent, cr, mode, ctx): node_descriptor.__init__(self, parent) self._context=ctx + self._size = 0L if mode in ('r', 'r+'): cntobj = parent.context._dirobj.pool.get('document.directory.content') data = cntobj.process_read(cr, parent.context.uid, parent, ctx) if data: + self._size = len(data) parent.content_length = len(data) StringIO.__init__(self, data) elif mode in ('w', 'w+'): @@ -1421,6 +1432,9 @@ class nodefd_content(StringIO, node_descriptor): raise IOError(errno.EINVAL, "Invalid file mode") self.mode = mode + def size(self): + return self._size + def close(self): # we now open a *separate* cursor, to update the data. # FIXME: this may be improved, for concurrency handling From 1c5c26a8afaed89d83fcaaa5aa6fb9a31d26a6c8 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:35 +0200 Subject: [PATCH 24/59] doc webdav: move to the node_descriptor API We need to support chunked transfers, as well as Ranges, so we should no longer use the node.get_data() but node.open_data('r'). bzr revid: p_christ@hol.gr-20101103112535-34sr4dhd3neunfv1 --- addons/document_webdav/dav_fs.py | 109 ++++++++++++++++++- addons/document_webdav/test/webdav_test1.yml | 2 +- addons/document_webdav/test_davclient.py | 9 +- 3 files changed, 114 insertions(+), 6 deletions(-) diff --git a/addons/document_webdav/dav_fs.py b/addons/document_webdav/dav_fs.py index 3e8e1c2b3e5..ebd3fa6b7cc 100644 --- a/addons/document_webdav/dav_fs.py +++ b/addons/document_webdav/dav_fs.py @@ -24,6 +24,7 @@ import base64 import sys import os import time +import errno from string import joinfields, split, lower import netsvc @@ -74,6 +75,89 @@ def _str2time(cre): cre = cre[:fdot] return time.mktime(time.strptime(cre,'%Y-%m-%d %H:%M:%S')) + frac +class BoundStream2(object): + """Wraps around a seekable buffer, reads a determined range of data + + Note that the supplied stream object MUST support a size() which + should return its data length (in bytes). + + A variation of the class in websrv_lib.py + """ + + def __init__(self, stream, offset=None, length=None, chunk_size=None): + self._stream = stream + self._offset = offset or 0 + self._length = length or self._stream.size() + self._rem_length = length + assert length and isinstance(length, (int, long)) + assert length and length >= 0, length + self._chunk_size = chunk_size + if offset is not None: + self._stream.seek(offset) + + def read(self, size=-1): + if not self._stream: + raise IOError(errno.EBADF, "read() without stream") + + if self._rem_length == 0: + return '' + elif self._rem_length < 0: + raise EOFError() + + rsize = self._rem_length + if size > 0 and size < rsize: + rsize = size + if self._chunk_size and self._chunk_size < rsize: + rsize = self._chunk_size + + data = self._stream.read(rsize) + self._rem_length -= len(data) + + return data + + def __len__(self): + return self._length + + def tell(self): + res = self._stream.tell() + if self._offset: + res -= self._offset + return res + + def __iter__(self): + return self + + def next(self): + return self.read(65536) + + def seek(self, pos, whence=os.SEEK_SET): + """ Seek, computing our limited range + """ + if whence == os.SEEK_SET: + if pos < 0 or pos > self._length: + raise IOError(errno.EINVAL,"Cannot seek") + self._stream.seek(pos - self._offset) + self._rem_length = self._length - pos + elif whence == os.SEEK_CUR: + if pos > 0: + if pos > self._rem_length: + raise IOError(errno.EINVAL,"Cannot seek past end") + elif pos < 0: + oldpos = self.tell() + if oldpos + pos < 0: + raise IOError(errno.EINVAL,"Cannot seek before start") + self._stream.seek(pos, os.SEEK_CUR) + self._rem_length -= pos + elif whence == os.SEEK_END: + if pos > 0: + raise IOError(errno.EINVAL,"Cannot seek past end") + else: + if self._length + pos < 0: + raise IOError(errno.EINVAL,"Cannot seek before start") + newpos = self._offset + self._length + pos + self._stream.seek(newpos, os.SEEK_SET) + self._rem_length = 0 - pos + class openerp_dav_handler(dav_interface): """ This class models a OpenERP interface for the DAV server @@ -396,11 +480,28 @@ class openerp_dav_handler(dav_interface): node = self.uri2object(cr, uid, pool, uri2) if not node: raise DAV_NotFound2(uri2) + # TODO: if node is a collection, for some specific set of + # clients ( web browsers; available in node context), + # we may return a pseydo-html page with the directory listing. try: + res = node.open_data(cr,'r') if rrange: - self.parent.log_error("Doc get_data cannot use range") - raise DAV_Error(409) - datas = node.get_data(cr) + assert isinstance(rrange, (tuple,list)) + start, end = map(long, rrange) + if not start: + start = 0 + assert start >= 0 + if end and end < start: + self.parent.log_error("Invalid range for data: %s-%s" %(start, end)) + raise DAV_Error(416, "Invalid range for data") + if end: + if end >= res.size(): + raise DAV_Error(416, "Requested data exceeds available size") + length = (end + 1) - start + else: + length = res.size() - start + res = BoundStream2(res, offset=start, length=length) + except TypeError,e: # for the collections that return this error, the DAV standard # says we'd better just return 200 OK with empty data @@ -413,7 +514,7 @@ class openerp_dav_handler(dav_interface): self.parent.log_error("GET exception: %s",str(e)) self.parent.log_message("Exc: %s", traceback.format_exc()) raise DAV_Error, 409 - return str(datas) # FIXME! + return res finally: if cr: cr.close() diff --git a/addons/document_webdav/test/webdav_test1.yml b/addons/document_webdav/test/webdav_test1.yml index a64c2efae4c..a0f15c10fc9 100644 --- a/addons/document_webdav/test/webdav_test1.yml +++ b/addons/document_webdav/test/webdav_test1.yml @@ -56,4 +56,4 @@ dc = te.DAVClient() dc.get_creds(self, cr, uid) tdp = addons.get_module_resource('document_webdav', 'test_davclient.py') - res = dc.gd_get(path=cr.dbname+'/Documents/test_davclient.py', compare=tdp) + res = dc.gd_get(path=cr.dbname+'/Documents/test_davclient.py', crange=(4,508), compare=tdp) diff --git a/addons/document_webdav/test_davclient.py b/addons/document_webdav/test_davclient.py index 35c870838ec..e5c3659d870 100644 --- a/addons/document_webdav/test_davclient.py +++ b/addons/document_webdav/test_davclient.py @@ -654,7 +654,9 @@ class DAVClient(object): hdrs['Range'] = 'bytes='+ (','.join(rs)) s, m, d = self._http_request(self.davpath + path, method='GET', hdrs=hdrs) assert s in (200, 206), "Bad status: %s" % s - ctype = m.getheader('Content-Type').split(';',1)[0] + ctype = m.getheader('Content-Type') + if ctype and ';' in ctype: + ctype = ctype.split(';',1)[0] if mime: assert ctype == mime, m.getheader('Content-Type') rrange = None @@ -668,6 +670,11 @@ class DAVClient(object): fd = open(compare, 'rb') d2 = fd.read() fd.close() + if crange: + if len(crange) > 1: + raise NotImplementedError + r = crange[0] + d2 = d2[r[0]:r[1]+1] assert d2 == d, "Data does not match" return ctype, rrange, d From 169b716aa4541b2cdfd497de9e0bc2c70d38ba99 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:35 +0200 Subject: [PATCH 25/59] document: a class of static content node_descriptor, API bzr revid: p_christ@hol.gr-20101103112535-e24cqzamf1w42219 --- addons/document/nodes.py | 52 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 1a903614028..84eff0ecf64 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -1460,4 +1460,56 @@ class nodefd_content(StringIO, node_descriptor): cr.close() StringIO.close(self) +class nodefd_static(StringIO, node_descriptor): + """ A descriptor to nodes with static data. + """ + def __init__(self, parent, cr, mode, ctx=None): + node_descriptor.__init__(self, parent) + self._context=ctx + self._size = 0L + + if mode in ('r', 'r+'): + data = parent.get_data(cr) + if data: + self._size = len(data) + parent.content_length = len(data) + StringIO.__init__(self, data) + elif mode in ('w', 'w+'): + StringIO.__init__(self, None) + # at write, we start at 0 (= overwrite), but have the original + # data available, in case of a seek() + elif mode == 'a': + StringIO.__init__(self, None) + else: + logging.getLogger('document.nodes').error("Incorrect mode %s specified", mode) + raise IOError(errno.EINVAL, "Invalid file mode") + self.mode = mode + + def size(self): + return self._size + + def close(self): + # we now open a *separate* cursor, to update the data. + # FIXME: this may be improved, for concurrency handling + if self.mode == 'r': + StringIO.close(self) + return + + par = self._get_parent() + uid = par.context.uid + cr = pooler.get_db(par.context.dbname).cursor() + try: + if self.mode in ('w', 'w+', 'r+'): + data = self.getvalue() + par.set_data(cr, data) + elif self.mode == 'a': + raise NotImplementedError + cr.commit() + except Exception: + logging.getLogger('document.nodes').exception('Cannot update db content #%d for close:', par.cnt_id) + raise + finally: + cr.close() + StringIO.close(self) + #eof \ No newline at end of file From 1bfecf18279281c3fb0da00834237b4eb90fa7ea Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:25:35 +0200 Subject: [PATCH 26/59] caldav: implement the node_descriptor api. Two commits ago, the webdav module started using this API instead of the get_data() one, so CalDAV must adapt now. bzr revid: p_christ@hol.gr-20101103112535-5cj7nnv75z8p0ew2 --- addons/caldav/caldav_node.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/addons/caldav/caldav_node.py b/addons/caldav/caldav_node.py index b735d853025..978022fcbb5 100644 --- a/addons/caldav/caldav_node.py +++ b/addons/caldav/caldav_node.py @@ -21,7 +21,7 @@ import time from document_webdav import nodes -from document.nodes import _str2time +from document.nodes import _str2time, nodefd_static import logging import StringIO from orm_utils import get_last_modified @@ -381,6 +381,9 @@ class node_calendar(nodes.node_class): res.append(child._get_caldav_calendar_data(cr)) return res + def open_data(self, cr, mode): + return nodefd_static(self, cr, mode) + def _get_caldav_calendar_description(self, cr): uid = self.context.uid calendar_obj = self.context._dirobj.pool.get('basic.calendar') @@ -458,14 +461,10 @@ class res_node_calendar(nodes.node_class): self.model = res_model self.res_id = res_id - def open(self, cr, mode=False): - if self.type in ('collection','database'): - return False - s = StringIO.StringIO(self.get_data(cr)) - s.name = self - return s + def open_data(self, cr, mode): + return nodefd_static(self, cr, mode) - def get_data(self, cr, fil_obj = None): + def get_data(self, cr, fil_obj=None): uid = self.context.uid calendar_obj = self.context._dirobj.pool.get('basic.calendar') context = self.context.context.copy() From 0c13466702f84a36b46737b665837c4044cd0bf3 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: [PATCH 27/59] document: fixes for lint checks bzr revid: p_christ@hol.gr-20101103112604-nh7xqcx5e17h3wjo --- addons/document/document_storage.py | 2 +- addons/document/nodes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index dd0247e52fb..18fec8fc18f 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -667,7 +667,7 @@ class document_storage(osv.osv): if ktype == 'file': try: os.unlink(fname) - except Exception, e: + except Exception: self._doclog.warning("Could not remove file %s, please remove manually.", fname, exc_info=True) else: self._doclog.warning("Unknown unlink key %s" % ktype) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 84eff0ecf64..a1ed691cc6f 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -1496,7 +1496,7 @@ class nodefd_static(StringIO, node_descriptor): return par = self._get_parent() - uid = par.context.uid + # uid = par.context.uid cr = pooler.get_db(par.context.dbname).cursor() try: if self.mode in ('w', 'w+', 'r+'): From 2645423e54fb8f5464548bb8806d27fd15a56def Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: [PATCH 28/59] doc webdav: Enable yaml test, lint fixes bzr revid: p_christ@hol.gr-20101103112604-qbrkbmg81aj1d2vx --- addons/document_webdav/__openerp__.py | 2 ++ addons/document_webdav/dav_fs.py | 16 ++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/addons/document_webdav/__openerp__.py b/addons/document_webdav/__openerp__.py index e6b8cfd3f94..d06affc9402 100644 --- a/addons/document_webdav/__openerp__.py +++ b/addons/document_webdav/__openerp__.py @@ -57,6 +57,8 @@ 'webdav_setup.xml', ], "demo_xml" : [], + "test": [ 'test/webdav_test1.yml', + ], "active": False, "installable": True } diff --git a/addons/document_webdav/dav_fs.py b/addons/document_webdav/dav_fs.py index ebd3fa6b7cc..c34a323fadb 100644 --- a/addons/document_webdav/dav_fs.py +++ b/addons/document_webdav/dav_fs.py @@ -20,19 +20,16 @@ ############################################################################## import pooler -import base64 -import sys import os import time import errno -from string import joinfields, split, lower import netsvc import urlparse -from DAV.constants import COLLECTION, OBJECT -from DAV.errors import * -from DAV.iface import * +from DAV.constants import COLLECTION #, OBJECT +from DAV.errors import DAV_Error, DAV_Forbidden, DAV_NotFound +from DAV.iface import dav_interface import urllib from DAV.davcmd import copyone, copytree, moveone, movetree, delone, deltree @@ -685,8 +682,7 @@ class openerp_dav_handler(dav_interface): node = False objname = uri2[-1] - ext = objname.find('.') >0 and objname.split('.')[1] or False - + ret = None if not node: dir_node = self.uri2object(cr, uid, pool, uri2[:-1]) @@ -769,7 +765,7 @@ class openerp_dav_handler(dav_interface): """ if uri[-1]=='/':uri=uri[:-1] res=delone(self,uri) - parent='/'.join(uri.split('/')[:-1]) + # parent='/'.join(uri.split('/')[:-1]) return res def deltree(self, uri): @@ -781,7 +777,7 @@ class openerp_dav_handler(dav_interface): """ if uri[-1]=='/':uri=uri[:-1] res=deltree(self, uri) - parent='/'.join(uri.split('/')[:-1]) + # parent='/'.join(uri.split('/')[:-1]) return res From aed35b4545f058b8aa43da11fdf7040c56cdbc46 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: [PATCH 29/59] document: fix security rules The behaviour didn't match the intended one as in the documentation. bzr revid: p_christ@hol.gr-20101103112604-slw7gb04wlw2amim --- addons/document/security/document_security.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/document/security/document_security.xml b/addons/document/security/document_security.xml index e1a60165150..c4ef727894a 100644 --- a/addons/document/security/document_security.xml +++ b/addons/document/security/document_security.xml @@ -16,7 +16,7 @@ - ['|',('user_id', '=', False), ('user_id', '=', user.id), '|', ('group_ids','=',False), ('group_ids','in',user.groups_id), '|', ('company_id','=',False), ('company_id','child_of',[user.company_id.id])] + ['|','|',('group_ids','in',user.groups_id), ('user_id', '=', user.id), '&', ('user_id', '=', False), ('group_ids','=',False), '|', ('company_id','=',False), ('company_id','child_of',[user.company_id.id])] Read public directories @@ -28,7 +28,7 @@ - [('user_id', '=', user.id), '|', ('group_ids','=',False), ('group_ids','in',user.groups_id), '|', ('company_id','=',False), ('company_id','child_of',[user.company_id.id]) ] + [ '|', ('user_id', '=', user.id), '&', ('group_ids','in',user.groups_id), ('user_id','=',False), '|', ('company_id','=',False), ('company_id','child_of',[user.company_id.id]) ] Document modify own directories From 21847b3864632e5e2f99f33b9ea516fa93d00134 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: [PATCH 30/59] doc webdav: workaround bug in KHTML's PUT method. This isn't our fault, but KDE is bound to remain broken for quite some time, we'd better support it. bzr revid: p_christ@hol.gr-20101103112604-78mdkva38dx1vm9k --- addons/document_webdav/webdav_server.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/addons/document_webdav/webdav_server.py b/addons/document_webdav/webdav_server.py index df2c4a871f6..52356596af0 100644 --- a/addons/document_webdav/webdav_server.py +++ b/addons/document_webdav/webdav_server.py @@ -36,10 +36,13 @@ from service.websrv_lib import HTTPDir, FixSendError, HttpOptions from BaseHTTPServer import BaseHTTPRequestHandler import urlparse import urllib +import re from string import atoi,split from DAV.errors import * # from DAV.constants import DAV_VERSION_1, DAV_VERSION_2 +khtml_re = re.compile(r' KHTML/([0-9\.]+) ') + def OpenDAVConfig(**kw): class OpenDAV: def __init__(self, **kw): @@ -131,6 +134,16 @@ class DAVHandler(HttpOptions, FixSendError, DAVRequestHandler): # the BufferingHttpServer will send Connection: close , while # the BaseHTTPRequestHandler will only accept int code. # workaround both of them. + if self.command == 'PROPFIND' and int(code) == 404: + kh = khtml_re.search(self.headers.get('User-Agent','')) + if kh and (kh.group(1) < '4.5'): + # There is an ugly bug in all khtml < 4.5.x, where the 404 + # response is treated as an immediate error, which would even + # break the flow of a subsequent PUT request. At the same time, + # the 200 response (rather than 207 with content) is treated + # as "path not exist", so we send this instead + # https://bugs.kde.org/show_bug.cgi?id=166081 + code = 200 BaseHTTPRequestHandler.send_response(self, int(code), message) def send_header(self, key, value): From 5d6eb5dea1d9a9c912b71ba14680f89d39b1cc38 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: [PATCH 31/59] document: more reasonable Unix permission bits For display purposes only, but helps us understand group-write logic. bzr revid: p_christ@hol.gr-20101103112604-vi0wy1duifni2clv --- addons/document/nodes.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index a1ed691cc6f..f025b56698e 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -491,9 +491,15 @@ def _uid2unixperms(perms, has_owner): """ Convert the uidperms and the owner flag to full unix bits """ res = 0 - res |= (perms & 0x07) << 6 - res |= (perms & 0x05) << 3 - if not has_owner: + if has_owner: + res |= (perms & 0x07) << 6 + res |= (perms & 0x05) << 3 + elif perms & 0x02: + res |= (perms & 0x07) << 6 + res |= (perms & 0x07) << 3 + else: + res |= (perms & 0x07) << 6 + res |= (perms & 0x05) << 3 res |= 0x05 return res From 4959480bc1518d0c9bf838c8dcd700dd36e40eb8 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:04 +0200 Subject: [PATCH 32/59] document: fix realstore, implement move op. in it. bzr revid: p_christ@hol.gr-20101103112604-pd10spyv6hh3demi --- addons/document/document_storage.py | 51 +++++++++++++++++------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 18fec8fc18f..64b8b067bae 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -26,6 +26,7 @@ import tools import base64 import errno import logging +import shutil from StringIO import StringIO import psycopg2 @@ -470,11 +471,11 @@ class document_storage(osv.osv): elif boo.type == 'realstore': path, npath = self.__prepare_realpath(cr, file_node, ira, boo.path, - do_create = (mode[1] in ('w','a')) ) + do_create = (mode[0] in ('w','a')) ) fpath = os.path.join(path, npath[-1]) - if (not os.path.exists(fpath)) and mode[1] == 'r': + if (not os.path.exists(fpath)) and mode[0] == 'r': raise IOError("File not found: %s" % fpath) - elif mode[1] in ('w', 'a') and not ira.store_fname: + elif mode[0] in ('w', 'a') and not ira.store_fname: store_fname = os.path.join(*npath) cr.execute('UPDATE ir_attachment SET store_fname = %s WHERE id = %s', (store_fname, ira.id)) @@ -747,26 +748,34 @@ class document_storage(osv.osv): # nothing to do for a rename, allow to change the db field return { 'parent_id': ndir_bro.id } elif sbro.type == 'realstore': - raise NotImplementedError("Cannot move in realstore, yet") # TODO - fname = fil_bo.store_fname + ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) + + path, opath = self.__prepare_realpath(cr, file_node, ira, sbro.path, do_create=False) + fname = ira.store_fname + if not fname: - return ValueError("Tried to rename a non-stored file") - path = sbro.path - oldpath = os.path.join(path, fname) + self._doclog.warning("Trying to rename a non-stored file") + if fname != os.path.join(*opath): + self._doclog.warning("inconsistency in realstore: %s != %s" , fname, repr(opath)) + + oldpath = os.path.join(path, opath[-1]) - for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'): - if ch in new_name: - raise ValueError("Invalid char %s in name %s" %(ch, new_name)) - - file_node.fix_ppath(cr, ira) - npath = file_node.full_path() or [] - dpath = [path,] - dpath.extend(npath[:-1]) - dpath.append(new_name) - newpath = os.path.join(*dpath) - # print "old, new paths:", oldpath, newpath - os.rename(oldpath, newpath) - return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name } + npath = [sbro.path,] + (ndir_bro.get_full_path() or []) + npath = filter(lambda x: x is not None, npath) + newdir = os.path.join(*npath) + if not os.path.isdir(newdir): + self._doclog.debug("Must create dir %s", newdir) + os.makedirs(newdir) + npath.append(opath[-1]) + newpath = os.path.join(*npath) + + self._doclog.debug("Going to move %s from %s to %s", opath[-1], oldpath, newpath) + shutil.move(oldpath, newpath) + + store_path = npath[1:] + [opath[-1],] + store_fname = os.path.join(*store_path) + + return { 'store_fname': store_fname } else: raise TypeError("No %s storage" % sbro.type) From a21dcd7e1c8623f4aaf3a3659e81247273f652e9 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:05 +0200 Subject: [PATCH 33/59] document: fix regressions at storage and node_descriptor Since the introduction of __len__, the descriptors were broken for the ftp module. Fix. bzr revid: p_christ@hol.gr-20101103112605-tu6n6529mbdc18ve --- addons/document/document_storage.py | 6 ++++-- addons/document/nodes.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 64b8b067bae..513da92bfcd 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -181,7 +181,8 @@ class nodefd_db(StringIO, nodes.node_descriptor): cr = ira_browse._cr # reuse the cursor of the browse object, just now cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,)) data = cr.fetchone()[0] - self._size = len(data) + if data: + self._size = len(data) StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) @@ -260,7 +261,8 @@ class nodefd_db64(StringIO, nodes.node_descriptor): if mode in ('r', 'r+'): data = base64.decodestring(ira_browse.db_datas) - self._size = len(data) + if data: + self._size = len(data) StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index f025b56698e..c0eef134f25 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -181,6 +181,14 @@ class node_descriptor(object): def __len__(self): return self.size() + def __nonzero__(self): + """ Ensure that a node_descriptor will never equal False + + Since we do define __len__ and __iter__ for us, we must avoid + being regarded as non-true objects. + """ + return True + def next(self, str): raise NotImplementedError From bc7871e2f94b2ff0a8309f4f96cfcc27f0436c3d Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:05 +0200 Subject: [PATCH 34/59] doc ftp: some assertions, close ftp sessions at YML tests bzr revid: p_christ@hol.gr-20101103112605-dq8jthkma78ccmcj --- .../document_ftp/ftpserver/abstracted_fs.py | 2 ++ addons/document_ftp/ftpserver/ftpserver.py | 1 + .../document_ftp/test/document_ftp_test2.yml | 31 +++++++++++++++++-- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 9c16e7dde55..5eddffef4cb 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -146,6 +146,7 @@ class abstracted_fs(object): ret = child.open_data(cr, mode) cr.commit() + assert ret, "Cannot create descriptor for %r: %r" % (child, ret) return ret except EnvironmentError: raise @@ -156,6 +157,7 @@ class abstracted_fs(object): try: child = node.create_child(cr, objname, data=None) ret = child.open_data(cr, mode) + assert ret, "cannot create descriptor for %r" % child cr.commit() return ret except EnvironmentError: diff --git a/addons/document_ftp/ftpserver/ftpserver.py b/addons/document_ftp/ftpserver/ftpserver.py index 710494bb9db..bd727427541 100644 --- a/addons/document_ftp/ftpserver/ftpserver.py +++ b/addons/document_ftp/ftpserver/ftpserver.py @@ -2223,6 +2223,7 @@ class FTPHandler(asynchat.async_chat): if self.restart_position: mode = 'r+' fd = self.try_as_current_user(self.fs.create, (datacr, datacr[2], mode + 'b')) + assert fd except FTPExceptionSent: self.fs.close_cr(datacr) return diff --git a/addons/document_ftp/test/document_ftp_test2.yml b/addons/document_ftp/test/document_ftp_test2.yml index da55c353767..5da4bcd83ef 100644 --- a/addons/document_ftp/test/document_ftp_test2.yml +++ b/addons/document_ftp/test/document_ftp_test2.yml @@ -7,6 +7,7 @@ from document_ftp import test_easyftp as te ftp = te.get_plain_ftp(timeout=2.0) assert ftp.sock and (ftp.lastresp == '220'), ftp.lastresp + ftp.close() - I read the list of databases at port 8021 and confirm our db is there @@ -15,6 +16,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_login(cr, uid, self) assert cr.dbname in ftp.nlst("/") + ftp.close() - I try to locate the default "Documents" folder in the db. - @@ -22,6 +24,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_login(cr, uid, self) ftp.cwd('Documents') + ftp.close() - I create a "test.txt" file at the server (directly). The file should have the "abcd" content @@ -32,6 +35,7 @@ ftp = te.get_ftp_folder(cr, uid, self, 'Documents') fdata = StringIO('abcd') ftp.storbinary('STOR test.txt', fdata) + ftp.close() - I look for the "test.txt" file at the server - @@ -39,6 +43,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents') assert ftp.nlst("test.txt") == ['test.txt'] + ftp.close() - I check that the content of "test.txt" is "abcd" - @@ -46,7 +51,9 @@ from document_ftp import test_easyftp as te from cStringIO import StringIO ftp = te.get_ftp_folder(cr, uid, self, 'Documents') - assert te.get_ftp_fulldata(ftp, "test.txt") == 'abcd' + gotdata = te.get_ftp_fulldata(ftp, "test.txt") + ftp.close() + assert gotdata == 'abcd', 'Data: %r' % gotdata - I append the string 'defgh' to "test.txt" - @@ -56,6 +63,7 @@ ftp = te.get_ftp_folder(cr, uid, self, 'Documents') fdata = StringIO('defgh') ftp.storbinary('APPE test.txt', fdata) + ftp.close() - I check that the content of "text.txt" is 'abcddefgh' - @@ -63,7 +71,9 @@ from document_ftp import test_easyftp as te from cStringIO import StringIO ftp = te.get_ftp_folder(cr, uid, self, 'Documents') - assert te.get_ftp_fulldata(ftp, "test.txt") == 'abcddefgh' + gotdata = te.get_ftp_fulldata(ftp, "test.txt") + ftp.close() + assert gotdata == 'abcddefgh', 'Data: %r' % gotdata - I try to cd into an non-existing folder 'Not-This' - @@ -77,7 +87,9 @@ except ftplib.error_perm: pass except OSError, err: + ftp.close() assert err.errno == 2, err.errno + ftp.close() - I create a "test2.txt" file through FTP. - @@ -87,6 +99,7 @@ ftp = te.get_ftp_folder(cr, uid, self, 'Documents') fdata = StringIO('abcd') ftp.storbinary('STOR test2.txt', fdata) + ftp.close() - I look for the "test2.txt" file at the server - @@ -101,6 +114,7 @@ from cStringIO import StringIO ftp = te.get_ftp_folder(cr, uid, self, 'Documents') ftp.delete('test2.txt') + ftp.close() - I check at the server that test2.txt is deleted - @@ -116,6 +130,7 @@ ftp = te.get_ftp_folder(cr, uid, self, 'Documents') fdata = StringIO('abcd') ftp.storbinary('STOR test2.txt', fdata) + ftp.close() - I delete the test2.txt from the server (RPC). - @@ -137,6 +152,7 @@ except ftplib.error_perm: # 550 error: 'path not exists' nlst_result = [] assert "test2.txt" not in nlst_result, "Files: %r" % nlst_result + ftp.close() - I create a "test-name.txt" file - @@ -146,6 +162,7 @@ ftp = te.get_ftp_folder(cr, uid, self, 'Documents') fdata = StringIO('abcd') ftp.storbinary('STOR test-name.txt', fdata) + ftp.close() - I rename the "test-name.txt" file through ftp. - @@ -153,6 +170,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents') ftp.rename("test-name.txt", "test-renamed.txt") + ftp.close() - I check that test-name.txt has been renamed. - @@ -166,6 +184,7 @@ except error_perm, e: pass assert ftp.nlst("test-renamed.txt") == ['test-renamed.txt'] + ftp.close() - I create a new folder 'Test-Folder2' through FTP - @@ -173,6 +192,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents') ftp.mkd("Test-Folder2") + ftp.close() - I create a file 'test3.txt' at the 'Test-Folder2' - @@ -182,6 +202,7 @@ ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2') fdata = StringIO('abcd') ftp.storbinary('STOR test3.txt', fdata) + ftp.close() - I try to retrieve test3.txt - @@ -189,6 +210,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2') assert ftp.nlst("test3.txt") == ['test3.txt'], "File test3.txt is not there!" + ftp.close() - I create a new folder, 'Test-Folder3', through FTP I try to move test3.txt to 'Test-Folder3' @@ -197,6 +219,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents') ftp.mkd("Test-Folder3") + ftp.close() # TODO move - I remove the 'Test-Folder3' @@ -205,6 +228,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents') ftp.rmd("Test-Folder3") + ftp.close() - I check that test3.txt is removed. - @@ -219,6 +243,7 @@ for i in range(0, 200): fdata.seek(0) ftp.storbinary('STOR test-name%s.txt' %i, fdata) + ftp.close() - I list the 200 files, check speed - @@ -245,6 +270,7 @@ ftp.delete('test3.txt') for i in range(0, 200): ftp.delete('test-name%s.txt' %i) + ftp.close() - I remove the 'Test-Folder2' - @@ -252,3 +278,4 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents') ftp.rmd("Test-Folder2") + ftp.close() From 3ef534b21db44f9f3622e0dc9a31e10c69ad8df4 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 3 Nov 2010 13:26:05 +0200 Subject: [PATCH 35/59] document: bump module version bzr revid: p_christ@hol.gr-20101103112605-7mv9bcl0g0jkpp2v --- addons/document/__openerp__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/__openerp__.py b/addons/document/__openerp__.py index 5741a7223e0..5d0088579a1 100644 --- a/addons/document/__openerp__.py +++ b/addons/document/__openerp__.py @@ -22,7 +22,7 @@ { 'name': 'Integrated Document Management System', - 'version': '2.0', + 'version': '2.1', 'category': 'Generic Modules/Others', 'description': """This is a complete document management system: * User Authentication From 12b48d880ca1b243b3d9ab8393f0bdcf34c58738 Mon Sep 17 00:00:00 2001 From: "atp (Open ERP)" Date: Thu, 4 Nov 2010 19:04:03 +0530 Subject: [PATCH 36/59] [FIX]: Changed some Icon in Scrum Dashboard and Fix problem of project_scrum demo data. bzr revid: atp@tinyerp.co.in-20101104133403-pbmogp8j2mfppykv --- addons/project_scrum/board_project_scrum_view.xml | 4 ++-- addons/project_scrum/project_scrum_demo.xml | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/addons/project_scrum/board_project_scrum_view.xml b/addons/project_scrum/board_project_scrum_view.xml index 2d146c94367..08acece9bbb 100644 --- a/addons/project_scrum/board_project_scrum_view.xml +++ b/addons/project_scrum/board_project_scrum_view.xml @@ -16,9 +16,9 @@ -