From a95319ace39a4da7b2b2932b076b3582c824f99a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 25 Jun 2010 16:51:39 +0300 Subject: [PATCH 001/121] Document: do not allow nodes without storage. Every file must be linked to a storage, through its parent directory. If not, it is just a programming error. bzr revid: p_christ@hol.gr-20100625135139-u5r0y4fouz2i1av6 --- addons/document/nodes.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index dcd864e6db0..983ec2ac951 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -738,12 +738,9 @@ class node_file(node_class): the browse object. """ # this is where storage kicks in.. stor = self.storage_id - if not stor: - data_obj = self.context._dirobj.pool.get('ir.model.data') - data_id = data_obj._get_id(cr, self.context.uid, 'document', 'storage_db') - if data_id: - stor = data_obj.browse(cr, self.context.uid, data_id, context=self.context.context).res_id - assert stor + assert stor, "No storage for file #%s" % self.file_id + # If storage is not set properly, we are just screwed here, don't + # try to get it from default. stobj = self.context._dirobj.pool.get('document.storage') return stobj.get_data(cr, self.context.uid,stor, self,self.context.context, fil_obj) From 6eef5d7e360353c60a56d0b1ab4c3484f80059d8 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:09 +0300 Subject: [PATCH 002/121] Document: rearrange columns definition of document.file Note: *must* use the server as patched, for the 'url' and 'type' columns. bzr revid: p_christ@hol.gr-20100627201809-p5wy2w368gqoyy3h --- addons/document/document.py | 41 +++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/addons/document/document.py b/addons/document/document.py index aa893b617c3..c5315bf0574 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -72,33 +72,32 @@ class document_file(osv.osv): return res _columns = { + # Columns from ir.attachment: + 'create_date': fields.datetime('Date Created', readonly=True), + 'create_uid': fields.many2one('res.users', 'Creator', readonly=True), + 'write_date': fields.datetime('Date Modified', readonly=True), + 'write_uid': fields.many2one('res.users', 'Last Modification User', readonly=True), + 'res_model': fields.char('Attached Model', size=64), # readonly? + 'res_id': fields.integer('Attached ID'), + + # If ir.attachment contained any data before document is installed, preserve + # the data, don't drop the column! + 'db_datas': fields.binary('Data', oldname='datas'), + 'datas': fields.function(_data_get, method=True, fnct_inv=_data_set, string='File Content', type="binary", nodrop=True), + + # Fields of document: 'user_id': fields.many2one('res.users', 'Owner', select=1), 'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'), # the directory id now is mandatory. It can still be computed automatically. 'parent_id': fields.many2one('document.directory', 'Directory', select=1), + 'index_content': fields.text('Indexed Content'), + 'partner_id':fields.many2one('res.partner', 'Partner', select=1), + 'company_id': fields.many2one('res.company', 'Company'), 'file_size': fields.integer('File Size', required=True), 'file_type': fields.char('Content Type', size=128), - # If ir.attachment contained any data before document is installed, preserve - # the data, don't drop the column! - 'db_datas': fields.binary('Data', oldname='datas'), - 'index_content': fields.text('Indexed Content'), - 'write_date': fields.datetime('Date Modified', readonly=True), - 'write_uid': fields.many2one('res.users', 'Last Modification User', readonly=True), - 'create_date': fields.datetime('Date Created', readonly=True), - 'create_uid': fields.many2one('res.users', 'Creator', readonly=True), - 'store_method': fields.selection([('db', 'Database'), ('fs', 'Filesystem'), ('link', 'Link')], "Storing Method"), - 'datas': fields.function(_data_get, method=True, fnct_inv=_data_set, string='File Content', type="binary", nodrop=True), - 'url': fields.char('File URL',size=64), + + # fields used for file storage 'store_fname': fields.char('Stored Filename', size=200), - 'res_model': fields.char('Attached Model', size=64), #res_model - 'res_id': fields.integer('Attached ID'), #res_id - 'partner_id':fields.many2one('res.partner', 'Partner', select=1), - 'type':fields.selection([ - ('url','URL'), - ('binary','Binary'), - - ],'Type', help="Type is used to separate URL and binary File"), - 'company_id': fields.many2one('res.company', 'Company'), } def __get_def_directory(self, cr, uid, context=None): @@ -109,8 +108,6 @@ class document_file(osv.osv): 'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c), 'user_id': lambda self, cr, uid, ctx:uid, 'file_size': lambda self, cr, uid, ctx:0, - 'store_method': lambda *args: 'db', - 'type': 'binary', 'parent_id': __get_def_directory } _sql_constraints = [ From c11ad0047d1c6da4592c161e2e4a1ed6a5ad07c2 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:13 +0300 Subject: [PATCH 003/121] Document: files are required to have a parent directory Or else, there is no storage for them. bzr revid: p_christ@hol.gr-20100627201813-y5gatb8w39jnxeux --- addons/document/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/document.py b/addons/document/document.py index c5315bf0574..dd48f7daf7f 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -89,7 +89,7 @@ class document_file(osv.osv): 'user_id': fields.many2one('res.users', 'Owner', select=1), 'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'), # the directory id now is mandatory. It can still be computed automatically. - 'parent_id': fields.many2one('document.directory', 'Directory', select=1), + 'parent_id': fields.many2one('document.directory', 'Directory', select=1, required=True), 'index_content': fields.text('Indexed Content'), 'partner_id':fields.many2one('res.partner', 'Partner', select=1), 'company_id': fields.many2one('res.company', 'Company'), From 823dc6d50d2a158c0c7428aa09e18a0a0d781a45 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:18 +0300 Subject: [PATCH 004/121] Document directory: cleanup fields, type, constraints. bzr revid: p_christ@hol.gr-20100627201818-wjlumw10u8g28g27 --- addons/document/document_directory.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index fee633b7169..d83d1b6b60e 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -39,16 +39,21 @@ class document_directory(osv.osv): 'write_uid': fields.many2one('res.users', 'Last Modification User', readonly=True), 'create_date': fields.datetime('Date Created', readonly=True), 'create_uid': fields.many2one('res.users', 'Creator', readonly=True), - 'file_type': fields.char('Content Type', size=32), 'domain': fields.char('Domain', size=128, help="Use a domain if you want to apply an automatic filter on visible resources."), 'user_id': fields.many2one('res.users', 'Owner'), 'storage_id': fields.many2one('document.storage', 'Storage'), 'group_ids': fields.many2many('res.groups', 'document_directory_group_rel', 'item_id', 'group_id', 'Groups'), - 'parent_id': fields.many2one('document.directory', 'Parent Item'), + 'parent_id': fields.many2one('document.directory', 'Parent Item', select=1), 'child_ids': fields.one2many('document.directory', 'parent_id', 'Children'), 'file_ids': fields.one2many('ir.attachment', 'parent_id', 'Files'), 'content_ids': fields.one2many('document.directory.content', 'directory_id', 'Virtual Files'), - 'type': fields.selection([('directory','Static Directory'),('ressource','Other Resources')], 'Type', required=True), + 'type': fields.selection([ + ('directory','Static Directory'), + ('ressource','Folders per resource'), + ('at_record','Single folder under resource')], + 'Type', required=True, select=1, + help="Defines directory's behaviour."), + 'ressource_type_id': fields.many2one('ir.model', 'Directories Mapped to Objects', help="Select an object here and Open ERP will create a mapping for each of these " \ "objects, using the given domain, when browsing through FTP."), @@ -60,7 +65,7 @@ class document_directory(osv.osv): 'ressource_tree': fields.boolean('Tree Structure', help="Check this if you want to use the same tree structure as the object selected in the system."), 'dctx_ids': fields.one2many('document.directory.dctx', 'dir_id', 'Context fields'), - 'company_id': fields.many2one('res.company', 'Company'), + 'company_id': fields.many2one('res.company', 'Company'), } @@ -102,7 +107,8 @@ class document_directory(osv.osv): } _sql_constraints = [ ('dirname_uniq', 'unique (name,parent_id,ressource_id,ressource_parent_type_id)', 'The directory name must be unique !'), - ('no_selfparent', 'check(parent_id <> id)', 'Directory cannot be parent of itself!') + ('no_selfparent', 'check(parent_id <> id)', 'Directory cannot be parent of itself!'), + ('dir_parented', 'check(parent_id IS NOT NULL OR storage_id IS NOT NULL)', 'Directory must have a parent or a storage') ] def name_get(self, cr, uid, ids, context={}): res = [] From 07d395df15c22037d64f0d7dceeec4d1328ce999 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:26 +0300 Subject: [PATCH 005/121] Document: cleanup the view (and help) of directory. bzr revid: p_christ@hol.gr-20100627201826-1873sa0et2q8vx62 --- addons/document/document_directory.py | 5 ++--- addons/document/document_view.xml | 29 ++++++++++++++++++--------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index d83d1b6b60e..2872491129d 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -54,9 +54,8 @@ class document_directory(osv.osv): 'Type', required=True, select=1, help="Defines directory's behaviour."), - 'ressource_type_id': fields.many2one('ir.model', 'Directories Mapped to Objects', - help="Select an object here and Open ERP will create a mapping for each of these " \ - "objects, using the given domain, when browsing through FTP."), + 'ressource_type_id': fields.many2one('ir.model', 'Resource model', + help="Select an object here and there will be one folder per record of that resource."), 'resource_field': fields.many2one('ir.model.fields', 'Name field', help='Field to be used as name on resource directories. If empty, the "name" will be used.'), 'ressource_parent_type_id': fields.many2one('ir.model', 'Parent Model', help="If you put an object here, this directory template will appear bellow all of these objects. " \ diff --git a/addons/document/document_view.xml b/addons/document/document_view.xml index 714c6d2c107..36e89074d94 100644 --- a/addons/document/document_view.xml +++ b/addons/document/document_view.xml @@ -75,23 +75,31 @@
- + - - - - - - - + + + + + + + + + + + + + + - + + + From d885f0c85170dccda304173cdfc4393db0d54b45 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:35 +0300 Subject: [PATCH 006/121] Document: expand tabs at view, one wording improvement. bzr revid: p_christ@hol.gr-20100627201835-c3qzi1w0e8yxcc07 --- addons/document/document_view.xml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/addons/document/document_view.xml b/addons/document/document_view.xml index 36e89074d94..6f79188622d 100644 --- a/addons/document/document_view.xml +++ b/addons/document/document_view.xml @@ -84,19 +84,19 @@ - - - - + + + + - - - + + + - + @@ -121,7 +121,7 @@ - - From 6429cbe667c1a0b8ec4147315b175b39f57b6603 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:42 +0300 Subject: [PATCH 007/121] Doc WebDAV: fix for unicode names in paths. example, when exporiting dynamic folders with unicode names. bzr revid: p_christ@hol.gr-20100627201842-bor8tt5jk4vtgv6s --- addons/document_webdav/webdav.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/addons/document_webdav/webdav.py b/addons/document_webdav/webdav.py index e904d2ecfb7..f2f57c77730 100644 --- a/addons/document_webdav/webdav.py +++ b/addons/document_webdav/webdav.py @@ -48,6 +48,8 @@ def mk_prop_response(self, uri, good_props, bad_props, doc): # write href information uparts=urlparse.urlparse(uri) fileloc=uparts[2] + if isinstance(fileloc, unicode): + fileloc = fileloc.encode('utf-8') href=doc.createElement("D:href") davpath = self._dataclass.parent.get_davpath() hurl = '%s://%s%s%s' % (uparts[0], uparts[1], davpath, urllib.quote(fileloc)) @@ -126,6 +128,8 @@ def mk_propname_response(self,uri,propnames,doc): # write href information uparts=urlparse.urlparse(uri) fileloc=uparts[2] + if isinstance(fileloc, unicode): + fileloc = fileloc.encode('utf-8') href=doc.createElement("D:href") davpath = self._dataclass.parent.get_davpath() hurl = '%s://%s%s%s' % (uparts[0], uparts[1], davpath, urllib.quote(fileloc)) From 653ed555b23db5aa4ec279c0a0d7e24332eec80a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:46 +0300 Subject: [PATCH 008/121] Document UI: improve the ir.attachment view for docs. Try to respect the layout of the bare ir.attachment view, simplify. bzr revid: p_christ@hol.gr-20100627201846-99a2ptcmnl6e1o62 --- addons/document/document_view.xml | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/addons/document/document_view.xml b/addons/document/document_view.xml index 6f79188622d..e088e51de3b 100644 --- a/addons/document/document_view.xml +++ b/addons/document/document_view.xml @@ -229,44 +229,44 @@ - - + + + + + - - + + - + - - - - + + + + + + - - - - - - - - - - + + + + + + + + + - - - - From daee2860fcff75ebfb9a993669440b80405221ed Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:49 +0300 Subject: [PATCH 009/121] Document: set the storage only for the root folder It is not needed to set the default storage for children folders, and that would break setups where the admin has overriden that storage. bzr revid: p_christ@hol.gr-20100627201849-uzemo5lcgytzmbnu --- addons/document/document_data.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/addons/document/document_data.xml b/addons/document/document_data.xml index c2bef5de34f..b93ed4d7089 100644 --- a/addons/document/document_data.xml +++ b/addons/document/document_data.xml @@ -17,7 +17,7 @@ Documents - + 0 @@ -25,7 +25,6 @@ My Folder - 0 @@ -35,7 +34,6 @@ ressource 1 - 0 @@ -57,7 +55,6 @@ Personal Folders ressource - 0 From 85dc0b2fc6b86e54ad8449ff0da5e96550009b77 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:18:57 +0300 Subject: [PATCH 010/121] Document: remove the ir.attachment.group_ids column At the review, it was decided that per document permissions is too much. bzr revid: p_christ@hol.gr-20100627201857-e0sw2sun1ttvnj32 --- addons/document/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/document.py b/addons/document/document.py index dd48f7daf7f..ec6c159f2b7 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -87,7 +87,7 @@ class document_file(osv.osv): # Fields of document: 'user_id': fields.many2one('res.users', 'Owner', select=1), - 'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'), + # 'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'), # the directory id now is mandatory. It can still be computed automatically. 'parent_id': fields.many2one('document.directory', 'Directory', select=1, required=True), 'index_content': fields.text('Indexed Content'), From e91d089e5b159f9a6e8767a408086306bd6170fc Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sun, 27 Jun 2010 23:19:01 +0300 Subject: [PATCH 011/121] Document UI: revise the ir.attachment search, tree forms bzr revid: p_christ@hol.gr-20100627201901-op4tjd1vknnuq3el --- addons/document/document_view.xml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/addons/document/document_view.xml b/addons/document/document_view.xml index e088e51de3b..3e14c09f8d8 100644 --- a/addons/document/document_view.xml +++ b/addons/document/document_view.xml @@ -285,11 +285,13 @@ search - + + + context="{'group_by':'partner_id'}" groups="base.group_extended"/> - + @@ -317,11 +319,12 @@ - - + + + From ef0272dd240c68ac9032b32c2c1caf25859470b5 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Mon, 28 Jun 2010 00:08:58 +0300 Subject: [PATCH 012/121] Document: fix context expr. that broke the Dashboard bzr revid: p_christ@hol.gr-20100627210858-9y9ncl76g1e4wjyy --- addons/document/report/document_report_view.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/report/document_report_view.xml b/addons/document/report/document_report_view.xml index 00743d768e7..630e3bb9f84 100644 --- a/addons/document/report/document_report_view.xml +++ b/addons/document/report/document_report_view.xml @@ -61,7 +61,7 @@ report.document.user form tree - {"search_default_user":uid} + {'search_default_user': 'user_id'} From 38a7613cb344fccbceb0f69b3077a969ea74695d Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Mon, 28 Jun 2010 00:09:01 +0300 Subject: [PATCH 013/121] Document: expand tabs at board xml. bzr revid: p_christ@hol.gr-20100627210901-klb7bu4ojnhrpzbx --- .../document/report/document_report_view.xml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/addons/document/report/document_report_view.xml b/addons/document/report/document_report_view.xml index 630e3bb9f84..d0ba1303a39 100644 --- a/addons/document/report/document_report_view.xml +++ b/addons/document/report/document_report_view.xml @@ -33,22 +33,22 @@ - + report.document.user.search report.document.user search - + - - - - - - - - + + + + + + + + From e908e8cd75932ff2523c5fa5a78501613cb55804 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Mon, 28 Jun 2010 00:09:04 +0300 Subject: [PATCH 014/121] Document board: bring the menus inside the 'knowledge' They should not be under the general 'Dashboards' menu. bzr revid: p_christ@hol.gr-20100627210904-1397ab657saukag9 --- addons/document/board_document_view.xml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/addons/document/board_document_view.xml b/addons/document/board_document_view.xml index 4d89f96cbfb..312c79d71ab 100644 --- a/addons/document/board_document_view.xml +++ b/addons/document/board_document_view.xml @@ -50,19 +50,18 @@ - - @@ -105,10 +104,10 @@ From 4fd7b06c3007b84fdbe743d42af91468df9a3ed8 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 16:56:32 +0300 Subject: [PATCH 015/121] Document: fix domain passing in node_res_dir Conflicts: document/nodes.py bzr revid: p_christ@hol.gr-20100629135632-8d5cp645woc8g3l5 --- addons/document/nodes.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 983ec2ac951..932cab7a32d 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -417,7 +417,15 @@ class node_res_dir(node_class): ctx.update(self.dctx) where = [] if self.domain: - where += safe_eval(self.domain, self.dctx) + app = safe_eval(self.domain, self.dctx) + if not app: + pass + elif isinstance(app, list): + where.extend(app) + elif isinstance(app, tuple): + where.append(app) + else: + raise RuntimeError("incorrect domain expr: %s" % self.domain) if self.resm_id: where.append(('id','=',self.resm_id)) From bba02abfddd63d40945a29f8ab31e0938506ad3a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 16:58:41 +0300 Subject: [PATCH 016/121] Document: fix the storage_id of node_file. Not all directories have a storage_id. Instead, we have to traverse the hierarchy up until we find one that has storage. bzr revid: p_christ@hol.gr-20100629135841-5myx5hyuijwv95si --- addons/document/nodes.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 932cab7a32d..c28b6d07b93 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -684,10 +684,14 @@ class node_file(node_class): # This only propagates the problem to get_data. Better # fix those files to point to the root dir. - if fil.parent_id: - self.storage_id = fil.parent_id.storage_id.id - else: - self.storage_id = None + self.storage_id = None + par = fil.parent_id + while par: + if par.storage_id and par.storage_id.id: + self.storage_id = par.storage_id.id + break + par = par.parent_id + def open(self, cr, mode=False): uid = self.context.uid From 694753824bdecab4fa8ae3b0df3e25379319d234 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:00:26 +0300 Subject: [PATCH 017/121] Document UI: rename the default storage entry. bzr revid: p_christ@hol.gr-20100629140026-01pahx5ue1ycaf3g --- addons/document/document_data.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/document_data.xml b/addons/document/document_data.xml index b93ed4d7089..f13213034e8 100644 --- a/addons/document/document_data.xml +++ b/addons/document/document_data.xml @@ -9,7 +9,7 @@ - Default DB storage + Database storage db From e068962cca72f8936f34e2f81555ef93214e05fc Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:00:29 +0300 Subject: [PATCH 018/121] Document UI: rename the parent_id of a dir . bzr revid: p_christ@hol.gr-20100629140029-fnx876jkkezchc3j --- addons/document/document_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index 2872491129d..a464ca61247 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -43,7 +43,7 @@ class document_directory(osv.osv): 'user_id': fields.many2one('res.users', 'Owner'), 'storage_id': fields.many2one('document.storage', 'Storage'), 'group_ids': fields.many2many('res.groups', 'document_directory_group_rel', 'item_id', 'group_id', 'Groups'), - 'parent_id': fields.many2one('document.directory', 'Parent Item', select=1), + 'parent_id': fields.many2one('document.directory', 'Parent Directory', select=1), 'child_ids': fields.one2many('document.directory', 'parent_id', 'Children'), 'file_ids': fields.one2many('ir.attachment', 'parent_id', 'Files'), 'content_ids': fields.one2many('document.directory.content', 'directory_id', 'Virtual Files'), From cf0e7cbfc6259e2ea23610d5a4e7911d28f6a6cb Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:00:32 +0300 Subject: [PATCH 019/121] Document UI: Revise document, directory forms as reviewed bzr revid: p_christ@hol.gr-20100629140032-kq39g7q5ef27xdty --- addons/document/document_view.xml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/addons/document/document_view.xml b/addons/document/document_view.xml index 3a178686561..7cbd01ad3eb 100644 --- a/addons/document/document_view.xml +++ b/addons/document/document_view.xml @@ -66,6 +66,7 @@ name="Storage Media" action="action_document_storage_form" id="menu_document_storage_media" + groups="base.group_extended" parent="menu_document_management_configuration"/> @@ -73,10 +74,10 @@ document.directory form - + - + @@ -250,10 +251,12 @@ - - - + + + + + From 514f433c59e6778afad469a55d032e1e3da435a0 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:00:40 +0300 Subject: [PATCH 020/121] Document: ir.attachment.res_model and res_id should be readonly for UI Changing them would require a wizard or something. bzr revid: p_christ@hol.gr-20100629140040-am61qng5pxfz2dxs --- addons/document/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/document/document.py b/addons/document/document.py index ec6c159f2b7..fb69b84e6a5 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -77,8 +77,8 @@ class document_file(osv.osv): 'create_uid': fields.many2one('res.users', 'Creator', readonly=True), 'write_date': fields.datetime('Date Modified', readonly=True), 'write_uid': fields.many2one('res.users', 'Last Modification User', readonly=True), - 'res_model': fields.char('Attached Model', size=64), # readonly? - 'res_id': fields.integer('Attached ID'), + 'res_model': fields.char('Attached Model', size=64, readonly=True), + 'res_id': fields.integer('Attached ID', readonly=True), # If ir.attachment contained any data before document is installed, preserve # the data, don't drop the column! From 4d99b09045d55fc56bab863c7cbfbfe8145b25c0 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:00:49 +0300 Subject: [PATCH 021/121] Document, FTP: text skeleton of trivial yaml tests. bzr revid: p_christ@hol.gr-20100629140049-dowvybyjh6bncyrb --- addons/document/test/document_test2.yml | 24 +++++++ .../document_ftp/test/document_ftp_test2.yml | 62 +++++++++++++++++++ .../document_ftp/test/document_ftp_test3.yml | 25 ++++++++ .../document_ftp/test/document_ftp_test4.yml | 34 ++++++++++ 4 files changed, 145 insertions(+) create mode 100644 addons/document/test/document_test2.yml create mode 100644 addons/document_ftp/test/document_ftp_test2.yml create mode 100644 addons/document_ftp/test/document_ftp_test3.yml create mode 100644 addons/document_ftp/test/document_ftp_test4.yml diff --git a/addons/document/test/document_test2.yml b/addons/document/test/document_test2.yml new file mode 100644 index 00000000000..5cd6547e687 --- /dev/null +++ b/addons/document/test/document_test2.yml @@ -0,0 +1,24 @@ +- + In order to test the document management +- + I make sure the default installation has some storage and root directory. +- + I create a "Testing" folder where all the test data will go. +- + I create an attachment into the root folder (w. empty fields, test that + defaults work) +- + I delete the attachment from the root folder +- + I create an attachment into the Testing folder. +- + I update the attachment with data, namely "abcd" +- + I rename the attachment. +- + I search the testing folder for attachments. +- + I create an attachment to a 3rd resource, eg. a res.country +- + I search for the res.country attachment +- diff --git a/addons/document_ftp/test/document_ftp_test2.yml b/addons/document_ftp/test/document_ftp_test2.yml new file mode 100644 index 00000000000..4f77fdbb723 --- /dev/null +++ b/addons/document_ftp/test/document_ftp_test2.yml @@ -0,0 +1,62 @@ +- + In order to test the document_ftp functionality +- + I open the 8021 port and see for ftp presence there +- + I read the list of databases at port 8021 and confirm our db is + there +- + I try to locate the default "Documents" folder in the db. +- + I create a "test.txt" file at the server (directly). The file + should have the "abcd" content +- + I look for the "test.txt" file at the server +- + I check that the content of "test.txt" is "abcd" +- + I try to cd into an non-existing folder 'Not-This' +- + I create a "test2.txt" file through FTP. +- + I look for the "test2.txt" file at the server +- + I look for the "test2.txt" file through FTP. +- + I delete the "test2.txt" file using FTP. +- + I check at the server that test2.txt is deleted +- + I create a test2.txt file again. +- + I delete the test2.txt from the server (RPC). +- + I check through FTP that test2.txt does not appear. +- + I create a "test-name.txt" file +- + I rename the "test-name.txt" file through ftp. +- + I check that test-name.txt has been renamed. +- + I create a new folder 'Test-Folder2' through FTP +- + I create a file 'test3.txt' at the 'Test-Folder2' +- + I try to retrieve test3.txt +- + I create a new folder, 'Test-Folder3', through FTP + I try to move test3.txt to 'Test-Folder3' +- + I remove the 'Test-Folder3' +- + I check that test3.txt is removed. +- + I create 1000 files through FTP +- + I list the 1000 files, check speed +- + I read the 1000 files, check speed +- + I move the 1000 files to 'Test-Folder2' +- \ No newline at end of file diff --git a/addons/document_ftp/test/document_ftp_test3.yml b/addons/document_ftp/test/document_ftp_test3.yml new file mode 100644 index 00000000000..c83af1b3c78 --- /dev/null +++ b/addons/document_ftp/test/document_ftp_test3.yml @@ -0,0 +1,25 @@ +- + In order to check international character functionality +- + I create in the server a folder called 'Δοκιμαστικός Φάκελλος' +- + I check that this folder exists at the server +- + I login with FTP and check that 'Δοκιμαστικός Φάκελλος' is there +- + I create a file named 'Δοκιμή' into that folder +- + I cerate another folder called 'δοκ. directory 2' in there +- + I remove the 'δοκ. directory 2' +- + I remove the 'Δοκιμή' file +- + I rename 'Δοκιμαστικός Φάκελλος' into 'άλλος' +- + I place a file 'file Φ3' in 'άλλος' +- + I rename the file into file+range(1..200) (large filename) +- + I delete the file with the large name +- diff --git a/addons/document_ftp/test/document_ftp_test4.yml b/addons/document_ftp/test/document_ftp_test4.yml new file mode 100644 index 00000000000..05ddef1c97b --- /dev/null +++ b/addons/document_ftp/test/document_ftp_test4.yml @@ -0,0 +1,34 @@ +- + In order to check dynamic folder functionality of document + FTP +- + I create two partners 'Partner1' and 'Partner2' +-| + I create three partner categories: 'none', 'pat1' and 'all' + I attach Partner1 to pat1, Partner1+Partner2 to 'all' +- + I create a resource folder of partners, by the (none, pat1, all) + categories. +- + I browse through ftp in the resource folder, checking that three + categories are there. +- + I create a 'partners' folder by the first resource one. +- + I check through FTP that the correct partners are listed at each + 'partners' folder. +- + I create an ir.attachment, attached (not related) to Partner1 + +- + I check that pat1/Partner1 folder has the file. + I check that all/Partner1 folder has the file +- + I place a file at the 'pat1'/Partner1 folder, through FTP +- + I check at the server that the file is attached to Partner1 +- + I check that all/Partner1 also has the file +- | + Bonus Piste: + I create a 'Partner3' under 'all' +- From 0c0a6dcd7fbdcd10bba0eabd7ccc91775d6a7b3b Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:01:34 +0300 Subject: [PATCH 022/121] Document FTP: one more test case, skeleton. bzr revid: p_christ@hol.gr-20100629140134-38zbsp7pvw1svgfv --- addons/document_ftp/test/document_ftp_test2.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/addons/document_ftp/test/document_ftp_test2.yml b/addons/document_ftp/test/document_ftp_test2.yml index 4f77fdbb723..1b871ec14bd 100644 --- a/addons/document_ftp/test/document_ftp_test2.yml +++ b/addons/document_ftp/test/document_ftp_test2.yml @@ -14,6 +14,10 @@ I look for the "test.txt" file at the server - I check that the content of "test.txt" is "abcd" +- + I append the string 'defgh' into "test.txt" +- + I check that the content of "text.txt" is 'abcdefgh' - I try to cd into an non-existing folder 'Not-This' - From 60283953f8fb961d8c5d158bcf97be933a66ff8f Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:03:03 +0300 Subject: [PATCH 023/121] Document: deprecate the node.open() functions These had been erroneously used by doc_ftp. The get_data() must be called instead. However, one day, a seekable object for the data should be returned. Conflicts: document/nodes.py bzr revid: p_christ@hol.gr-20100629140303-6lkfcz8lmvde79dt --- addons/document/nodes.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index c28b6d07b93..e59243de9d5 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -694,16 +694,7 @@ class node_file(node_class): def open(self, cr, mode=False): - uid = self.context.uid - if self.type in ('collection','database'): - return False - fobj = self.context._dirobj.pool.get('ir.attachment').browse(cr, uid, self.file_id, context=self.context.context) - if fobj.store_method and fobj.store_method== 'fs' : - s = StringIO.StringIO(self.get_data(cr, fobj)) - else: - s = StringIO.StringIO(base64.decodestring(fobj.db_datas or '')) - s.name = self - return s + raise DeprecationWarning("Who called this?") def rm(self, cr): uid = self.context.uid @@ -799,14 +790,7 @@ class node_content(node_class): self.act_id = act_id def open(self, cr, mode=False): - uid = self.context.uid - if self.type in ('collection','database'): - return False - pool = self.context._dirobj.pool - res = getattr(pool.get('document.directory.content'), 'process_read')(cr, uid, self) - res = StringIO.StringIO(res) - res.name = self - return res + raise DeprecationWarning() def fill_fields(self, cr, dctx = None): """ Try to read the object and fill missing fields, like mimetype, From ad31f812bc437b24f3ad0d25795b8b6de980d8ea Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:03:11 +0300 Subject: [PATCH 024/121] Document_ftp: Don't use the open() method, but the get_data() bzr revid: p_christ@hol.gr-20100629140311-7bvmbrc2t7gjyn1g --- .../document_ftp/ftpserver/abstracted_fs.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 60b3c24c7a9..7d4eccb05fd 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -311,15 +311,11 @@ class abstracted_fs: raise OSError(1, 'Operation not permited.') # Reading operation cr = pooler.get_db(node.context.dbname).cursor() - res = False - #try: - if node.type not in ('collection','database'): - res = node.open(cr, mode) - #except: - # pass - cr.close() - if not res: - raise OSError(1, 'Operation not permited.') + try: + res = StringIO.StringIO(node.get_data(cr)) + res.name = node + finally: + cr.close() return res # ok, but need test more @@ -423,10 +419,14 @@ class abstracted_fs: return None try: db,pool = pooler.get_db_and_pool(dbname) - except: + except Exception: raise OSError(1, 'Operation not permited.') cr = db.cursor() - uid = security.login(dbname, self.username, self.password) + try: + uid = security.login(dbname, self.username, self.password) + except Exception: + cr.close() + raise if not uid: raise OSError(2, 'Authentification Required.') return cr, uid, pool From d835a1419b2761b9cb178b47c86cc4ee287bc7c9 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:03:28 +0300 Subject: [PATCH 025/121] Document FTP: don't touch the databases, on db_list Trying a login on each db listed would result in the database loaded by the pooler. This behaviour is not really welcome. However, some clients (like kio_ftp) may still want to descend into the databases listed, in which case we cannot avoid to load the db(s). bzr revid: p_christ@hol.gr-20100629140328-2kqlfpo3oyhu0ufi --- addons/document_ftp/ftpserver/abstracted_fs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 7d4eccb05fd..a44008dfcfa 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -428,6 +428,7 @@ class abstracted_fs: cr.close() raise if not uid: + cr.close() raise OSError(2, 'Authentification Required.') return cr, uid, pool @@ -445,10 +446,10 @@ class abstracted_fs: result = [] for db in self.db_list(): try: - uid = security.login(db, self.username, self.password) - if uid: - result.append(false_node(db)) - except osv.except_osv: + result.append(false_node(db)) + except osv.except_osv: + import traceback + traceback.print_exc() pass return result cr = pooler.get_db(path.context.dbname).cursor() From d130db349e6caedc789f71064b2dde3888ece7f9 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:15 +0300 Subject: [PATCH 026/121] Document FTP: use pythonic logging in abstract_fs bzr revid: p_christ@hol.gr-20100629140515-u73gs9ks4tiylhuo --- .../document_ftp/ftpserver/abstracted_fs.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index a44008dfcfa..b9e20214045 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -4,6 +4,7 @@ import time from tarfile import filemode import StringIO import base64 +import logging import glob import fnmatch @@ -16,11 +17,6 @@ from osv import osv from document.nodes import node_res_dir, node_res_obj import stat -def log(message): - logger = netsvc.Logger() - logger.notifyChannel('DMS', netsvc.LOG_ERROR, message) - - def _get_month_name(month): month=int(month) if month==1:return 'Jan' @@ -104,7 +100,7 @@ class content_wrapper(StringIO.StringIO): StringIO.StringIO.close(self, *args, **kwargs) -class abstracted_fs: +class abstracted_fs(object): """A class used to interact with the file system, providing a high level, cross-platform interface compatible with both Windows and UNIX style filesystems. @@ -140,8 +136,8 @@ class abstracted_fs: if res and len(res): self.db_name_list.append(db_name) cr.commit() - except Exception, e: - log(e) + except Exception: + self._log.warning('Cannot use db "%s"', db_name) finally: if cr is not None: cr.close() @@ -154,6 +150,7 @@ class abstracted_fs: self.root = None self.cwd = '/' self.rnfr = None + self._log = logging.getLogger('FTP.fs') # --- Pathname / conversion utilities @@ -299,7 +296,7 @@ class abstracted_fs: s = file_wrapper('', cid, node.context.dbname, uid, ) return s except Exception,e: - log(e) + self._log.exception('Cannot create item %s at node %s', objname, repr(node)) raise OSError(1, 'Operation not permited.') finally: if cr: @@ -399,7 +396,7 @@ class abstracted_fs: pool.get('document.directory').create(cr, uid, val) cr.commit() except Exception,e: - log(e) + self._log.exception('Cannot create dir "%s" at node %s', basename, repr(node)) raise OSError(1, 'Operation not permited.') finally: if cr: cr.close() @@ -448,8 +445,6 @@ class abstracted_fs: try: result.append(false_node(db)) except osv.except_osv: - import traceback - traceback.print_exc() pass return result cr = pooler.get_db(path.context.dbname).cursor() @@ -513,6 +508,7 @@ class abstracted_fs: """ cr = False try: + # FIXME! wrong code here, doesn't use the node API dst_basename = _to_unicode(dst_basename) cr = pooler.get_db(src.context.dbname).cursor() uid = src.context.uid @@ -632,7 +628,7 @@ class abstracted_fs: else: raise OSError(1, 'Operation not permited.') except Exception,err: - log(err) + self._log.exception('Cannot rename "%s" to "%s" at "%s"', src, dst_basename, dst_basedir) raise OSError(1,'Operation not permited.') finally: if cr: cr.close() From 74d60c861f8a8ea7bc04a1dd135db699a94bdbc3 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:19 +0300 Subject: [PATCH 027/121] Document: the node context shouldn't contain 'lang' Node context should be as neutral as possible, in order to help caching. bzr revid: p_christ@hol.gr-20100629140519-fgxcr2wm5ctg8ywl --- addons/document/document_directory.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index a464ca61247..1b62418106c 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -195,17 +195,8 @@ class document_directory(osv.osv): """ if not context: context = {} - lang = context.get('lang',False) - if not lang: - user = self.pool.get('res.users').browse(cr, uid, uid) - lang = user.context_lang - context['lang'] = lang - try: #just instrumentation - return nodes.get_node_context(cr, uid, context).get_uri(cr, uri) - except Exception,e: - print "exception: ",e - raise + return nodes.get_node_context(cr, uid, context).get_uri(cr, uri) def _locate_child(self, cr, uid, root_id, uri,nparent, ncontext): From 2406e4a93d0654995bd89f44f28a4bac039a0a70 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:22 +0300 Subject: [PATCH 028/121] Document API: descriptors for data access. Trying to pass the full data of a node through get_data()/set_data() is sub-optimal and partially incompatible with some protocols. Instead, define a way to use transparent descriptors of the data. bzr revid: p_christ@hol.gr-20100629140522-8lot7kk0mm83u5c2 --- addons/document/nodes.py | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index e59243de9d5..d78fdc8af92 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -68,8 +68,42 @@ class node_context(object): duri = duri[1:] return ndir +class node_descriptor(object): + """A file-like interface to the data contents of a node. + + This class is NOT a node, but an /open descriptor/ for some + node. It can hold references to a cursor or a file object, + because the life of a node_descriptor will be the open period + of the data. + It should also take care of locking, with any native mechanism + or using the db. + For the implementation, it would be OK just to wrap around file, + StringIO or similar class. The node_descriptor is only needed to + provide the link to the parent /node/ object. + """ + def __init__(self, parent, **kwargs): + assert isinstance(parent, node_class) + self.name = parent.displayname + self.__parent = parent + + def open(self, **kwargs): + raise NotImplementedError + + def close(self): + raise NotImplementedError + def read(self, size=None): + raise NotImplementedError + + def seek(self, offset, whence=None): + raise NotImplementedError + + def tell(self): + raise NotImplementedError + + def write(self, str): + raise NotImplementedError class node_class(object): """ this is a superclass for our inodes @@ -123,6 +157,20 @@ class node_class(object): def get_data(self,cr): raise TypeError('no data for %s'% self.type) + + def open_data(self, cr, mode): + """ Open a node_descriptor object for this node. + + @param the mode of open, eg 'r', 'w', 'a', like file.open() + + This operation may lock the data for this node (and accross + other node hierarchies), until the descriptor is close()d. If + the node is locked, subsequent opens (depending on mode) may + immediately fail with an exception (which?). + For this class, there is no data, so no implementation. Each + child class that has data should override this. + """ + raise TypeError('no data for %s' % self.type) def _get_storage(self,cr): raise RuntimeError("no storage for base class") From 075affaccccd021dd14bc2c4b7a08f3dbf7a8fa7 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:25 +0300 Subject: [PATCH 029/121] Document API: implement the node_file.open_data() bzr revid: p_christ@hol.gr-20100629140525-a54b2p51b8co2zsf --- addons/document/nodes.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index d78fdc8af92..75db03a2c1a 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -19,11 +19,11 @@ # ############################################################################## -import base64 -import StringIO +# import base64 +# import StringIO from osv import osv, fields from osv.orm import except_orm -import urlparse +# import urlparse import pooler from tools.safe_eval import safe_eval @@ -741,8 +741,13 @@ class node_file(node_class): par = par.parent_id - def open(self, cr, mode=False): - raise DeprecationWarning("Who called this?") + def open_data(self, cr, mode): + stor = self.storage_id + assert stor, "No storage for file #%s" % self.file_id + # If storage is not set properly, we are just screwed here, don't + # try to get it from default. + stobj = self.context._dirobj.pool.get('document.storage') + return stobj.get_file(cr, self.context.uid, stor, self, mode=mode, context=self.context.context) def rm(self, cr): uid = self.context.uid From 54f9817335817d9947372192ed17ed72b0c25805 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:27 +0300 Subject: [PATCH 030/121] Document API: don't accept extra args in node_descriptor. bzr revid: p_christ@hol.gr-20100629140527-sz2sjgrv4cjw18uk --- addons/document/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 75db03a2c1a..1c14e287118 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -82,7 +82,7 @@ class node_descriptor(object): provide the link to the parent /node/ object. """ - def __init__(self, parent, **kwargs): + def __init__(self, parent): assert isinstance(parent, node_class) self.name = parent.displayname self.__parent = parent From f3cb7cd41539de507f800f10826f003017a40c8a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:30 +0300 Subject: [PATCH 031/121] Document storage: implement node_descriptors. bzr revid: p_christ@hol.gr-20100629140530-btusqmlz2lyadfbx --- addons/document/document_storage.py | 102 ++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 90be2bc47df..b2db4eb36a6 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -24,7 +24,9 @@ from osv import osv, fields import os import tools import base64 +import errno import logging +from StringIO import StringIO from tools.misc import ustr from tools.translate import _ @@ -34,6 +36,7 @@ from osv.orm import except_orm import random import string import netsvc +import nodes from content_index import cntIndex DMS_ROOT_PATH = tools.config.get('document_path', os.path.join(tools.config.get('root_path'), 'filestore')) @@ -84,6 +87,64 @@ def create_directory(path): os.makedirs(path) return dir_name +class nodefd_file(nodes.node_descriptor): + """ A descriptor to a real file + + Inheriting directly from file doesn't work, since file exports + some read-only attributes (like 'name') that we don't like. + """ + def __init__(self, parent, path, mode): + nodes.node_descriptor.__init__(self, parent) + self.__file = open(path, mode) + + for attr in ('closed', 'read', 'write', 'seek', 'tell'): + setattr(self,attr, getattr(self.__file, attr)) + + def close(self): + # TODO: locking in init, close() + self.__file.close() + + +class nodefd_db(StringIO, nodes.node_descriptor): + """ A descriptor to db data + """ + def __init__(self, parent, ira_browse, mode): + nodes.node_descriptor.__init__(self, parent) + if mode.endswith('b'): + mode = mode[:-1] + + if mode == 'r': + StringIO.__init__(self, ira_browse.db_datas) + elif mode == 'w': + StringIO.__init__(self, ira_browse.db_datas) + # at write, we start at 0 (= overwrite), but have the original + # data available, in case of a seek() + elif mode == 'a': + StringIO.__init__(self, None) + else: + logging.getLogger('document.storage').error("Incorrect mode %s specified", mode) + raise IOError(errno.EINVAL, "Invalid file mode") + self.mode = mode + + def close(self): + # we now open a *separate* cursor, to update the data. + # FIXME: this may be improved, for concurrency handling + uid = self.__parent.context.uid + cr = pooler.get_db(self.__parent.context.dbname).cursor() + if mode == 'w': + out = self.getvalue() + cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', + (out, len(out), self.__parent.file_id)) + elif mode == 'a': + out = self.getvalue() + cr.execute("UPDATE ir_attachment " \ + "SET db_datas = COALESCE(db_datas,'') || %s, " \ + " file_size = COALESCE(file_size, 0) + %d " \ + " WHERE id = %s", + (out, len(out), self.__parent.file_id)) + cr.commit() + cr.close() + StringIO.close(self) class document_storage(osv.osv): """ The primary object for data storage. @@ -147,6 +208,47 @@ class document_storage(osv.osv): ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) return self.__get_data_3(cr, uid, boo, ira, context) + def get_file(self, cr, uid, id, file_node, mode, context=None): + if context is None: + context = {} + boo = self.browse(cr, uid, id, context) + if not boo.online: + raise RuntimeError('media offline') + + ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context) + if boo.type == 'filestore': + if not ira.store_fname: + # On a migrated db, some files may have the wrong storage type + # try to fix their directory. + if ira.file_size: + self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id) + raise IOError(errno.ENOENT, 'No file can be located') + fpath = os.path.join(boo.path, ira.store_fname) + if self._debug: + self._doclog.debug("Trying to read \"%s\".."% fpath) + return nodefd_file(file_node, path=fpath, mode=mode) + + elif boo.type == 'db': + # TODO: we need a better api for large files + if self._debug: + self._doclog.debug("Trying to obtain db_datas for ir.attachment[%d]", ira.id) + return nodefd_db(file_node, ira_browse=ira, mode=mode) + + elif boo.type == 'realstore': + if not ira.store_fname: + # On a migrated db, some files may have the wrong storage type + # try to fix their directory. + if ira.file_size: + self._doclog.warning("ir.attachment #%d does not have a filename, trying the name." %ira.id) + sfname = ira.name + fpath = os.path.join(boo.path,ira.store_fname or ira.name) + if not os.path.exists(fpath): + raise IOError("File not found: %s" % fpath) + return nodefd_file(file_node, path=fpath, mode=mode) + + else: + raise TypeError("No %s storage" % boo.type) + def __get_data_3(self, cr, uid, boo, ira, context): if not boo.online: raise RuntimeError('media offline') From b2a10c1b20a8e7b416d56a9bb1ac9cfef47e3a97 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 29 Jun 2010 17:05:32 +0300 Subject: [PATCH 032/121] Document FTP: adapt to node_descriptors API bzr revid: p_christ@hol.gr-20100629140532-xgpjex7h0q1g0cff --- .../document_ftp/ftpserver/abstracted_fs.py | 126 ++++++------------ addons/document_ftp/ftpserver/ftpserver.py | 2 +- 2 files changed, 39 insertions(+), 89 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index b9e20214045..2b61ffd86e9 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -14,7 +14,8 @@ import netsvc import os from service import security from osv import osv -from document.nodes import node_res_dir, node_res_obj +#from document.nodes import node_res_dir, node_res_obj +from document.nodes import get_node_context import stat def _get_month_name(month): @@ -55,50 +56,6 @@ def _to_decode(s): return s.decode('ascii') except UnicodeError: return s - - -class file_wrapper(StringIO.StringIO): - def __init__(self, sstr='', ressource_id=False, dbname=None, uid=1, name=''): - StringIO.StringIO.__init__(self, sstr) - self.ressource_id = ressource_id - self.name = name - self.dbname = dbname - self.uid = uid - def close(self, *args, **kwargs): - db,pool = pooler.get_db_and_pool(self.dbname) - cr = db.cursor() - cr.commit() - try: - val = self.getvalue() - val2 = { - 'datas': base64.encodestring(val), - 'file_size': len(val), - } - pool.get('ir.attachment').write(cr, self.uid, [self.ressource_id], val2) - finally: - cr.commit() - cr.close() - StringIO.StringIO.close(self, *args, **kwargs) - -class content_wrapper(StringIO.StringIO): - def __init__(self, dbname, uid, pool, node, name=''): - StringIO.StringIO.__init__(self, '') - self.dbname = dbname - self.uid = uid - self.node = node - self.pool = pool - self.name = name - def close(self, *args, **kwargs): - db,pool = pooler.get_db_and_pool(self.dbname) - cr = db.cursor() - cr.commit() - try: - getattr(self.pool.get('document.directory.content'), 'process_write')(cr, self.uid, self.node, self.getvalue()) - finally: - cr.commit() - cr.close() - StringIO.StringIO.close(self, *args, **kwargs) - class abstracted_fs(object): """A class used to interact with the file system, providing a high @@ -186,14 +143,16 @@ class abstracted_fs(object): # Ok def ftp2fs(self, path_orig, data): - path = self.ftpnorm(path_orig) + path = self.ftpnorm(path_orig) if not data or (path and path=='/'): return None path2 = filter(None,path.split('/'))[1:] - (cr, uid, pool) = data + (cr, uid) = data if len(path2): - path2[-1]=_to_unicode(path2[-1]) - res = pool.get('document.directory').get_object(cr, uid, path2[:]) + path2[-1]=_to_unicode(path2[-1]) + + ctx = get_node_context(cr, uid, {}) + res = ctx.get_uri(cr, path2[:]) if not res: raise OSError(2, 'Not such file or directory.') return res @@ -302,15 +261,13 @@ class abstracted_fs(object): if cr: cr.close() - # Ok def open(self, node, mode): if not node: raise OSError(1, 'Operation not permited.') # Reading operation cr = pooler.get_db(node.context.dbname).cursor() try: - res = StringIO.StringIO(node.get_data(cr)) - res.name = node + res = node.open_data(cr, mode) finally: cr.close() return res @@ -401,7 +358,6 @@ class abstracted_fs(object): finally: if cr: cr.close() - # Ok def close_cr(self, data): if data: data[0].close() @@ -415,7 +371,7 @@ class abstracted_fs(object): if dbname not in self.db_list(): return None try: - db,pool = pooler.get_db_and_pool(dbname) + db = pooler.get_db(dbname) except Exception: raise OSError(1, 'Operation not permited.') cr = db.cursor() @@ -427,9 +383,22 @@ class abstracted_fs(object): if not uid: cr.close() raise OSError(2, 'Authentification Required.') - return cr, uid, pool + return cr, uid - # Ok + def get_node_cr_uid(self, node): + """ Get cr, uid, pool from a node + """ + db = pooler.get_db(node.context.dbname) + return db.cursor(), node.context.uid + + def get_node_cr(self, node): + """ Get the cursor for the database of a node + + The cursor is the only thing that a node will not store + persistenly, so we have to obtain a new one for each call. + """ + return self.get_node_cr_uid(node)[0] + def listdir(self, path): """List the content of a directory.""" class false_node(object): @@ -447,32 +416,21 @@ class abstracted_fs(object): except osv.except_osv: pass return result - cr = pooler.get_db(path.context.dbname).cursor() + cr = self.get_node_cr(path) res = path.children(cr) cr.close() return res - # Ok def rmdir(self, node): """Remove the specified directory.""" assert node - cr = pooler.get_db(node.context.dbname).cursor() - uid = node.context.uid - pool = pooler.get_pool(node.context.dbname) - object = node.context._dirobj.browse(cr, uid, node.dir_id) - if not object: - raise OSError(2, 'Not such file or directory.') - if object._table_name == 'document.directory': - if node.children(cr): - raise OSError(39, 'Directory not empty.') - res = pool.get('document.directory').unlink(cr, uid, [object.id]) - else: - raise OSError(1, 'Operation not permited.') + cr = self.get_node_cr(node) + try: + node.rmcol(cr) + cr.commit() + finally: + cr.close() - cr.commit() - cr.close() - - # Ok def remove(self, node): assert node if node.type == 'collection': @@ -484,20 +442,12 @@ class abstracted_fs(object): def rmfile(self, node): """Remove the specified file.""" assert node - if node.type == 'collection': - return self.rmdir(node) - uid = node.context.uid - pool = pooler.get_pool(node.context.dbname) - cr = pooler.get_db(node.context.dbname).cursor() - object = pool.get('ir.attachment').browse(cr, uid, node.file_id) - if not object: - raise OSError(2, 'Not such file or directory.') - if object._table_name == 'ir.attachment': - res = pool.get('ir.attachment').unlink(cr, uid, [object.id]) - else: - raise OSError(1, 'Operation not permited.') - cr.commit() - cr.close() + cr = self.get_node_cr(node) + try: + node.rm(cr) + cr.commit() + finally: + cr.close() # Ok def rename(self, src, dst_basedir, dst_basename): diff --git a/addons/document_ftp/ftpserver/ftpserver.py b/addons/document_ftp/ftpserver/ftpserver.py index 3a85b3061a7..c02323dbc53 100644 --- a/addons/document_ftp/ftpserver/ftpserver.py +++ b/addons/document_ftp/ftpserver/ftpserver.py @@ -793,7 +793,7 @@ class DTPHandler(asyncore.dispatcher): if self.transfer_finished: self.cmd_channel.respond("226 Transfer complete.") if self.file_obj: - fname = self.cmd_channel.fs.fs2ftp(self.file_obj.name) + fname = self.file_obj.name self.cmd_channel.log('"%s" %s.' %(fname, action)) else: tot_bytes = self.get_transmitted_bytes() From 3f4d6dd95dc134db7d5b637f78584588d7bbf0eb Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:30 +0300 Subject: [PATCH 033/121] Document: fix menu parents (after root moved to knowledge) bzr revid: p_christ@hol.gr-20100701175130-5ya6cxp5s4xb3gyi --- addons/document/board_document_view.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/board_document_view.xml b/addons/document/board_document_view.xml index 312c79d71ab..c8b155f5a4d 100644 --- a/addons/document/board_document_view.xml +++ b/addons/document/board_document_view.xml @@ -52,7 +52,7 @@ From ad1253c96bb0f36c9ad50cce959cc5e1f1e3e07a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:30 +0300 Subject: [PATCH 034/121] Document: start some trivial tests. bzr revid: p_christ@hol.gr-20100701175130-bs2rx4w81ck6gh9l --- addons/document/test/document_test2.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/addons/document/test/document_test2.yml b/addons/document/test/document_test2.yml index 5cd6547e687..e386db760c4 100644 --- a/addons/document/test/document_test2.yml +++ b/addons/document/test/document_test2.yml @@ -2,8 +2,18 @@ In order to test the document management - I make sure the default installation has some storage and root directory. +- + !assert {model: document.storage, id: storage_default }: + - id != False +- + !assert {model: document.directory, id: dir_root}: + - storage_id != False - I create a "Testing" folder where all the test data will go. +- + !record {model: document.directory, id: dir_tests }: + name: 'Testing' + parent_id: dir_root - I create an attachment into the root folder (w. empty fields, test that defaults work) From 6d64f88d5536ae14026556729433ea27b5eece61 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:31 +0300 Subject: [PATCH 035/121] Document: fix content indexer, where no filename is supplied bzr revid: p_christ@hol.gr-20100701175131-pbp7lsswuh8muyou --- addons/document/content_index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/addons/document/content_index.py b/addons/document/content_index.py index e7f88834647..d6012e60091 100644 --- a/addons/document/content_index.py +++ b/addons/document/content_index.py @@ -153,7 +153,10 @@ class contentIndex(object): if realfname : fname = realfname else: - bname,ext = os.path.splitext(filename) + try: + bname,ext = os.path.splitext(filename or 'test.tmp') + except Exception: + bname, ext = filename, 'tmp' fd, fname = tempfile.mkstemp(suffix=ext) os.write(fd, content) os.close(fd) From 6aef03991d174112c5787502e825ee4895f52bf0 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:31 +0300 Subject: [PATCH 036/121] Document: tests bzr revid: p_christ@hol.gr-20100701175131-5acgsuetw1qx0juu --- addons/document/test/document_test2.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/addons/document/test/document_test2.yml b/addons/document/test/document_test2.yml index e386db760c4..637fc2d9e48 100644 --- a/addons/document/test/document_test2.yml +++ b/addons/document/test/document_test2.yml @@ -17,12 +17,29 @@ - I create an attachment into the root folder (w. empty fields, test that defaults work) +- + !record {model: ir.attachment, id: file_test1 }: + name: Test file - I delete the attachment from the root folder +- + !delete {model: ir.attachment, id: file_test1, search: } - I create an attachment into the Testing folder. +- + !record {model: ir.attachment, id: file_test2 }: + name: Test file 2 + parent_id: dir_tests - I update the attachment with data, namely "abcd" +- + !record {model: ir.attachment, id: file_test2 }: + datas: abcd +- + I test that the datas of the attachment are correct +- + !assert {model: ir.attachment, id: file_test2 }: + - datas == 'abcd' - I rename the attachment. - From 8307ff013916ffcaf8c7f0c8a796df77a64bfcd4 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:31 +0300 Subject: [PATCH 037/121] Document: workaround case when text file consists of non-unicode. The content indexer would be trying to process the garbage of the file and fail. bzr revid: p_christ@hol.gr-20100701175131-b83jg498xjuosmd9 --- addons/document/document_storage.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index b2db4eb36a6..26ba5dea772 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -378,16 +378,21 @@ class document_storage(osv.osv): self._doclog.debug('Cannot index file:', exc_info=True) pass + try: + icont_u = ustr(icont) + except UnicodeError: + icont_u = '' + # a hack: /assume/ that the calling write operation will not try # to write the fname and size, and update them in the db concurrently. # We cannot use a write() here, because we are already in one. cr.execute('UPDATE ir_attachment SET store_fname = %s, file_size = %s, index_content = %s, file_type = %s WHERE id = %s', - (store_fname, filesize, ustr(icont), mime, file_node.file_id)) + (store_fname, filesize, icont_u, mime, file_node.file_id)) file_node.content_length = filesize file_node.content_type = mime return True except Exception, e : - self._doclog.warning( "Couldn't save data:", exc_info=True) + self._doclog.warning("Couldn't save data:", exc_info=True) # should we really rollback once we have written the actual data? # at the db case (only), that rollback would be safe raise except_orm(_('Error at doc write!'), str(e)) From 1e99bfc52c55dd20114b9c5586a8e6fe0c921dd2 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:31 +0300 Subject: [PATCH 038/121] Document: test cases bzr revid: p_christ@hol.gr-20100701175131-jltfosk4l5uvzo5l --- addons/document/test/document_test2.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/addons/document/test/document_test2.yml b/addons/document/test/document_test2.yml index 637fc2d9e48..a8962ac1ae8 100644 --- a/addons/document/test/document_test2.yml +++ b/addons/document/test/document_test2.yml @@ -34,18 +34,35 @@ I update the attachment with data, namely "abcd" - !record {model: ir.attachment, id: file_test2 }: - datas: abcd + datas: "abcd" - I test that the datas of the attachment are correct - !assert {model: ir.attachment, id: file_test2 }: - - datas == 'abcd' + - datas == "abcd\n" - I rename the attachment. +- + !record {model: ir.attachment, id: file_test2 }: + name: Test renamed 2 - I search the testing folder for attachments. +- + !python {model: ir.attachment}: | + ids = self.search(cr, uid, [('parent_id.name','=', 'Testing'), ('name','=','Test renamed 2')]) + assert ids == [ ref("file_test2") ], ids - I create an attachment to a 3rd resource, eg. a res.country +- + !record {model: ir.attachment, id: attach_3rd }: + name: 'Res country attachment.txt' + datas: 'defg' + res_model: res.country + res_id: !eval ref("base.za") - I search for the res.country attachment +- + !python {model: ir.attachment}: | + ids = self.search(cr, uid, [('res_model', '=', 'res.country'), ('res_id', '=', ref("base.za"))]) + assert ids == [ ref("attach_3rd")], ids - From 28464cb07c33a759653abda9310fd9e3e99ea4ef Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:31 +0300 Subject: [PATCH 039/121] Document tests: need to remove each document created in tests There is commit commands inside document, to prevent from partially writting data (eg. filestore and the db). Thus, the atomicity of the transaction is broken. bzr revid: p_christ@hol.gr-20100701175131-8v8xyosvz8nly0pi --- addons/document/test/document_test2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/addons/document/test/document_test2.yml b/addons/document/test/document_test2.yml index a8962ac1ae8..3a3b573e4e9 100644 --- a/addons/document/test/document_test2.yml +++ b/addons/document/test/document_test2.yml @@ -65,4 +65,6 @@ !python {model: ir.attachment}: | ids = self.search(cr, uid, [('res_model', '=', 'res.country'), ('res_id', '=', ref("base.za"))]) assert ids == [ ref("attach_3rd")], ids +- + !delete {moel: ir.attachment, id: attach_3rd, search: } - From 11c5171326d8eba7d897843ab3182878ac3cdb8f Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:31 +0300 Subject: [PATCH 040/121] Draft api for document.write(parent_id: ...) If the parent directory for some document node changes, we may have a major change in the stored content, so an API is needed for that. Conflicts: document/nodes.py bzr revid: p_christ@hol.gr-20100701175131-ldt3b0er5a7tja0u --- addons/document/document.py | 26 +++++++++++++++++++++++++- addons/document/nodes.py | 26 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/addons/document/document.py b/addons/document/document.py index fb69b84e6a5..27bd9aa3520 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -28,6 +28,7 @@ import os import pooler import netsvc +from osv.orm import except_orm #import StringIO from psycopg2 import Binary @@ -153,7 +154,30 @@ class document_file(osv.osv): return False if not self._check_duplication(cr, uid, vals, ids, 'write'): raise osv.except_osv(_('ValidateError'), _('File name must be unique!')) - result = super(document_file, self).write(cr, uid, ids, vals, context=context) + if 'parent_id' in vals: + # perhaps this file is changing directory + nctx = nodes.get_node_context(cr,uid,context) + dirobj = self.pool.get('document.directory') + dbro = dirobj.browse(cr, uid, vals['parent_id'], context=context) + ids2 = [] + result = False + for fbro in self.browse(cr, uid, ids, context=context): + if fbro.parent_id != vals['parent_id']: + fnode = nodes.node_file(None,None,nctx,fbro) + res = fnode.move_to(cr, fbro, dbro, True) + if isinstance(res, dict): + vals2 = vals.copy() + vals2.update(res) + wid = res.get('id', fbro.id) + result = super(document_file,self).write(cr,uid,wid,vals2,context=context) + # TODO: how to handle/merge several results? + elif res == True: + ids2.append(fbro.id) + elif res == False: + pass + ids = ids2 + if len(ids): + result = super(document_file,self).write(cr, uid, ids, vals, context=context) cr.commit() return result diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 1c14e287118..4ed36a45024 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -209,6 +209,32 @@ class node_class(object): def get_dav_eprop(self, cr, ns, prop): return None + def move_to(self, cr, fil_obj, ndir_obj, in_write=False): + """ Move this node to a new parent directory. + fil_obj, can be None, is the browse object for the file, if already + available. + ndir_obj must be the browse object to the new doc.directory location, + where this node should be moved to. + in_write: When called by write(), we shouldn't attempt to write the + object, but instead return the dict of vals (avoid re-entrance). + If false, we should write all data to the object, here, as if the + caller won't do anything after calling move_to() + + Return value: + True: the node is moved, the caller can update other values, too. + False: the node is either removed or fully updated, the caller + must discard the fil_obj, not attempt to write any more to it. + dict: values to write back to the object. *May* contain a new id! + + Depending on src and target storage, implementations of this function + could do various things. + Should also consider node<->content, dir<->dir moves etc. + + Move operations, as instructed from APIs (eg. request from DAV) could + use this function. + """ + raise NotImplementedError + def rm(self, cr): raise RuntimeError("Not Implemented") From 530932640babacde4973fd47f2a9632c1d70bd3d Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 041/121] Document: fix setting parent of document_file. There is some code that calls the 'move_to()' in case the document's parent (aka. directory) is changed. Fix notation there. bzr revid: p_christ@hol.gr-20100701175132-8ex94b3pdwkz4lq6 --- addons/document/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/document/document.py b/addons/document/document.py index 27bd9aa3520..428425be930 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -120,7 +120,7 @@ class document_file(osv.osv): res_model = vals.get('res_model', False) res_id = vals.get('res_id', 0) if op == 'write': - for file in self.browse(cr, uid, ids): + for file in self.browse(cr, uid, ids): # FIXME fields_only if not name: name = file.name if not parent_id: @@ -162,7 +162,7 @@ class document_file(osv.osv): ids2 = [] result = False for fbro in self.browse(cr, uid, ids, context=context): - if fbro.parent_id != vals['parent_id']: + if fbro.parent_id.id != vals['parent_id']: fnode = nodes.node_file(None,None,nctx,fbro) res = fnode.move_to(cr, fbro, dbro, True) if isinstance(res, dict): From 4052a14c95a0c7019a572acbc3d5a76a264de671 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 042/121] Document: amend the API for the move_to(), allow rename It is better to have the same API for both move and rename. bzr revid: p_christ@hol.gr-20100701175132-qgbf7p4izkh844vg --- addons/document/nodes.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 4ed36a45024..7c258ca0355 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -209,12 +209,14 @@ class node_class(object): def get_dav_eprop(self, cr, ns, prop): return None - def move_to(self, cr, fil_obj, ndir_obj, in_write=False): + def move_to(self, cr, fil_obj, ndir_obj, new_name=False, in_write=False): """ Move this node to a new parent directory. - fil_obj, can be None, is the browse object for the file, if already - available. - ndir_obj must be the browse object to the new doc.directory location, - where this node should be moved to. + @param fil_obj, can be None, is the browse object for the file, + if already available. + @param ndir_obj must be the browse object to the new doc.directory + location, where this node should be moved to. + @param new_name a name to rename this node to. If omitted, the old + name is preserved in_write: When called by write(), we shouldn't attempt to write the object, but instead return the dict of vals (avoid re-entrance). If false, we should write all data to the object, here, as if the From abebb5c31194e86af9bf6326f4f6a4980f17be1c Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 043/121] Document_ftp: *only* serve databases that have the module installed. The document_ftp is not really a database-specific server, but a server-wide implementation. So, it could normally serve all the databases of the server, once one of them has it installed. Limit that and instead only serve the dbs that have the module marked as installed. bzr revid: p_christ@hol.gr-20100701175132-dydehiw5m6nnvcwy --- addons/document_ftp/ftpserver/abstracted_fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 2b61ffd86e9..48c57cfc57b 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -88,7 +88,7 @@ class abstracted_fs(object): if not cr.fetchone(): continue - cr.execute("select id from ir_module_module where name like 'document%' and state='installed' ") + cr.execute("SELECT id FROM ir_module_module WHERE name = 'document_ftp' AND state='installed' ") res = cr.fetchone() if res and len(res): self.db_name_list.append(db_name) From 4913a2f0dba67d082e225bafa97b4c576e4f8cb7 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 044/121] Document: fix nodefd_db.close() Some vars were missing/inaccessible, put the queries in a try block. bzr revid: p_christ@hol.gr-20100701175132-389l8frlqqg7hgr3 --- addons/document/document_storage.py | 37 +++++++++++++++++------------ addons/document/nodes.py | 3 +++ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 26ba5dea772..08d3d82893b 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -35,6 +35,7 @@ from osv.orm import except_orm import random import string +import pooler import netsvc import nodes from content_index import cntIndex @@ -129,21 +130,27 @@ class nodefd_db(StringIO, nodes.node_descriptor): def close(self): # we now open a *separate* cursor, to update the data. # FIXME: this may be improved, for concurrency handling - uid = self.__parent.context.uid - cr = pooler.get_db(self.__parent.context.dbname).cursor() - if mode == 'w': - out = self.getvalue() - cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', - (out, len(out), self.__parent.file_id)) - elif mode == 'a': - out = self.getvalue() - cr.execute("UPDATE ir_attachment " \ - "SET db_datas = COALESCE(db_datas,'') || %s, " \ - " file_size = COALESCE(file_size, 0) + %d " \ - " WHERE id = %s", - (out, len(out), self.__parent.file_id)) - cr.commit() - cr.close() + par = self._get_parent() + uid = par.context.uid + cr = pooler.get_db(par.context.dbname).cursor() + try: + if self.mode == 'w': + out = self.getvalue() + cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', + (out, len(out), par.file_id)) + elif self.mode == 'a': + out = self.getvalue() + cr.execute("UPDATE ir_attachment " \ + "SET db_datas = COALESCE(db_datas,'') || %s, " \ + " file_size = COALESCE(file_size, 0) + %d " \ + " WHERE id = %s", + (out, len(out), par.file_id)) + cr.commit() + except Exception, e: + logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id) + raise + finally: + cr.close() StringIO.close(self) class document_storage(osv.osv): diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 7c258ca0355..1b516694aaf 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -87,6 +87,9 @@ class node_descriptor(object): self.name = parent.displayname self.__parent = parent + def _get_parent(self): + return self.__parent + def open(self, **kwargs): raise NotImplementedError From d52f451bcadd20d36532c837e0164569287764e1 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 045/121] Document: tests improvements. bzr revid: p_christ@hol.gr-20100701175132-ojsjqidcfegz9py8 --- addons/document/test/document_test2.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/addons/document/test/document_test2.yml b/addons/document/test/document_test2.yml index 3a3b573e4e9..fa5e98d3bdb 100644 --- a/addons/document/test/document_test2.yml +++ b/addons/document/test/document_test2.yml @@ -40,6 +40,8 @@ - !assert {model: ir.attachment, id: file_test2 }: - datas == "abcd\n" + - file_size == 5 + - file_type == 'text/plain' - I rename the attachment. - @@ -56,6 +58,7 @@ - !record {model: ir.attachment, id: attach_3rd }: name: 'Res country attachment.txt' + parent_id: dir_tests datas: 'defg' res_model: res.country res_id: !eval ref("base.za") From 493fe42f13f4ea99027af259462a54cd1dbddbd3 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 046/121] Document: improve the API of move_to() bzr revid: p_christ@hol.gr-20100701175132-s9y160v1etxt1ugp --- addons/document/nodes.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 1b516694aaf..14d8b16d987 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -212,14 +212,15 @@ class node_class(object): def get_dav_eprop(self, cr, ns, prop): return None - def move_to(self, cr, fil_obj, ndir_obj, new_name=False, in_write=False): + def move_to(self, cr, ndir_node, new_name=False, fil_obj=None, ndir_obj=None, in_write=False): """ Move this node to a new parent directory. + @param ndir_node the collection that this node should be moved under + @param new_name a name to rename this node to. If omitted, the old + name is preserved @param fil_obj, can be None, is the browse object for the file, if already available. @param ndir_obj must be the browse object to the new doc.directory location, where this node should be moved to. - @param new_name a name to rename this node to. If omitted, the old - name is preserved in_write: When called by write(), we shouldn't attempt to write the object, but instead return the dict of vals (avoid re-entrance). If false, we should write all data to the object, here, as if the From a286f1ae6d7fa6e07ca44439f3c57bae7d39688a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 047/121] Document API: fns to get nodes from orm objects. ORM objects should not directly create nodes, but instead ask the context to provide them (because they could be cached). bzr revid: p_christ@hol.gr-20100701175132-hfnvj5czdwuaswwg --- addons/document/nodes.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 14d8b16d987..acbf4ed8dcd 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -67,6 +67,27 @@ class node_context(object): return False duri = duri[1:] return ndir + + def get_dir_node(cr, dbro): + """Create (or locate) a node for a directory + @param dbro a browse object of document.directory + """ + fullpath = self._dirobj.get_full_path(cr, self.uid, dbro.id, self.context) + if dbro.type == 'directory': + return node_dir(fullpath, None ,self, dbro) + elif dbro.type == 'ressource': + assert dbro.ressource_parent_type_id == False + return node_res_dir(fullparh, None, self, dbro) + else: + raise ValueError("dir node for %s type", dbro.type) + + def get_file_node(cr, fbro): + """ Create or locate a node for a static file + @param fbro a browse object of an ir.attachment + """ + # TODO: fill the parent + return node_file(None,None,self,fbro) + class node_descriptor(object): """A file-like interface to the data contents of a node. From 0153d03c825974985ad5486f7ae9229cf0814853 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:32 +0300 Subject: [PATCH 048/121] Document storage: support more file open modes. We may be called with 'w+', 'r+' modes, support them. However, we still don't support the 'a+' mode. bzr revid: p_christ@hol.gr-20100701175132-v21yzk2h335emum1 --- addons/document/document_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 08d3d82893b..606600e17cb 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -114,10 +114,10 @@ class nodefd_db(StringIO, nodes.node_descriptor): if mode.endswith('b'): mode = mode[:-1] - if mode == 'r': - StringIO.__init__(self, ira_browse.db_datas) - elif mode == 'w': + if mode in ('r', 'r+'): StringIO.__init__(self, ira_browse.db_datas) + elif mode in ('w', 'w+'): + StringIO.__init__(self, None) # at write, we start at 0 (= overwrite), but have the original # data available, in case of a seek() elif mode == 'a': @@ -134,7 +134,7 @@ class nodefd_db(StringIO, nodes.node_descriptor): uid = par.context.uid cr = pooler.get_db(par.context.dbname).cursor() try: - if self.mode == 'w': + if self.mode in ('w', 'w+', 'r+'): out = self.getvalue() cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', (out, len(out), par.file_id)) From 2c5082001e8d6379fa36dd34cf070575ffcb31b1 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 049/121] Document: use the new move_to() api, protect setting of file_size. bzr revid: p_christ@hol.gr-20100701175133-9prni6r0kspt27ku --- addons/document/document.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/addons/document/document.py b/addons/document/document.py index 428425be930..221c9680aa7 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -159,12 +159,13 @@ class document_file(osv.osv): nctx = nodes.get_node_context(cr,uid,context) dirobj = self.pool.get('document.directory') dbro = dirobj.browse(cr, uid, vals['parent_id'], context=context) + dnode = nctx.get_dir_node(cr, dbro) ids2 = [] result = False for fbro in self.browse(cr, uid, ids, context=context): if fbro.parent_id.id != vals['parent_id']: - fnode = nodes.node_file(None,None,nctx,fbro) - res = fnode.move_to(cr, fbro, dbro, True) + fnode = nctx.get_file_node(cr, fbro) + res = fnode.move_to(cr, dnode, fbro, dbro, True) if isinstance(res, dict): vals2 = vals.copy() vals2.update(res) @@ -176,9 +177,11 @@ class document_file(osv.osv): elif res == False: pass ids = ids2 + if 'file_size' in vals: # only write that field using direct SQL calls + del vals['file_size'] if len(ids): result = super(document_file,self).write(cr, uid, ids, vals, context=context) - cr.commit() + cr.commit() # ? return result def create(self, cr, uid, vals, context=None): @@ -219,11 +222,15 @@ class document_file(osv.osv): else: datas = vals.get('datas', False) - vals['file_size'] = datas and len(datas) or 0 + if datas: + vals['file_size'] = len(datas) + else: + if vals.get('file_size'): + del vals['file_size'] if not self._check_duplication(cr, uid, vals): raise osv.except_osv(_('ValidateError'), _('File name must be unique!')) result = super(document_file, self).create(cr, uid, vals, context) - cr.commit() + cr.commit() # ? return result def unlink(self, cr, uid, ids, context={}): From 7c26eb67fb646e02848435e768fd9c08a63529c9 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 050/121] Document storage: the default db encoding is base64 in bytea The storage class has to consider that. Also, prepare a class for raw bytea storage. bzr revid: p_christ@hol.gr-20100701175133-xxgu3cdlff0u7086 --- addons/document/document_storage.py | 64 ++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 606600e17cb..290faac0229 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -115,7 +115,9 @@ class nodefd_db(StringIO, nodes.node_descriptor): mode = mode[:-1] if mode in ('r', 'r+'): - StringIO.__init__(self, ira_browse.db_datas) + cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s', ira_browse.id) + data = cr.fetchone()[0] + StringIO.__init__(self, data) elif mode in ('w', 'w+'): StringIO.__init__(self, None) # at write, we start at 0 (= overwrite), but have the original @@ -136,13 +138,13 @@ class nodefd_db(StringIO, nodes.node_descriptor): try: if self.mode in ('w', 'w+', 'r+'): out = self.getvalue() - cr.execute('UPDATE ir_attachment SET db_datas = %s, file_size=%d WHERE id = %s', + cr.execute("UPDATE ir_attachment SET db_datas = decode(%s,'escape'), file_size=%s WHERE id = %s", (out, len(out), par.file_id)) elif self.mode == 'a': out = self.getvalue() cr.execute("UPDATE ir_attachment " \ - "SET db_datas = COALESCE(db_datas,'') || %s, " \ - " file_size = COALESCE(file_size, 0) + %d " \ + "SET db_datas = COALESCE(db_datas,'') || decode(%s, 'escape'), " \ + " file_size = COALESCE(file_size, 0) + %s " \ " WHERE id = %s", (out, len(out), par.file_id)) cr.commit() @@ -153,6 +155,58 @@ class nodefd_db(StringIO, nodes.node_descriptor): cr.close() StringIO.close(self) +class nodefd_db64(StringIO, nodes.node_descriptor): + """ A descriptor to db data, base64 (the old way) + + It stores the data in base64 encoding at the db. Not optimal, but + the transparent compression of Postgres will save the day. + """ + def __init__(self, parent, ira_browse, mode): + nodes.node_descriptor.__init__(self, parent) + if mode.endswith('b'): + mode = mode[:-1] + + if mode in ('r', 'r+'): + StringIO.__init__(self, base64.decodestring(ira_browse.db_datas)) + elif mode in ('w', 'w+'): + StringIO.__init__(self, None) + # at write, we start at 0 (= overwrite), but have the original + # data available, in case of a seek() + elif mode == 'a': + StringIO.__init__(self, None) + else: + logging.getLogger('document.storage').error("Incorrect mode %s specified", mode) + raise IOError(errno.EINVAL, "Invalid file mode") + self.mode = mode + + def close(self): + # we now open a *separate* cursor, to update the data. + # FIXME: this may be improved, for concurrency handling + par = self._get_parent() + uid = par.context.uid + cr = pooler.get_db(par.context.dbname).cursor() + try: + if self.mode in ('w', 'w+', 'r+'): + out = self.getvalue() + cr.execute('UPDATE ir_attachment SET db_datas = %s::bytea, file_size=%s WHERE id = %s', + (base64.encodestring(out), len(out), par.file_id)) + elif self.mode == 'a': + out = self.getvalue() + # Yes, we're obviously using the wrong representation for storing our + # data as base64-in-bytea + cr.execute("UPDATE ir_attachment " \ + "SET db_datas = encode( (COALESCE(decode(encode(db_datas,'escape'),'base64'),'') || decode(%s, 'base64')),'base64')::bytea , " \ + " file_size = COALESCE(file_size, 0) + %s " \ + " WHERE id = %s", + (base64.encodestring(out), len(out), par.file_id)) + cr.commit() + except Exception, e: + logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id) + raise + finally: + cr.close() + StringIO.close(self) + class document_storage(osv.osv): """ The primary object for data storage. Each instance of this object is a storage media, in which our application @@ -239,7 +293,7 @@ class document_storage(osv.osv): # TODO: we need a better api for large files if self._debug: self._doclog.debug("Trying to obtain db_datas for ir.attachment[%d]", ira.id) - return nodefd_db(file_node, ira_browse=ira, mode=mode) + return nodefd_db64(file_node, ira_browse=ira, mode=mode) elif boo.type == 'realstore': if not ira.store_fname: From c9f313dc218d1164f7133af2b4e3a7b8261110c6 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 051/121] Document: isolate the partner_id auto-detection in a fn() When a document is created, some code tries to auto-detect the related partner_id from res_model. This should better be put in a function, and reused in other places. bzr revid: p_christ@hol.gr-20100701175133-6aq2tx9lkuskvedh --- addons/document/document.py | 41 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/addons/document/document.py b/addons/document/document.py index 221c9680aa7..dbd4ec3f5b0 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -194,26 +194,10 @@ class document_file(osv.osv): vals['res_id'] = context.get('default_res_id', False) if not vals.get('res_model', False) and context.get('default_res_model', False): vals['res_model'] = context.get('default_res_model', False) - if vals.get('res_id', False) and vals.get('res_model', False): - obj_model = self.pool.get(vals['res_model']) - result = obj_model.read(cr, uid, [vals['res_id']], ['name', 'partner_id', 'address_id'], context=context) - if len(result): - obj = result[0] - if obj_model._name == 'res.partner': - vals['partner_id'] = obj['id'] - elif obj.get('address_id', False): - if isinstance(obj['address_id'], tuple) or isinstance(obj['address_id'], list): - address_id = obj['address_id'][0] - else: - address_id = obj['address_id'] - address = self.pool.get('res.partner.address').read(cr, uid, [address_id], context=context) - if len(address): - vals['partner_id'] = address[0]['partner_id'][0] or False - elif obj.get('partner_id', False): - if isinstance(obj['partner_id'], tuple) or isinstance(obj['partner_id'], list): - vals['partner_id'] = obj['partner_id'][0] - else: - vals['partner_id'] = obj['partner_id'] + if vals.get('res_id', False) and vals.get('res_model', False) \ + and not vals.get('partner_id', False): + vals['partner_id'] = __get_partner_id(cr, uid, \ + vals['res_model'], vals['res_id'], context) datas = None if vals.get('link', False) : @@ -233,6 +217,23 @@ class document_file(osv.osv): cr.commit() # ? return result + def __get_partner_id(self, cr, uid, res_model, res_id, context): + """ A helper to retrieve the associated partner from any res_model+id + It is a hack that will try to discover if the mentioned record is + clearly associated with a partner record. + """ + if False: + obj_model = self.pool.get(vals['res_model']) + if obj_model._name == 'res.partner': + return res_id + elif 'partner_id' in obj_model._columns and obj_model._columns['partner_id']._obj == 'res.partner': + bro = obj_model.browse(self, cr, uid, res_id, context=context) + return bro.partner_id.id + elif 'address_id' in obj_model._columns and obj_model._columns['address_id']._obj == 'res.partner.address': + bro = obj_model.browse(self, cr, uid, res_id, context=context) + return bro.address_id.partner_id.id + return False + def unlink(self, cr, uid, ids, context={}): stor = self.pool.get('document.storage') unres = [] From 575889d33ed2932d32c67a5454758d2d00dcab7b Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 052/121] Document API: have equality operators for nodes, context These will be used for advanced operations (eg. move) of nodes. Conflicts: document/nodes.py bzr revid: p_christ@hol.gr-20100701175133-gbsdly2fwcwtr0dy --- addons/document/nodes.py | 73 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index acbf4ed8dcd..e4f2a0bf615 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -57,6 +57,22 @@ class node_context(object): assert self._dirobj self.rootdir = False #self._dirobj._get_root_directory(cr,uid,context) + def __eq__(self, other): + if not type(other) == node_context: + return False + if self.dbname != other.dbname: + return False + if self.uid != other.uid: + return False + if self.context != other.context: + return False + if self.rootdir != other.rootdir: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + def get_uri(self, cr, uri): """ Although this fn passes back to doc.dir, it is needed since it is a potential caching point """ @@ -68,7 +84,7 @@ class node_context(object): duri = duri[1:] return ndir - def get_dir_node(cr, dbro): + def get_dir_node(self, cr, dbro): """Create (or locate) a node for a directory @param dbro a browse object of document.directory """ @@ -81,7 +97,7 @@ class node_context(object): else: raise ValueError("dir node for %s type", dbro.type) - def get_file_node(cr, fbro): + def get_file_node(self, cr, fbro): """ Create or locate a node for a static file @param fbro a browse object of an ir.attachment """ @@ -152,6 +168,12 @@ class node_class(object): self.dctx = parent.dctx.copy() self.displayname = 'Object' + def __eq__(self, other): + return NotImplemented + + def __ne__(self, other): + return not self.__eq__(other) + def full_path(self): """ Return the components of the full path for some node. @@ -369,7 +391,17 @@ class node_dir(node_database): print e pass - + def __eq__(self, other): + if not self.context == other.context: + return False + if type(self) != type(other): + return False + # Two directory nodes, for the same document.directory, may have a + # different context! (dynamic folders) + if self.dctx != other.dctx: + return False + return self.dir_id == other.dir_id + def get_data(self, cr): res = '' for child in self.children(cr): @@ -493,6 +525,17 @@ class node_res_dir(node_class): for dfld in dirr.dctx_ids: self.dctx_dict['dctx_' + dfld.field] = dfld.expr + def __eq__(self, other): + if not self.context == other.context: + return False + if type(self) != type(other): + return False + # Two nodes, for the same document.directory, may have a + # different context! (dynamic folders) + if self.dctx != other.dctx: + return False + return self.dir_id == other.dir_id + def children(self, cr, domain=None): return self._child_get(cr, domain=domain) @@ -596,6 +639,21 @@ class node_res_obj(node_class): else: self.res_id = res_id + def __eq__(self, other): + if not self.context == other.context: + return False + if type(self) != type(other): + return False + if not self.res_model == other.res_model: + return False + if not self.res_id == other.res_id: + return False + if self.domain != other.domain: + return False + if self.dctx != other.dctx: + return False + return self.dir_id == other.dir_id + def children(self, cr, domain=None): return self._child_get(cr, domain=domain) + self._file_get(cr) @@ -793,6 +851,15 @@ class node_file(node_class): break par = par.parent_id + def __eq__(self, other): + if not self.context == other.context: + return False + if type(self) != type(other): + return False + if self.dctx != other.dctx: + return False + return self.file_id == other.file_id + def open_data(self, cr, mode): stor = self.storage_id From 1819f137e974f75cf746a672efd6942091f1179a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 053/121] Document: storage must be set before _set_data No attempt to use a default one. Conflicts: document/nodes.py bzr revid: p_christ@hol.gr-20100701175133-783ril5hg7rls246 --- addons/document/nodes.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index e4f2a0bf615..998ea7706b2 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -934,12 +934,7 @@ class node_file(node_class): the browse object. """ # this is where storage kicks in.. stor = self.storage_id - if not stor: - data_obj = self.context._dirobj.pool.get('ir.model.data') - data_id = data_obj._get_id(cr, self.context.uid, 'document', 'storage_db') - if data_id: - stor = data_obj.browse(cr, self.context.uid, data_id, context=self.context.context).res_id - assert stor + assert stor, "No storage for file #%s" % self.file_id stobj = self.context._dirobj.pool.get('document.storage') return stobj.set_data(cr, self.context.uid,stor, self, data, self.context.context, fil_obj) From 9cdbcbfee1d356ebaab68e17e75aa0bf1eb16b5d Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 054/121] Document API: fix equality operators They must first check for the type of the 'other' side, or attribute errors may occur. bzr revid: p_christ@hol.gr-20100701175133-mjauo28qv6vkqfuq --- addons/document/nodes.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 998ea7706b2..639d0e9b9b8 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -392,10 +392,10 @@ class node_dir(node_database): pass def __eq__(self, other): - if not self.context == other.context: - return False if type(self) != type(other): return False + if not self.context == other.context: + return False # Two directory nodes, for the same document.directory, may have a # different context! (dynamic folders) if self.dctx != other.dctx: @@ -526,10 +526,10 @@ class node_res_dir(node_class): self.dctx_dict['dctx_' + dfld.field] = dfld.expr def __eq__(self, other): - if not self.context == other.context: - return False if type(self) != type(other): return False + if not self.context == other.context: + return False # Two nodes, for the same document.directory, may have a # different context! (dynamic folders) if self.dctx != other.dctx: @@ -640,10 +640,10 @@ class node_res_obj(node_class): self.res_id = res_id def __eq__(self, other): - if not self.context == other.context: - return False if type(self) != type(other): return False + if not self.context == other.context: + return False if not self.res_model == other.res_model: return False if not self.res_id == other.res_id: @@ -852,10 +852,10 @@ class node_file(node_class): par = par.parent_id def __eq__(self, other): - if not self.context == other.context: - return False if type(self) != type(other): return False + if not self.context == other.context: + return False if self.dctx != other.dctx: return False return self.file_id == other.file_id From 54b9e746ac4dad02e17d98c8febf5c899baf2ab0 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 055/121] Document API: trivial implementation of simple file rename bzr revid: p_christ@hol.gr-20100701175133-k0gnyan62wo2gs7n --- addons/document/nodes.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 639d0e9b9b8..a77c35b4cdf 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -941,6 +941,44 @@ class node_file(node_class): def _get_ttag(self,cr): return 'file-%d' % self.file_id + def move_to(self, cr, ndir_node, new_name=False, fil_obj=None, ndir_obj=None, in_write=False): + if ndir_node.context != self.context: + raise NotImplementedError("Cannot move files between contexts") + + doc_obj = self.context._dirobj.pool.get('ir.attachment') + if not fil_obj: + dbro = doc_obj.browse(cr, self.context.uid, self.file_id, context=self.context.context) + else: + dbro = fil_obj + assert dbro.id == self.file_id + + if not dbro: + raise IndexError("Cannot locate doc %d", self.file_id) + + if (not self.parent): + # there *must* be a parent node for this one + self.parent = self.context.get_dir_node(cr, dbro.parent_id.id) + + if self.parent != ndir_node: + logger.debug('Cannot move file %r from %r to %r', self, self.parent, ndir_node) + raise NotImplementedError('Cannot move file to another dir') + + ret = {} + if new_name and (new_name != dbro.name): + stobj = self.context._dirobj.pool.get('document.storage') + r2 = stobj.simple_rename(cr, self.context.uid, self, new_name, self.context.context) + ret.update(r2) + + del dbro + + if not in_write: + # We have to update the data ourselves + if ret: + doc_obj.write(cr, self.context.uid, [self.file_id,], ret, self.context.context) + ret = True + + return ret + class node_content(node_class): our_type = 'content' def __init__(self, path, parent, context, cnt, dctx = None, act_id=None): From 49bf479c188533b93d74f1e0f873f8d71b6ebd75 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:33 +0300 Subject: [PATCH 056/121] Document storage: implementation of simple rename Conflicts: document/document_storage.py bzr revid: p_christ@hol.gr-20100701175133-ti0itn58w9az88w6 --- addons/document/document_storage.py | 38 ++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 290faac0229..aeea5f99011 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -480,7 +480,7 @@ class document_storage(osv.osv): path = storage_bo.path return ( storage_bo.id, 'file', os.path.join(path, fname)) else: - raise TypeError("No %s storage" % boo.type) + raise TypeError("No %s storage" % storage_bo.type) def do_unlink(self, cr, uid, unres): for id, ktype, fname in unres: @@ -494,6 +494,42 @@ class document_storage(osv.osv): return True + def simple_rename(self, cr, uid, file_node, new_name, context=None): + """ A preparation for a file rename. + It will not affect the database, but merely check and perhaps + rename the realstore file. + + @return the dict of values that can safely be be stored in the db. + """ + sbro = self.browse(cr, uid, file_node.storage_id, context=context) + assert sbro, "The file #%d didn't provide storage" % file_node.file_id + + if sbro.type in ('filestore', 'db'): + # nothing to do for a rename, allow to change the db field + return { 'name': new_name, 'datas_fname': new_name } + elif sbro.type == 'realstore': + fname = fil_bo.store_fname + if not fname: + return ValueError("Tried to rename a non-stored file") + path = storage_bo.path + oldpath = os.path.join(path, fname) + + for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'): + if ch in new_name: + raise ValueError("Invalid char %s in name %s" %(ch, new_name)) + + file_node.fix_ppath(cr, ira) + npath = file_node.full_path() or [] + dpath = [path,] + dpath.extend(npath[:-1]) + dpath.append(new_name) + newpath = os.path.join(*dpath) + # print "old, new paths:", oldpath, newpath + os.rename(oldpath, newpath) + return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name } + else: + raise TypeError("No %s storage" % boo.type) + document_storage() From fb3f520268ce6437cdee1529b5bd974426af1e42 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Thu, 1 Jul 2010 20:51:34 +0300 Subject: [PATCH 057/121] Document ftp: use create, coll_create, rename from node API bzr revid: p_christ@hol.gr-20100701175134-g5wkaweznslhajgm --- .../document_ftp/ftpserver/abstracted_fs.py | 247 +++--------------- 1 file changed, 30 insertions(+), 217 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 48c57cfc57b..cb711c9d7fd 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -187,73 +187,31 @@ class abstracted_fs(object): # --- Wrapper methods around open() and tempfile.mkstemp - # Ok def create(self, node, objname, mode): + """ Create a children file-node under node, open it + @return open node_descriptor of the created node + """ objname = _to_unicode(objname) - cr = None + cr = self.get_node_cr(node) try: - uid = node.context.uid - pool = pooler.get_pool(node.context.dbname) - cr = pooler.get_db(node.context.dbname).cursor() child = node.child(cr, objname) if child: - if child.type in ('collection','database'): + if child.type not in ('file','content'): raise OSError(1, 'Operation not permited.') - if child.type == 'content': - s = content_wrapper(node.context.dbname, uid, pool, child) - return s - fobj = pool.get('ir.attachment') - ext = objname.find('.') >0 and objname.split('.')[1] or False - # TODO: test if already exist and modify in this case if node.type=file - ### checked already exits - object2 = False - if isinstance(node, node_res_obj): - object2 = node and pool.get(node.context.context['res_model']).browse(cr, uid, node.context.context['res_id']) or False - - cid = False - object = node.context._dirobj.browse(cr, uid, node.dir_id) - where = [('name','=',objname)] - if object and (object.type in ('directory')) or object2: - where.append(('parent_id','=',object.id)) - else: - where.append(('parent_id','=',False)) + ret = child.open_data(cr, mode) + cr.close() + return ret + except OSError: + cr.close() + raise + except Exception,e: + self._log.exception('Cannot locate item %s at node %s', objname, repr(node)) + pass - if object2: - where += [('res_id','=',object2.id),('res_model','=',object2._name)] - cids = fobj.search(cr, uid, where) - if len(cids): - cid = cids[0] - - if not cid: - val = { - 'name': objname, - 'datas_fname': objname, - 'parent_id' : node.dir_id, - 'datas': '', - 'file_size': 0L, - 'file_type': ext, - 'store_method' : (object.storage_id.type == 'filestore' and 'fs')\ - or (object.storage_id.type == 'db' and 'db') - } - if object and (object.type in ('directory')) or not object2: - val['parent_id']= object and object.id or False - partner = False - if object2: - if 'partner_id' in object2 and object2.partner_id.id: - partner = object2.partner_id.id - if object2._name == 'res.partner': - partner = object2.id - val.update( { - 'res_model': object2._name, - 'partner_id': partner, - 'res_id': object2.id - }) - cid = fobj.create(cr, uid, val, context={}) - cr.commit() - - s = file_wrapper('', cid, node.context.dbname, uid, ) - return s + try: + child = node.create_child(cr, objname, data=None) + return child.open_data(cr, mode) except Exception,e: self._log.exception('Cannot create item %s at node %s', objname, repr(node)) raise OSError(1, 'Operation not permited.') @@ -265,7 +223,7 @@ class abstracted_fs(object): if not node: raise OSError(1, 'Operation not permited.') # Reading operation - cr = pooler.get_db(node.context.dbname).cursor() + cr = self.get_node_cr(node) try: res = node.open_data(cr, mode) finally: @@ -279,24 +237,12 @@ class abstracted_fs(object): name. Unlike mkstemp it returns an object with a file-like interface. """ - raise 'Not Yet Implemented' -# class FileWrapper: -# def __init__(self, fd, name): -# self.file = fd -# self.name = name -# def __getattr__(self, attr): -# return getattr(self.file, attr) -# -# text = not 'b' in mode -# # max number of tries to find out a unique file name -# tempfile.TMP_MAX = 50 -# fd, name = tempfile.mkstemp(suffix, prefix, dir, text=text) -# file = os.fdopen(fd, mode) -# return FileWrapper(file, name) + raise NotImplementedError text = not 'b' in mode # for unique file , maintain version if duplicate file if dir: + # TODO cr = dir.cr uid = dir.uid pool = pooler.get_pool(node.context.dbname) @@ -330,28 +276,12 @@ class abstracted_fs(object): cr = False if not node: raise OSError(1, 'Operation not permited.') + + cr, uid = self.get_node_cr(node) try: basename =_to_unicode(basename) - cr = pooler.get_db(node.context.dbname).cursor() - uid = node.context.uid - pool = pooler.get_pool(node.context.dbname) - object2 = False - if isinstance(node, node_res_obj): - object2 = node and pool.get(node.context.context['res_model']).browse(cr, uid, node.context.context['res_id']) or False - obj = node.context._dirobj.browse(cr, uid, node.dir_id) - if obj and (obj.type == 'ressource') and not object2: - raise OSError(1, 'Operation not permited.') - val = { - 'name': basename, - 'ressource_parent_type_id': obj and obj.ressource_type_id.id or False, - 'ressource_id': object2 and object2.id or False, - 'parent_id' : False - } - if (obj and (obj.type in ('directory'))) or not object2: - val['parent_id'] = obj and obj.id or False - # Check if it alreayd exists ! - pool.get('document.directory').create(cr, uid, val) - cr.commit() + cdir = node.create_child_collection(cr, basename) + self._log.debug("Created child dir: %r", cdir) except Exception,e: self._log.exception('Cannot create dir "%s" at node %s', basename, repr(node)) raise OSError(1, 'Operation not permited.') @@ -449,134 +379,17 @@ class abstracted_fs(object): finally: cr.close() - # Ok def rename(self, src, dst_basedir, dst_basename): - """ - Renaming operation, the effect depends on the src: + """ Renaming operation, the effect depends on the src: * A file: read, create and remove * A directory: change the parent and reassign childs to ressource """ - cr = False + cr = self.get_node_cr(src) try: - # FIXME! wrong code here, doesn't use the node API - dst_basename = _to_unicode(dst_basename) - cr = pooler.get_db(src.context.dbname).cursor() - uid = src.context.uid - if src.type == 'collection': - obj2 = False - dst_obj2 = False - pool = pooler.get_pool(src.context.dbname) - if isinstance(src, node_res_obj): - obj2 = src and pool.get(src.context.context['res_model']).browse(cr, uid, src.context.context['res_id']) or False - obj = src.context._dirobj.browse(cr, uid, src.dir_id) - if isinstance(dst_basedir, node_res_obj): - dst_obj2 = dst_basedir and pool.get(dst_basedir.context.context['res_model']).browse(cr, uid, dst_basedir.context.context['res_id']) or False - dst_obj = dst_basedir.context._dirobj.browse(cr, uid, dst_basedir.dir_id) - if obj._table_name <> 'document.directory': - raise OSError(1, 'Operation not permited.') - result = { - 'directory': [], - 'attachment': [] - } - # Compute all childs to set the new ressource ID - child_ids = [src] - while len(child_ids): - node = child_ids.pop(0) - child_ids += node.children(cr) - if node.type == 'collection': - object2 = False - if isinstance(node, node_res_obj): - object2 = node and pool.get(node.context.context['res_model']).browse(cr, uid, node.context.context['res_id']) or False - object = node.context._dirobj.browse(cr, uid, node.dir_id) - result['directory'].append(object.id) - if (not object.ressource_id) and object2: - raise OSError(1, 'Operation not permited.') - elif node.type == 'file': - result['attachment'].append(object.id) - - if obj2 and not obj.ressource_id: - raise OSError(1, 'Operation not permited.') - - if (dst_obj and (dst_obj.type in ('directory'))) or not dst_obj2: - parent_id = dst_obj and dst_obj.id or False - else: - parent_id = False - - - if dst_obj2: - ressource_type_id = pool.get('ir.model').search(cr, uid, [('model','=',dst_obj2._name)])[0] - ressource_id = dst_obj2.id - ressource_model = dst_obj2._name - if dst_obj2._name == 'res.partner': - partner_id = dst_obj2.id - else: - partner_id = pool.get(dst_obj2._name).fields_get(cr, uid, ['partner_id']) and dst_obj2.partner_id.id or False - else: - ressource_type_id = False - ressource_id = False - ressource_model = False - partner_id = False - pool.get('document.directory').write(cr, uid, result['directory'], { - 'name' : dst_basename, - 'ressource_id': ressource_id, - 'ressource_parent_type_id': ressource_type_id, - 'parent_id' : parent_id - }) - val = { - 'res_id': ressource_id, - 'res_model': ressource_model, - 'partner_id': partner_id - } - pool.get('ir.attachment').write(cr, uid, result['attachment'], val) - if (not val['res_id']) and result['attachment']: - cr.execute('update ir_attachment set res_id=NULL where id in ('+','.join(map(str,result['attachment']))+')') - - cr.commit() - - elif src.type == 'file': - pool = pooler.get_pool(src.context.dbname) - obj = pool.get('ir.attachment').browse(cr, uid, src.file_id) - dst_obj2 = False - if isinstance(dst_basedir, node_res_obj): - dst_obj2 = dst_basedir and pool.get(dst_basedir.context.context['res_model']).browse(cr, uid, dst_basedir.context.context['res_id']) or False - dst_obj = dst_basedir.context._dirobj.browse(cr, uid, dst_basedir.dir_id) - - val = { - 'partner_id':False, - #'res_id': False, - 'res_model': False, - 'name': dst_basename, - 'datas_fname': dst_basename, - } - - if (dst_obj and (dst_obj.type in ('directory','ressource'))) or not dst_obj2: - val['parent_id'] = dst_obj and dst_obj.id or False - else: - val['parent_id'] = False - - if dst_obj2: - val['res_model'] = dst_obj2._name - val['res_id'] = dst_obj2.id - if dst_obj2._name == 'res.partner': - val['partner_id'] = dst_obj2.id - else: - val['partner_id'] = pool.get(dst_obj2._name).fields_get(cr, uid, ['partner_id']) and dst_obj2.partner_id.id or False - elif obj.res_id: - # I had to do that because writing False to an integer writes 0 instead of NULL - # change if one day we decide to improve osv/fields.py - cr.execute('update ir_attachment set res_id=NULL where id=%s', (obj.id,)) - - pool.get('ir.attachment').write(cr, uid, [obj.id], val) - cr.commit() - elif src.type=='content': - src_file = self.open(src,'r') - dst_file = self.create(dst_basedir, dst_basename, 'w') - dst_file.write(src_file.getvalue()) - dst_file.close() - src_file.close() - cr.commit() - else: - raise OSError(1, 'Operation not permited.') + nname = _to_unicode(dst_basename) + ret = src.move_to(cr, dst_basedir, new_name=nname) + # API shouldn't wait for us to write the object + assert (ret is True) or (ret is False) except Exception,err: self._log.exception('Cannot rename "%s" to "%s" at "%s"', src, dst_basename, dst_basedir) raise OSError(1,'Operation not permited.') From 35c932d76a63bed6210943efdbd1b7b162ca828d Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 01:42:49 +0300 Subject: [PATCH 058/121] Document FTP: write Yaml tests, have special helper module. bzr revid: p_christ@hol.gr-20100701224249-fmble80t8nlg2azq --- .../document_ftp/test/document_ftp_test2.yml | 151 +++++++++++++++++- addons/document_ftp/test_easyftp.py | 66 ++++++++ 2 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 addons/document_ftp/test_easyftp.py diff --git a/addons/document_ftp/test/document_ftp_test2.yml b/addons/document_ftp/test/document_ftp_test2.yml index 1b871ec14bd..e0b3b72b320 100644 --- a/addons/document_ftp/test/document_ftp_test2.yml +++ b/addons/document_ftp/test/document_ftp_test2.yml @@ -2,65 +2,212 @@ In order to test the document_ftp functionality - I open the 8021 port and see for ftp presence there +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_plain_ftp() + assert ftp.sock and (ftp.lastresp == '220'), ftp.lastresp - I read the list of databases at port 8021 and confirm our db is there +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_login(cr, uid, self) + assert cr.dbname in ftp.nlst("/") - I try to locate the default "Documents" folder in the db. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_login(cr, uid, self) + ftp.cwd('/Documents') - I create a "test.txt" file at the server (directly). The file should have the "abcd" content +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + fdata = StringIO('abcd') + ftp.storbinary('STOR test.txt', fdata) - I look for the "test.txt" file at the server +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + assert ftp.nlst("test.txt") == ['test.txt'] - I check that the content of "test.txt" is "abcd" +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + assert te.get_ftp_fulldata(ftp, "test.txt") == 'abcd' - I append the string 'defgh' into "test.txt" - - I check that the content of "text.txt" is 'abcdefgh' + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + fdata = StringIO('defgh') + ftp.storbinary('APPE test.txt', fdata) +- + I check that the content of "text.txt" is 'abcddefgh' +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + assert te.get_ftp_fulldata(ftp, "test.txt") == 'abcddefgh' - I try to cd into an non-existing folder 'Not-This' +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_login(cr, uid, self) + try: + ftp.cwd('/Not-This') + assert False, "We should't be able to change here" + except OSError, err: + assert err.errno == 2, err.errno - I create a "test2.txt" file through FTP. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + fdata = StringIO('abcd') + ftp.storbinary('STOR test2.txt', fdata) - I look for the "test2.txt" file at the server - - I look for the "test2.txt" file through FTP. + !python {model: ir.attachment }: | + ids = self.search(cr, uid, [('name', '=', 'test2.txt')]) + assert ids - I delete the "test2.txt" file using FTP. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + ftp.delete('test2.txt') - I check at the server that test2.txt is deleted +- + !python {model: ir.attachment }: | + ids = self.search(cr, uid, [('name', '=', 'test2.txt')]) + assert not ids - I create a test2.txt file again. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + fdata = StringIO('abcd') + ftp.storbinary('STOR test2.txt', fdata) - I delete the test2.txt from the server (RPC). +- + !delete { model: ir.attachment, id:, search: "[('name','=','test2.txt')]" } - I check through FTP that test2.txt does not appear. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + assert ftp.nlst("test2.txt") == [] - I create a "test-name.txt" file +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + fdata = StringIO('abcd') + ftp.storbinary('STOR test-name.txt', fdata) - I rename the "test-name.txt" file through ftp. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + ftp.rename("test-name.txt", "test-renamed.txt") - I check that test-name.txt has been renamed. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + assert ftp.nlst("test-name.txt") == [] + assert ftp.nlst("test-renamed.txt") == ['test-renamed.txt'] - I create a new folder 'Test-Folder2' through FTP +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + ftp.mkd("Test-Folder2") - I create a file 'test3.txt' at the 'Test-Folder2' +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2') + fdata = StringIO('abcd') + ftp.storbinary('STOR test3.txt', fdata) - I try to retrieve test3.txt +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2') + assert ftp.nlst("test3.txt") == ['test3.txt'] - I create a new folder, 'Test-Folder3', through FTP I try to move test3.txt to 'Test-Folder3' +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents') + ftp.mkd("Test-Folder2") + # TODO move - I remove the 'Test-Folder3' - I check that test3.txt is removed. - I create 1000 files through FTP +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + from cStringIO import StringIO + ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2') + fdata = StringIO('abcd') + # TODO speed + for i in range(0, 1000): + fdata.seek(0) + ftp.storbinary('STOR test-name%s.txt' %i, fdata) - I list the 1000 files, check speed +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2') + # TODO speed + assert len(ftp.nlst() > 1000) - I read the 1000 files, check speed + # TODO - I move the 1000 files to 'Test-Folder2' + # TODO - \ No newline at end of file diff --git a/addons/document_ftp/test_easyftp.py b/addons/document_ftp/test_easyftp.py new file mode 100644 index 00000000000..eb709c79e31 --- /dev/null +++ b/addons/document_ftp/test_easyftp.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Tiny SPRL (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" This is a testing module, which exports some functions for the YAML tests. + Instead of repeating the same FTP code all over, we prefer to have + it in this file +""" + +from ftplib import FTP +from tools.misc import detect_ip_addr +from tools import config + +def get_plain_ftp(): + ftp = FTP() + host = config.get('ftp_server_host', '127.0.0.1') + port = config.get('ftp_server_port','8021') + ftp.connect(host,port) + return ftp + +def get_ftp_login(cr, uid, ormobj): + ftp = get_plain_ftp() + user = ormobj.pool.get('res.users').read(cr, uid, uid) + ftp.login(user.get('login',''),user.get('password','')) + ftp.cwd("/" + cr.dbname) + return ftp + +def get_ftp_anonymous(cr): + ftp = get_plain_ftp() + ftp.login('anonymous', 'the-test') + ftp.cwd("/") + return ftp + +def get_ftp_folder(cr, uid, ormobj, foldername): + ftp = get_ftp_login(cr, uid, ormobj) + ftp.cwd("/" + cr.dbname+"/"+foldername) + return ftp + +def get_ftp_fulldata(ftp, fname, limit=8192): + from functools import partial + data = [] + def ffp(data, ndata): + if len(data)+ len(ndata) > limit: + raise IndexError('Data over the limit') + data.append(ndata) + ftp.retrbinary('RETR %s' % fname, partial(ffp,data)) + return ''.join(data) + +#eof \ No newline at end of file From 5da9c2660533a6ecfabf3359d900da3b4bdb24b3 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 01:43:08 +0300 Subject: [PATCH 059/121] Document FTP: fix syntax error in mkdir. bzr revid: p_christ@hol.gr-20100701224308-g95nx5ckh8mgjgsb --- addons/document_ftp/ftpserver/abstracted_fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index cb711c9d7fd..710d49f7dc4 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -277,7 +277,7 @@ class abstracted_fs(object): if not node: raise OSError(1, 'Operation not permited.') - cr, uid = self.get_node_cr(node) + cr = self.get_node_cr(node) try: basename =_to_unicode(basename) cdir = node.create_child_collection(cr, basename) From 88b0420b3e06a06029755a9593b41be260ec4a0e Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 01:43:11 +0300 Subject: [PATCH 060/121] Document ftp: do commit the cursors, at mkdir and rename cr.close() is not enough for data to go to db. bzr revid: p_christ@hol.gr-20100701224311-ef8q4qbmb8b9j6p1 --- addons/document_ftp/ftpserver/abstracted_fs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 710d49f7dc4..5bb0f674fa2 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -282,6 +282,7 @@ class abstracted_fs(object): basename =_to_unicode(basename) cdir = node.create_child_collection(cr, basename) self._log.debug("Created child dir: %r", cdir) + cr.commit() except Exception,e: self._log.exception('Cannot create dir "%s" at node %s', basename, repr(node)) raise OSError(1, 'Operation not permited.') @@ -390,6 +391,7 @@ class abstracted_fs(object): ret = src.move_to(cr, dst_basedir, new_name=nname) # API shouldn't wait for us to write the object assert (ret is True) or (ret is False) + cr.commit() except Exception,err: self._log.exception('Cannot rename "%s" to "%s" at "%s"', src, dst_basename, dst_basedir) raise OSError(1,'Operation not permited.') From 24fa87a6ae74b41dab764d172e3caea65f843025 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 11:11:09 +0300 Subject: [PATCH 061/121] Document ftp: tests for dynamic folders. We pass a few of them, but some functional issues are apparent. bzr revid: p_christ@hol.gr-20100702081109-p6zl07eayj65rqs8 --- .../document_ftp/test/document_ftp_test4.yml | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/addons/document_ftp/test/document_ftp_test4.yml b/addons/document_ftp/test/document_ftp_test4.yml index 05ddef1c97b..3bc915fc780 100644 --- a/addons/document_ftp/test/document_ftp_test4.yml +++ b/addons/document_ftp/test/document_ftp_test4.yml @@ -1,16 +1,54 @@ - In order to check dynamic folder functionality of document + FTP -- - I create two partners 'Partner1' and 'Partner2' --| +- | + I create two partners 'Partner1' and 'Partner2'. I create three partner categories: 'none', 'pat1' and 'all' I attach Partner1 to pat1, Partner1+Partner2 to 'all' +- + !record {model: res.partner.category, id: tpat_categ_none }: + name: 'No partners' +- + !record {model: res.partner.category, id: tpat_categ_pat1 }: + name: 'Pat 1' +- + !record {model: res.partner.category, id: tpat_categ_all }: + name: 'All Partner1+2' +- + !record {model: res.partner, id: tpartner1 }: + name: Partner 1 + category_id: + - tpat_categ_pat1 + - tpat_categ_all +- + !record {model: res.partner, id: tpartner_2 }: + name: 'Partner 2' + category_id: + - tpat_categ_all - I create a resource folder of partners, by the (none, pat1, all) categories. +- + !record {model: document.directory, id: dir_tests2 }: + name: Partners Testing + parent_id: document.dir_root + type: ressource + ressource_type_id: base.model_res_partner_category + domain: [] +- + I commit (because FTP operations are on different transaction) +- + !python {model: document.directory, id: }: | + cr.commit() - I browse through ftp in the resource folder, checking that three categories are there. +- + !python {model: ir.attachment}: | + from document_ftp import test_easyftp as te + ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Partners Testing') + dirs = ftp.nlst() + dirs.sort() + assert dirs == [ 'All Partner1+2', 'No partners', 'Pat 1' ] - I create a 'partners' folder by the first resource one. - From be2f5723c9edb0b8650f79c55219d3d66a177633 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 11:45:35 +0300 Subject: [PATCH 062/121] Document, ftp: switch to new series of tests. bzr revid: p_christ@hol.gr-20100702084535-6yz0f5xfsvh4jl9b --- addons/document/__openerp__.py | 2 +- addons/document_ftp/__openerp__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/addons/document/__openerp__.py b/addons/document/__openerp__.py index a260a083d74..63b09121309 100644 --- a/addons/document/__openerp__.py +++ b/addons/document/__openerp__.py @@ -53,7 +53,7 @@ ], 'demo_xml': [ 'document_demo.xml','board_document_demo.xml'], 'test': [ - 'test/document_test.yml', + 'test/document_test2.yml', ], 'installable': True, 'active': False, diff --git a/addons/document_ftp/__openerp__.py b/addons/document_ftp/__openerp__.py index a2b88d0a2a2..42afbe53ed8 100644 --- a/addons/document_ftp/__openerp__.py +++ b/addons/document_ftp/__openerp__.py @@ -39,7 +39,9 @@ ], 'demo_xml': [], 'test': [ - 'test/document_ftp_test.yml', + 'test/document_ftp_test2.yml', + # 'test/document_ftp_test2.yml', + 'test/document_ftp_test4.yml', ], 'installable': True, 'active': False, From a72a73acdd69ae0f21dfd3b795e9fe3111181676 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 11:54:01 +0300 Subject: [PATCH 063/121] Document: remove trailing whitespace bzr revid: p_christ@hol.gr-20100702085401-8c8hwv9831jcwzts --- addons/document/nodes.py | 274 +++++++++++++++++++-------------------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index a77c35b4cdf..472823a2386 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ############################################################################## -# +# # OpenERP, Open Source Management Solution # Copyright (C) 2004-2010 Tiny SPRL (). # @@ -15,7 +15,7 @@ # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . +# along with this program. If not, see . # ############################################################################## @@ -49,7 +49,7 @@ class node_context(object): context """ cached_roots = {} - def __init__(self, cr, uid, context=None): + def __init__(self, cr, uid, context=None): self.dbname = cr.dbname self.uid = uid self.context = context @@ -69,21 +69,21 @@ class node_context(object): if self.rootdir != other.rootdir: return False return True - + def __ne__(self, other): return not self.__eq__(other) def get_uri(self, cr, uri): """ Although this fn passes back to doc.dir, it is needed since - it is a potential caching point """ - (ndir, duri) = self._dirobj._locate_child(cr, self.uid, self.rootdir, uri, None, self) - while duri: - ndir = ndir.child(cr, duri[0]) + it is a potential caching point """ + (ndir, duri) = self._dirobj._locate_child(cr, self.uid, self.rootdir, uri, None, self) + while duri: + ndir = ndir.child(cr, duri[0]) if not ndir: - return False - duri = duri[1:] + return False + duri = duri[1:] return ndir - + def get_dir_node(self, cr, dbro): """Create (or locate) a node for a directory @param dbro a browse object of document.directory @@ -107,7 +107,7 @@ class node_context(object): class node_descriptor(object): """A file-like interface to the data contents of a node. - + This class is NOT a node, but an /open descriptor/ for some node. It can hold references to a cursor or a file object, because the life of a node_descriptor will be the open period @@ -123,13 +123,13 @@ class node_descriptor(object): assert isinstance(parent, node_class) self.name = parent.displayname self.__parent = parent - + def _get_parent(self): return self.__parent def open(self, **kwargs): raise NotImplementedError - + def close(self): raise NotImplementedError @@ -147,7 +147,7 @@ class node_descriptor(object): class node_class(object): """ this is a superclass for our inodes - It is an API for all code that wants to access the document files. + It is an API for all code that wants to access the document files. Nodes have attributes which contain usual file properties """ our_type = 'baseclass' @@ -167,7 +167,7 @@ class node_class(object): if parent: self.dctx = parent.dctx.copy() self.displayname = 'Object' - + def __eq__(self, other): return NotImplemented @@ -176,7 +176,7 @@ class node_class(object): def full_path(self): """ Return the components of the full path for some - node. + node. The returned list only contains the names of nodes. """ if self.parent: @@ -200,15 +200,15 @@ class node_class(object): def path_get(self): print "node_class.path_get()" return False - + def get_data(self,cr): raise TypeError('no data for %s'% self.type) - + def open_data(self, cr, mode): """ Open a node_descriptor object for this node. - + @param the mode of open, eg 'r', 'w', 'a', like file.open() - + This operation may lock the data for this node (and accross other node hierarchies), until the descriptor is close()d. If the node is locked, subsequent opens (depending on mode) may @@ -223,7 +223,7 @@ class node_class(object): def get_etag(self,cr): """ Get a tag, unique per object + modification. - + see. http://tools.ietf.org/html/rfc2616#section-13.3.3 """ return self._get_ttag(cr) + ':' + self._get_wtag(cr) @@ -233,14 +233,14 @@ class node_class(object): wtime = time.mktime(time.strptime(self.write_date,'%Y-%m-%d %H:%M:%S')) else: wtime = time.time() return str(wtime) - + def _get_ttag(self,cr): """ Get a unique tag for this type/id of object. Must be overriden, so that each node is uniquely identified. """ print "node_class.get_ttag()",self raise RuntimeError("get_etag stub()") - + def get_dav_props(self, cr): """ If this class has special behaviour for GroupDAV etc, export its capabilities """ @@ -260,25 +260,25 @@ class node_class(object): @param ndir_node the collection that this node should be moved under @param new_name a name to rename this node to. If omitted, the old name is preserved - @param fil_obj, can be None, is the browse object for the file, + @param fil_obj, can be None, is the browse object for the file, if already available. - @param ndir_obj must be the browse object to the new doc.directory + @param ndir_obj must be the browse object to the new doc.directory location, where this node should be moved to. in_write: When called by write(), we shouldn't attempt to write the object, but instead return the dict of vals (avoid re-entrance). If false, we should write all data to the object, here, as if the caller won't do anything after calling move_to() - + Return value: True: the node is moved, the caller can update other values, too. - False: the node is either removed or fully updated, the caller + False: the node is either removed or fully updated, the caller must discard the fil_obj, not attempt to write any more to it. dict: values to write back to the object. *May* contain a new id! - + Depending on src and target storage, implementations of this function could do various things. Should also consider node<->content, dir<->dir moves etc. - + Move operations, as instructed from APIs (eg. request from DAV) could use this function. """ @@ -295,17 +295,17 @@ class node_class(object): class node_database(node_class): """ A node representing the database directory - + """ our_type = 'database' def __init__(self, path=[], parent=False, context=None): super(node_database,self).__init__(path, parent, context) def children(self, cr, domain=None): - res = self._child_get(cr, domain=domain) + self._file_get(cr) + res = self._child_get(cr, domain=domain) + self._file_get(cr) return res - def child(self, cr, name, domain=None): + def child(self, cr, name, domain=None): res = self._child_get(cr, name, domain=None) if res: return res[0] @@ -319,7 +319,7 @@ class node_database(node_class): uid = self.context.uid ctx = self.context.context.copy() ctx.update(self.dctx) - where = [('parent_id','=',parent_id)] + where = [('parent_id','=',parent_id)] if name: where.append(('name','=',name)) if not domain: @@ -327,16 +327,16 @@ class node_database(node_class): where2 = where + domain + [('type', '=', 'directory')] ids = dirobj.search(cr, uid, where2, context=ctx) - res = [] - for dirr in dirobj.browse(cr, uid, ids, context=ctx): + res = [] + for dirr in dirobj.browse(cr, uid, ids, context=ctx): res.append(node_dir(dirr.name, self, self.context,dirr)) - + where2 = where + domain + [('type', '=', 'ressource'), ('ressource_parent_type_id','=',False)] - ids = dirobj.search(cr, uid, where2, context=ctx) - for dirr in dirobj.browse(cr, uid, ids, context=ctx): + ids = dirobj.search(cr, uid, where2, context=ctx) + for dirr in dirobj.browse(cr, uid, ids, context=ctx): res.append(node_res_dir(dirr.name, self, self.context, dirr)) - - fil_obj = dirobj.pool.get('ir.attachment') + + fil_obj = dirobj.pool.get('ir.attachment') ids = fil_obj.search(cr, uid, where, context=ctx) if ids: for fil in fil_obj.browse(cr, uid, ids, context=ctx): @@ -360,7 +360,7 @@ class node_database(node_class): def _get_ttag(self,cr): return 'db-%s' % cr.dbname - + class node_dir(node_database): our_type = 'collection' @@ -396,7 +396,7 @@ class node_dir(node_database): return False if not self.context == other.context: return False - # Two directory nodes, for the same document.directory, may have a + # Two directory nodes, for the same document.directory, may have a # different context! (dynamic folders) if self.dctx != other.dctx: return False @@ -404,26 +404,26 @@ class node_dir(node_database): def get_data(self, cr): res = '' - for child in self.children(cr): - res += child.get_data(cr) + for child in self.children(cr): + res += child.get_data(cr) return res - + def _file_get(self, cr, nodename=False): - return super(node_dir,self)._file_get(cr, nodename, self.dir_id) + return super(node_dir,self)._file_get(cr, nodename, self.dir_id) + - def _child_get(self, cr, name=None, domain=None): return super(node_dir,self)._child_get(cr, name, self.dir_id, domain=domain) def rmcol(self, cr): uid = self.context.uid - directory = self.context._dirobj.browse(cr, uid, self.dir_id) - res = False + directory = self.context._dirobj.browse(cr, uid, self.dir_id) + res = False if not directory: - raise OSError(2, 'Not such file or directory.') + raise OSError(2, 'Not such file or directory.') if directory._table_name=='document.directory': if self.children(cr): raise OSError(39, 'Directory not empty.') @@ -433,12 +433,12 @@ class node_dir(node_database): return res def create_child_collection(self, cr, objname): - object2 = False + object2 = False dirobj = self.context._dirobj uid = self.context.uid ctx = self.context.context.copy() ctx.update(self.dctx) - obj = dirobj.browse(cr, uid, self.dir_id) + obj = dirobj.browse(cr, uid, self.dir_id) if obj and (obj.type == 'ressource') and not object2: raise OSError(1, 'Operation not permited.') @@ -448,11 +448,11 @@ class node_dir(node_database): 'ressource_parent_type_id': obj and obj.ressource_type_id.id or False, 'ressource_id': object2 and object2.id or False, 'parent_id' : obj and obj.id or False - } - + } + return dirobj.create(cr, uid, val) - - + + def create_child(self, cr, path, data): """ API function to create a child file object and node Return the node_* created @@ -487,8 +487,8 @@ class node_dir(node_database): if self.write_date: wtime = time.mktime(time.strptime(self.write_date, '%Y-%m-%d %H:%M:%S')) else: wtime = time.time() - return str(wtime) - + return str(wtime) + def _get_ttag(self,cr): return 'dir-%d' % self.dir_id @@ -513,13 +513,13 @@ class node_res_dir(node_class): self.resm_id = dirr.ressource_id self.namefield = dirr.resource_field.name or 'name' self.displayname = dirr.name - # Important: the domain is evaluated using the *parent* dctx! + # Important: the domain is evaluated using the *parent* dctx! self.domain = dirr.domain self.ressource_tree = dirr.ressource_tree # and then, we add our own vars in the dctx: if dctx: self.dctx.update(dctx) - + # and then, we prepare a dctx dict, for deferred evaluation: self.dctx_dict = {} for dfld in dirr.dctx_ids: @@ -530,7 +530,7 @@ class node_res_dir(node_class): return False if not self.context == other.context: return False - # Two nodes, for the same document.directory, may have a + # Two nodes, for the same document.directory, may have a # different context! (dynamic folders) if self.dctx != other.dctx: return False @@ -539,7 +539,7 @@ class node_res_dir(node_class): def children(self, cr, domain=None): return self._child_get(cr, domain=domain) - def child(self,cr, name, domain=None): + def child(self,cr, name, domain=None): res = self._child_get(cr, name, domain=domain) if res: return res[0] @@ -548,7 +548,7 @@ class node_res_dir(node_class): def _child_get(self, cr, name = None, domain=None): """ return virtual children of resource, based on the foreign object. - + Note that many objects use NULL for a name, so we should better call the name_search(),name_get() set of methods """ @@ -572,28 +572,28 @@ class node_res_dir(node_class): raise RuntimeError("incorrect domain expr: %s" % self.domain) if self.resm_id: where.append(('id','=',self.resm_id)) - + if name: where.append((self.namefield,'=',name)) - + # print "Where clause for %s" % self.res_model, where if self.ressource_tree: object2 = False if self.resm_id: - object2 = dirobj.pool.get(self.res_model).browse(cr, uid, self.resm_id) or False - if obj._parent_name in obj.fields_get(cr, uid): + object2 = dirobj.pool.get(self.res_model).browse(cr, uid, self.resm_id) or False + if obj._parent_name in obj.fields_get(cr, uid): where.append((obj._parent_name,'=',object2 and object2.id or False)) - + resids = obj.search(cr, uid, where, context=ctx) - res = [] + res = [] for bo in obj.browse(cr, uid, resids, context=ctx): if not bo: continue name = getattr(bo, self.namefield) if not name: continue - # Yes! we can't do better but skip nameless records. - + # Yes! we can't do better but skip nameless records. + res.append(node_res_obj(name, self.dir_id, self, self.context, self.res_model, bo)) return res @@ -610,7 +610,7 @@ class node_res_obj(node_class): def __init__(self, path, dir_id, parent, context, res_model, res_bo, res_id = None): super(node_res_obj,self).__init__(path, parent,context) assert parent - #todo: more info from dirr + #todo: more info from dirr self.dir_id = dir_id self.mimetype = 'application/x-directory' # 'httpd/unix-directory' @@ -620,15 +620,15 @@ class node_res_obj(node_class): self.content_length = 0 self.res_model = res_model self.domain = parent.domain - self.displayname = path - self.dctx_dict = parent.dctx_dict - if res_bo: - self.res_id = res_bo.id + self.displayname = path + self.dctx_dict = parent.dctx_dict + if res_bo: + self.res_id = res_bo.id dc2 = self.context.context dc2.update(self.dctx) dc2['res_model'] = res_model dc2['res_id'] = res_bo.id - dc2['this'] = res_bo + dc2['this'] = res_bo for fld,expr in self.dctx_dict.items(): try: self.dctx[fld] = safe_eval(expr, dc2) @@ -654,11 +654,11 @@ class node_res_obj(node_class): return False return self.dir_id == other.dir_id - def children(self, cr, domain=None): + def children(self, cr, domain=None): return self._child_get(cr, domain=domain) + self._file_get(cr) - def child(self, cr, name, domain=None): - res = self._child_get(cr, name, domain=domain) + def child(self, cr, name, domain=None): + res = self._child_get(cr, name, domain=domain) if res: return res[0] res = self._file_get(cr, name) @@ -713,7 +713,7 @@ class node_res_obj(node_class): return ('vevent-collection','http://groupdav.org/') return None - def _child_get(self, cr, name=None, domain=None): + def _child_get(self, cr, name=None, domain=None): dirobj = self.context._dirobj uid = self.context.uid ctx = self.context.context.copy() @@ -721,38 +721,38 @@ class node_res_obj(node_class): directory = dirobj.browse(cr, uid, self.dir_id) obj = dirobj.pool.get(self.res_model) where = [] - res = [] + res = [] if name: - where.append(('name','=',name)) - + where.append(('name','=',name)) + # Directory Structure display in tree structure - if self.res_id and directory.ressource_tree: - where1 = [] - if obj._parent_name in obj.fields_get(cr, uid): + if self.res_id and directory.ressource_tree: + where1 = [] + if obj._parent_name in obj.fields_get(cr, uid): where1 = where + [(obj._parent_name, '=', self.res_id)] - resids = obj.search(cr, uid, where1, context=ctx) + resids = obj.search(cr, uid, where1, context=ctx) for bo in obj.browse(cr, uid, resids, context=ctx): namefield = directory.resource_field.name or 'name' if not bo: continue res_name = getattr(bo, namefield) if not res_name: - continue - res.append(node_res_obj(res_name, self.dir_id, self, self.context, self.res_model, res_bo = bo)) - - - where2 = where + [('parent_id','=',self.dir_id) ] - ids = dirobj.search(cr, uid, where2, context=ctx) + continue + res.append(node_res_obj(res_name, self.dir_id, self, self.context, self.res_model, res_bo = bo)) + + + where2 = where + [('parent_id','=',self.dir_id) ] + ids = dirobj.search(cr, uid, where2, context=ctx) for dirr in dirobj.browse(cr, uid, ids, context=ctx): if dirr.type == 'directory': res.append(node_res_obj(dirr.name, dirr.id, self, self.context, self.res_model, res_bo = None, res_id = self.res_id)) elif dirr.type == 'ressource': # child resources can be controlled by properly set dctx res.append(node_res_dir(dirr.name,self,self.context, dirr, {'active_id': self.res_id})) - - - + + + fil_obj = dirobj.pool.get('ir.attachment') where3 = where2 + [('res_model', '=', self.res_model), ('res_id','=',self.res_id)] # print "where clause for dir_obj", where2 @@ -760,20 +760,20 @@ class node_res_obj(node_class): if ids: for fil in fil_obj.browse(cr, uid, ids, context=ctx): res.append(node_file(fil.name, self, self.context, fil)) - - - # Get Child Ressource Directories - if directory.ressource_type_id and directory.ressource_type_id.id: - where4 = where + [('ressource_parent_type_id','=',directory.ressource_type_id.id)] + + + # Get Child Ressource Directories + if directory.ressource_type_id and directory.ressource_type_id.id: + where4 = where + [('ressource_parent_type_id','=',directory.ressource_type_id.id)] where5 = where4 + [('ressource_id','=',0)] - dirids = dirobj.search(cr,uid, where5) + dirids = dirobj.search(cr,uid, where5) where5 = where4 + [('ressource_id','=',self.res_id)] - dirids = dirids + dirobj.search(cr,uid, where5) + dirids = dirids + dirobj.search(cr,uid, where5) for dirr in dirobj.browse(cr, uid, dirids, context=ctx): if dirr.type == 'directory' and not dirr.parent_id: res.append(node_res_obj(dirr.name, dirr.id, self, self.context, self.res_model, res_bo = None, res_id = self.res_id)) if dirr.type == 'ressource': - res.append(node_res_dir(dirr.name, self, self.context, dirr, {'active_id': self.res_id})) + res.append(node_res_dir(dirr.name, self, self.context, dirr, {'active_id': self.res_id})) return res def create_child_collection(self, cr, objname): @@ -781,26 +781,26 @@ class node_res_obj(node_class): uid = self.context.uid ctx = self.context.context.copy() ctx.update(self.dctx) - res_obj = dirobj.pool.get(self.context.context['res_model']) - - object2 = res_obj.browse(cr, uid, self.context.context['res_id']) or False - - obj = dirobj.browse(cr, uid, self.dir_id) + res_obj = dirobj.pool.get(self.context.context['res_model']) + + object2 = res_obj.browse(cr, uid, self.context.context['res_id']) or False + + obj = dirobj.browse(cr, uid, self.dir_id) if obj and (obj.type == 'ressource') and not object2: raise OSError(1, 'Operation not permited.') - + val = { 'name': objname, 'ressource_parent_type_id': obj and obj.ressource_type_id.id or False, 'ressource_id': object2 and object2.id or False, 'parent_id' : False - } - if (obj and (obj.type in ('directory'))) or not object2: - val['parent_id'] = obj and obj.id or False - + } + if (obj and (obj.type in ('directory'))) or not object2: + val['parent_id'] = obj and obj.id or False + return dirobj.create(cr, uid, val) - + def create_child(self, cr, path, data): """ API function to create a child file object and node Return the node_* created @@ -840,7 +840,7 @@ class node_file(node_class): self.write_date = fil.write_date or fil.create_date self.content_length = fil.file_size self.displayname = fil.name - + # This only propagates the problem to get_data. Better # fix those files to point to the root dir. self.storage_id = None @@ -873,17 +873,17 @@ class node_file(node_class): uid = self.context.uid document_obj = self.context._dirobj.pool.get('ir.attachment') if self.type in ('collection','database'): - return False - document = document_obj.browse(cr, uid, self.file_id, context=self.context.context) + return False + document = document_obj.browse(cr, uid, self.file_id, context=self.context.context) res = False if document and document._table_name == 'ir.attachment': res = document_obj.unlink(cr, uid, [document.id]) - return res + return res def fix_ppath(self, cr, fbro): """Sometimes we may init this w/o path, parent. This function fills the missing path from the file browse object - + Note: this may be an expensive operation, do on demand. However, once caching is in, we might want to do that at init time and keep this object anyway @@ -892,7 +892,7 @@ class node_file(node_class): return assert fbro uid = self.context.uid - + dirpath = [] if fbro.parent_id: dirobj = self.context._dirobj.pool.get('document.directory') @@ -901,14 +901,14 @@ class node_file(node_class): dirpath.append(fbro.datas_fname) else: dirpath.append(fbro.name) - + if len(dirpath)>1: self.path = dirpath else: - self.path = dirpath[0] - + self.path = dirpath[0] + def get_data(self, cr, fil_obj = None): - """ Retrieve the data for some file. + """ Retrieve the data for some file. fil_obj may optionally be specified, and should be a browse object for the file. This is useful when the caller has already initiated the browse object. """ @@ -928,7 +928,7 @@ class node_file(node_class): return self.content_length def set_data(self, cr, data, fil_obj = None): - """ Store data at some file. + """ Store data at some file. fil_obj may optionally be specified, and should be a browse object for the file. This is useful when the caller has already initiated the browse object. """ @@ -944,7 +944,7 @@ class node_file(node_class): def move_to(self, cr, ndir_node, new_name=False, fil_obj=None, ndir_obj=None, in_write=False): if ndir_node.context != self.context: raise NotImplementedError("Cannot move files between contexts") - + doc_obj = self.context._dirobj.pool.get('ir.attachment') if not fil_obj: dbro = doc_obj.browse(cr, self.context.uid, self.file_id, context=self.context.context) @@ -954,29 +954,29 @@ class node_file(node_class): if not dbro: raise IndexError("Cannot locate doc %d", self.file_id) - + if (not self.parent): # there *must* be a parent node for this one self.parent = self.context.get_dir_node(cr, dbro.parent_id.id) - + if self.parent != ndir_node: logger.debug('Cannot move file %r from %r to %r', self, self.parent, ndir_node) raise NotImplementedError('Cannot move file to another dir') - + ret = {} if new_name and (new_name != dbro.name): stobj = self.context._dirobj.pool.get('document.storage') r2 = stobj.simple_rename(cr, self.context.uid, self, new_name, self.context.context) ret.update(r2) - + del dbro - + if not in_write: # We have to update the data ourselves if ret: doc_obj.write(cr, self.context.uid, [self.file_id,], ret, self.context.context) ret = True - + return ret class node_content(node_class): @@ -997,14 +997,14 @@ class node_content(node_class): def open(self, cr, mode=False): raise DeprecationWarning() - + def fill_fields(self, cr, dctx = None): """ Try to read the object and fill missing fields, like mimetype, dates etc. This function must be different from the constructor, because it uses the db cursor. """ - + cr.execute('SELECT DISTINCT mimetype FROM document_directory_content_type WHERE active AND code = %s;', (self.extension,)) res = cr.fetchall() From 6ebec0292d26e73742cb62203b9a0ccf649d76d1 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 12:27:32 +0300 Subject: [PATCH 064/121] Document API: fix node_dir._file_get() Directory _file_get was in the wrong place, causing the root folder to fail, once it was associated with dynamic context. bzr revid: p_christ@hol.gr-20100702092732-jx8tisemq7iqxtb3 --- addons/document/nodes.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index 472823a2386..ca223bb9486 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -343,19 +343,8 @@ class node_database(node_class): res.append(node_file(fil.name, self, self.context, fil)) return res - def _file_get(self,cr, nodename=False, directory_id=False): + def _file_get(self,cr, nodename=False): res = [] - cntobj = self.context._dirobj.pool.get('document.directory.content') - uid = self.context.uid - ctx = self.context.context.copy() - ctx.update(self.dctx) - where = [('directory_id','=',directory_id) ] - ids = cntobj.search(cr, uid, where, context=ctx) - for content in cntobj.browse(cr, uid, ids, context=ctx): - res3 = cntobj._file_get(cr, self, nodename, content) - if res3: - res.extend(res3) - return res def _get_ttag(self,cr): @@ -408,12 +397,21 @@ class node_dir(node_database): res += child.get_data(cr) return res - - def _file_get(self, cr, nodename=False): - return super(node_dir,self)._file_get(cr, nodename, self.dir_id) - + res = super(node_dir,self)._file_get(cr, nodename) + + cntobj = self.context._dirobj.pool.get('document.directory.content') + uid = self.context.uid + ctx = self.context.context.copy() + ctx.update(self.dctx) + where = [('directory_id','=',self.dir_id) ] + ids = cntobj.search(cr, uid, where, context=ctx) + for content in cntobj.browse(cr, uid, ids, context=ctx): + res3 = cntobj._file_get(cr, self, nodename, content) + if res3: + res.extend(res3) + return res def _child_get(self, cr, name=None, domain=None): return super(node_dir,self)._child_get(cr, name, self.dir_id, domain=domain) From dce6c870841df6ff43397bed17aea43e2da06350 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 2 Jul 2010 13:38:58 +0300 Subject: [PATCH 065/121] Document ftp: stop early if not in a node. This will rather help debug some cases, where we try operations on the "/" path. bzr revid: p_christ@hol.gr-20100702103858-8jrqfzkd8g65vrzr --- addons/document_ftp/ftpserver/abstracted_fs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 5bb0f674fa2..0766d42d501 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -319,6 +319,7 @@ class abstracted_fs(object): def get_node_cr_uid(self, node): """ Get cr, uid, pool from a node """ + assert node db = pooler.get_db(node.context.dbname) return db.cursor(), node.context.uid From 3f4c75919c7b2b6a5b2a7b72a80b37adc39ec1d3 Mon Sep 17 00:00:00 2001 From: "Harry (OpenERP)" Date: Sat, 3 Jul 2010 00:50:58 +0300 Subject: [PATCH 066/121] Document: fix creating of node_res_obj without data . bzr revid: p_christ@hol.gr-20100702215058-5h49fw2dzox3022h --- addons/document/nodes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index ca223bb9486..d9ec0a3b607 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -820,7 +820,8 @@ class node_res_obj(node_class): fil_id = fil_obj.create(cr, uid, val, context=ctx) fil = fil_obj.browse(cr, uid, fil_id, context=ctx) fnode = node_file(path, self, self.context, fil) - fnode.set_data(cr, data, fil) + if data is not None: + fnode.set_data(cr, data, fil) return fnode def _get_ttag(self,cr): From 50070dfa133163638a0b3a48a4eb8aabd7225643 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sat, 3 Jul 2010 00:51:02 +0300 Subject: [PATCH 067/121] Document API: have node.get_uri(), unix permissions and user/group These attributes will help the FTP display the files correctly. bzr revid: p_christ@hol.gr-20100702215102-0yx9y2psp32ha33u --- addons/document/nodes.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/addons/document/nodes.py b/addons/document/nodes.py index d9ec0a3b607..3599b75a15f 100644 --- a/addons/document/nodes.py +++ b/addons/document/nodes.py @@ -93,7 +93,7 @@ class node_context(object): return node_dir(fullpath, None ,self, dbro) elif dbro.type == 'ressource': assert dbro.ressource_parent_type_id == False - return node_res_dir(fullparh, None, self, dbro) + return node_res_dir(fullpath, None, self, dbro) else: raise ValueError("dir node for %s type", dbro.type) @@ -161,6 +161,9 @@ class node_class(object): self.mimetype = 'application/octet-stream' self.create_date = None self.write_date = None + self.unixperms = 0660 + self.uuser = 'user' + self.ugroup = 'group' self.content_length = 0 # dynamic context: self.dctx = {} @@ -197,6 +200,16 @@ class node_class(object): print "node_class.child()" return None + def get_uri(self, cr, uri): + duri = uri + ndir = self + while duri: + ndir = ndir.child(cr, duri[0]) + if not ndir: + return False + duri = duri[1:] + return ndir + def path_get(self): print "node_class.path_get()" return False @@ -300,6 +313,7 @@ class node_database(node_class): our_type = 'database' def __init__(self, path=[], parent=False, context=None): super(node_database,self).__init__(path, parent, context) + self.unixperms = 040750 def children(self, cr, domain=None): res = self._child_get(cr, domain=domain) + self._file_get(cr) @@ -365,6 +379,7 @@ class node_dir(node_database): # TODO: the write date should be MAX(file.write).. self.write_date = dirr and (dirr.write_date or dirr.create_date) or False self.content_length = 0 + self.unixperms = 040750 if dctx: self.dctx.update(dctx) dc2 = self.context.context @@ -507,6 +522,7 @@ class node_res_dir(node_class): # TODO: the write date should be MAX(file.write).. self.write_date = dirr.write_date or dirr.create_date self.content_length = 0 + self.unixperms = 040750 self.res_model = dirr.ressource_type_id and dirr.ressource_type_id.model or False self.resm_id = dirr.ressource_id self.namefield = dirr.resource_field.name or 'name' @@ -616,6 +632,7 @@ class node_res_obj(node_class): # TODO: the write date should be MAX(file.write).. self.write_date = parent.write_date self.content_length = 0 + self.unixperms = 040750 self.res_model = res_model self.domain = parent.domain self.displayname = path @@ -986,6 +1003,7 @@ class node_content(node_class): self.create_date = False self.write_date = False self.content_length = False + self.unixperms = 0640 self.extension = cnt.extension self.report_id = cnt.report_id and cnt.report_id.id #self.mimetype = cnt.extension. From e29f5bef650ec823ab2f4c6d67cf494949a73d20 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sat, 3 Jul 2010 00:52:49 +0300 Subject: [PATCH 068/121] Document FTP: rewrite the code, resolve nodes only once. A big patch: nodes should not be calculated multiple times from the path string, within an operation. Instead, they are resolved only once in a node object (may reuse the cwd_node, if possible) and then op relative to that node. bzr revid: p_christ@hol.gr-20100702215249-kzucr3i1p5f6vvnt --- .../document_ftp/ftpserver/abstracted_fs.py | 442 +++++++++--------- addons/document_ftp/ftpserver/ftpserver.py | 275 +++++------ 2 files changed, 339 insertions(+), 378 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 0766d42d501..60974896b41 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -5,6 +5,7 @@ from tarfile import filemode import StringIO import base64 import logging +import errno import glob import fnmatch @@ -70,11 +71,20 @@ class abstracted_fs(object): - (str) root: the user home directory. - (str) cwd: the current working directory. - (str) rnfr: source file to be renamed. + """ + def __init__(self): + self.root = None + self.cwd = '/' + self.cwd_node = None + self.rnfr = None + self._log = logging.getLogger('FTP.fs') + # Ok def db_list(self): - #return pooler.pool_dic.keys() + """Get the list of available databases, with FTPd support + """ s = netsvc.ExportService.getService('db') result = s.exp_list(document=True) self.db_name_list = [] @@ -102,97 +112,58 @@ class abstracted_fs(object): # pooler.close_db(db_name) return self.db_name_list - # Ok - def __init__(self): - self.root = None - self.cwd = '/' - self.rnfr = None - self._log = logging.getLogger('FTP.fs') - - # --- Pathname / conversion utilities - - # Ok def ftpnorm(self, ftppath): """Normalize a "virtual" ftp pathname (tipically the raw string - coming from client) depending on the current working directory. + coming from client). - Example (having "/foo" as current working directory): - 'x' -> '/foo/x' - - Note: directory separators are system independent ("/"). - Pathname returned is always absolutized. + Pathname returned is relative!. """ - if os.path.isabs(ftppath): - p = os.path.normpath(ftppath) - else: - p = os.path.normpath(os.path.join(self.cwd, ftppath)) + p = os.path.normpath(ftppath) # normalize string in a standard web-path notation having '/' - # as separator. + # as separator. xrg: is that really in the spec? p = p.replace("\\", "/") # os.path.normpath supports UNC paths (e.g. "//a/b/c") but we # don't need them. In case we get an UNC path we collapse # redundant separators appearing at the beginning of the string while p[:2] == '//': p = p[1:] - # Anti path traversal: don't trust user input, in the event - # that self.cwd is not absolute, return "/" as a safety measure. - # This is for extra protection, maybe not really necessary. - if not os.path.isabs(p): - p = "/" + if p == '.': + return '' return p - # Ok - def ftp2fs(self, path_orig, data): - path = self.ftpnorm(path_orig) - if not data or (path and path=='/'): - return None - path2 = filter(None,path.split('/'))[1:] - (cr, uid) = data - if len(path2): - path2[-1]=_to_unicode(path2[-1]) - - ctx = get_node_context(cr, uid, {}) - res = ctx.get_uri(cr, path2[:]) - if not res: - raise OSError(2, 'Not such file or directory.') - return res + def get_cwd(self): + """ return the cwd, decoded in utf""" + return _to_decode(self.cwd) + + def ftp2fs(self, path_orig, data): + raise DeprecationWarning() - # Ok def fs2ftp(self, node): + """ Return the string path of a node, in ftp form + """ res='/' if node: paths = node.full_path() - paths = map(lambda x: '/' +x, paths) - res = os.path.normpath(''.join(paths)) - res = res.replace("\\", "/") - while res[:2] == '//': - res = res[1:] - res = '/' + node.context.dbname + '/' + _to_decode(res) + res = '/' + node.context.dbname + '/' + \ + _to_decode(os.path.join(*paths)) - #res = node and ('/' + node.cr.dbname + '/' + _to_decode(self.ftpnorm(node.path))) or '/' return res - # Ok def validpath(self, path): """Check whether the path belongs to user's home directory. - Expected argument is a "real" filesystem pathname. - - If path is a symbolic link it is resolved to check its real - destination. - - Pathnames escaping from user's root directory are considered - not valid. + Expected argument is a datacr tuple """ - return path and True or False + # TODO: are we called for "/" ? + return isinstance(path, tuple) and path[1] and True or False # --- Wrapper methods around open() and tempfile.mkstemp - def create(self, node, objname, mode): + def create(self, datacr, objname, mode): """ Create a children file-node under node, open it @return open node_descriptor of the created node """ objname = _to_unicode(objname) - cr = self.get_node_cr(node) + cr , node, rem = datacr try: child = node.child(cr, objname) if child: @@ -200,10 +171,8 @@ class abstracted_fs(object): raise OSError(1, 'Operation not permited.') ret = child.open_data(cr, mode) - cr.close() return ret - except OSError: - cr.close() + except EnvironmentError: raise except Exception,e: self._log.exception('Cannot locate item %s at node %s', objname, repr(node)) @@ -215,19 +184,13 @@ class abstracted_fs(object): except Exception,e: self._log.exception('Cannot create item %s at node %s', objname, repr(node)) raise OSError(1, 'Operation not permited.') - finally: - if cr: - cr.close() - def open(self, node, mode): - if not node: + def open(self, datacr, mode): + if not (datacr and datacr[1]): raise OSError(1, 'Operation not permited.') # Reading operation - cr = self.get_node_cr(node) - try: - res = node.open_data(cr, mode) - finally: - cr.close() + cr, node, rem = datacr + res = node.open_data(cr, mode) return res # ok, but need test more @@ -258,26 +221,26 @@ class abstracted_fs(object): # Ok - def chdir(self, path): - if not path: + def chdir(self, datacr): + if (not datacr) or datacr == (None, None, None): self.cwd = '/' + self.cwd_node = None return None - if path.type in ('collection','database'): - self.cwd = self.fs2ftp(path) - elif path.type in ('file'): - parent_path = path.full_path()[:-1] - self.cwd = os.path.normpath(''.join(parent_path)) - else: - raise OSError(1, 'Operation not permited.') + if not datacr[1]: + raise OSError(1, 'Operation not permitted') + if datacr[1].type not in ('collection','database'): + raise OSError(2, 'Path is not a directory') + self.cwd = '/'+datacr[1].context.dbname + '/' + self.cwd += '/'.join(datacr[1].full_path()) + self.cwd_node = datacr[1] # Ok - def mkdir(self, node, basename): + def mkdir(self, datacr, basename): """Create the specified directory.""" - cr = False + cr, node, rem = datacr or (None, None, None) if not node: raise OSError(1, 'Operation not permited.') - cr = self.get_node_cr(node) try: basename =_to_unicode(basename) cdir = node.create_child_collection(cr, basename) @@ -290,31 +253,87 @@ class abstracted_fs(object): if cr: cr.close() def close_cr(self, data): - if data: + if data and data[0]: data[0].close() return True - def get_cr(self, path): - path = self.ftpnorm(path) - if path=='/': - return None - dbname = path.split('/')[1] - if dbname not in self.db_list(): - return None - try: - db = pooler.get_db(dbname) - except Exception: - raise OSError(1, 'Operation not permited.') - cr = db.cursor() - try: - uid = security.login(dbname, self.username, self.password) - except Exception: - cr.close() - raise - if not uid: - cr.close() - raise OSError(2, 'Authentification Required.') - return cr, uid + def get_cr(self, pathname): + raise DeprecationWarning() + + def get_crdata(self, line, mode='file'): + """ Get database cursor, node and remainder data, for commands + + This is the helper function that will prepare the arguments for + any of the subsequent commands. + It returns a tuple in the form of: + @code ( cr, node, rem_path=None ) + + @param line An absolute or relative ftp path, as passed to the cmd. + @param mode A word describing the mode of operation, so that this + function behaves properly in the different commands. + """ + path = self.ftpnorm(line) + if self.cwd_node is None: + if not os.path.isabs(path): + path = os.path.join(self.root, path) + + if path == '/' and mode in ('list', 'cwd'): + return (None, None, None ) + + path = os.path.normpath(path) # again, for '/db/../ss' + if path == '.': path = '' + + if os.path.isabs(path) and self.cwd_node is not None \ + and path.startswith(self.cwd): + # make relative, so that cwd_node is used again + path = path[len(self.cwd):] + if path.startswith('/'): + path = path[1:] + + p_parts = path.split('/') # hard-code the unix sep here, by spec. + + assert '..' not in p_parts + + rem_path = None + if mode in ('create',): + rem_path = p_parts[-1] + p_parts = p_parts[:-1] + + if os.path.isabs(path): + # we have to start from root, again + p_parts = p_parts[1:] + dbname = p_parts[0] + if dbname not in self.db_list(): + return None + try: + db = pooler.get_db(dbname) + except Exception: + raise OSError(1, 'Database cannot be used.') + cr = db.cursor() + try: + uid = security.login(dbname, self.username, self.password) + except Exception: + cr.close() + raise + if not uid: + cr.close() + raise OSError(2, 'Authentification Required.') + n = get_node_context(cr, uid, {}) + node = n.get_uri(cr, p_parts[1:]) + return (cr, node, rem_path) + else: + # we never reach here if cwd_node is not set + if p_parts and p_parts[-1] == '': + p_parts = p_parts[:-1] + cr, uid = self.get_node_cr_uid(self.cwd_node) + if p_parts: + node = self.cwd_node.get_uri(cr, p_parts) + else: + node = self.cwd_node + if node is False and mode not in ('???'): + cr.close() + raise IOError(errno.ENOENT, 'Path does not exist') + return (cr, node, rem_path) def get_node_cr_uid(self, node): """ Get cr, uid, pool from a node @@ -331,16 +350,21 @@ class abstracted_fs(object): """ return self.get_node_cr_uid(node)[0] - def listdir(self, path): + def listdir(self, datacr): """List the content of a directory.""" class false_node(object): - write_date = None - create_date = None + write_date = 0.0 + create_date = 0.0 + unixperms = 040550 + content_length = 0L + uuser = 'root' + ugroup = 'root' type = 'database' + def __init__(self, db): - self.path = '/'+db + self.path = db - if path is None: + if datacr[1] is None: result = [] for db in self.db_list(): try: @@ -348,76 +372,57 @@ class abstracted_fs(object): except osv.except_osv: pass return result - cr = self.get_node_cr(path) - res = path.children(cr) - cr.close() + cr, node, rem = datacr + res = node.children(cr) return res - def rmdir(self, node): + def rmdir(self, datacr): """Remove the specified directory.""" + cr, node, rem = datacr assert node cr = self.get_node_cr(node) - try: - node.rmcol(cr) - cr.commit() - finally: - cr.close() + node.rmcol(cr) + cr.commit() - def remove(self, node): - assert node - if node.type == 'collection': - return self.rmdir(node) - elif node.type == 'file': - return self.rmfile(node) + def remove(self, datacr): + assert datacr[1] + if datacr[1].type == 'collection': + return self.rmdir(datacr) + elif datacr[1].type == 'file': + return self.rmfile(datacr) raise OSError(1, 'Operation not permited.') - def rmfile(self, node): + def rmfile(self, datacr): """Remove the specified file.""" - assert node - cr = self.get_node_cr(node) - try: - node.rm(cr) - cr.commit() - finally: - cr.close() + assert datacr[1] + cr = datacr[0] + datacr[1].rm(cr) + cr.commit() - def rename(self, src, dst_basedir, dst_basename): + def rename(self, src, datacr): """ Renaming operation, the effect depends on the src: * A file: read, create and remove * A directory: change the parent and reassign childs to ressource """ - cr = self.get_node_cr(src) + cr = datacr[0] try: - nname = _to_unicode(dst_basename) - ret = src.move_to(cr, dst_basedir, new_name=nname) + nname = _to_unicode(datacr[2]) + ret = src.move_to(cr, datacr[1], new_name=nname) # API shouldn't wait for us to write the object assert (ret is True) or (ret is False) cr.commit() except Exception,err: self._log.exception('Cannot rename "%s" to "%s" at "%s"', src, dst_basename, dst_basedir) raise OSError(1,'Operation not permited.') - finally: - if cr: cr.close() - - - # Nearly Ok def stat(self, node): - r = list(os.stat('/')) - if self.isfile(node): - r[0] = 33188 - r[6] = self.getsize(node) - r[7] = self.getmtime(node) - r[8] = self.getmtime(node) - r[9] = self.getmtime(node) - return os.stat_result(r) - lstat = stat + raise NotImplementedError() # --- Wrapper methods around os.path.* # Ok def isfile(self, node): - if node and (node.type not in ('collection','database')): + if node and (node.type in ('file','content')): return True return False @@ -426,7 +431,6 @@ class abstracted_fs(object): """Return True if path is a symbolic link.""" return False - # Ok def isdir(self, node): """Return True if path is a directory.""" if node is None: @@ -435,19 +439,20 @@ class abstracted_fs(object): return True return False - # Ok - def getsize(self, node): + def getsize(self, datacr): """Return the size of the specified file in bytes.""" - result = 0L - if node.type=='file': - result = node.content_length or 0L - return result + if not (datacr and datacr[1]): + return 0L + if datacr[1].type in ('file', 'content'): + return datacr[1].content_length or 0L + return 0L # Ok - def getmtime(self, node): + def getmtime(self, datacr): """Return the last modified time as a number of seconds since the epoch.""" + node = datacr[1] if node.write_date or node.create_date: dt = (node.write_date or node.create_date)[:19] result = time.mktime(time.strptime(dt, '%Y-%m-%d %H:%M:%S')) @@ -468,7 +473,9 @@ class abstracted_fs(object): """Return True if path refers to an existing path, including a broken or circular symbolic link. """ + raise DeprecationWarning() return path and True or False + exists = lexists # Ok, can be improved @@ -487,23 +494,21 @@ class abstracted_fs(object): # note: the following operations are no more blocking - # Ok - def get_list_dir(self, path): + def get_list_dir(self, datacr): """"Return an iterator object that yields a directory listing in a form suitable for LIST command. """ - if self.isdir(path): - listing = self.listdir(path) + if not datacr: + return None + elif self.isdir(datacr[1]): + listing = self.listdir(datacr) #listing.sort() - return self.format_list(path and path.path or '/', listing) + return self.format_list(datacr[0], datacr[1], listing) # if path is a file or a symlink we return information about it - elif self.isfile(path): - basedir, filename = os.path.split(path.path) - self.lstat(path) # raise exc in case of problems - return self.format_list(basedir, [path]) + elif self.isfile(datacr[1]): + par = datacr[1].parent + return self.format_list(datacr[0], par, [datacr[1]]) - - # Ok def get_stat_dir(self, rawline, datacr): """Return an iterator object that yields a list of files matching a dirname pattern non-recursively in a form @@ -526,13 +531,12 @@ class abstracted_fs(object): listing.sort() return self.format_list(basedir, listing) - # Ok - def format_list(self, basedir, listing, ignore_err=True): + def format_list(self, cr, parent_node, listing, ignore_err=True): """Return an iterator object that yields the entries of given directory emulating the "/bin/ls -lA" UNIX command output. - - (str) basedir: the absolute dirname. - - (list) listing: the names of the entries in basedir + - (str) basedir: the parent directory node. Can be None + - (list) listing: a list of nodes - (bool) ignore_err: when False raise exception if os.lstat() call fails. @@ -548,35 +552,35 @@ class abstracted_fs(object): drwxrwxrwx 1 owner group 0 Aug 31 18:50 e-books -rw-rw-rw- 1 owner group 380 Sep 02 3:40 module.py """ - for file in listing: - try: - st = self.lstat(file) - except os.error: - if ignore_err: - continue - raise - perms = filemode(st.st_mode) # permissions - nlinks = st.st_nlink # number of links to inode - if not nlinks: # non-posix system, let's use a bogus value - nlinks = 1 - size = st.st_size # file size - uname = "owner" - gname = "group" + for node in listing: + perms = filemode(node.unixperms) # permissions + nlinks = 1 + size = node.content_length or 0L + uname = node.uuser + gname = node.ugroup # stat.st_mtime could fail (-1) if last mtime is too old # in which case we return the local time as last mtime try: - mname=_get_month_name(time.strftime("%m", time.localtime(st.st_mtime))) - mtime = mname+' '+time.strftime("%d %H:%M", time.localtime(st.st_mtime)) + st_mtime = node.write_date or 0.0 + if isinstance(st_mtime, basestring): + st_mtime = time.strptime(st_mtime, '%Y-%m-%d %H:%M:%S') + elif isinstance(st_mtime, float): + st_mtime = time.localtime(st_mtime) + mname=_get_month_name(time.strftime("%m", st_mtime )) + mtime = mname+' '+time.strftime("%d %H:%M", st_mtime) except ValueError: mname=_get_month_name(time.strftime("%m")) mtime = mname+' '+time.strftime("%d %H:%M") + fpath = node.path + if isinstance(fpath, (list, tuple)): + fpath = fpath[-1] # formatting is matched with proftpd ls output - path=_to_decode(file.path) #file.path.encode('ascii','replace').replace('?','_') + path=_to_decode(fpath) yield "%s %3s %-8s %-8s %8s %s %s\r\n" %(perms, nlinks, uname, gname, - size, mtime, path.split('/')[-1]) + size, mtime, path) # Ok - def format_mlsx(self, basedir, listing, perms, facts, ignore_err=True): + def format_mlsx(self, cr, basedir, listing, perms, facts, ignore_err=True): """Return an iterator object that yields the entries of a given directory or of a single file in a form suitable with MLSD and MLST commands. @@ -609,15 +613,9 @@ class abstracted_fs(object): if 'd' in perms: permdir += 'p' type = size = perm = modify = create = unique = mode = uid = gid = "" - for file in listing: - try: - st = self.stat(file) - except OSError: - if ignore_err: - continue - raise + for node in listing: # type + perm - if stat.S_ISDIR(st.st_mode): + if self.isdir(node): if 'type' in facts: type = 'type=dir;' if 'perm' in facts: @@ -628,29 +626,37 @@ class abstracted_fs(object): if 'perm' in facts: perm = 'perm=%s;' %permfile if 'size' in facts: - size = 'size=%s;' %st.st_size # file size + size = 'size=%s;' % (node.content_length or 0L) # last modification time if 'modify' in facts: try: - modify = 'modify=%s;' %time.strftime("%Y%m%d%H%M%S", - time.localtime(st.st_mtime)) + st_mtime = node.write_date or 0.0 + if isinstance(st_mtime, basestring): + st_mtime = time.strptime(st_mtime, '%Y-%m-%d %H:%M:%S') + elif isinstance(st_mtime, float): + st_mtime = time.localtime(st_mtime) + modify = 'modify=%s;' %time.strftime("%Y%m%d%H%M%S", st_mtime) except ValueError: # stat.st_mtime could fail (-1) if last mtime is too old modify = "" if 'create' in facts: # on Windows we can provide also the creation time try: - create = 'create=%s;' %time.strftime("%Y%m%d%H%M%S", - time.localtime(st.st_ctime)) + st_ctime = node.create_date or 0.0 + if isinstance(st_ctime, basestring): + st_ctime = time.strptime(st_ctime, '%Y-%m-%d %H:%M:%S') + elif isinstance(st_mtime, float): + st_ctime = time.localtime(st_ctime) + create = 'create=%s;' %time.strftime("%Y%m%d%H%M%S",st_ctime) except ValueError: create = "" # UNIX only if 'unix.mode' in facts: - mode = 'unix.mode=%s;' %oct(st.st_mode & 0777) + mode = 'unix.mode=%s;' %oct(node.unixperms & 0777) if 'unix.uid' in facts: - uid = 'unix.uid=%s;' %st.st_uid + uid = 'unix.uid=%s;' % node.uuser if 'unix.gid' in facts: - gid = 'unix.gid=%s;' %st.st_gid + gid = 'unix.gid=%s;' % node.ugroup # We provide unique fact (see RFC-3659, chapter 7.5.2) on # posix platforms only; we get it by mixing st_dev and # st_ino values which should be enough for granting an @@ -659,10 +665,12 @@ class abstracted_fs(object): # Implementors who want to provide unique fact on other # platforms should use some platform-specific method (e.g. # on Windows NTFS filesystems MTF records could be used). - if 'unique' in facts: - unique = "unique=%x%x;" %(st.st_dev, st.st_ino) - path=_to_decode(file.path) - path = path and path.split('/')[-1] or None + # if 'unique' in facts: todo + # unique = "unique=%x%x;" %(st.st_dev, st.st_ino) + path = node.path + if isinstance (path, (list, tuple)): + path = path[-1] + path=_to_decode(path) yield "%s%s%s%s%s%s%s%s%s %s\r\n" %(type, size, perm, modify, create, mode, uid, gid, unique, path) diff --git a/addons/document_ftp/ftpserver/ftpserver.py b/addons/document_ftp/ftpserver/ftpserver.py index c02323dbc53..f92fbb34613 100644 --- a/addons/document_ftp/ftpserver/ftpserver.py +++ b/addons/document_ftp/ftpserver/ftpserver.py @@ -1138,19 +1138,11 @@ class AbstractedFS: # note: the following operations are no more blocking - def get_list_dir(self, path): + def get_list_dir(self, datacr): """"Return an iterator object that yields a directory listing in a form suitable for LIST command. """ - if self.isdir(path): - listing = self.listdir(path) - listing.sort() - return self.format_list(path, listing) - # if path is a file or a symlink we return information about it - else: - basedir, filename = os.path.split(path) - self.lstat(path) # raise exc in case of problems - return self.format_list(basedir, [filename]) + raise DeprecationWarning() def get_stat_dir(self, rawline): """Return an iterator object that yields a list of files @@ -1591,26 +1583,8 @@ class FTPHandler(asynchat.async_chat): def __check_path(self, cmd, line): """Check whether a path is valid.""" - # For the following commands we have to make sure that the real - # path destination belongs to the user's root directory. - # If provided path is a symlink we follow its final destination - # to do so. - if cmd in ('APPE','CWD','DELE','MDTM','NLST','MLSD','MLST','RETR', - 'RMD','SIZE','STOR','XCWD','XRMD'): - datacr = None - datacr = self.fs.get_cr(line) - try: - if not self.fs.validpath(self.fs.ftp2fs(line, datacr)): - line = self.fs.ftpnorm(line) - err = '"%s" points to a path which is outside ' \ - "the user's root directory" %line - self.respond("550 %s." %err) - self.log('FAIL %s "%s". %s.' %(cmd, line, err)) - self.fs.close_cr(datacr) - return False - except: - pass - self.fs.close_cr(datacr) + + # Always true, we will only check later, once we have a cursor return True def __check_perm(self, cmd, line, datacr): @@ -1623,6 +1597,7 @@ class FTPHandler(asynchat.async_chat): 'RNFR':'f', 'MKD':'m', 'XMKD':'m', 'STOR':'w'} + raise NotImplementedError if cmd in map: if cmd == 'STAT' and not line: return True @@ -2032,56 +2007,65 @@ class FTPHandler(asynchat.async_chat): # - Some older FTP clients erroneously issue /bin/ls-like LIST # formats in which case we fall back on cwd as default. if not line or line.lower() in ('-a', '-l', '-al', '-la'): - line = self.fs.cwd + line = '' + datacr = None try: - data = None - data = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, data) - line = self.fs.ftpnorm(line) - iterator = self.run_as_current_user(self.fs.get_list_dir, path) + datacr = self.fs.get_crdata(line, mode='list') + iterator = self.run_as_current_user(self.fs.get_list_dir, datacr) + except IOError, err: + self.fs.close_cr(datacr) + self.respond('550 %s.'% err.strerror) + return except OSError, err: - self.fs.close_cr(data) + self.fs.close_cr(datacr) why = _strerror(err) self.log('FAIL LIST "%s". %s.' %(line, why)) self.respond('550 %s.' %why) - else: - self.fs.close_cr(data) - self.log('OK LIST "%s". Transfer starting.' %line) + return + try: + self.log('OK LIST "%s". Transfer starting.' % line) producer = BufferedIteratorProducer(iterator) self.push_dtp_data(producer, isproducer=True) + finally: + self.fs.close_cr(datacr) + def ftp_NLST(self, line): """Return a list of files in the specified directory in a compact form to the client. """ if not line: - line = self.fs.cwd + line = '' + + datacr = None try: - data = None - data = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, data) - line = self.fs.ftpnorm(line) - if self.fs.isdir(path): - listing = self.run_as_current_user(self.fs.listdir, path) - listing = map(lambda x:os.path.split(x.path)[1], listing) + datacr = self.fs.get_crdata(line, mode='list') + if not datacr: + datacr = ( None, None, None ) + if self.fs.isdir(datacr[1]): + listing = self.run_as_current_user(self.fs.listdir, datacr) + listing = [ x.path[-1] for x in listing ] else: # if path is a file we just list its name - self.fs.lstat(path) # raise exc in case of problems - basedir, filename = os.path.split(line) - listing = [filename] + listing = [datacr[1].path[-1],] + except IOError, err: + self.fs.close_cr(datacr) + self.respond('550 %s.'% err.strerror) + return except OSError, err: - self.fs.close_cr(data) + self.fs.close_cr(datacr) why = _strerror(err) self.log('FAIL NLST "%s". %s.' %(line, why)) self.respond('550 %s.' %why) - else: - self.fs.close_cr(data) - data = '' - if listing: - listing.sort() - data = '\r\n'.join(listing) + '\r\n' - self.log('OK NLST "%s". Transfer starting.' %line) - self.push_dtp_data(data) + return + + self.fs.close_cr(datacr) + data = '' + if listing: + listing.sort() + data = '\r\n'.join(listing) + '\r\n' + self.log('OK NLST "%s". Transfer starting.' %line) + self.push_dtp_data(data) # --- MLST and MLSD commands @@ -2096,18 +2080,15 @@ class FTPHandler(asynchat.async_chat): """ # if no argument, fall back on cwd as default if not line: - line = self.fs.cwd + line = '' + datacr = None try: - datacr = None - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, datacr) - line = self.fs.ftpnorm(line) - basedir, basename = os.path.split(path) + datacr = self.fs.get_crdata(line, mode='list') perms = self.authorizer.get_perms(self.username) - iterator = self.run_as_current_user(self.fs.format_mlsx, basedir, - [basename], perms, self.current_facts, ignore_err=False) + iterator = self.run_as_current_user(self.fs.format_mlsx, datacr[0], datacr[1].parent, + [datacr[1],], perms, self.current_facts, ignore_err=False) data = ''.join(iterator) - except OSError, err: + except EnvironmentError, err: self.fs.close_cr(datacr) why = _strerror(err) self.log('FAIL MLST "%s". %s.' %(line, why)) @@ -2129,19 +2110,18 @@ class FTPHandler(asynchat.async_chat): """ # if no argument, fall back on cwd as default if not line: - line = self.fs.cwd + line = '' + + datacr = None try: - datacr = None - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, datacr) - line = self.fs.ftpnorm(line) + datacr = self.fs.get_crdata(line, mode='list') # RFC-3659 requires 501 response code if path is not a directory - if not self.fs.isdir(path): + if not self.fs.isdir(datacr[1]): err = 'No such directory' self.log('FAIL MLSD "%s". %s.' %(line, err)) self.respond("501 %s." %err) return - listing = self.run_as_current_user(self.fs.listdir, path) + listing = self.run_as_current_user(self.fs.listdir, datacr) except OSError, err: self.fs.close_cr(datacr) why = _strerror(err) @@ -2150,7 +2130,7 @@ class FTPHandler(asynchat.async_chat): else: self.fs.close_cr(datacr) perms = self.authorizer.get_perms(self.username) - iterator = self.fs.format_mlsx(path, listing, perms, + iterator = self.fs.format_mlsx(datacr[0], datacr[1], listing, perms, self.current_facts) producer = BufferedIteratorProducer(iterator) self.log('OK MLSD "%s". Transfer starting.' %line) @@ -2160,12 +2140,10 @@ class FTPHandler(asynchat.async_chat): """Retrieve the specified file (transfer from the server to the client) """ + datacr = None try: - datacr = None - datacr = self.fs.get_cr(line) - file = self.fs.ftp2fs(line, datacr) - line = self.fs.ftpnorm(line) - fd = self.run_as_current_user(self.fs.open, file, 'rb') + datacr = self.fs.get_crdata(line, mode='file') + fd = self.run_as_current_user(self.fs.open, datacr, 'rb') except OSError, err: self.fs.close_cr(datacr) why = _strerror(err) @@ -2174,7 +2152,7 @@ class FTPHandler(asynchat.async_chat): return except IOError, err: self.fs.close_cr(datacr) - why = _strerror(err) + why = err.strerror self.log('FAIL RETR "%s". %s.' %(line, why)) self.respond('550 %s.' %why) return @@ -2187,7 +2165,7 @@ class FTPHandler(asynchat.async_chat): # the REST. ok = 0 try: - assert not self.restart_position > self.fs.getsize(file) + assert not self.restart_position > self.fs.getsize(datacr) fd.seek(self.restart_position) ok = 1 except AssertionError: @@ -2217,14 +2195,9 @@ class FTPHandler(asynchat.async_chat): else: cmd = 'STOR' - line = self.fs.ftpnorm(line) - basedir,basename = os.path.split(line) - datacr = None try: - datacr = self.fs.get_cr(line) - file = self.fs.ftp2fs(basedir, datacr) - + datacr = self.fs.get_crdata(line,mode='create') except OSError, err: self.fs.close_cr(datacr) why = _strerror(err) @@ -2235,7 +2208,7 @@ class FTPHandler(asynchat.async_chat): if self.restart_position: mode = 'r+' try: - fd = self.run_as_current_user(self.fs.create, file, basename, mode + 'b') + fd = self.run_as_current_user(self.fs.create, datacr, datacr[2], mode + 'b') except IOError, err: self.fs.close_cr(datacr) why = _strerror(err) @@ -2251,7 +2224,7 @@ class FTPHandler(asynchat.async_chat): # specified in the REST. ok = 0 try: - assert not self.restart_position > self.fs.getsize(self.fs.ftp2fs(line, datacr)) + assert not self.restart_position > self.fs.getsize(datacr) fd.seek(self.restart_position) ok = 1 except AssertionError: @@ -2293,15 +2266,12 @@ class FTPHandler(asynchat.async_chat): self.respond("450 Can't STOU while REST request is pending.") return - datacr = None - datacr = self.fs.get_cr(line) if line: - line = self.fs.ftpnorm(line) - basedir,prefix = os.path.split(line) - basedir = self.fs.ftp2fs(basedir, datacr) - #prefix = prefix + '.' + datacr = self.fs.get_crdata(line, mode='create') + # TODO else: + # TODO basedir = self.fs.ftp2fs(self.fs.cwd, datacr) prefix = 'ftpd.' try: @@ -2500,21 +2470,23 @@ class FTPHandler(asynchat.async_chat): def ftp_PWD(self, line): """Return the name of the current working directory to the client.""" - self.respond('257 "%s" is the current directory.' %self.fs.cwd) + cwd = self.fs.get_cwd() + self.respond('257 "%s" is the current directory.' % cwd) def ftp_CWD(self, line): """Change the current working directory.""" - # TODO: a lot of FTP servers go back to root directory if no + # check: a lot of FTP servers go back to root directory if no # arg is provided but this is not specified in RFC-959. # Search for official references about this behaviour. - if not line: - line = '/' datacr = None try: - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, datacr) - self.run_as_current_user(self.fs.chdir, path) - except OSError, err: + datacr = self.fs.get_crdata(line,'cwd') + self.run_as_current_user(self.fs.chdir, datacr) + cwd = self.fs.get_cwd() + self.log('OK CWD "%s".' % cwd) + self.respond('250 "%s" is the current directory.' % cwd) + except EnvironmentError, err: + self.log("Could not cwd: %s" % err) if err.errno==2: why = 'Authentication Required or Failed' self.log('FAIL CWD "%s". %s.' %(self.fs.ftpnorm(line), why)) @@ -2523,9 +2495,6 @@ class FTPHandler(asynchat.async_chat): why = _strerror(err) self.log('FAIL CWD "%s". %s.' %(self.fs.ftpnorm(line), why)) self.respond('550 %s.' %why) - else: - self.log('OK CWD "%s".' %self.fs.cwd) - self.respond('250 "%s" is the current directory.' %self.fs.cwd) self.fs.close_cr(datacr) def ftp_CDUP(self, line): @@ -2554,17 +2523,15 @@ class FTPHandler(asynchat.async_chat): """ datacr = None try: - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, datacr) - line = self.fs.ftpnorm(line) - if self.fs.isdir(path): - why = "%s is not retrievable" %line - self.log('FAIL SIZE "%s". %s.' %(line, why)) - self.respond("550 %s." %why) - self.fs.close_cr(datacr) - return - size = self.run_as_current_user(self.fs.getsize, path) - except OSError, err: + datacr = self.fs.get_crdata(line, mode='file') + #if self.fs.isdir(datacr[1]): + # why = "%s is not retrievable" %line + # self.log('FAIL SIZE "%s". %s.' %(line, why)) + # self.respond("550 %s." %why) + # self.fs.close_cr(datacr) + # return + size = self.run_as_current_user(self.fs.getsize, datacr) + except EnvironmentError, err: why = _strerror(err) self.log('FAIL SIZE "%s". %s.' %(line, why)) self.respond('550 %s.' %why) @@ -2579,35 +2546,31 @@ class FTPHandler(asynchat.async_chat): """ datacr = None try: - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, datacr) - line = self.fs.ftpnorm(line) - if not self.fs.isfile(self.fs.realpath(path)): + datacr = self.fs.get_crdata(line) + if not self.fs.isfile(datacr): why = "%s is not retrievable" %line self.log('FAIL MDTM "%s". %s.' %(line, why)) self.respond("550 %s." %why) self.fs.close_cr(datacr) return - lmt = self.run_as_current_user(self.fs.getmtime, path) + lmt = self.run_as_current_user(self.fs.getmtime, datacr) + lmt = time.strftime("%Y%m%d%H%M%S", time.localtime(lmt)) + self.respond("213 %s" %lmt) + self.log('OK MDTM "%s".' %line) except OSError, err: why = _strerror(err) self.log('FAIL MDTM "%s". %s.' %(line, why)) self.respond('550 %s.' %why) - else: - lmt = time.strftime("%Y%m%d%H%M%S", time.localtime(lmt)) - self.respond("213 %s" %lmt) - self.log('OK MDTM "%s".' %line) self.fs.close_cr(datacr) def ftp_MKD(self, line): """Create the specified directory.""" - datacr = None - line = self.fs.ftpnorm(line) - basedir,basename = os.path.split(line) try: - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(basedir, datacr) - self.run_as_current_user(self.fs.mkdir, path, basename) + datacr = self.fs.get_crdata(line, mode='create') + self.run_as_current_user(self.fs.mkdir, datacr, datacr[2]) + except IOError, err: + self.log('FAIL MKD "%s". %s.' %(line, err.strerror)) + self.respond('550 %s.' % err.strerror) except OSError, err: why = _strerror(err) self.log('FAIL MKD "%s". %s.' %(line, why)) @@ -2621,16 +2584,14 @@ class FTPHandler(asynchat.async_chat): """Remove the specified directory.""" datacr = None try: - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, datacr) - line = self.fs.ftpnorm(line) - if self.fs.realpath(path) == self.fs.realpath(self.fs.root): + datacr = self.fs.get_crdata(line, mode='delete') + if not datacr[1]: msg = "Can't remove root directory." self.respond("550 %s" %msg) self.log('FAIL MKD "/". %s' %msg) self.fs.close_cr(datacr) return - self.run_as_current_user(self.fs.rmdir, path) + self.run_as_current_user(self.fs.rmdir, datacr) except OSError, err: why = _strerror(err) self.log('FAIL RMD "%s". %s.' %(line, why)) @@ -2644,10 +2605,8 @@ class FTPHandler(asynchat.async_chat): """Delete the specified file.""" datacr = None try: - datacr = self.fs.get_cr(line) - path = self.fs.ftp2fs(line, datacr) - line = self.fs.ftpnorm(line) - self.run_as_current_user(self.fs.remove, path) + datacr = self.fs.get_crdata(line, mode='delete') + self.run_as_current_user(self.fs.remove, datacr) except OSError, err: why = _strerror(err) self.log('FAIL DELE "%s". %s.' %(line, why)) @@ -2662,15 +2621,13 @@ class FTPHandler(asynchat.async_chat): here, see RNTO command)""" datacr = None try: - datacr = self.fs.get_cr(line) - line = self.fs.ftpnorm(line) - path = self.fs.ftp2fs(line, datacr) - if not self.fs.lexists(path): + datacr = self.fs.get_crdata(line, mode='rfnr') + if not datacr[1]: self.respond("550 No such file or directory.") - elif self.fs.realpath(path) == self.fs.realpath(self.fs.root): + elif not datacr[1]: self.respond("550 Can't rename the home directory.") else: - self.fs.rnfr = line + self.fs.rnfr = datacr[1] self.respond("350 Ready for destination name.") except: self.respond("550 Can't find the file or directory.") @@ -2686,21 +2643,17 @@ class FTPHandler(asynchat.async_chat): datacr = None try: try: - datacr = self.fs.get_cr(line) - src = self.fs.ftp2fs(self.fs.rnfr, datacr) - line = self.fs.ftpnorm(line) - basedir,basename = os.path.split(line) - dst = self.fs.ftp2fs(basedir, datacr) - self.run_as_current_user(self.fs.rename, src, dst,basename) - except OSError, err: + datacr = self.fs.get_crdata(line,'create') + oldname = '/'.join(self.fs.rnfr.path) + self.run_as_current_user(self.fs.rename, self.fs.rnfr, datacr) + self.fs.rnfr = None + self.log('OK RNFR/RNTO "%s ==> %s".' %(oldname, line)) + self.respond("250 Renaming ok.") + except EnvironmentError, err: why = _strerror(err) self.log('FAIL RNFR/RNTO "%s ==> %s". %s.' \ %(self.fs.ftpnorm(self.fs.rnfr), line, why)) self.respond('550 %s.' %why) - else: - self.log('OK RNFR/RNTO "%s ==> %s".' \ - %(self.fs.ftpnorm(self.fs.rnfr), line)) - self.respond("250 Renaming ok.") finally: self.fs.rnfr = None self.fs.close_cr(datacr) From 0828a06de49509334308597ffa1bda929dcf7460 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sat, 3 Jul 2010 00:53:02 +0300 Subject: [PATCH 069/121] Document ftp: don't close the cursor at the abstract_fs Since the get_crdata() has created cursors outside the scope of this function, it is not wise to close them inside. bzr revid: p_christ@hol.gr-20100702215302-y6fsu4kz05g1a9v7 --- addons/document_ftp/ftpserver/abstracted_fs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 60974896b41..ae6ca935f64 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -249,8 +249,6 @@ class abstracted_fs(object): except Exception,e: self._log.exception('Cannot create dir "%s" at node %s', basename, repr(node)) raise OSError(1, 'Operation not permited.') - finally: - if cr: cr.close() def close_cr(self, data): if data and data[0]: From 45b7a3adc3a8e3669d15598d67f99513856fbab0 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Sat, 3 Jul 2010 00:53:04 +0300 Subject: [PATCH 070/121] Document FTP: improve the tests Now, a non-existing path will return 550, an exception. bzr revid: p_christ@hol.gr-20100702215304-7l2pit3ceih3wvo9 --- addons/document_ftp/test/document_ftp_test2.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/addons/document_ftp/test/document_ftp_test2.yml b/addons/document_ftp/test/document_ftp_test2.yml index e0b3b72b320..a5eaa13521f 100644 --- a/addons/document_ftp/test/document_ftp_test2.yml +++ b/addons/document_ftp/test/document_ftp_test2.yml @@ -145,8 +145,13 @@ - !python {model: ir.attachment}: | from document_ftp import test_easyftp as te + from ftplib import error_perm ftp = te.get_ftp_folder(cr, uid, self, 'Documents') - assert ftp.nlst("test-name.txt") == [] + try: + res = ftp.nlst("test-name.txt") + assert res == [] + except error_perm, e: + pass assert ftp.nlst("test-renamed.txt") == ['test-renamed.txt'] - I create a new folder 'Test-Folder2' through FTP @@ -203,7 +208,7 @@ from document_ftp import test_easyftp as te ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2') # TODO speed - assert len(ftp.nlst() > 1000) + assert len(ftp.nlst()) > 1000 - I read the 1000 files, check speed # TODO From 606df7368c8fdea6bc54d787832069fbb2304822 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 6 Jul 2010 09:01:55 +0300 Subject: [PATCH 071/121] Document ftp: better handling of wrong paths. bzr revid: p_christ@hol.gr-20100706060155-nyljhua59ahax7sx --- addons/document_ftp/ftpserver/abstracted_fs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index ae6ca935f64..b5465c9873d 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -300,9 +300,11 @@ class abstracted_fs(object): if os.path.isabs(path): # we have to start from root, again p_parts = p_parts[1:] + if not p_parts: + raise IOError(errno.EPERM, 'Cannot perform operation at root dir') dbname = p_parts[0] if dbname not in self.db_list(): - return None + return IOError(errno.ENOENT,'Invalid database path') try: db = pooler.get_db(dbname) except Exception: From e8bf4a4f18fe70d52192f8893105ba9291abef4a Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 6 Jul 2010 15:10:45 +0300 Subject: [PATCH 072/121] Document: fix the get_partner_id functionality Was blocking document.create() bzr revid: p_christ@hol.gr-20100706121045-mblekyw8nlkiy1r0 --- addons/document/document.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/addons/document/document.py b/addons/document/document.py index dbd4ec3f5b0..3dc84529d8d 100644 --- a/addons/document/document.py +++ b/addons/document/document.py @@ -196,7 +196,7 @@ class document_file(osv.osv): vals['res_model'] = context.get('default_res_model', False) if vals.get('res_id', False) and vals.get('res_model', False) \ and not vals.get('partner_id', False): - vals['partner_id'] = __get_partner_id(cr, uid, \ + vals['partner_id'] = self.__get_partner_id(cr, uid, \ vals['res_model'], vals['res_id'], context) datas = None @@ -222,16 +222,15 @@ class document_file(osv.osv): It is a hack that will try to discover if the mentioned record is clearly associated with a partner record. """ - if False: - obj_model = self.pool.get(vals['res_model']) - if obj_model._name == 'res.partner': - return res_id - elif 'partner_id' in obj_model._columns and obj_model._columns['partner_id']._obj == 'res.partner': - bro = obj_model.browse(self, cr, uid, res_id, context=context) - return bro.partner_id.id - elif 'address_id' in obj_model._columns and obj_model._columns['address_id']._obj == 'res.partner.address': - bro = obj_model.browse(self, cr, uid, res_id, context=context) - return bro.address_id.partner_id.id + obj_model = self.pool.get(res_model) + if obj_model._name == 'res.partner': + return res_id + elif 'partner_id' in obj_model._columns and obj_model._columns['partner_id']._obj == 'res.partner': + bro = obj_model.browse(self, cr, uid, res_id, context=context) + return bro.partner_id.id + elif 'address_id' in obj_model._columns and obj_model._columns['address_id']._obj == 'res.partner.address': + bro = obj_model.browse(self, cr, uid, res_id, context=context) + return bro.address_id.partner_id.id return False def unlink(self, cr, uid, ids, context={}): From 67386ec78b3fec4bfde55135fa88a2322c3a4804 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 6 Jul 2010 15:10:47 +0300 Subject: [PATCH 073/121] Document: fix typo in test.yml bzr revid: p_christ@hol.gr-20100706121047-p2nex1m3g86nirt6 --- addons/document/test/document_test2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document/test/document_test2.yml b/addons/document/test/document_test2.yml index fa5e98d3bdb..a83cfa23bfe 100644 --- a/addons/document/test/document_test2.yml +++ b/addons/document/test/document_test2.yml @@ -69,5 +69,5 @@ ids = self.search(cr, uid, [('res_model', '=', 'res.country'), ('res_id', '=', ref("base.za"))]) assert ids == [ ref("attach_3rd")], ids - - !delete {moel: ir.attachment, id: attach_3rd, search: } + !delete {model: ir.attachment, id: attach_3rd, search: } - From 61b09c7fc9be040308c7d1fdb73bed9818aa6827 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 6 Jul 2010 15:10:50 +0300 Subject: [PATCH 074/121] Document ftp: fix NLST, path resolution. bzr revid: p_christ@hol.gr-20100706121050-5u0i2hsm2phz0i30 --- addons/document_ftp/ftpserver/abstracted_fs.py | 12 ++++++++---- addons/document_ftp/ftpserver/ftpserver.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index b5465c9873d..6731908d1cf 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -275,8 +275,8 @@ class abstracted_fs(object): if not os.path.isabs(path): path = os.path.join(self.root, path) - if path == '/' and mode in ('list', 'cwd'): - return (None, None, None ) + if path == '/' and mode in ('list', 'cwd'): + return (None, None, None ) path = os.path.normpath(path) # again, for '/db/../ss' if path == '.': path = '' @@ -299,12 +299,14 @@ class abstracted_fs(object): if os.path.isabs(path): # we have to start from root, again - p_parts = p_parts[1:] + while p_parts[0] == '': + p_parts = p_parts[1:] + # self._log.debug("Path parts: %r ", p_parts) if not p_parts: raise IOError(errno.EPERM, 'Cannot perform operation at root dir') dbname = p_parts[0] if dbname not in self.db_list(): - return IOError(errno.ENOENT,'Invalid database path') + raise IOError(errno.ENOENT,'Invalid database path') try: db = pooler.get_db(dbname) except Exception: @@ -320,6 +322,7 @@ class abstracted_fs(object): raise OSError(2, 'Authentification Required.') n = get_node_context(cr, uid, {}) node = n.get_uri(cr, p_parts[1:]) + # self._log.debug("get_crdata(abs): %r" % ( (cr, node, rem_path),)) return (cr, node, rem_path) else: # we never reach here if cwd_node is not set @@ -333,6 +336,7 @@ class abstracted_fs(object): if node is False and mode not in ('???'): cr.close() raise IOError(errno.ENOENT, 'Path does not exist') + # self._log.debug("get_crdata(rel): %r" % ( (cr, node, rem_path),)) return (cr, node, rem_path) def get_node_cr_uid(self, node): diff --git a/addons/document_ftp/ftpserver/ftpserver.py b/addons/document_ftp/ftpserver/ftpserver.py index f92fbb34613..75ad5b00883 100644 --- a/addons/document_ftp/ftpserver/ftpserver.py +++ b/addons/document_ftp/ftpserver/ftpserver.py @@ -2043,11 +2043,17 @@ class FTPHandler(asynchat.async_chat): if not datacr: datacr = ( None, None, None ) if self.fs.isdir(datacr[1]): - listing = self.run_as_current_user(self.fs.listdir, datacr) - listing = [ x.path[-1] for x in listing ] + nodelist = self.run_as_current_user(self.fs.listdir, datacr) else: # if path is a file we just list its name - listing = [datacr[1].path[-1],] + nodelist = [datacr[1],] + + listing = [] + for nl in nodelist: + if isinstance(nl.path, (list, tuple)): + listing.append(nl.path[-1]) + else: + listing.append(nl.path) # assume string except IOError, err: self.fs.close_cr(datacr) self.respond('550 %s.'% err.strerror) From 950058dae348bfb856ce5cf9c120822d7a147f6f Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Tue, 6 Jul 2010 15:10:53 +0300 Subject: [PATCH 075/121] Document FTP: fix yaml tests. bzr revid: p_christ@hol.gr-20100706121053-5a9z29ub8hrnpqo9 --- addons/document_ftp/test/document_ftp_test2.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/addons/document_ftp/test/document_ftp_test2.yml b/addons/document_ftp/test/document_ftp_test2.yml index a5eaa13521f..8b7d538c00b 100644 --- a/addons/document_ftp/test/document_ftp_test2.yml +++ b/addons/document_ftp/test/document_ftp_test2.yml @@ -21,7 +21,7 @@ !python {model: ir.attachment}: | from document_ftp import test_easyftp as te ftp = te.get_ftp_login(cr, uid, self) - ftp.cwd('/Documents') + ftp.cwd('Documents') - I create a "test.txt" file at the server (directly). The file should have the "abcd" content @@ -69,10 +69,13 @@ - !python {model: ir.attachment}: | from document_ftp import test_easyftp as te + import ftplib ftp = te.get_ftp_login(cr, uid, self) try: ftp.cwd('/Not-This') assert False, "We should't be able to change here" + except ftplib.error_perm: + pass except OSError, err: assert err.errno == 2, err.errno - From 1927431e0bf256503fbdcf282008a7b45f449559 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 7 Jul 2010 02:27:00 +0300 Subject: [PATCH 076/121] Document: remove debug calls from trunk code. That code would only work in trunk-xrg. bzr revid: p_christ@hol.gr-20100706232700-x56f57gqhzupfphq --- addons/document/document_storage.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index aeea5f99011..809c3c83be8 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -285,14 +285,10 @@ class document_storage(osv.osv): self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id) raise IOError(errno.ENOENT, 'No file can be located') fpath = os.path.join(boo.path, ira.store_fname) - if self._debug: - self._doclog.debug("Trying to read \"%s\".."% fpath) return nodefd_file(file_node, path=fpath, mode=mode) elif boo.type == 'db': # TODO: we need a better api for large files - if self._debug: - self._doclog.debug("Trying to obtain db_datas for ir.attachment[%d]", ira.id) return nodefd_db64(file_node, ira_browse=ira, mode=mode) elif boo.type == 'realstore': From eb96e0f256b9130f0f1b814be6415d6b35a46d23 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Wed, 7 Jul 2010 02:29:28 +0300 Subject: [PATCH 077/121] Document ftp: solve the tests paradox the FTP server has been limited to only serve databases that have the document_ftp module "installed". However, this would break running the tests, because the module would be marked as "to upgrade". bzr revid: p_christ@hol.gr-20100706232928-5ckj97p61nfrq3rn --- addons/document_ftp/ftpserver/abstracted_fs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/document_ftp/ftpserver/abstracted_fs.py b/addons/document_ftp/ftpserver/abstracted_fs.py index 6731908d1cf..7667b7ee473 100644 --- a/addons/document_ftp/ftpserver/abstracted_fs.py +++ b/addons/document_ftp/ftpserver/abstracted_fs.py @@ -98,7 +98,7 @@ class abstracted_fs(object): if not cr.fetchone(): continue - cr.execute("SELECT id FROM ir_module_module WHERE name = 'document_ftp' AND state='installed' ") + cr.execute("SELECT id FROM ir_module_module WHERE name = 'document_ftp' AND state IN ('installed', 'to upgrade') ") res = cr.fetchone() if res and len(res): self.db_name_list.append(db_name) From 03865bc917b6fa7ee3e3964411722ccdc75b52c2 Mon Sep 17 00:00:00 2001 From: "P. Christeas" Date: Fri, 9 Jul 2010 01:50:42 +0300 Subject: [PATCH 078/121] Document: Less dir, storage options, cleanup views. The 'virtual' storage is now hidden. An extra 'db64' type, hidden, is supported in the code, for v5-compatible databases (until we migrate the db_datas column of them). The 'at_record' directory type is gone. A module in extra-addons will permit the compatible behaviour, for those experts. bzr revid: p_christ@hol.gr-20100708225042-fp3zcwmw4zp3ee04 --- addons/document/document_directory.py | 2 +- addons/document/document_storage.py | 39 +++++++++++++++++++++++---- addons/document/document_view.xml | 9 +++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/addons/document/document_directory.py b/addons/document/document_directory.py index 1b62418106c..9913946eee4 100644 --- a/addons/document/document_directory.py +++ b/addons/document/document_directory.py @@ -50,7 +50,7 @@ class document_directory(osv.osv): 'type': fields.selection([ ('directory','Static Directory'), ('ressource','Folders per resource'), - ('at_record','Single folder under resource')], + ], 'Type', required=True, select=1, help="Defines directory's behaviour."), diff --git a/addons/document/document_storage.py b/addons/document/document_storage.py index 809c3c83be8..b1b884a9d98 100644 --- a/addons/document/document_storage.py +++ b/addons/document/document_storage.py @@ -232,7 +232,7 @@ class document_storage(osv.osv): 'group_ids': fields.many2many('res.groups', 'document_storage_group_rel', 'item_id', 'group_id', 'Groups'), 'dir_ids': fields.one2many('document.directory', 'parent_id', 'Directories'), 'type': fields.selection([('db', 'Database'), ('filestore', 'Internal File storage'), - ('realstore', 'External file storage'), ('virtual', 'Virtual storage')], 'Type', required=True), + ('realstore','External file storage'),], 'Type', required=True), 'path': fields.char('Path', size=250, select=1, help="For file storage, the root path of the storage"), 'online': fields.boolean('Online', help="If not checked, media is currently offline and its contents not available", required=True), 'readonly': fields.boolean('Read Only', help="If set, media is for reading only"), @@ -270,6 +270,8 @@ class document_storage(osv.osv): return self.__get_data_3(cr, uid, boo, ira, context) def get_file(self, cr, uid, id, file_node, mode, context=None): + """ Return a file-like object for the contents of some node + """ if context is None: context = {} boo = self.browse(cr, uid, id, context) @@ -289,6 +291,9 @@ class document_storage(osv.osv): elif boo.type == 'db': # TODO: we need a better api for large files + return nodefd_db(file_node, ira_browse=ira, mode=mode) + + elif boo.type == 'db64': return nodefd_db64(file_node, ira_browse=ira, mode=mode) elif boo.type == 'realstore': @@ -303,6 +308,9 @@ class document_storage(osv.osv): raise IOError("File not found: %s" % fpath) return nodefd_file(file_node, path=fpath, mode=mode) + elif boo.type == 'virtual': + raise ValueError('Virtual storage does not support static files') + else: raise TypeError("No %s storage" % boo.type) @@ -318,13 +326,21 @@ class document_storage(osv.osv): return None fpath = os.path.join(boo.path, ira.store_fname) return file(fpath, 'rb').read() - elif boo.type == 'db': + elif boo.type == 'db64': # TODO: we need a better api for large files if ira.db_datas: out = base64.decodestring(ira.db_datas) else: out = '' return out + elif boo.type == 'db': + # We do an explicit query, to avoid type transformations. + cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s', (ira.id,)) + res = cr.fetchone() + if res: + return res[0] + else: + return '' elif boo.type == 'realstore': if not ira.store_fname: # On a migrated db, some files may have the wrong storage type @@ -339,6 +355,10 @@ class document_storage(osv.osv): return None else: raise IOError("File not found: %s" % fpath) + + elif boo.type == 'virtual': + raise ValueError('Virtual storage does not support static files') + else: raise TypeError("No %s storage" % boo.type) @@ -387,7 +407,12 @@ class document_storage(osv.osv): raise except_orm(_('Error!'), str(e)) elif boo.type == 'db': filesize = len(data) - # will that work for huge data? TODO + # will that work for huge data? + cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s', + (data, file_node.file_id)) + elif boo.type == 'db64': + filesize = len(data) + # will that work for huge data? out = base64.encodestring(data) cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s', (out, file_node.file_id)) @@ -419,6 +444,10 @@ class document_storage(osv.osv): except Exception,e : self._doclog.warning("Couldn't save data:", exc_info=True) raise except_orm(_('Error!'), str(e)) + + elif boo.type == 'virtual': + raise ValueError('Virtual storage does not support static files') + else: raise TypeError("No %s storage" % boo.type) @@ -467,7 +496,7 @@ class document_storage(osv.osv): return None path = storage_bo.path return (storage_bo.id, 'file', os.path.join(path, fname)) - elif storage_bo.type == 'db': + elif storage_bo.type in ('db', 'db64'): return None elif storage_bo.type == 'realstore': fname = fil_bo.store_fname @@ -500,7 +529,7 @@ class document_storage(osv.osv): sbro = self.browse(cr, uid, file_node.storage_id, context=context) assert sbro, "The file #%d didn't provide storage" % file_node.file_id - if sbro.type in ('filestore', 'db'): + if sbro.type in ('filestore', 'db', 'db64'): # nothing to do for a rename, allow to change the db field return { 'name': new_name, 'datas_fname': new_name } elif sbro.type == 'realstore': diff --git a/addons/document/document_view.xml b/addons/document/document_view.xml index f342fb7cf56..4f72245ce40 100644 --- a/addons/document/document_view.xml +++ b/addons/document/document_view.xml @@ -16,7 +16,7 @@ - + @@ -85,15 +85,14 @@ + + - + - - -