[MERGE] merged new api of document module

bzr revid: hmo@tinyerp.com-20100720130513-3t2cbyvns3hlj6qn

addons/document/__openerp__.py

@@ -53,7 +53,7 @@
     ],
     'demo_xml': [ 'document_demo.xml','board_document_demo.xml'],
     'test': [
-        'test/document_test.yml',
+        'test/document_test2.yml',
     ],
     'installable': True,
     'active': False,

addons/document/board_document_view.xml

@@ -53,7 +53,7 @@
         <menuitem id="menu_reporting" name="Reporting" sequence="2" parent="knowledge.menu_document"/>
          <menuitem
             name="Dashboard"
-            id="menu_board_document"
+            id="menu_reports_document"
             parent="menu_reporting"
             sequence="0"
             icon="terp-graph"
@@ -61,10 +61,10 @@
 
 
          <menuitem
-            parent="menu_board_document"
+            parent="menu_reports_document"
             action="open_board_document_manager"
             sequence="1"
-            id="menu_board_document_manager"
+            id="menu_reports_document_manager"
             icon="terp-graph"/>
 
         <record model="ir.ui.view" id="board_document_manager_form1">
@@ -107,10 +107,10 @@
         </record>
 
         <menuitem
-            parent="menu_board_document"
+            parent="menu_reports_document"
             action="open_board_document_manager1"
             sequence="1"
-            id="menu_board_document_manager1"
+            id="menu_reports_document_manager1"
             icon="terp-graph"/>
      </data>
  </openerp>

addons/document/content_index.py

@@ -153,7 +153,10 @@ class contentIndex(object):
                 if realfname :
                     fname = realfname
                 else:
-                    bname,ext = os.path.splitext(filename)
+                    try:
+                        bname,ext = os.path.splitext(filename or 'test.tmp')
+                    except Exception:
+                        bname, ext = filename, 'tmp'
                     fd, fname = tempfile.mkstemp(suffix=ext)
                     os.write(fd, content)
                     os.close(fd)

addons/document/directory_content.py

@@ -58,7 +58,9 @@ class document_directory_content(osv.osv):
         'suffix': fields.char('Suffix', size=16),
         'report_id': fields.many2one('ir.actions.report.xml', 'Report'),
         'extension': fields.selection(_extension_get, 'Document Type', required=True, size=4),
-        'include_name': fields.boolean('Include Record Name', help="Check this field if you want that the name of the file start by the record name."),
+        'include_name': fields.boolean('Include Record Name', 
+                help="Check this field if you want that the name of the file to contain the record name." \
+                    "\nIf set, the directory will have to be a resource one."),
         'directory_id': fields.many2one('document.directory', 'Directory'),
     }
     _defaults = {
@@ -88,12 +90,19 @@ class document_directory_content(osv.osv):
             tname = (content.prefix or '') + (content.suffix or '') + (content.extension or '')
         if tname.find('/'):
             tname=tname.replace('/', '_')
+        act_id = False
+        if 'dctx_res_id' in node.dctx:
+            act_id = node.dctx['dctx_res_id']
+        elif hasattr(node, 'res_id'):
+            act_id = node.res_id
+        else:
+            act_id = node.context.context.get('res_id',False)
         if not nodename:
-            n = nodes.node_content(tname, node, node.context,content)
+            n = nodes.node_content(tname, node, node.context,content, act_id=act_id)
             res2.append( n)
         else:
             if nodename == tname:
-                n = nodes.node_content(tname, node, node.context,content)
+                n = nodes.node_content(tname, node, node.context,content, act_id=act_id)
                 n.fill_fields(cr)
                 res2.append(n)
         return res2
@@ -108,6 +117,10 @@ class document_directory_content(osv.osv):
             raise Exception("Invalid content: %s" % node.extension)
         report = self.pool.get('ir.actions.report.xml').browse(cr, uid, node.report_id)
         srv = netsvc.Service._services['report.'+report.report_name]
-        pdf,pdftype = srv.create(cr, uid, [node.context.context['res_id']], {}, {})        
+        ctx = node.context.context.copy()
+        ctx.update(node.dctx)
+        pdf,pdftype = srv.create(cr, uid, [node.act_id,], {}, context=ctx)
         return pdf
 document_directory_content()
+
+#eof

addons/document/document.py

@@ -28,6 +28,7 @@ import os
 
 import pooler
 import netsvc
+from osv.orm import except_orm
 #import StringIO
 
 from psycopg2 import Binary
@@ -46,7 +47,10 @@ class document_file(osv.osv):
 
     def _data_get(self, cr, uid, ids, name, arg, context):
         fbrl = self.browse(cr, uid, ids, context=context)
-        nctx = nodes.get_node_context(cr, uid, context)
+        nctx = nodes.get_node_context(cr, uid, context={})
+        # nctx will /not/ inherit the caller's context. Most of
+        # it would be useless, anyway (like active_id, active_model, 
+        # bin_size etc.)
         result = {}
         bin_size = context.get('bin_size', False)
         for fbro in fbrl:
@@ -66,39 +70,38 @@ class document_file(osv.osv):
         if not value:
             return True
         fbro = self.browse(cr, uid, id, context=context)
-        nctx = nodes.get_node_context(cr, uid, context)
+        nctx = nodes.get_node_context(cr, uid, context={})
         fnode = nodes.node_file(None, None, nctx, fbro)
         res = fnode.set_data(cr, base64.decodestring(value), fbro)
         return res
 
     _columns = {
-        'user_id': fields.many2one('res.users', 'Owner', select=1),
-        'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'),
-        # the directory id now is mandatory. It can still be computed automatically.
-        'parent_id': fields.many2one('document.directory', 'Directory', select=1),
-        'file_size': fields.integer('File Size', required=True),
-        'file_type': fields.char('Content Type', size=128),
+        # Columns from ir.attachment:
+        'create_date': fields.datetime('Date Created', readonly=True),
+        'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
+        'write_date': fields.datetime('Date Modified', readonly=True),
+        'write_uid':  fields.many2one('res.users', 'Last Modification User', readonly=True),
+        'res_model': fields.char('Attached Model', size=64, readonly=True),
+        'res_id': fields.integer('Attached ID', readonly=True),
+
         # If ir.attachment contained any data before document is installed, preserve
         # the data, don't drop the column!
         'db_datas': fields.binary('Data', oldname='datas'),
-        'index_content': fields.text('Indexed Content'),
-        'write_date': fields.datetime('Date Modified', readonly=True),
-        'write_uid':  fields.many2one('res.users', 'Last Modification User', readonly=True),
-        'create_date': fields.datetime('Date Created', readonly=True),
-        'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
-        'store_method': fields.selection([('db', 'Database'), ('fs', 'Filesystem'), ('link', 'Link')], "Storing Method"),
         'datas': fields.function(_data_get, method=True, fnct_inv=_data_set, string='File Content', type="binary", nodrop=True),
-        'url': fields.char('File URL',size=64),
-        'store_fname': fields.char('Stored Filename', size=200),
-        'res_model': fields.char('Attached Model', size=64), #res_model
-        'res_id': fields.integer('Attached ID'), #res_id
-        'partner_id':fields.many2one('res.partner', 'Partner', select=1),
-        'type':fields.selection([
-            ('url','URL'),
-            ('binary','Binary'),
 
-        ],'Type', help="Type is used to separate URL and binary File"),
+        # Fields of document:
+        'user_id': fields.many2one('res.users', 'Owner', select=1),
+        # 'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'),
+        # the directory id now is mandatory. It can still be computed automatically.
+        'parent_id': fields.many2one('document.directory', 'Directory', select=1, required=True),
+        'index_content': fields.text('Indexed Content'),
+        'partner_id':fields.many2one('res.partner', 'Partner', select=1),
         'company_id': fields.many2one('res.company', 'Company'),
+        'file_size': fields.integer('File Size', required=True),
+        'file_type': fields.char('Content Type', size=128),
+        
+        # fields used for file storage
+        'store_fname': fields.char('Stored Filename', size=200),
     }
 
     def __get_def_directory(self, cr, uid, context=None):
@@ -109,8 +112,6 @@ class document_file(osv.osv):
         'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c),
         'user_id': lambda self, cr, uid, ctx:uid,
         'file_size': lambda self, cr, uid, ctx:0,
-        'store_method': lambda *args: 'db',
-        'type': 'binary',
         'parent_id': __get_def_directory
     }
     _sql_constraints = [
@@ -122,7 +123,7 @@ class document_file(osv.osv):
         res_model = vals.get('res_model', False)
         res_id = vals.get('res_id', 0)
         if op == 'write':
-            for file in self.browse(cr, uid, ids):
+            for file in self.browse(cr, uid, ids): # FIXME fields_only
                 if not name:
                     name = file.name
                 if not parent_id:
@@ -156,8 +157,44 @@ class document_file(osv.osv):
             return False
         if not self._check_duplication(cr, uid, vals, ids, 'write'):
             raise osv.except_osv(_('ValidateError'), _('File name must be unique!'))
-        result = super(document_file, self).write(cr, uid, ids, vals, context=context)
-        cr.commit()
+        
+        # if nodes call this write(), they must skip the code below
+        from_node = context and context.get('__from_node', False)
+        if (('parent_id' in vals) or ('name' in vals)) and not from_node:
+            # perhaps this file is renaming or changing directory
+            nctx = nodes.get_node_context(cr,uid,context={})
+            dirobj = self.pool.get('document.directory')
+            if 'parent_id' in vals:
+                dbro = dirobj.browse(cr, uid, vals['parent_id'], context=context)
+                dnode = nctx.get_dir_node(cr, dbro)
+            else:
+                dbro = None
+                dnode = None
+            ids2 = []
+            result = False
+            for fbro in self.browse(cr, uid, ids, context=context):
+                if ('parent_id' not in vals or fbro.parent_id.id == vals['parent_id']) \
+                    and ('name' not in vals or fbro.name == vals['name']) :
+                        ids2.append(fbro.id)
+                        continue
+                fnode = nctx.get_file_node(cr, fbro)
+                res = fnode.move_to(cr, dnode or fnode.parent, vals.get('name', fbro.name), fbro, dbro, True)
+                if isinstance(res, dict):
+                    vals2 = vals.copy()
+                    vals2.update(res)
+                    wid = res.get('id', fbro.id)
+                    result = super(document_file,self).write(cr,uid,wid,vals2,context=context)
+                    # TODO: how to handle/merge several results?
+                elif res == True:
+                    ids2.append(fbro.id)
+                elif res == False:
+                    pass
+            ids = ids2
+        if 'file_size' in vals: # only write that field using direct SQL calls
+            del vals['file_size']
+        if len(ids) and len(vals):
+            result = super(document_file,self).write(cr, uid, ids, vals, context=context)
+        cr.commit() # ?
         return result
 
     def create(self, cr, uid, vals, context=None):
@@ -170,26 +207,10 @@ class document_file(osv.osv):
             vals['res_id'] = context.get('default_res_id', False)
         if not vals.get('res_model', False) and context.get('default_res_model', False):
             vals['res_model'] = context.get('default_res_model', False)
-        if vals.get('res_id', False) and vals.get('res_model', False):
-            obj_model = self.pool.get(vals['res_model'])
-            result = obj_model.read(cr, uid, [vals['res_id']], ['name', 'partner_id', 'address_id'], context=context)
-            if len(result):
-                obj = result[0]
-                if obj_model._name == 'res.partner':
-                    vals['partner_id'] = obj['id']
-                elif obj.get('address_id', False):
-                    if isinstance(obj['address_id'], tuple) or isinstance(obj['address_id'], list):
-                        address_id = obj['address_id'][0]
-                    else:
-                        address_id = obj['address_id']
-                    address = self.pool.get('res.partner.address').read(cr, uid, [address_id], context=context)
-                    if len(address):
-                        vals['partner_id'] = address[0]['partner_id'][0] or False
-                elif obj.get('partner_id', False):
-                    if isinstance(obj['partner_id'], tuple) or isinstance(obj['partner_id'], list):
-                        vals['partner_id'] = obj['partner_id'][0]
-                    else:
-                        vals['partner_id'] = obj['partner_id']
+        if vals.get('res_id', False) and vals.get('res_model', False) \
+                and not vals.get('partner_id', False):
+            vals['partner_id'] = self.__get_partner_id(cr, uid, \
+                vals['res_model'], vals['res_id'], context)
 
         datas = None
         if vals.get('link', False) :
@@ -198,13 +219,33 @@ class document_file(osv.osv):
         else:
             datas = vals.get('datas', False)
 
-        vals['file_size'] = datas and len(datas) or 0
+        if datas:
+            vals['file_size'] = len(datas)
+        else:
+            if vals.get('file_size'):
+                del vals['file_size']
         if not self._check_duplication(cr, uid, vals):
             raise osv.except_osv(_('ValidateError'), _('File name must be unique!'))
         result = super(document_file, self).create(cr, uid, vals, context)
-        cr.commit()
+        cr.commit() # ?
         return result
 
+    def __get_partner_id(self, cr, uid, res_model, res_id, context):
+        """ A helper to retrieve the associated partner from any res_model+id
+            It is a hack that will try to discover if the mentioned record is
+            clearly associated with a partner record.
+        """
+        obj_model = self.pool.get(res_model)
+        if obj_model._name == 'res.partner':
+            return res_id
+        elif 'partner_id' in obj_model._columns and obj_model._columns['partner_id']._obj == 'res.partner':
+            bro = obj_model.browse(cr, uid, res_id, context=context)
+            return bro.partner_id.id
+        elif 'address_id' in obj_model._columns and obj_model._columns['address_id']._obj == 'res.partner.address':
+            bro = obj_model.browse(cr, uid, res_id, context=context)
+            return bro.address_id.partner_id.id
+        return False
+
     def unlink(self, cr, uid, ids, context={}):
         stor = self.pool.get('document.storage')
         unres = []

addons/document/document_data.xml

@@ -9,7 +9,7 @@
     </record>
     
     <record model="document.storage" id="storage_default">
-        <field name="name">Default DB storage</field>
+        <field name="name">Database storage</field>
         <field name="type">db</field>
         <field name="user_id" ref="base.user_admin"/>
     </record>
@@ -17,7 +17,7 @@
     <record model="document.directory" id="dir_root">
         <field name="name">Documents</field>
         <field name="user_id" ref="base.user_admin"/>
-	<field name="storage_id" ref="storage_default"/>
+        <field name="storage_id" ref="storage_default"/>
         <field name="ressource_id">0</field>
     </record>
 
@@ -25,7 +25,6 @@
         <field name="name">My Folder</field>
         <field name="parent_id" ref="dir_root"/>
         <field name="user_id" ref="base.user_admin"/>
-	<field name="storage_id" ref="storage_default"/>
         <field name="ressource_id">0</field>
  
     </record>
@@ -35,7 +34,6 @@
         <field name="parent_id" ref="dir_root"/>
         <field name="type">ressource</field>
         <field name="ressource_tree">1</field>
-	<field name="storage_id" ref="storage_default"/>
         <field name="ressource_id">0</field>
 
         <field name="ressource_type_id" search="[('model','=','res.partner.category')]" />
@@ -57,7 +55,6 @@
         <field name="name">Personal Folders</field>
         <field name="parent_id" ref="dir_root"/>
         <field name="type">ressource</field>
-	<field name="storage_id" ref="storage_default"/>
         <field name="ressource_type_id" ref="base.model_res_users" />
         <field name="ressource_id">0</field>
 

addons/document/document_directory.py

@@ -39,20 +39,27 @@ class document_directory(osv.osv):
         'write_uid':  fields.many2one('res.users', 'Last Modification User', readonly=True),
         'create_date': fields.datetime('Date Created', readonly=True),
         'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
-        'file_type': fields.char('Content Type', size=32),
         'domain': fields.char('Domain', size=128, help="Use a domain if you want to apply an automatic filter on visible resources."),
         'user_id': fields.many2one('res.users', 'Owner'),
         'storage_id': fields.many2one('document.storage', 'Storage'),
         'group_ids': fields.many2many('res.groups', 'document_directory_group_rel', 'item_id', 'group_id', 'Groups'),
-        'parent_id': fields.many2one('document.directory', 'Parent Item'),
+        'parent_id': fields.many2one('document.directory', 'Parent Directory', select=1),
         'child_ids': fields.one2many('document.directory', 'parent_id', 'Children'),
         'file_ids': fields.one2many('ir.attachment', 'parent_id', 'Files'),
         'content_ids': fields.one2many('document.directory.content', 'directory_id', 'Virtual Files'),
-        'type': fields.selection([('directory','Static Directory'),('ressource','Other Resources')], 'Type', required=True),
-        'ressource_type_id': fields.many2one('ir.model', 'Directories Mapped to Objects',
-            help="Select an object here and Open ERP will create a mapping for each of these " \
-                 "objects, using the given domain, when browsing through FTP."),
+        'type': fields.selection([ 
+            ('directory','Static Directory'),
+            ('ressource','Folders per resource'),
+            ],
+            'Type', required=True, select=1,
+            help="Defines directory's behaviour."),
+        
+        'ressource_type_id': fields.many2one('ir.model', 'Resource model',
+            help="Select an object here and there will be one folder per record of that resource."),
         'resource_field': fields.many2one('ir.model.fields', 'Name field', help='Field to be used as name on resource directories. If empty, the "name" will be used.'),
+        'resource_find_all': fields.boolean('Find all resources', required=True,
+                help="If true, all attachments that match this resource will " \
+                    " be located. If false, only ones that have this as parent." ),
         'ressource_parent_type_id': fields.many2one('ir.model', 'Parent Model',
             help="If you put an object here, this directory template will appear bellow all of these objects. " \
                  "Don't put a parent directory if you select a parent model."),
@@ -60,7 +67,7 @@ class document_directory(osv.osv):
         'ressource_tree': fields.boolean('Tree Structure',
             help="Check this if you want to use the same tree structure as the object selected in the system."),
         'dctx_ids': fields.one2many('document.directory.dctx', 'dir_id', 'Context fields'),
-        'company_id': fields.many2one('res.company', 'Company'),        
+        'company_id': fields.many2one('res.company', 'Company'),
     }
 
 
@@ -99,10 +106,12 @@ class document_directory(osv.osv):
         'type': lambda *args: 'directory',
         'ressource_id': lambda *a: 0,        
         'storage_id': _get_def_storage,
+        'resource_find_all': True,
     }
     _sql_constraints = [
         ('dirname_uniq', 'unique (name,parent_id,ressource_id,ressource_parent_type_id)', 'The directory name must be unique !'),
-        ('no_selfparent', 'check(parent_id <> id)', 'Directory cannot be parent of itself!')
+        ('no_selfparent', 'check(parent_id <> id)', 'Directory cannot be parent of itself!'),
+        ('dir_parented', 'check(parent_id IS NOT NULL OR storage_id IS NOT NULL)', 'Directory must have a parent or a storage')
     ]
     def name_get(self, cr, uid, ids, context={}):
         res = []
@@ -190,18 +199,25 @@ class document_directory(osv.osv):
         """
         if not context:
                 context = {}
-        lang = context.get('lang',False)
-        if not lang:
-            user = self.pool.get('res.users').browse(cr, uid, uid)
-            lang = user.context_lang 
-            context['lang'] = lang
             
-        try: #just instrumentation
-                return nodes.get_node_context(cr, uid, context).get_uri(cr, uri)
-        except Exception,e:
-                print "exception: ",e
-                raise
+        return nodes.get_node_context(cr, uid, context).get_uri(cr, uri)
 
+    def get_dir_permissions(self, cr, uid, ids ):
+        """Check what permission user 'uid' has on directory 'id'
+        """
+        assert len(ids) == 1
+        id = ids[0]
+        
+        cr.execute( "SELECT count(dg.item_id) AS needs, count(ug.uid) AS has " \
+                " FROM document_directory_group_rel dg " \
+                "   LEFT OUTER JOIN res_groups_users_rel ug " \
+                "   ON (dg.group_id = ug.gid AND ug.uid = %s) " \
+                " WHERE dg.item_id = %s ", (uid, id))
+        needs, has = cr.fetchone()
+        if needs and not has:
+            return 1  # still allow to descend into.
+        else:
+            return 15
 
     def _locate_child(self, cr, uid, root_id, uri,nparent, ncontext):
         """ try to locate the node in uri,
@@ -274,6 +290,8 @@ class document_directory(osv.osv):
             raise osv.except_osv(_('ValidateError'), _('Directory name contains special characters!'))
         return super(document_directory,self).create(cr, uid, vals, context)
 
+    # TODO def unlink(...
+
 document_directory()
 
 class document_directory_dctx(osv.osv):

addons/document/document_storage.py

@@ -24,7 +24,10 @@ from osv import osv, fields
 import os
 import tools
 import base64
+import errno
 import logging
+from StringIO import StringIO
+import psycopg2
 
 from tools.misc import ustr
 from tools.translate import _
@@ -33,7 +36,9 @@ from osv.orm import except_orm
 
 import random
 import string
+import pooler
 import netsvc
+import nodes
 from content_index import cntIndex
 
 DMS_ROOT_PATH = tools.config.get('document_path', os.path.join(tools.config.get('root_path'), 'filestore'))
@@ -84,6 +89,222 @@ def create_directory(path):
     os.makedirs(path)
     return dir_name
 
+class nodefd_file(nodes.node_descriptor):
+    """ A descriptor to a real file
+
+    Inheriting directly from file doesn't work, since file exports
+    some read-only attributes (like 'name') that we don't like.
+    """
+    def __init__(self, parent, path, mode):
+        nodes.node_descriptor.__init__(self, parent)
+        self.__file = open(path, mode)
+        if mode.endswith('b'):
+            mode = mode[:-1]
+        self.mode = mode
+        
+        for attr in ('closed', 'read', 'write', 'seek', 'tell'):
+            setattr(self,attr, getattr(self.__file, attr))
+
+    def close(self):
+        # TODO: locking in init, close()
+        fname = self.__file.name
+        self.__file.close()
+
+        if self.mode in ('w', 'w+', 'r+'):
+            par = self._get_parent()
+            cr = pooler.get_db(par.context.dbname).cursor()
+            icont = ''
+            mime = ''
+            filename = par.path
+            if isinstance(filename, (tuple, list)):
+                filename = '/'.join(filename)
+            
+            try:
+                mime, icont = cntIndex.doIndex(None, filename=filename,
+                        content_type=None, realfname=fname)
+            except Exception:
+                logging.getLogger('document.storage').debug('Cannot index file:', exc_info=True)
+                pass
+
+            try:
+                icont_u = ustr(icont)
+            except UnicodeError:
+                icont_u = ''
+
+            try:
+                fsize = os.stat(fname).st_size
+                cr.execute("UPDATE ir_attachment " \
+                            " SET index_content = %s, file_type = %s, " \
+                            " file_size = %s " \
+                            "  WHERE id = %s",
+                            (icont_u, mime, fsize, par.file_id))
+                par.content_length = fsize
+                par.content_type = mime
+                cr.commit()
+                cr.close()
+            except Exception:
+                logging.getLogger('document.storage').warning('Cannot save file indexed content:', exc_info=True)
+
+        elif self.mode in ('a', 'a+' ):
+            try:
+                par = self._get_parent()
+                cr = pooler.get_db(par.context.dbname).cursor()
+                fsize = os.stat(fname).st_size
+                cr.execute("UPDATE ir_attachment SET file_size = %s " \
+                            "  WHERE id = %s",
+                            (fsize, par.file_id))
+                par.content_length = fsize
+                cr.commit()
+                cr.close()
+            except Exception:
+                logging.getLogger('document.storage').warning('Cannot save file appended content:', exc_info=True)
+
+
+
+class nodefd_db(StringIO, nodes.node_descriptor):
+    """ A descriptor to db data
+    """
+    def __init__(self, parent, ira_browse, mode):
+        nodes.node_descriptor.__init__(self, parent)
+        if mode.endswith('b'):
+            mode = mode[:-1]
+        
+        if mode in ('r', 'r+'):
+            cr = ira_browse._cr # reuse the cursor of the browse object, just now
+            cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,))
+            data = cr.fetchone()[0]
+            StringIO.__init__(self, data)
+        elif mode in ('w', 'w+'):
+            StringIO.__init__(self, None)
+            # at write, we start at 0 (= overwrite), but have the original
+            # data available, in case of a seek()
+        elif mode == 'a':
+            StringIO.__init__(self, None)
+        else:
+            logging.getLogger('document.storage').error("Incorrect mode %s specified", mode)
+            raise IOError(errno.EINVAL, "Invalid file mode")
+        self.mode = mode
+
+    def close(self):
+        # we now open a *separate* cursor, to update the data.
+        # FIXME: this may be improved, for concurrency handling
+        par = self._get_parent()
+        uid = par.context.uid
+        cr = pooler.get_db(par.context.dbname).cursor()
+        try:
+            if self.mode in ('w', 'w+', 'r+'):
+                data = self.getvalue()
+                icont = ''
+                mime = ''
+                filename = par.path
+                if isinstance(filename, (tuple, list)):
+                    filename = '/'.join(filename)
+            
+                try:
+                    mime, icont = cntIndex.doIndex(data, filename=filename,
+                            content_type=None, realfname=None)
+                except Exception:
+                    logging.getLogger('document.storage').debug('Cannot index file:', exc_info=True)
+                    pass
+
+                try:
+                    icont_u = ustr(icont)
+                except UnicodeError:
+                    icont_u = ''
+
+                out = psycopg2.Binary(data)
+                cr.execute("UPDATE ir_attachment " \
+                            "SET db_datas = %s, file_size=%s, " \
+                            " index_content= %s, file_type=%s " \
+                            " WHERE id = %s",
+                    (out, len(data), icont_u, mime, par.file_id))
+            elif self.mode == 'a':
+                data = self.getvalue()
+                out = psycopg2.Binary(data)
+                cr.execute("UPDATE ir_attachment " \
+                    "SET db_datas = COALESCE(db_datas,'') || %s, " \
+                    "    file_size = COALESCE(file_size, 0) + %s " \
+                    " WHERE id = %s",
+                    (out, len(data), par.file_id))
+            cr.commit()
+        except Exception, e:
+            logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id)
+            raise
+        finally:
+            cr.close()
+        StringIO.close(self)
+
+class nodefd_db64(StringIO, nodes.node_descriptor):
+    """ A descriptor to db data, base64 (the old way)
+    
+        It stores the data in base64 encoding at the db. Not optimal, but
+        the transparent compression of Postgres will save the day.
+    """
+    def __init__(self, parent, ira_browse, mode):
+        nodes.node_descriptor.__init__(self, parent)
+        if mode.endswith('b'):
+            mode = mode[:-1]
+        
+        if mode in ('r', 'r+'):
+            StringIO.__init__(self, base64.decodestring(ira_browse.db_datas))
+        elif mode in ('w', 'w+'):
+            StringIO.__init__(self, None)
+            # at write, we start at 0 (= overwrite), but have the original
+            # data available, in case of a seek()
+        elif mode == 'a':
+            StringIO.__init__(self, None)
+        else:
+            logging.getLogger('document.storage').error("Incorrect mode %s specified", mode)
+            raise IOError(errno.EINVAL, "Invalid file mode")
+        self.mode = mode
+
+    def close(self):
+        # we now open a *separate* cursor, to update the data.
+        # FIXME: this may be improved, for concurrency handling
+        par = self._get_parent()
+        uid = par.context.uid
+        cr = pooler.get_db(par.context.dbname).cursor()
+        try:
+            if self.mode in ('w', 'w+', 'r+'):
+                data = self.getvalue()
+                icont = ''
+                mime = ''
+                filename = par.path
+                if isinstance(filename, (tuple, list)):
+                    filename = '/'.join(filename)
+            
+                try:
+                    mime, icont = cntIndex.doIndex(data, filename=filename,
+                            content_type=None, realfname=None)
+                except Exception:
+                    logging.getLogger('document.storage').debug('Cannot index file:', exc_info=True)
+                    pass
+
+                try:
+                    icont_u = ustr(icont)
+                except UnicodeError:
+                    icont_u = ''
+
+                cr.execute('UPDATE ir_attachment SET db_datas = %s::bytea, file_size=%s, ' \
+                        'index_content = %s, file_type = %s ' \
+                        'WHERE id = %s',
+                        (base64.encodestring(out), len(out), icont_u, mime, par.file_id))
+            elif self.mode == 'a':
+                out = self.getvalue()
+                # Yes, we're obviously using the wrong representation for storing our
+                # data as base64-in-bytea
+                cr.execute("UPDATE ir_attachment " \
+                    "SET db_datas = encode( (COALESCE(decode(encode(db_datas,'escape'),'base64'),'') || decode(%s, 'base64')),'base64')::bytea , " \
+                    "    file_size = COALESCE(file_size, 0) + %s " \
+                    " WHERE id = %s",
+                    (base64.encodestring(out), len(out), par.file_id))
+            cr.commit()
+        except Exception, e:
+            logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id)
+            raise
+        finally:
+            cr.close()
+        StringIO.close(self)
 
 class document_storage(osv.osv):
     """ The primary object for data storage.
@@ -110,7 +331,7 @@ class document_storage(osv.osv):
         'group_ids': fields.many2many('res.groups', 'document_storage_group_rel', 'item_id', 'group_id', 'Groups'),
         'dir_ids': fields.one2many('document.directory', 'parent_id', 'Directories'),
         'type': fields.selection([('db', 'Database'), ('filestore', 'Internal File storage'),
-            ('realstore', 'External file storage'), ('virtual', 'Virtual storage')], 'Type', required=True),
+                ('realstore','External file storage'),], 'Type', required=True),
         'path': fields.char('Path', size=250, select=1, help="For file storage, the root path of the storage"),
         'online': fields.boolean('Online', help="If not checked, media is currently offline and its contents not available", required=True),
         'readonly': fields.boolean('Read Only', help="If set, media is for reading only"),
@@ -133,6 +354,49 @@ class document_storage(osv.osv):
         ('path_uniq', 'UNIQUE(type,path)', "The storage path must be unique!")
         ]
 
+    def __get_random_fname(self, path):
+        flag = None
+        # This can be improved
+        if os.path.isdir(path):
+            for dirs in os.listdir(path):
+                if os.path.isdir(os.path.join(path, dirs)) and len(os.listdir(os.path.join(path, dirs))) < 4000:
+                    flag = dirs
+                    break
+        flag = flag or create_directory(path)
+        filename = random_name()
+        return os.path.join(flag, filename)
+
+    def __prepare_realpath(self, cr, file_node, ira, store_path, do_create=True):
+        """ Cleanup path for realstore, create dirs if needed
+        
+            @param file_node  the node
+            @param ira    ir.attachment browse of the file_node
+            @param store_path the path of the parent storage object, list
+            @param do_create  create the directories, if needed
+            
+            @return tuple(path "/var/filestore/real/dir/", npath ['dir','fname.ext'] )
+        """
+        file_node.fix_ppath(cr, ira)
+        npath = file_node.full_path() or []
+        # npath may contain empty elements, for root directory etc.
+        npath = filter(lambda x: x is not None, npath)
+
+        # if self._debug:
+        #     self._doclog.debug('Npath: %s', npath)
+        for n in npath:
+            if n == '..':
+                raise ValueError("Invalid '..' element in path")
+            for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?',):
+                if ch in n:
+                    raise ValueError("Invalid char %s in path %s" %(ch, n))
+        dpath = [store_path,]
+        dpath += npath[:-1]
+        path = os.path.join(*dpath)
+        if not os.path.isdir(path):
+            self._doclog.debug("Create dirs: %s", path)
+            os.makedirs(path)
+        return path, npath
+
     def get_data(self, cr, uid, id, file_node, context=None, fil_obj=None):
         """ retrieve the contents of some file_node having storage_id = id
             optionally, fil_obj could point to the browse object of the file
@@ -141,15 +405,71 @@ class document_storage(osv.osv):
         if not context:
             context = {}
         boo = self.browse(cr, uid, id, context)
+        if not boo.online:
+            raise IOError(errno.EREMOTE, 'medium offline')
+        
         if fil_obj:
             ira = fil_obj
         else:
             ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
         return self.__get_data_3(cr, uid, boo, ira, context)
 
-    def __get_data_3(self, cr, uid, boo, ira, context):
+    def get_file(self, cr, uid, id, file_node, mode, context=None):
+        """ Return a file-like object for the contents of some node
+        """
+        if context is None:
+            context = {}
+        boo = self.browse(cr, uid, id, context)
         if not boo.online:
-            raise RuntimeError('media offline')
+            raise IOError(errno.EREMOTE, 'medium offline')
+        
+        if boo.readonly and mode not in ('r', 'rb'):
+            raise IOError(errno.EPERM, "Readonly medium")
+        
+        ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
+        if boo.type == 'filestore':
+            if not ira.store_fname:
+                # On a migrated db, some files may have the wrong storage type
+                # try to fix their directory.
+                if mode in ('r','r+'):
+                    if ira.file_size:
+                        self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id)
+                    raise IOError(errno.ENOENT, 'No file can be located')
+                else:
+                    store_fname = self.__get_random_fname(boo.path)
+                    cr.execute('UPDATE ir_attachment SET store_fname = %s WHERE id = %s',
+                                (store_fname, ira.id))
+                    fpath = os.path.join(boo.path, store_fname)
+            else:
+                fpath = os.path.join(boo.path, ira.store_fname)
+            return nodefd_file(file_node, path=fpath, mode=mode)
+
+        elif boo.type == 'db':
+            # TODO: we need a better api for large files
+            return nodefd_db(file_node, ira_browse=ira, mode=mode)
+
+        elif boo.type == 'db64':
+            return nodefd_db64(file_node, ira_browse=ira, mode=mode)
+
+        elif boo.type == 'realstore':
+            path, npath = self.__prepare_realpath(cr, file_node, ira, boo.path,
+                            do_create = (mode[1] in ('w','a'))  )
+            fpath = os.path.join(path, npath[-1])
+            if (not os.path.exists(fpath)) and mode[1] == 'r':
+                raise IOError("File not found: %s" % fpath)
+            elif mode[1] in ('w', 'a') and not ira.store_fname:
+                store_fname = os.path.join(*npath)
+                cr.execute('UPDATE ir_attachment SET store_fname = %s WHERE id = %s',
+                                (store_fname, ira.id))
+            return nodefd_file(file_node, path=fpath, mode=mode)
+
+        elif boo.type == 'virtual':
+            raise ValueError('Virtual storage does not support static files')
+        
+        else:
+            raise TypeError("No %s storage" % boo.type)
+
+    def __get_data_3(self, cr, uid, boo, ira, context):
         if boo.type == 'filestore':
             if not ira.store_fname:
                 # On a migrated db, some files may have the wrong storage type
@@ -159,13 +479,21 @@ class document_storage(osv.osv):
                 return None
             fpath = os.path.join(boo.path, ira.store_fname)
             return file(fpath, 'rb').read()
-        elif boo.type == 'db':
+        elif boo.type == 'db64':
             # TODO: we need a better api for large files
             if ira.db_datas:
                 out = base64.decodestring(ira.db_datas)
             else:
                 out = ''
             return out
+        elif boo.type == 'db':
+            # We do an explicit query, to avoid type transformations.
+            cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s', (ira.id,))
+            res = cr.fetchone()
+            if res:
+                return res[0]
+            else:
+                return ''
         elif boo.type == 'realstore':
             if not ira.store_fname:
                 # On a migrated db, some files may have the wrong storage type
@@ -179,7 +507,11 @@ class document_storage(osv.osv):
             elif not ira.store_fname:
                 return None
             else:
-                raise IOError("File not found: %s" % fpath)
+                raise IOError(errno.ENOENT, "File not found: %s" % fpath)
+
+        elif boo.type == 'virtual':
+            raise ValueError('Virtual storage does not support static files')
+
         else:
             raise TypeError("No %s storage" % boo.type)
 
@@ -197,30 +529,25 @@ class document_storage(osv.osv):
             ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
 
         if not boo.online:
-            raise RuntimeError('media offline')
+            raise IOError(errno.EREMOTE, 'medium offline')
+        
+        if boo.readonly:
+            raise IOError(errno.EPERM, "Readonly medium")
+
         self._doclog.debug( "Store data for ir.attachment #%d" % ira.id)
         store_fname = None
         fname = None
         if boo.type == 'filestore':
             path = boo.path
             try:
-                flag = None
-                # This can be improved  
-                if os.path.isdir(path):
-                    for dirs in os.listdir(path):
-                        if os.path.isdir(os.path.join(path, dirs)) and len(os.listdir(os.path.join(path, dirs))) < 4000:
-                            flag = dirs
-                            break
-                flag = flag or create_directory(path)
-                filename = random_name()
-                fname = os.path.join(path, flag, filename)
+                store_fname = self.__get_random_fname(path)
+                fname = os.path.join(path, store_fname)
                 fp = file(fname, 'wb')
                 fp.write(data)
                 fp.close()
                 self._doclog.debug( "Saved data to %s" % fname)
                 filesize = len(data) # os.stat(fname).st_size
-                store_fname = os.path.join(flag, filename)
-
+                
                 # TODO Here, an old file would be left hanging.
 
             except Exception, e:
@@ -228,27 +555,19 @@ class document_storage(osv.osv):
                 raise except_orm(_('Error!'), str(e))
         elif boo.type == 'db':
             filesize = len(data)
-            # will that work for huge data? TODO
+            # will that work for huge data?
+            out = psycopg2.Binary(data)
+            cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s',
+                (out, file_node.file_id))
+        elif boo.type == 'db64':
+            filesize = len(data)
+            # will that work for huge data?
             out = base64.encodestring(data)
             cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s',
                 (out, file_node.file_id))
         elif boo.type == 'realstore':
             try:
-                file_node.fix_ppath(cr, ira)
-                npath = file_node.full_path() or []
-                # npath may contain empty elements, for root directory etc.
-                for i, n in enumerate(npath):
-                    if n == None:
-                        del npath[i]
-                for n in npath:
-                    for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'):
-                        if ch in n:
-                            raise ValueError("Invalid char %s in path %s" %(ch, n))
-                dpath = [boo.path,]
-                dpath += npath[:-1]
-                path = os.path.join(*dpath)
-                if not os.path.isdir(path):
-                    os.makedirs(path)
+                path, npath = self.__prepare_realpath(cr, file_node, ira, boo.path, do_create=True)
                 fname = os.path.join(path, npath[-1])
                 fp = file(fname,'wb')
                 fp.write(data)
@@ -260,6 +579,10 @@ class document_storage(osv.osv):
             except Exception,e :
                 self._doclog.warning("Couldn't save data:", exc_info=True)
                 raise except_orm(_('Error!'), str(e))
+
+        elif boo.type == 'virtual':
+            raise ValueError('Virtual storage does not support static files')
+
         else:
             raise TypeError("No %s storage" % boo.type)
 
@@ -276,16 +599,21 @@ class document_storage(osv.osv):
                 self._doclog.debug('Cannot index file:', exc_info=True)
                 pass
 
+            try:
+                icont_u = ustr(icont)
+            except UnicodeError:
+                icont_u = ''
+
             # a hack: /assume/ that the calling write operation will not try
             # to write the fname and size, and update them in the db concurrently.
             # We cannot use a write() here, because we are already in one.
             cr.execute('UPDATE ir_attachment SET store_fname = %s, file_size = %s, index_content = %s, file_type = %s WHERE id = %s',
-                (store_fname, filesize, ustr(icont), mime, file_node.file_id))
+                (store_fname, filesize, icont_u, mime, file_node.file_id))
             file_node.content_length = filesize
             file_node.content_type = mime
             return True
         except Exception, e :
-            self._doclog.warning( "Couldn't save data:", exc_info=True)
+            self._doclog.warning("Couldn't save data:", exc_info=True)
             # should we really rollback once we have written the actual data?
             # at the db case (only), that rollback would be safe
             raise except_orm(_('Error at doc write!'), str(e))
@@ -295,7 +623,10 @@ class document_storage(osv.osv):
         files that have to be removed, too. """
 
         if not storage_bo.online:
-            raise RuntimeError('media offline')
+            raise IOError(errno.EREMOTE, 'medium offline')
+        
+        if storage_bo.readonly:
+            raise IOError(errno.EPERM, "Readonly medium")
 
         if storage_bo.type == 'filestore':
             fname = fil_bo.store_fname
@@ -303,7 +634,7 @@ class document_storage(osv.osv):
                 return None
             path = storage_bo.path
             return (storage_bo.id, 'file', os.path.join(path, fname))
-        elif storage_bo.type == 'db':
+        elif storage_bo.type in ('db', 'db64'):
             return None
         elif storage_bo.type == 'realstore':
             fname = fil_bo.store_fname
@@ -312,7 +643,7 @@ class document_storage(osv.osv):
             path = storage_bo.path
             return ( storage_bo.id, 'file', os.path.join(path, fname))
         else:
-            raise TypeError("No %s storage" % boo.type)
+            raise TypeError("No %s storage" % storage_bo.type)
 
     def do_unlink(self, cr, uid, unres):
         for id, ktype, fname in unres:
@@ -326,6 +657,102 @@ class document_storage(osv.osv):
 
         return True
 
+    def simple_rename(self, cr, uid, file_node, new_name, context=None):
+        """ A preparation for a file rename.
+            It will not affect the database, but merely check and perhaps
+            rename the realstore file.
+            
+            @return the dict of values that can safely be be stored in the db.
+        """
+        sbro = self.browse(cr, uid, file_node.storage_id, context=context)
+        assert sbro, "The file #%d didn't provide storage" % file_node.file_id
+
+        if not sbro.online:
+            raise IOError(errno.EREMOTE, 'medium offline')
+        
+        if sbro.readonly:
+            raise IOError(errno.EPERM, "Readonly medium")
+
+        if sbro.type in ('filestore', 'db', 'db64'):
+            # nothing to do for a rename, allow to change the db field
+            return { 'name': new_name, 'datas_fname': new_name }
+        elif sbro.type == 'realstore':
+            ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
+
+            path, npath = self.__prepare_realpath(cr, file_node, ira, sbro.path, do_create=False)
+            fname = ira.store_fname
+
+            if not fname:
+                self._doclog.warning("Trying to rename a non-stored file")
+            if fname != os.path.join(*npath):
+                self._doclog.warning("inconsistency in realstore: %s != %s" , fname, repr(npath))
+
+            oldpath = os.path.join(path, npath[-1])
+            newpath = os.path.join(path, new_name)
+            os.rename(oldpath, newpath)
+            store_path = npath[:-1]
+            store_path.append(new_name)
+            store_fname = os.path.join(*store_path)
+            return { 'name': new_name, 'datas_fname': new_name, 'store_fname': store_fname }
+        else:
+            raise TypeError("No %s storage" % boo.type)
+
+    def simple_move(self, cr, uid, file_node, ndir_bro, context=None):
+        """ A preparation for a file move.
+            It will not affect the database, but merely check and perhaps
+            move the realstore file.
+            
+            @param ndir_bro a browse object of document.directory, where this
+                    file should move to.
+            @return the dict of values that can safely be be stored in the db.
+        """
+        sbro = self.browse(cr, uid, file_node.storage_id, context=context)
+        assert sbro, "The file #%d didn't provide storage" % file_node.file_id
+
+        if not sbro.online:
+            raise IOError(errno.EREMOTE, 'medium offline')
+        
+        if sbro.readonly:
+            raise IOError(errno.EPERM, "Readonly medium")
+
+        par = ndir_bro
+        psto = None
+        while par:
+            if par.storage_id:
+                psto = par.storage_id.id
+                break
+            par = par.parent_id
+        if file_node.storage_id != psto:
+            self._doclog.debug('Cannot move file %r from %r to %r', file_node, file_node.parent, ndir_bro.name)
+            raise NotImplementedError('Cannot move files between storage media')
+
+        if sbro.type in ('filestore', 'db', 'db64'):
+            # nothing to do for a rename, allow to change the db field
+            return { 'parent_id': ndir_bro.id }
+        elif sbro.type == 'realstore':
+            raise NotImplementedError("Cannot move in realstore, yet") # TODO
+            fname = fil_bo.store_fname
+            if not fname:
+                return ValueError("Tried to rename a non-stored file")
+            path = storage_bo.path
+            oldpath = os.path.join(path, fname)
+            
+            for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'):
+                if ch in new_name:
+                    raise ValueError("Invalid char %s in name %s" %(ch, new_name))
+                
+            file_node.fix_ppath(cr, ira)
+            npath = file_node.full_path() or []
+            dpath = [path,]
+            dpath.extend(npath[:-1])
+            dpath.append(new_name)
+            newpath = os.path.join(*dpath)
+            # print "old, new paths:", oldpath, newpath
+            os.rename(oldpath, newpath)
+            return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name }
+        else:
+            raise TypeError("No %s storage" % boo.type)
+
 
 document_storage()
 

addons/document/document_view.xml

@@ -16,7 +16,7 @@
                     <field name="online"/>
                     <field name="readonly"/>
                 </group>
-                <group colspan="2" attrs="{'invisible':[('type','=','db')]}">
+                <group colspan="2" attrs="{'invisible':[('type','in',['db', 'db64'])]}">
                     <field name="path"/>
                 </group>
             </form>
@@ -61,6 +61,7 @@
         name="Storage Media"
         action="action_document_storage_form"
         id="menu_document_storage_media"
+        groups="base.group_extended"
         parent="menu_document_management_configuration"/>
 
     <record model="ir.ui.view" id="view_document_directory_form">
@@ -68,27 +69,35 @@
         <field name="model">document.directory</field>
         <field name="type">form</field>
         <field name="arch" type="xml">
-            <form string="Directories">
+            <form string="Directories" col="6">
                 <field name="name" select="1" colspan="4"/>
-                <field name="user_id"/>
                 <field name="parent_id"/>
-                <field name="storage_id" />
+                <field name="storage_id" widget="selection" />
+                <field name="user_id"/>
                 <field name="company_id" groups="base.group_multi_company" widget="selection"/>
                 <notebook colspan="4">
                 <page string="Definition">
                     <separator string="Directory Type" colspan="4"/>
                     <field name="type"/>
-                    <field name="ressource_type_id"  on_change="onchange_content_id(ressource_type_id)" attrs="{'required': [('type','=','ressource')], 'readonly': [('type','=','static')]}"/>
-                    <newline/>
-                    <field name="domain" attrs="{'required': [('type','=','ressource')], 'readonly': [('type','=','static')]}"/>
-                    <field name="ressource_tree"/>
-                    <field name="resource_field"     domain="[('model_id','=',ressource_type_id), ('ttype', 'in', ('char', 'selection', 'date', 'datetime'))]"/>
-                    <field name="ressource_id" select="2" readonly="1"/>
-                    <field name="ressource_parent_type_id"/>
+                    <group colspan="4" col="4" attrs="{'invisible': [('type','!=','ressource')]}">
+                        <field name="ressource_type_id"  on_change="onchange_content_id(ressource_type_id)" 
+                            attrs="{'required': [('type','=','ressource')] }"/>
+                        <field name="resource_find_all" groups="base.group_extended" />
+                        <newline/>
+                        <field name="resource_field"     domain="[('model_id','=',ressource_type_id), ('ttype', 'in', ('char', 'selection', 'date', 'datetime'))]"/>
+                        <field name="ressource_tree"/>
+                        <newline/>
+                        <field name="domain" attrs="{'required': [('type','=','ressource')], 'readonly': [('type','=','static')]}"/>
+                    </group>
+                    <group colspan="4" col="4">
+                        <field name="ressource_parent_type_id"/>
+                        <field name="ressource_id" select="2" readonly="1"/>
+                    </group>
+                    
                 </page>
-                <page string="Generated Files">
+                <page string="Generated Files" groups="base.group_extended">
                     <label colspan="4" string="For each entry here, virtual files will appear in this folder." />
-                    <field name="content_ids" nolabel="1" colspan="4" attrs="{'readonly': [('ressource_type_id','=',False)]}">
+                    <field name="content_ids" nolabel="1" colspan="4" >
                         <form string="Contents">
                             <field name="name"/>
                             <field name="sequence"/>
@@ -108,6 +117,7 @@
                     </field>
                 </page>
                 <page string="Dynamic context" groups="base.group_extended">
+                    <label colspan="4" string="Define words in the context, for all child directories and files" />
                     <field name="dctx_ids" nolabel="1" colspan="4">
                         <tree string="Fields" editable="bottom">
                             <field name="field"/>
@@ -120,6 +130,8 @@
                     </field>
                 </page>
                 <page string="Security">
+                    <label colspan="4" string="Only members of these groups will have access to this directory and its files." />
+                    <label colspan="4" string="These groups, however, do NOT apply to children directories, which must define their own groups." />
                     <field name="group_ids" colspan="4" nolabel="1"/>
                 </page>
                 </notebook>
@@ -213,44 +225,46 @@
         <field name="arch" type="xml">
             <form string="Documents">
                 <group colspan="4" col="6">
-                    <field name="name" select="1" />
-                    <field name="type"/>
+                    <field name="name" select="1" colspan="4" />
                     <field name="parent_id"/>
+                    <newline/>
+                    <field name="user_id"/>
                     <field name="company_id" groups="base.group_multi_company" widget="selection"/>
                 </group>
                 <notebook colspan="4">
                 <page string="Attachment">
                     <group col="2" colspan="2">
                         <separator string="Data" colspan="2"/>
+                        <field name="type"/>
+                        <newline />
                         <group col="2" colspan="2" attrs="{'invisible':[('type','=','url')]}">
-                        <field name="datas" filename="datas_fname"/>
-                        <field name="datas_fname" select="1"/>
+                            <field name="datas" filename="datas_fname"/>
+                            <field name="datas_fname" select="1"/>
                         </group>
                         <group col="2" colspan="2" attrs="{'invisible':[('type','=','binary')]}">
-                        <field name="url" widget="url"/>
+                            <field name="url" widget="url"/>
                         </group>
                     </group>
                     <group col="2" colspan="2">
-                        <separator string="Relation" colspan="2"/>
-                         <field name="res_name" readonly="1"/>
-                         <field name="partner_id"/>
-                          <field name="user_id"/>
-                    </group>
-                    <group col="4" colspan="4">
-                        <separator string="History" colspan="4"/>
-                        <group col="4" colspan="4">
-                        <field name="create_uid"/>
-                        <field name="create_date"/>
-                        </group>
-                        <group col="4" colspan="4">
-                        <field name="write_uid"/>
-                        <field name="write_date"/>
+                    <field name="res_id" invisible="True"/>
+                        <group col="2" colspan="2" attrs="{'invisible': [('res_id','=',0)]}">
+                            <separator string="Attached To" colspan="2"/>
+                            <field name="res_model" readonly="True" invisible="True"/>
+                            <field name="res_name" readonly="1"/>
                         </group>
+                        <separator string="Related to" colspan="2"/>
+                            <field name="partner_id"/>
+                    </group>
+                    <group col="2" colspan="2" groups="base.group_extended">
+                        <separator string="Created" colspan="2"/>
+                        <field name="create_uid" readonly="1"/>
+                        <field name="create_date" readonly="1"/>
+                    </group>
+                    <group col="2" colspan="2" groups="base.group_extended">
+                        <separator string="Modified" colspan="2"/>
+                        <field name="write_uid" readonly="1"/>
+                        <field name="write_date" readonly="1"/>
                     </group>
-
-                </page>
-                <page string="Security">
-                    <field name="group_ids" colspan="4" nolabel="1"/>
                 </page>
                 <page string="Indexed Content" groups="base.group_extended">
                     <field name="index_content" colspan="4" nolabel="1"/>
@@ -269,11 +283,13 @@
             <field name="type">search</field>
             <field name="arch" type="xml">
              <search string="IR Attachment">
-             <filter icon="terp-go-month" string="Recent"
-                    help="less 1 month modified/created attachments"
+             <filter icon="terp-go-month" string="Recent Month"
+                    help="Attachment modified/created last 30 days"
                     domain="[('create_date','&lt;=', time.strftime('%%Y-%%m-%%d')),('create_date','&gt;',(datetime.date.today()-datetime.timedelta(days=30)).strftime('%%Y-%%m-%%d'))]"
                 />
                  <separator orientation="vertical"/>
+                 <field name="name"/>
+                 <field name="parent_id" />
                  <field name="user_id">
                     <filter icon="terp-personal"
                         domain="[('user_id','=', False)]"
@@ -284,9 +300,9 @@
                 <newline/>
                 <group expand="0" string="Group By...">
                     <filter string="Partner" icon="terp-personal" domain="[]"
-                       context="{'group_by':'partner_id'}" />
+                       context="{'group_by':'partner_id'}"  groups="base.group_extended"/>
                     <filter string="Directory" icon="terp-folder-green" domain="[]" context="{'group_by':'parent_id'}"/>
-                    <filter string="Type"  domain="[]" context="{'group_by':'type'}"/>
+                    <filter string="Type"  domain="[]" context="{'group_by':'type'}"  groups="base.group_extended"/>
                     <filter string="Owner" icon="terp-personal" domain="[]" context="{'group_by':'user_id'}"/>
                     <filter string="Company" icon="terp-personal" domain="[]" context="{'group_by':'company_id'}" groups="base.group_multi_company"/>
                 </group>
@@ -301,11 +317,12 @@
         <field name="arch" type="xml">
             <tree colors="blue:type in ('url',)">
                 <field name="name"/>
-                <field name="type"/>
-                <field name="datas_fname"/>
+                <field name="parent_id" />
                 <field name="user_id"/>
                 <field name="create_date"/>
                 <field name="write_date"/>
+                <field name="partner_id" groups="base.group_extended" />
+                <field name="type"  groups="base.group_extended"/>
             </tree>
 
         </field>

addons/document/report/document_report_view.xml

@@ -33,22 +33,22 @@
             </field>
         </record>
 
-		<record id="view_report_document_user_search" model="ir.ui.view">
+        <record id="view_report_document_user_search" model="ir.ui.view">
             <field name="name">report.document.user.search</field>
             <field name="model">report.document.user</field>
             <field name="type">search</field>
             <field name="arch" type="xml">
                 <search string="All users files">
-                	<group col="12" colspan="4">
+                        <group col="12" colspan="4">
                         <separator orientation="vertical"/>
-						<filter icon="terp-go-year" string="This Year" domain="[('name','=',time.localtime()[0])]" help="All Months Files"/>
-						<filter icon="terp-go-month" string="This Month" domain="[('month','=',time.localtime()[1])]" help="This Months Files"/>
-						<separator orientation="vertical"/>
-						<field name="name" select="1"/>
-						<field name="month" select="1"/>
-	                    <field name="user" select="1"/>
-	                    <field name="directory" select="1"/>
-	           		</group>
+                            <filter icon="terp-go-year" string="This Year" domain="[('name','=',time.localtime()[0])]" help="All Months Files"/>
+                            <filter icon="terp-go-month" string="This Month" domain="[('month','=',time.localtime()[1])]" help="This Months Files"/>
+                            <separator orientation="vertical"/>
+                            <field name="name" select="1"/>
+                            <field name="month" select="1"/>
+                            <field name="user" select="1"/>
+                            <field name="directory" select="1"/>
+                        </group>
                </search>
             </field>
         </record>
@@ -61,7 +61,7 @@
             <field name="res_model">report.document.user</field>
             <field name="view_type">form</field>
             <field name="view_mode">tree</field>
-            <field name="context">{"search_default_user":uid}</field>
+            <field name="context">{'search_default_user': 'user_id'}</field>
             <field name="search_view_id" ref="view_report_document_user_search"/>
         </record>
 

addons/document/test/document_test2.yml

@@ -0,0 +1,72 @@
+- 
+    In order to test the document management
+-
+    I make sure the default installation has some storage and root directory.
+-
+    !assert {model: document.storage, id: storage_default }:
+      - id != False
+-
+    !assert {model: document.directory, id: dir_root}:
+      - storage_id != False
+-
+    I create a "Testing" folder where all the test data will go.
+-
+    !record {model: document.directory, id: dir_tests }:
+        name: 'Testing'
+        parent_id: dir_root
+-
+    I create an attachment into the root folder (w. empty fields, test that
+    defaults work)
+-
+    !record {model: ir.attachment, id: file_test1 }:
+        name: Test file
+-
+    I delete the attachment from the root folder
+-
+    !delete {model: ir.attachment, id: file_test1, search: }
+-
+    I create an attachment into the Testing folder.
+-
+    !record {model: ir.attachment, id: file_test2 }:
+        name: Test file 2
+        parent_id: dir_tests
+-
+    I update the attachment with data, namely "abcd"
+-
+    !record {model: ir.attachment, id: file_test2 }:
+        datas: "abcd"
+-
+    I test that the datas of the attachment are correct
+-
+    !assert {model: ir.attachment, id: file_test2 }:
+        - datas == "abcd\n"
+        - file_size == 5
+        - file_type == 'text/plain'
+-
+    I rename the attachment.
+-
+    !record {model: ir.attachment, id: file_test2 }:
+        name: Test renamed 2
+-
+    I search the testing folder for attachments.
+-
+    !python {model: ir.attachment}: |
+        ids = self.search(cr, uid, [('parent_id.name','=', 'Testing'), ('name','=','Test renamed 2')])
+        assert ids == [ ref("file_test2") ], ids
+-
+    I create an attachment to a 3rd resource, eg. a res.country
+-
+    !record {model: ir.attachment, id: attach_3rd }:
+        name: 'Res country attachment.txt'
+        parent_id: dir_tests
+        datas: 'defg'
+        res_model: res.country
+        res_id: !eval ref("base.za")
+-
+    I search for the res.country attachment
+-
+    !python {model: ir.attachment}: |
+        ids = self.search(cr, uid, [('res_model', '=', 'res.country'), ('res_id', '=', ref("base.za"))])
+        assert ids == [ ref("attach_3rd")], ids
+-
+    !delete {model: ir.attachment, id: attach_3rd, search: }

addons/document_ftp/__openerp__.py

@@ -26,20 +26,23 @@
     'category': 'Generic Modules/Others',
     'description': """This is a support FTP Interface with document management system.
     With this module you would not only be able to access documents through open erp
-    but you would also be able to connect with them through the file system using the FTP protocol.
+    but you would also be able to connect with them through the file system using the 
+    a FTP client.
 """,
     'author': 'Tiny',
     'website': 'http://www.openerp.com',
     'depends': ['base', 'document'],
     'init_xml': [],
     'update_xml': [
-        'wizard/ftp_configuration_view.xml',  
-        'wizard/ftp_browse_view.xml',        
+        'wizard/ftp_configuration_view.xml',
+        'wizard/ftp_browse_view.xml',
         'security/ir.model.access.csv'
     ],
     'demo_xml': [],
     'test': [
-        'test/document_ftp_test.yml',
+        'test/document_ftp_test2.yml',
+        # 'test/document_ftp_test2.yml',
+        'test/document_ftp_test4.yml',
     ],
     'installable': True,
     'active': False,

addons/document_ftp/ftpserver/ftpserver.py

@@ -221,6 +221,33 @@ def _strerror(err):
     else:
         return err.strerror
 
+def _to_unicode(s):
+    try:
+        return s.decode('utf-8')
+    except UnicodeError:
+        pass
+    try:
+        return s.decode('latin')
+    except UnicodeError:
+        pass
+    try:
+        return s.encode('ascii')
+    except UnicodeError:
+        return s
+
+def _to_decode(s):
+    try:
+        return s.encode('utf-8')
+    except UnicodeError:
+        pass
+    try:
+        return s.encode('latin')
+    except UnicodeError:
+        pass
+    try:
+        return s.decode('ascii')
+    except UnicodeError:
+        return s
 
 # --- library defined exceptions
 
@@ -793,7 +820,7 @@ class DTPHandler(asyncore.dispatcher):
         if self.transfer_finished:
             self.cmd_channel.respond("226 Transfer complete.")
             if self.file_obj:
-                fname = self.cmd_channel.fs.fs2ftp(self.file_obj.name)
+                fname = self.file_obj.name
                 self.cmd_channel.log('"%s" %s.' %(fname, action))
         else:
             tot_bytes = self.get_transmitted_bytes()
@@ -1138,19 +1165,11 @@ class AbstractedFS:
 
     # note: the following operations are no more blocking
 
-    def get_list_dir(self, path):
+    def get_list_dir(self, datacr):
         """"Return an iterator object that yields a directory listing
         in a form suitable for LIST command.
         """
-        if self.isdir(path):
-            listing = self.listdir(path)
-            listing.sort()
-            return self.format_list(path, listing)
-        # if path is a file or a symlink we return information about it
-        else:
-            basedir, filename = os.path.split(path)
-            self.lstat(path)  # raise exc in case of problems
-            return self.format_list(basedir, [filename])
+        raise DeprecationWarning()
 
     def get_stat_dir(self, rawline):
         """Return an iterator object that yields a list of files
@@ -1335,6 +1354,11 @@ class AbstractedFS:
 
 # --- FTP
 
+class FTPExceptionSent(Exception):
+    """An FTP exception that FTPHandler has processed
+    """
+    pass
+
 class FTPHandler(asynchat.async_chat):
     """Implements the FTP server Protocol Interpreter (see RFC-959),
     handling commands received from the client on the control channel.
@@ -1434,6 +1458,14 @@ class FTPHandler(asynchat.async_chat):
         self._epsvall = False
         self.__in_dtp_queue = None
         self.__out_dtp_queue = None
+        
+        self.__errno_responses = {
+                errno.EPERM: 553,
+                errno.EINVAL: 504,
+                errno.ENOENT: 550,
+                errno.EREMOTE: 450,
+                errno.EEXIST: 521,
+                }
 
         # mlsx facts attributes
         self.current_facts = ['type', 'perm', 'size', 'modify']
@@ -1591,26 +1623,8 @@ class FTPHandler(asynchat.async_chat):
 
     def __check_path(self, cmd, line):
         """Check whether a path is valid."""
-        # For the following commands we have to make sure that the real
-        # path destination belongs to the user's root directory.
-        # If provided path is a symlink we follow its final destination
-        # to do so.
-        if cmd in ('APPE','CWD','DELE','MDTM','NLST','MLSD','MLST','RETR',
-                   'RMD','SIZE','STOR','XCWD','XRMD'):
-            datacr = None
-            datacr = self.fs.get_cr(line)
-            try:
-                if not self.fs.validpath(self.fs.ftp2fs(line, datacr)):
-                    line = self.fs.ftpnorm(line)
-                    err = '"%s" points to a path which is outside ' \
-                          "the user's root directory" %line
-                    self.respond("550 %s." %err)
-                    self.log('FAIL %s "%s". %s.' %(cmd, line, err))
-                    self.fs.close_cr(datacr)
-                    return False
-            except:
-                pass
-            self.fs.close_cr(datacr)
+        
+        # Always true, we will only check later, once we have a cursor
         return True
 
     def __check_perm(self, cmd, line, datacr):
@@ -1623,6 +1637,7 @@ class FTPHandler(asynchat.async_chat):
                'RNFR':'f',
                'MKD':'m', 'XMKD':'m',
                'STOR':'w'}
+        raise NotImplementedError
         if cmd in map:
             if cmd == 'STAT' and not line:
                 return True
@@ -1813,6 +1828,58 @@ class FTPHandler(asynchat.async_chat):
 
         # --- connection
 
+    def try_as_current_user(self, function, args=None, kwargs=None, line=None, errno_resp=None):
+        """run function as current user, auto-respond in exceptions
+           @param args,kwargs the arguments, in list and dict respectively
+           @param errno_resp a dictionary of responses to IOError, OSError
+        """
+        if errno_resp:
+            eresp = self.__errno_responses.copy()
+            eresp.update(errno_resp)
+        else:
+            eresp = self.__errno_responses
+
+        uline = ''
+        if line:
+            uline = ' "%s"' % _to_unicode(line)
+        try:
+            if args is None:
+                args = ()
+            if kwargs is None:
+                kwargs = {}
+            return self.run_as_current_user(function, *args, **kwargs)
+        except NotImplementedError, err:
+            cmdname = function.__name__
+            why = err.args[0] or 'Not implemented'
+            self.log('FAIL %s() not implemented:  %s.' %(cmdname, why))
+            self.respond('502 %s.' %why)
+            raise FTPExceptionSent(why)
+        except EnvironmentError, err:
+            cmdname = function.__name__
+            try:
+                logline(traceback.format_exc())
+            except Exception:
+                pass
+            ret_code = eresp.get(err.errno, '451')
+            why = (err.strerror) or 'Error in command'
+            self.log('FAIL %s() %s errno=%s:  %s.' %(cmdname, uline, err.errno, why))
+            self.respond('%s %s.' % (str(ret_code), why))
+        
+            raise FTPExceptionSent(why)
+        except Exception, e:
+            cmdname = function.__name__
+            try:
+                logerror(traceback.format_exc())
+            except Exception:
+                pass
+            why = (err.args and err.args[0]) or 'Exception'
+            self.log('FAIL %s() %s Exception:  %s.' %(cmdname, uline, why))
+            self.respond('451 %s.' % why)
+            raise FTPExceptionSent(why)
+
+    def get_crdata2(self, *args, **kwargs):
+        return self.try_as_current_user(self.fs.get_crdata, args, kwargs, line=args[0])
+
     def _make_eport(self, ip, port):
         """Establish an active data channel with remote client which
         issued a PORT or EPRT command.
@@ -2032,56 +2099,58 @@ class FTPHandler(asynchat.async_chat):
         # - Some older FTP clients erroneously issue /bin/ls-like LIST
         #   formats in which case we fall back on cwd as default.
         if not line or line.lower() in ('-a', '-l', '-al', '-la'):
-            line = self.fs.cwd
+            line = ''
+        datacr = None
         try:
-            data = None
-            data = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, data)
-            line = self.fs.ftpnorm(line)
-            iterator = self.run_as_current_user(self.fs.get_list_dir, path)
-        except OSError, err:
-            self.fs.close_cr(data)
-            why = _strerror(err)
-            self.log('FAIL LIST "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
-        else:
-            self.fs.close_cr(data)
-            self.log('OK LIST "%s". Transfer starting.' %line)
+            datacr = self.get_crdata2(line, mode='list')
+            iterator = self.try_as_current_user(self.fs.get_list_dir, (datacr,))
+        except FTPExceptionSent:
+            self.fs.close_cr(datacr)
+            return
+        
+        try:
+            self.log('OK LIST "%s". Transfer starting.' % line)
             producer = BufferedIteratorProducer(iterator)
             self.push_dtp_data(producer, isproducer=True)
+        finally:
+            self.fs.close_cr(datacr)
+
 
     def ftp_NLST(self, line):
         """Return a list of files in the specified directory in a
         compact form to the client.
         """
         if not line:
-            line = self.fs.cwd
+            line = ''
+
+        datacr = None
         try:
-            data = None
-            data = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, data)
-            line = self.fs.ftpnorm(line)
-            if self.fs.isdir(path):
-                listing = self.run_as_current_user(self.fs.listdir, path)
-                listing = map(lambda x:os.path.split(x.path)[1], listing)
+            datacr = self.get_crdata2(line, mode='list')
+            if not datacr:
+                datacr = ( None, None, None )
+            if self.fs.isdir(datacr[1]):
+                nodelist = self.try_as_current_user(self.fs.listdir, (datacr,))
             else:
                 # if path is a file we just list its name
-                self.fs.lstat(path)  # raise exc in case of problems
-                basedir, filename = os.path.split(line)
-                listing = [filename]
-        except OSError, err:
-            self.fs.close_cr(data)
-            why = _strerror(err)
-            self.log('FAIL NLST "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
-        else:
-            self.fs.close_cr(data)
-            data = ''
-            if listing:
-                listing.sort()
-                data = '\r\n'.join(listing) + '\r\n'
-            self.log('OK NLST "%s". Transfer starting.' %line)
-            self.push_dtp_data(data)
+                nodelist = [datacr[1],]
+            
+            listing = []
+            for nl in nodelist:
+                if isinstance(nl.path, (list, tuple)):
+                    listing.append(nl.path[-1])
+                else:
+                    listing.append(nl.path)    # assume string
+        except FTPExceptionSent:
+            self.fs.close_cr(datacr)
+            return
+        
+        self.fs.close_cr(datacr)
+        data = ''
+        if listing:
+            listing.sort()
+            data =  ''.join([ _to_decode(x) + '\r\n' for x in listing ])
+        self.log('OK NLST "%s". Transfer starting.' %line)
+        self.push_dtp_data(data)
 
         # --- MLST and MLSD commands
 
@@ -2096,22 +2165,17 @@ class FTPHandler(asynchat.async_chat):
         """
         # if no argument, fall back on cwd as default
         if not line:
-            line = self.fs.cwd
+            line = ''
+        datacr = None
         try:
-            datacr = None
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, datacr)
-            line = self.fs.ftpnorm(line)
-            basedir, basename = os.path.split(path)
+            datacr = self.get_crdata2(line, mode='list')
             perms = self.authorizer.get_perms(self.username)
-            iterator = self.run_as_current_user(self.fs.format_mlsx, basedir,
-                       [basename], perms, self.current_facts, ignore_err=False)
+            iterator = self.try_as_current_user(self.fs.format_mlsx, (datacr[0], datacr[1].parent,
+                       [datacr[1],], perms, self.current_facts), {'ignore_err':False})
             data = ''.join(iterator)
-        except OSError, err:
+        except FTPExceptionSent:
             self.fs.close_cr(datacr)
-            why = _strerror(err)
-            self.log('FAIL MLST "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
+            return
         else:
             self.fs.close_cr(datacr)
             # since TVFS is supported (see RFC-3659 chapter 6), a fully
@@ -2129,28 +2193,25 @@ class FTPHandler(asynchat.async_chat):
         """
         # if no argument, fall back on cwd as default
         if not line:
-            line = self.fs.cwd
+            line = ''
+        
+        datacr = None
         try:
-            datacr = None
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, datacr)
-            line = self.fs.ftpnorm(line)
+            datacr = self.get_crdata2(line, mode='list')
             # RFC-3659 requires 501 response code if path is not a directory
-            if not self.fs.isdir(path):
+            if not self.fs.isdir(datacr[1]):
                 err = 'No such directory'
                 self.log('FAIL MLSD "%s". %s.' %(line, err))
                 self.respond("501 %s." %err)
                 return
-            listing = self.run_as_current_user(self.fs.listdir, path)
-        except OSError, err:
+            listing = self.try_as_current_user(self.fs.listdir, (datacr,))
+        except FTPExceptionSent:
             self.fs.close_cr(datacr)
-            why = _strerror(err)
-            self.log('FAIL MLSD "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
+            return
         else:
             self.fs.close_cr(datacr)
             perms = self.authorizer.get_perms(self.username)
-            iterator = self.fs.format_mlsx(path, listing, perms,
+            iterator = self.fs.format_mlsx(datacr[0], datacr[1], listing, perms,
                        self.current_facts)
             producer = BufferedIteratorProducer(iterator)
             self.log('OK MLSD "%s". Transfer starting.' %line)
@@ -2160,23 +2221,12 @@ class FTPHandler(asynchat.async_chat):
         """Retrieve the specified file (transfer from the server to the
         client)
         """
+        datacr = None
         try:
-            datacr = None
-            datacr = self.fs.get_cr(line)
-            file = self.fs.ftp2fs(line, datacr)
-            line = self.fs.ftpnorm(line)
-            fd = self.run_as_current_user(self.fs.open, file, 'rb')            
-        except OSError, err:
+            datacr = self.get_crdata2(line, mode='file')
+            fd = self.try_as_current_user(self.fs.open, (datacr, 'rb'))
+        except FTPExceptionSent:
             self.fs.close_cr(datacr)
-            why = _strerror(err)
-            self.log('FAIL RETR "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
-            return
-        except IOError, err:
-            self.fs.close_cr(datacr)
-            why = _strerror(err)
-            self.log('FAIL RETR "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
             return
 
         if self.restart_position:
@@ -2187,7 +2237,7 @@ class FTPHandler(asynchat.async_chat):
             # the REST.
             ok = 0
             try:
-                assert not self.restart_position > self.fs.getsize(file)
+                assert not self.restart_position > self.fs.getsize(datacr)
                 fd.seek(self.restart_position)
                 ok = 1
             except AssertionError:
@@ -2217,30 +2267,14 @@ class FTPHandler(asynchat.async_chat):
         else:
             cmd = 'STOR'
 
-        line = self.fs.ftpnorm(line)
-        basedir,basename = os.path.split(line)
-
         datacr = None
         try:
-            datacr = self.fs.get_cr(line)
-            file = self.fs.ftp2fs(basedir, datacr)
-
-        except OSError, err:
+            datacr = self.get_crdata2(line,mode='create')
+            if self.restart_position:
+                mode = 'r+'
+            fd = self.try_as_current_user(self.fs.create, (datacr, datacr[2], mode + 'b'))
+        except FTPExceptionSent:
             self.fs.close_cr(datacr)
-            why = _strerror(err)
-            self.log('FAIL %s "%s". %s.' %(cmd, line, why))
-            self.respond('550 %s.' %why)
-            return
-
-        if self.restart_position:
-            mode = 'r+'
-        try:
-            fd = self.run_as_current_user(self.fs.create, file, basename, mode + 'b')
-        except IOError, err:
-            self.fs.close_cr(datacr)
-            why = _strerror(err)
-            self.log('FAIL %s "%s". %s.' %(cmd, line, why))
-            self.respond('550 %s.' %why)
             return
 
         if self.restart_position:
@@ -2251,7 +2285,7 @@ class FTPHandler(asynchat.async_chat):
             # specified in the REST.
             ok = 0
             try:
-                assert not self.restart_position > self.fs.getsize(self.fs.ftp2fs(line, datacr))
+                assert not self.restart_position > self.fs.getsize(datacr)
                 fd.seek(self.restart_position)
                 ok = 1
             except AssertionError:
@@ -2293,21 +2327,21 @@ class FTPHandler(asynchat.async_chat):
             self.respond("450 Can't STOU while REST request is pending.")
             return
 
-        datacr = None
-        datacr = self.fs.get_cr(line)
 
         if line:
-            line = self.fs.ftpnorm(line)
-            basedir,prefix = os.path.split(line)
-            basedir = self.fs.ftp2fs(basedir, datacr)
-            #prefix = prefix + '.'
+            datacr = self.get_crdata2(line, mode='create')
+            # TODO
         else:
+            # TODO
             basedir = self.fs.ftp2fs(self.fs.cwd, datacr)
             prefix = 'ftpd.'
         try:
-            fd = self.run_as_current_user(self.fs.mkstemp, prefix=prefix,
-                                          dir=basedir)
-        except IOError, err:
+            fd = self.try_as_current_user(self.fs.mkstemp, kwargs={'prefix':prefix,
+                                          'dir': basedir}, line=line )
+        except FTPExceptionSent:
+            self.fs.close_cr(datacr)
+            return
+        except IOError, err: # TODO
             # hitted the max number of tries to find out file with
             # unique name
             if err.errno == errno.EEXIST:
@@ -2500,33 +2534,25 @@ class FTPHandler(asynchat.async_chat):
 
     def ftp_PWD(self, line):
         """Return the name of the current working directory to the client."""
-        self.respond('257 "%s" is the current directory.' %self.fs.cwd)
+        cwd = self.fs.get_cwd()
+        self.respond('257 "%s" is the current directory.' % cwd)
 
     def ftp_CWD(self, line):
         """Change the current working directory."""
-        # TODO: a lot of FTP servers go back to root directory if no
+        # check: a lot of FTP servers go back to root directory if no
         # arg is provided but this is not specified in RFC-959.
         # Search for official references about this behaviour.
-        if not line:
-            line = '/'
         datacr = None
         try:
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, datacr)
-            self.run_as_current_user(self.fs.chdir, path)
-        except OSError, err:
-            if err.errno==2:
-                why = 'Authentication Required or Failed'
-                self.log('FAIL CWD "%s". %s.' %(self.fs.ftpnorm(line), why))
-                self.respond('530 %s.' %why)
-            else:
-                why = _strerror(err)
-                self.log('FAIL CWD "%s". %s.' %(self.fs.ftpnorm(line), why))
-                self.respond('550 %s.' %why)
-        else:
-            self.log('OK CWD "%s".' %self.fs.cwd)
-            self.respond('250 "%s" is the current directory.' %self.fs.cwd)
-        self.fs.close_cr(datacr)
+            datacr = self.get_crdata2(line,'cwd')
+            self.try_as_current_user(self.fs.chdir, (datacr,), line=line, errno_resp={2: 530})
+            cwd = self.fs.get_cwd()
+            self.log('OK CWD "%s".' % cwd)
+            self.respond('250 "%s" is the current directory.' % cwd)
+        except FTPExceptionSent:
+            return
+        finally:
+            self.fs.close_cr(datacr)
 
     def ftp_CDUP(self, line):
         """Change into the parent directory."""
@@ -2554,20 +2580,17 @@ class FTPHandler(asynchat.async_chat):
         """
         datacr = None
         try:
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, datacr)
-            line = self.fs.ftpnorm(line)
-            if self.fs.isdir(path):
-                why = "%s is not retrievable" %line
-                self.log('FAIL SIZE "%s". %s.' %(line, why))
-                self.respond("550 %s." %why)
-                self.fs.close_cr(datacr)
-                return
-            size = self.run_as_current_user(self.fs.getsize, path)
-        except OSError, err:
-            why = _strerror(err)
-            self.log('FAIL SIZE "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
+            datacr = self.get_crdata2(line, mode='file')
+            #if self.fs.isdir(datacr[1]):
+            #    why = "%s is not retrievable" %line
+            #    self.log('FAIL SIZE "%s". %s.' %(line, why))
+            #    self.respond("550 %s." %why)
+            #    self.fs.close_cr(datacr)
+            #    return
+            size = self.try_as_current_user(self.fs.getsize,(datacr,), line=line)
+        except FTPExceptionSent:
+            self.fs.close_cr(datacr)
+            return
         else:
             self.respond("213 %s" %size)
             self.log('OK SIZE "%s".' %line)
@@ -2578,40 +2601,35 @@ class FTPHandler(asynchat.async_chat):
         3307 style timestamp (YYYYMMDDHHMMSS) as defined in RFC-3659.
         """
         datacr = None
+            
         try:
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, datacr)
-            line = self.fs.ftpnorm(line)
-            if not self.fs.isfile(self.fs.realpath(path)):
-                why = "%s is not retrievable" %line
-                self.log('FAIL MDTM "%s". %s.' %(line, why))
-                self.respond("550 %s." %why)
-                self.fs.close_cr(datacr)
-                return
-            lmt = self.run_as_current_user(self.fs.getmtime, path)
-        except OSError, err:
-            why = _strerror(err)
-            self.log('FAIL MDTM "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
-        else:
+            if line.find('/', 1) < 0:
+                # root or db, just return local
+                lmt = None
+            else:
+                datacr = self.get_crdata2(line)
+                if not datacr:
+                    raise IOError(errno.ENOENT, "%s is not retrievable" %line)
+                #if not self.fs.isfile(datacr[1]):
+                #    raise IOError(errno.EPERM, "%s is not a regular file" % line)
+                
+                lmt = self.try_as_current_user(self.fs.getmtime, (datacr,), line=line)
             lmt = time.strftime("%Y%m%d%H%M%S", time.localtime(lmt))
             self.respond("213 %s" %lmt)
             self.log('OK MDTM "%s".' %line)
-        self.fs.close_cr(datacr)
+        except FTPExceptionSent:
+            return
+        finally:
+            self.fs.close_cr(datacr)
 
     def ftp_MKD(self, line):
         """Create the specified directory."""
-        datacr = None
-        line = self.fs.ftpnorm(line)
-        basedir,basename = os.path.split(line)
         try:
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(basedir, datacr)
-            self.run_as_current_user(self.fs.mkdir, path, basename)
-        except OSError, err:
-            why = _strerror(err)
-            self.log('FAIL MKD "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
+            datacr = self.get_crdata2(line, mode='create')
+            self.try_as_current_user(self.fs.mkdir, (datacr, datacr[2]), line=line)
+        except FTPExceptionSent:
+            self.fs.close_cr(datacr)
+            return
         else:
             self.log('OK MKD "%s".' %line)
             self.respond("257 Directory created.")
@@ -2621,40 +2639,30 @@ class FTPHandler(asynchat.async_chat):
         """Remove the specified directory."""
         datacr = None
         try:
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, datacr)
-            line = self.fs.ftpnorm(line)
-            if self.fs.realpath(path) == self.fs.realpath(self.fs.root):
+            datacr = self.get_crdata2(line, mode='delete')
+            if not datacr[1]:
                 msg = "Can't remove root directory."
-                self.respond("550 %s" %msg)
+                self.respond("553 %s" %msg)
                 self.log('FAIL MKD "/". %s' %msg)
                 self.fs.close_cr(datacr)
                 return
-            self.run_as_current_user(self.fs.rmdir, path)
-        except OSError, err:
-            why = _strerror(err)
-            self.log('FAIL RMD "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
-        else:
+            self.try_as_current_user(self.fs.rmdir, (datacr,), line=line)
             self.log('OK RMD "%s".' %line)
             self.respond("250 Directory removed.")
+        except FTPExceptionSent:
+            pass
         self.fs.close_cr(datacr)
 
     def ftp_DELE(self, line):
         """Delete the specified file."""
         datacr = None
         try:
-            datacr = self.fs.get_cr(line)
-            path = self.fs.ftp2fs(line, datacr)
-            line = self.fs.ftpnorm(line)
-            self.run_as_current_user(self.fs.remove, path)
-        except OSError, err:
-            why = _strerror(err)
-            self.log('FAIL DELE "%s". %s.' %(line, why))
-            self.respond('550 %s.' %why)
-        else:
+            datacr = self.get_crdata2(line, mode='delete')
+            self.try_as_current_user(self.fs.remove, (datacr,), line=line)
             self.log('OK DELE "%s".' %line)
             self.respond("250 File removed.")
+        except FTPExceptionSent:
+            pass
         self.fs.close_cr(datacr)
 
     def ftp_RNFR(self, line):
@@ -2662,18 +2670,16 @@ class FTPHandler(asynchat.async_chat):
         here, see RNTO command)"""
         datacr = None
         try:
-            datacr = self.fs.get_cr(line)
-            line = self.fs.ftpnorm(line)
-            path = self.fs.ftp2fs(line, datacr)
-            if not self.fs.lexists(path):
+            datacr = self.get_crdata2(line, mode='rfnr')
+            if not datacr[1]:
                 self.respond("550 No such file or directory.")
-            elif self.fs.realpath(path) == self.fs.realpath(self.fs.root):
-                self.respond("550 Can't rename the home directory.")
+            elif not datacr[1]:
+                self.respond("553 Can't rename the home directory.")
             else:
-                self.fs.rnfr = line
+                self.fs.rnfr = datacr[1]
                 self.respond("350 Ready for destination name.")
-        except:
-            self.respond("550 Can't find the file or directory.")
+        except FTPExceptionSent:
+            pass
         self.fs.close_cr(datacr)
 
     def ftp_RNTO(self, line):
@@ -2685,22 +2691,17 @@ class FTPHandler(asynchat.async_chat):
             return
         datacr = None
         try:
-            try:
-                datacr = self.fs.get_cr(line)
-                src = self.fs.ftp2fs(self.fs.rnfr, datacr)
-                line = self.fs.ftpnorm(line)
-                basedir,basename = os.path.split(line)
-                dst = self.fs.ftp2fs(basedir, datacr)
-                self.run_as_current_user(self.fs.rename, src, dst,basename)
-            except OSError, err:
-                why = _strerror(err)
-                self.log('FAIL RNFR/RNTO "%s ==> %s". %s.' \
-                         %(self.fs.ftpnorm(self.fs.rnfr), line, why))
-                self.respond('550 %s.' %why)
-            else:
-                self.log('OK RNFR/RNTO "%s ==> %s".' \
-                         %(self.fs.ftpnorm(self.fs.rnfr), line))
-                self.respond("250 Renaming ok.")
+            datacr = self.get_crdata2(line,'create')
+            oldname = self.fs.rnfr.path
+            if isinstance(oldname, (list, tuple)):
+                oldname = '/'.join(oldname)
+            self.try_as_current_user(self.fs.rename, (self.fs.rnfr, datacr), line=line)
+            self.fs.rnfr = None
+            self.log('OK RNFR/RNTO "%s ==> %s".' % \
+                    (_to_unicode(oldname), _to_unicode(line)))
+            self.respond("250 Renaming ok.")
+        except FTPExceptionSent:
+            pass
         finally:
             self.fs.rnfr = None
             self.fs.close_cr(datacr)
@@ -2789,9 +2790,9 @@ class FTPHandler(asynchat.async_chat):
             datacr = None
             try:
                 datacr = self.fs.get_cr(line)
-                iterator = self.run_as_current_user(self.fs.get_stat_dir, line, datacr)
-            except OSError, err:
-                self.respond('550 %s.' %_strerror(err))
+                iterator = self.try_as_current_user(self.fs.get_stat_dir, (line, datacr), line=line)
+            except FTPExceptionSent:
+                pass
             else:
                 self.push('213-Status of "%s":\r\n' %self.fs.ftpnorm(line))
                 self.push_with_producer(BufferedIteratorProducer(iterator))

addons/document_ftp/test/document_ftp_test2.yml

@@ -0,0 +1,220 @@
+-
+    In order to test the document_ftp functionality
+-
+    I open the 8021 port and see for ftp presence there
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_plain_ftp(timeout=2.0)
+        assert ftp.sock and (ftp.lastresp == '220'), ftp.lastresp
+-
+    I read the list of databases at port 8021 and confirm our db is
+    there
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_login(cr, uid, self)
+        assert cr.dbname in ftp.nlst("/")
+-
+    I try to locate the default "Documents" folder in the db.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_login(cr, uid, self)
+        ftp.cwd('Documents')
+-
+    I create a "test.txt" file at the server (directly). The file
+    should have the "abcd" content
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        fdata = StringIO('abcd')
+        ftp.storbinary('STOR test.txt', fdata)
+-
+    I look for the "test.txt" file at the server
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        assert ftp.nlst("test.txt") == ['test.txt']
+-
+    I check that the content of "test.txt" is "abcd"
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        assert te.get_ftp_fulldata(ftp, "test.txt") == 'abcd'
+-
+    I append the string 'defgh' into "test.txt"
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        fdata = StringIO('defgh')
+        ftp.storbinary('APPE test.txt', fdata)
+-
+    I check that the content of "text.txt" is 'abcddefgh'
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        assert te.get_ftp_fulldata(ftp, "test.txt") == 'abcddefgh'
+-
+    I try to cd into an non-existing folder 'Not-This'
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        import ftplib
+        ftp = te.get_ftp_login(cr, uid, self)
+        try:
+            ftp.cwd('/Not-This')
+            assert False, "We should't be able to change here"
+        except ftplib.error_perm:
+            pass
+        except OSError, err:
+            assert err.errno == 2, err.errno
+-
+    I create a "test2.txt" file through FTP.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        fdata = StringIO('abcd')
+        ftp.storbinary('STOR test2.txt', fdata)
+-
+    I look for the "test2.txt" file at the server
+-
+    !python {model: ir.attachment }: |
+        ids = self.search(cr, uid, [('name', '=', 'test2.txt')])
+        assert ids
+-
+    I delete the "test2.txt" file using FTP.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        ftp.delete('test2.txt')
+-
+    I check at the server that test2.txt is deleted
+-
+    !python {model: ir.attachment }: |
+        ids = self.search(cr, uid, [('name', '=', 'test2.txt')])
+        assert not ids
+-
+    I create a test2.txt file again.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        fdata = StringIO('abcd')
+        ftp.storbinary('STOR test2.txt', fdata)
+-
+    I delete the test2.txt from the server (RPC).
+-
+    !delete { model: ir.attachment, id:,  search: "[('name','=','test2.txt')]" }
+-
+    I check through FTP that test2.txt does not appear.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        assert ftp.nlst("test2.txt") == []
+-
+    I create a "test-name.txt" file
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        fdata = StringIO('abcd')
+        ftp.storbinary('STOR test-name.txt', fdata)
+-
+    I rename the "test-name.txt" file through ftp.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        ftp.rename("test-name.txt", "test-renamed.txt")
+-
+    I check that test-name.txt has been renamed.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from ftplib import error_perm
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        try:
+            res = ftp.nlst("test-name.txt")
+            assert res == []
+        except error_perm, e:
+            pass
+        assert ftp.nlst("test-renamed.txt") == ['test-renamed.txt']
+-
+    I create a new folder 'Test-Folder2' through FTP
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        ftp.mkd("Test-Folder2")
+-
+    I create a file 'test3.txt' at the 'Test-Folder2'
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2')
+        fdata = StringIO('abcd')
+        ftp.storbinary('STOR test3.txt', fdata)
+-
+    I try to retrieve test3.txt
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2')
+        assert ftp.nlst("test3.txt") == ['test3.txt']
+-
+    I create a new folder, 'Test-Folder3', through FTP
+    I try to move test3.txt to 'Test-Folder3'
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents')
+        ftp.mkd("Test-Folder2")
+        # TODO move
+-
+    I remove the 'Test-Folder3'
+-
+    I check that test3.txt is removed.
+-
+    I create 200 files through FTP
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2')
+        fdata = StringIO('abcd')
+        # TODO speed
+        for i in range(0, 200):
+            fdata.seek(0)
+            ftp.storbinary('STOR test-name%s.txt' %i, fdata)
+-
+    I list the 200 files, check speed
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Test-Folder2')
+        # TODO speed
+        assert len(ftp.nlst()) > 200
+-
+    I read the 200 files, check speed
+    # TODO
+-
+    I move the 200 files to 'Test-Folder2'
+    # TODO

addons/document_ftp/test/document_ftp_test3.yml

@@ -0,0 +1,82 @@
+-
+    In order to check international character functionality
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_plain_ftp(timeout=1.0)
+-
+    I create in the server a folder called 'Äïêéìáóôéêüò ÖÜêåëëïò'
+-
+    !record {model: document.directory, id: dir_itests }:
+        name: 'Äïêéìáóôéêüò ÖÜêåëëïò'
+        parent_id: document.dir_root
+-
+    And then I create another folder, under it, through FTP
+-
+    !python {model: ir.attachment}: |
+        cr.commit()
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò')
+        ftp.mkd("ÖÜêåëëïò áðü êÜôù")
+-
+    I check that this folder exists at the server
+-
+    !assert {model: document.directory, id: , search: "[('name','=','ÖÜêåëëïò áðü êÜôù')]" }:
+      - parent_id != False
+-
+    I login with FTP and check that 'Äïêéìáóôéêüò ÖÜêåëëïò' is there
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò/ÖÜêåëëïò áðü êÜôù')
+-
+    I create a file named 'ÄïêéìÞ' into that folder
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò/ÖÜêåëëïò áðü êÜôù')
+        fdata = StringIO('êåßìåíï ìå utf-8')
+        ftp.storbinary('STOR ÄïêéìÞ.txt', fdata)
+-
+    I remove the 'ÄïêéìÞ.txt' file
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò/ÖÜêåëëïò áðü êÜôù')
+        ftp.delete('ÄïêéìÞ.txt')
+-
+    I rename 'ÖÜêåëëïò áðü êÜôù' into 'Üëëïò'
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò')
+        ftp.rename("ÖÜêåëëïò áðü êÜôù", "Üëëïò")
+-
+    I place a file 'file Ö3' in 'Üëëïò'
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò/Üëëïò')
+        fdata = StringIO('êé Üëëï êåßìåíï')
+        ftp.storbinary('STOR file Ö3.txt', fdata)
+-
+    I rename the file into file+range(1..200) (large filename)
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò/Üëëïò')
+        vuvuzela = 'b'+''.join('z' * 200)+'!'
+        ftp.rename("file Ö3.txt", vuvuzela)
+-
+    I delete the file with the large name
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Äïêéìáóôéêüò ÖÜêåëëïò/Üëëïò')
+        vuvuzela = 'b'+''.join('z' * 200)+'!'
+        ftp.delete(vuvuzela)

addons/document_ftp/test/document_ftp_test4.yml

@@ -0,0 +1,120 @@
+-
+    In order to check dynamic folder functionality of document + FTP
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_plain_ftp(timeout=1.0)
+- |
+    I create two partners 'Partner1' and 'Partner2'.
+    I create three partner categories: 'none', 'pat1' and 'all'
+    I attach Partner1 to pat1, Partner1+Partner2 to 'all'
+-
+    !record {model: res.partner.category, id: tpat_categ_none }:
+        name: 'No partners'
+-
+    !record {model: res.partner.category, id: tpat_categ_pat1 }:
+        name: 'Pat 1'
+-
+    !record {model: res.partner.category, id: tpat_categ_all }:
+        name: 'All Partner1+2'
+-
+    !record {model: res.partner, id: tpartner1 }:
+        name: Partner 1
+        category_id:
+            - tpat_categ_pat1
+            - tpat_categ_all
+-
+    !record {model: res.partner, id: tpartner_2 }:
+        name: 'Partner 2'
+        category_id:
+            - tpat_categ_all
+-
+    I create a resource folder of partners, by the (none, pat1, all) 
+    categories.
+-
+    !record {model: document.directory, id: dir_tests2 }:
+        name: Partners Testing
+        parent_id: document.dir_root
+        type: ressource
+        ressource_type_id: base.model_res_partner_category
+        domain: [] # TODO
+-
+    I commit (because FTP operations are on different transaction)
+-
+    !python {model: document.directory, id: }: |
+        cr.commit()
+-
+    I browse through ftp in the resource folder, checking that three
+    categories are there.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Partners Testing')
+        dirs = ftp.nlst()
+        for dir in [ 'All Partner1+2', 'No partners', 'Pat 1' ]:
+            assert dir in dirs, "Dir %s not in folder" % dir
+-
+    I create a 'partners' folder by the first resource one.
+-
+    !record {model: document.directory, id: dir_respart1 }:
+        name: Partners of Test
+        parent_id: dir_tests2
+        type: ressource
+        ressource_type_id: base.model_res_partner
+        domain: "[('category_id','in',[active_id])]"
+        ressource_parent_type_id : base.model_res_partner_category
+-
+    !python {model: document.directory, id: }: |
+        cr.commit()
+-
+    I check through FTP that the correct partners are listed at each
+    'partners' folder.
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Partners Testing')
+        correct = { 'All Partner1+2': [ 'Partner 1', 'Partner 2' ],
+                'No partners': [],
+                'Pat 1': ['Partner 1',] }
+        for dir in correct:
+            res = ftp.nlst(dir+'/Partners of Test')
+            assert res == correct[dir], "Dir %s falsely contains %s" %(dir, res)
+-
+    I create an ir.attachment, attached (not related) to Partner1
+-
+    !record {model: ir.attachment, id: file_test1 }:
+        name: File of pat1
+        res_model: res.partner
+        res_id:  !eval ref("tpartner1")
+-
+    I check that pat1/Partner1 folder has the file.
+    I check that all/Partner1 folder has the file
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Partners Testing')
+        dirs = [ 'All Partner1+2', 'Pat 1' ]
+        for dir in dirs:
+            res = ftp.nlst(dir+'/Partners of Test/Partner 1')
+            assert 'File of pat1' in res, "Dir %s contains only %s" %(dir, res)
+-
+    I place a file at the 'pat1'/Partner1 folder, through FTP
+-
+    !python {model: ir.attachment}: |
+        from document_ftp import test_easyftp as te
+        from cStringIO import StringIO
+        ftp = te.get_ftp_folder(cr, uid, self, 'Documents/Partners Testing/Pat 1/Partners of Test/Partner 1')
+        fdata = StringIO('abcd')
+        ftp.storbinary('STOR pat1-dynamic.txt', fdata)
+-
+    I check at the server that the file is attached to Partner1
+-
+    !assert {model: ir.attachment, id: , search: "[('name','=','pat1-dynamic.txt')]" }:
+      - parent_id.name == 'Partners of Test'
+      - res_model == 'res.partner'
+      - res_id !=  False
+-
+    I check that all/Partner1 also has the file
+- |
+    Bonus Piste:
+    I create a 'Partner3' under 'all'

addons/document_ftp/test_easyftp.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#    
+#    OpenERP, Open Source Management Solution
+#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.     
+#
+##############################################################################
+
+""" This is a testing module, which exports some functions for the YAML tests.
+    Instead of repeating the same FTP code all over, we prefer to have
+    it in this file
+"""
+
+from ftplib import FTP
+from tools.misc import detect_ip_addr
+from tools import config
+
+def get_plain_ftp(timeout=10.0):
+    ftp = FTP()
+    host = config.get('ftp_server_host', '127.0.0.1')
+    port = config.get('ftp_server_port','8021')
+    ftp.connect(host,port, timeout=timeout)
+    return ftp
+
+def get_ftp_login(cr, uid, ormobj):
+    ftp = get_plain_ftp()
+    user = ormobj.pool.get('res.users').read(cr, uid, uid)
+    ftp.login(user.get('login',''),user.get('password',''))
+    ftp.cwd("/" + cr.dbname)
+    return ftp
+
+def get_ftp_anonymous(cr):
+    ftp = get_plain_ftp()
+    ftp.login('anonymous', 'the-test')
+    ftp.cwd("/")
+    return ftp
+
+def get_ftp_folder(cr, uid, ormobj, foldername):
+    ftp = get_ftp_login(cr, uid, ormobj)
+    ftp.cwd("/" + cr.dbname+"/"+foldername)
+    return ftp
+
+def get_ftp_fulldata(ftp, fname, limit=8192):
+    from functools import partial
+    data = []
+    def ffp(data, ndata):
+        if len(data)+ len(ndata) > limit:
+            raise IndexError('Data over the limit')
+        data.append(ndata)
+    ftp.retrbinary('RETR %s' % fname, partial(ffp,data))
+    return ''.join(data)
+
+#eof

addons/document_webdav/dav_fs.py

@@ -46,6 +46,15 @@ CACHE_SIZE=20000
 urlparse.uses_netloc.append('webdav')
 urlparse.uses_netloc.append('webdavs')
 
+class DAV_NotFound2(DAV_NotFound):
+    """404 exception, that accepts our list uris
+    """
+    def __init__(self, *args):
+        if len(args) and isinstance(args[0], (tuple, list)):
+            path = ''.join([ '/' + x for x in args[0]])
+            args = (path, )
+        DAV_NotFound.__init__(self, *args)
+
 class openerp_dav_handler(dav_interface):
     """
     This class models a OpenERP interface for the DAV server
@@ -74,6 +83,38 @@ class openerp_dav_handler(dav_interface):
         cr.close()     
         return props
 
+    def _try_function(self, funct, args, opname='run function', cr=None, 
+	    default_exc=DAV_Forbidden):
+        """ Try to run a function, and properly convert exceptions to DAV ones.
+        
+            @objname the name of the operation being performed
+            @param cr if given, the cursor to close at exceptions
+        """
+        
+        try:
+            funct(*args)
+        except DAV_Error:
+            if cr: cr.close()
+            raise
+        except NotImplementedError, e:
+            if cr: cr.close()
+            import traceback
+            self.parent.log_error("Cannot %s: %s", opname, str(e))
+            self.parent.log_message("Exc: %s",traceback.format_exc())
+            # see par 9.3.1 of rfc
+            raise DAV_Error(403, str(e) or 'Not supported at this path')
+        except EnvironmentError, err:
+            if cr: cr.close()
+            import traceback
+            self.parent.log_error("Cannot %s: %s", opname, err.strerror)
+            self.parent.log_message("Exc: %s",traceback.format_exc())
+            raise default_exc(err.strerror)
+        except Exception,e:
+            import traceback
+            self.parent.log_error("Cannot create %s: %s", opname, str(e))
+            self.parent.log_message("Exc: %s",traceback.format_exc())
+            raise default_exc("Operation failed")
+
     def _get_dav_lockdiscovery(self, uri):
         raise DAV_NotFound
 
@@ -192,7 +233,7 @@ class openerp_dav_handler(dav_interface):
         
         if not node:
             if cr: cr.close()
-            raise DAV_NotFound(uri2)
+            raise DAV_NotFound2(uri2)
         else:
             fp = node.full_path()
             if fp and len(fp):
@@ -255,26 +296,25 @@ class openerp_dav_handler(dav_interface):
                 raise DAV_Error, 409
             node = self.uri2object(cr, uid, pool, uri2)   
             if not node:
-                raise DAV_NotFound(uri2)
+                raise DAV_NotFound2(uri2)
             try:
                 if rrange:
                     self.parent.log_error("Doc get_data cannot use range")
                     raise DAV_Error(409)
                 datas = node.get_data(cr)
             except TypeError,e:
-                import traceback                
-                self.parent.log_error("GET typeError: %s", str(e))
-                self.parent.log_message("Exc: %s",traceback.format_exc())
-                raise DAV_Forbidden
+                # for the collections that return this error, the DAV standard
+                # says we'd better just return 200 OK with empty data
+                return ''
             except IndexError,e :
                 self.parent.log_error("GET IndexError: %s", str(e))
-                raise DAV_NotFound(uri2)
+                raise DAV_NotFound2(uri2)
             except Exception,e:
                 import traceback
                 self.parent.log_error("GET exception: %s",str(e))
                 self.parent.log_message("Exc: %s", traceback.format_exc())
                 raise DAV_Error, 409
-            return datas
+            return str(datas) # FIXME!
         finally:
             if cr: cr.close()    
 
@@ -288,7 +328,7 @@ class openerp_dav_handler(dav_interface):
                 return COLLECTION
             node = self.uri2object(cr,uid,pool, uri2)
             if not node:
-                raise DAV_NotFound(uri2)
+                raise DAV_NotFound2(uri2)
             if node.type in ('collection','database'):
                 return COLLECTION
             return OBJECT
@@ -304,7 +344,7 @@ class openerp_dav_handler(dav_interface):
         node = self.uri2object(cr, uid, pool, uri2)
         if not node:
             cr.close()
-            raise DAV_NotFound(uri2)
+            raise DAV_NotFound2(uri2)
         cr.close()
         return node.displayname
 
@@ -320,7 +360,7 @@ class openerp_dav_handler(dav_interface):
         node = self.uri2object(cr, uid, pool, uri2)
         if not node:
             cr.close()
-            raise DAV_NotFound(uri2)
+            raise DAV_NotFound2(uri2)
         result = node.content_length or 0
         cr.close()
         return str(result)
@@ -337,7 +377,7 @@ class openerp_dav_handler(dav_interface):
         node = self.uri2object(cr, uid, pool, uri2)
         if not node:
             cr.close()
-            raise DAV_NotFound(uri2)
+            raise DAV_NotFound2(uri2)
         result = node.get_etag(cr)
         cr.close()
         return str(result)
@@ -352,7 +392,7 @@ class openerp_dav_handler(dav_interface):
         try:            
             node = self.uri2object(cr, uid, pool, uri2)
             if not node:
-                raise DAV_NotFound(uri2)
+                raise DAV_NotFound2(uri2)
             if node.write_date:
                 return time.mktime(time.strptime(node.write_date,'%Y-%m-%d %H:%M:%S'))
             else:
@@ -369,7 +409,7 @@ class openerp_dav_handler(dav_interface):
         try:            
             node = self.uri2object(cr, uid, pool, uri2)
             if not node:
-                raise DAV_NotFound(uri2)            
+                raise DAV_NotFound2(uri2)
             if node.create_date:
                 result = time.mktime(time.strptime(node.create_date,'%Y-%m-%d %H:%M:%S'))
             else:
@@ -387,7 +427,7 @@ class openerp_dav_handler(dav_interface):
         try:            
             node = self.uri2object(cr, uid, pool, uri2)
             if not node:
-                raise DAV_NotFound(uri2)
+                raise DAV_NotFound2(uri2)
             result = str(node.mimetype)
             return result
             #raise DAV_NotFound, 'Could not find %s' % path
@@ -395,20 +435,28 @@ class openerp_dav_handler(dav_interface):
             if cr: cr.close()    
     
     def mkcol(self,uri):
-        """ create a new collection """                  
+        """ create a new collection
+            see par. 9.3 of rfc4918
+        """
         self.parent.log_message('MKCOL: %s' % uri)
-        uri = self.uri2local(uri)[1:]
-        if uri[-1]=='/':uri=uri[:-1]
-        parent = '/'.join(uri.split('/')[:-1])        
-        parent = self.baseuri + parent
-        uri = self.baseuri + uri        
-        cr, uid, pool,dbname, uri2 = self.get_cr(uri)
+        cr, uid, pool, dbname, uri2 = self.get_cr(uri)
+        if not uri2[-1]:
+            cr.close()
+            raise DAV_Error(409, "Cannot create nameless collection")
         if not dbname:
+            cr.close()
             raise DAV_Error, 409
         node = self.uri2object(cr,uid,pool, uri2[:-1])
-        if node:
-            node.create_child_collection(cr, uri2[-1])  
-            cr.commit()      
+        if not node:
+            cr.close()
+            raise DAV_Error(409, "Parent path %s does not exist" % uri2[:-1])
+        nc = node.child(cr, uri2[-1])
+        if nc:
+            cr.close()
+            raise DAV_Error(405, "Path already exists")
+        self._try_function(node.create_child_collection, (cr, uri2[-1]),
+                    "create col %s" % uri2[-1], cr=cr)
+        cr.commit()
         cr.close()
         return True
 
@@ -430,22 +478,13 @@ class openerp_dav_handler(dav_interface):
         if not node:
             dir_node = self.uri2object(cr, uid, pool, uri2[:-1])
             if not dir_node:
+                cr.close()
                 raise DAV_NotFound('Parent folder not found')
-            try:
-                dir_node.create_child(cr, objname, data)
-            except Exception,e:
-                import traceback
-                self.parent.log_error("Cannot create %s: %s", objname, str(e))
-                self.parent.log_message("Exc: %s",traceback.format_exc())
-                raise DAV_Forbidden
+
+            self._try_function(dir_node.create_child, (cr, objname, data), 
+                    "create %s" % objname, cr=cr)
         else:
-            try:
-                node.set_data(cr, data)
-            except Exception,e:
-                import traceback
-                self.parent.log_error("Cannot save %s: %s", objname, str(e))
-                self.parent.log_message("Exc: %s",traceback.format_exc())
-                raise DAV_Forbidden
+            self._try_function(node.set_data, (cr, data), "save %s" % objname, cr=cr)
             
         cr.commit()
         cr.close()
@@ -618,7 +657,6 @@ class openerp_dav_handler(dav_interface):
         advanced systems we might also have to copy properties from
         the source to the destination.
         """
-        print " copy a collection."
         return self.mkcol(dst)
 
 

addons/document_webdav/webdav.py

@@ -48,6 +48,8 @@ def mk_prop_response(self, uri, good_props, bad_props, doc):
     # write href information
     uparts=urlparse.urlparse(uri)
     fileloc=uparts[2]
+    if isinstance(fileloc, unicode):
+        fileloc = fileloc.encode('utf-8')
     href=doc.createElement("D:href")
     davpath = self._dataclass.parent.get_davpath()
     hurl = '%s://%s%s%s' % (uparts[0], uparts[1], davpath, urllib.quote(fileloc))
@@ -126,6 +128,8 @@ def mk_propname_response(self,uri,propnames,doc):
     # write href information
     uparts=urlparse.urlparse(uri)
     fileloc=uparts[2]
+    if isinstance(fileloc, unicode):
+        fileloc = fileloc.encode('utf-8')
     href=doc.createElement("D:href")
     davpath = self._dataclass.parent.get_davpath()
     hurl = '%s://%s%s%s' % (uparts[0], uparts[1], davpath, urllib.quote(fileloc))

addons/document_webdav/webdav_server.py

@@ -78,6 +78,18 @@ class DAVHandler(FixSendError,DAVRequestHandler):
         self.baseuri = "http://%s:%d/"% (self.server.server_name, self.server.server_port)
         self.IFACE_CLASS  = openerp_dav_handler(self, self.verbose)
         
+    def copymove(self, CLASS):
+        """ Our uri scheme removes the /webdav/ component from there, so we
+        need to mangle the header, too.
+        """
+        dest = self.headers['Destination']
+        up = urlparse.urlparse(urllib.unquote(self.headers['Destination']))
+        if up.path.startswith(self.davpath):
+            self.headers['Destination'] = up.path[len(self.davpath):]
+        else:
+            raise DAV_Forbidden("Not allowed to copy/move outside webdav path")
+        DAVRequestHandler.copymove(self, CLASS)
+
     def get_davpath(self):
         return self.davpath
     
@@ -175,6 +187,11 @@ class DAVHandler(FixSendError,DAVRequestHandler):
 
         self.send_body(None, '201', 'Created', '', headers=headers)
 
+    def do_DELETE(self):
+        try:
+            DAVRequestHandler.do_DELETE(self)
+        except DAV_Error, (ec, dd):
+            return self.send_status(ec)
 
 from service.http_server import reg_http_service,OpenERPAuthProvider