[MERGE]

bzr revid: hmo@tinyerp.com-20100711060646-gwp6039vsln1x30a

addons/document/__openerp__.py

@@ -53,7 +53,7 @@
     ],
     'demo_xml': [ 'document_demo.xml','board_document_demo.xml'],
     'test': [
-        'test/document_test.yml',
+        'test/document_test2.yml',
     ],
     'installable': True,
     'active': False,

addons/document/board_document_view.xml

@@ -53,7 +53,7 @@
         <menuitem id="menu_reporting" name="Reporting" sequence="2" parent="knowledge.menu_document"/>
          <menuitem
             name="Dashboard"
-            id="menu_board_document"
+            id="menu_reports_document"
             parent="menu_reporting"
             sequence="0"
             icon="terp-graph"
@@ -61,10 +61,10 @@
 
 
          <menuitem
-            parent="menu_board_document"
+            parent="menu_reports_document"
             action="open_board_document_manager"
             sequence="1"
-            id="menu_board_document_manager"
+            id="menu_reports_document_manager"
             icon="terp-graph"/>
 
         <record model="ir.ui.view" id="board_document_manager_form1">
@@ -107,10 +107,10 @@
         </record>
 
         <menuitem
-            parent="menu_board_document"
+            parent="menu_reports_document"
             action="open_board_document_manager1"
             sequence="1"
-            id="menu_board_document_manager1"
+            id="menu_reports_document_manager1"
             icon="terp-graph"/>
      </data>
  </openerp>

addons/document/content_index.py

@@ -153,7 +153,10 @@ class contentIndex(object):
                 if realfname :
                     fname = realfname
                 else:
-                    bname,ext = os.path.splitext(filename)
+                    try:
+                        bname,ext = os.path.splitext(filename or 'test.tmp')
+                    except Exception:
+                        bname, ext = filename, 'tmp'
                     fd, fname = tempfile.mkstemp(suffix=ext)
                     os.write(fd, content)
                     os.close(fd)

addons/document/document.py

@@ -28,6 +28,7 @@ import os
 
 import pooler
 import netsvc
+from osv.orm import except_orm
 #import StringIO
 
 from psycopg2 import Binary
@@ -72,33 +73,32 @@ class document_file(osv.osv):
         return res
 
     _columns = {
-        'user_id': fields.many2one('res.users', 'Owner', select=1),
-        'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'),
-        # the directory id now is mandatory. It can still be computed automatically.
-        'parent_id': fields.many2one('document.directory', 'Directory', select=1),
-        'file_size': fields.integer('File Size', required=True),
-        'file_type': fields.char('Content Type', size=128),
+        # Columns from ir.attachment:
+        'create_date': fields.datetime('Date Created', readonly=True),
+        'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
+        'write_date': fields.datetime('Date Modified', readonly=True),
+        'write_uid':  fields.many2one('res.users', 'Last Modification User', readonly=True),
+        'res_model': fields.char('Attached Model', size=64, readonly=True),
+        'res_id': fields.integer('Attached ID', readonly=True),
+
         # If ir.attachment contained any data before document is installed, preserve
         # the data, don't drop the column!
         'db_datas': fields.binary('Data', oldname='datas'),
-        'index_content': fields.text('Indexed Content'),
-        'write_date': fields.datetime('Date Modified', readonly=True),
-        'write_uid':  fields.many2one('res.users', 'Last Modification User', readonly=True),
-        'create_date': fields.datetime('Date Created', readonly=True),
-        'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
-        'store_method': fields.selection([('db', 'Database'), ('fs', 'Filesystem'), ('link', 'Link')], "Storing Method"),
         'datas': fields.function(_data_get, method=True, fnct_inv=_data_set, string='File Content', type="binary", nodrop=True),
-        'url': fields.char('File URL',size=64),
-        'store_fname': fields.char('Stored Filename', size=200),
-        'res_model': fields.char('Attached Model', size=64), #res_model
-        'res_id': fields.integer('Attached ID'), #res_id
-        'partner_id':fields.many2one('res.partner', 'Partner', select=1),
-        'type':fields.selection([
-            ('url','URL'),
-            ('binary','Binary'),
 
-        ],'Type', help="Type is used to separate URL and binary File"),
+        # Fields of document:
+        'user_id': fields.many2one('res.users', 'Owner', select=1),
+        # 'group_ids': fields.many2many('res.groups', 'document_group_rel', 'item_id', 'group_id', 'Groups'),
+        # the directory id now is mandatory. It can still be computed automatically.
+        'parent_id': fields.many2one('document.directory', 'Directory', select=1, required=True),
+        'index_content': fields.text('Indexed Content'),
+        'partner_id':fields.many2one('res.partner', 'Partner', select=1),
         'company_id': fields.many2one('res.company', 'Company'),
+        'file_size': fields.integer('File Size', required=True),
+        'file_type': fields.char('Content Type', size=128),
+        
+        # fields used for file storage
+        'store_fname': fields.char('Stored Filename', size=200),
     }
 
     def __get_def_directory(self, cr, uid, context=None):
@@ -109,8 +109,6 @@ class document_file(osv.osv):
         'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c),
         'user_id': lambda self, cr, uid, ctx:uid,
         'file_size': lambda self, cr, uid, ctx:0,
-        'store_method': lambda *args: 'db',
-        'type': 'binary',
         'parent_id': __get_def_directory
     }
     _sql_constraints = [
@@ -122,7 +120,7 @@ class document_file(osv.osv):
         res_model = vals.get('res_model', False)
         res_id = vals.get('res_id', 0)
         if op == 'write':
-            for file in self.browse(cr, uid, ids):
+            for file in self.browse(cr, uid, ids): # FIXME fields_only
                 if not name:
                     name = file.name
                 if not parent_id:
@@ -156,8 +154,41 @@ class document_file(osv.osv):
             return False
         if not self._check_duplication(cr, uid, vals, ids, 'write'):
             raise osv.except_osv(_('ValidateError'), _('File name must be unique!'))
-        result = super(document_file, self).write(cr, uid, ids, vals, context=context)
-        cr.commit()
+        if ('parent_id' in vals) or ('name' in vals):
+            # perhaps this file is renaming or changing directory
+            nctx = nodes.get_node_context(cr,uid,context)
+            dirobj = self.pool.get('document.directory')
+            if 'parent_id' in vals:
+                dbro = dirobj.browse(cr, uid, vals['parent_id'], context=context)
+                dnode = nctx.get_dir_node(cr, dbro)
+            else:
+                dbro = None
+                dnode = None
+            ids2 = []
+            result = False
+            for fbro in self.browse(cr, uid, ids, context=context):
+                if ('parent_id' not in vals or fbro.parent_id.id == vals['parent_id']) \
+                    and ('name' not in vals or fbro.name == vals['name']) :
+                        ids2.append(fbro.id)
+                        continue
+                fnode = nctx.get_file_node(cr, fbro)
+                res = fnode.move_to(cr, dnode or fnode.parent, vals.get('name', fbro.name), fbro, dbro, True)
+                if isinstance(res, dict):
+                    vals2 = vals.copy()
+                    vals2.update(res)
+                    wid = res.get('id', fbro.id)
+                    result = super(document_file,self).write(cr,uid,wid,vals2,context=context)
+                    # TODO: how to handle/merge several results?
+                elif res == True:
+                    ids2.append(fbro.id)
+                elif res == False:
+                    pass
+            ids = ids2
+        if 'file_size' in vals: # only write that field using direct SQL calls
+            del vals['file_size']
+        if len(ids) and len(vals):
+            result = super(document_file,self).write(cr, uid, ids, vals, context=context)
+        cr.commit() # ?
         return result
 
     def create(self, cr, uid, vals, context=None):
@@ -170,26 +201,10 @@ class document_file(osv.osv):
             vals['res_id'] = context.get('default_res_id', False)
         if not vals.get('res_model', False) and context.get('default_res_model', False):
             vals['res_model'] = context.get('default_res_model', False)
-        if vals.get('res_id', False) and vals.get('res_model', False):
-            obj_model = self.pool.get(vals['res_model'])
-            result = obj_model.read(cr, uid, [vals['res_id']], ['name', 'partner_id', 'address_id'], context=context)
-            if len(result):
-                obj = result[0]
-                if obj_model._name == 'res.partner':
-                    vals['partner_id'] = obj['id']
-                elif obj.get('address_id', False):
-                    if isinstance(obj['address_id'], tuple) or isinstance(obj['address_id'], list):
-                        address_id = obj['address_id'][0]
-                    else:
-                        address_id = obj['address_id']
-                    address = self.pool.get('res.partner.address').read(cr, uid, [address_id], context=context)
-                    if len(address):
-                        vals['partner_id'] = address[0]['partner_id'][0] or False
-                elif obj.get('partner_id', False):
-                    if isinstance(obj['partner_id'], tuple) or isinstance(obj['partner_id'], list):
-                        vals['partner_id'] = obj['partner_id'][0]
-                    else:
-                        vals['partner_id'] = obj['partner_id']
+        if vals.get('res_id', False) and vals.get('res_model', False) \
+                and not vals.get('partner_id', False):
+            vals['partner_id'] = self.__get_partner_id(cr, uid, \
+                vals['res_model'], vals['res_id'], context)
 
         datas = None
         if vals.get('link', False) :
@@ -198,13 +213,33 @@ class document_file(osv.osv):
         else:
             datas = vals.get('datas', False)
 
-        vals['file_size'] = datas and len(datas) or 0
+        if datas:
+            vals['file_size'] = len(datas)
+        else:
+            if vals.get('file_size'):
+                del vals['file_size']
         if not self._check_duplication(cr, uid, vals):
             raise osv.except_osv(_('ValidateError'), _('File name must be unique!'))
         result = super(document_file, self).create(cr, uid, vals, context)
-        cr.commit()
+        cr.commit() # ?
         return result
 
+    def __get_partner_id(self, cr, uid, res_model, res_id, context):
+        """ A helper to retrieve the associated partner from any res_model+id
+            It is a hack that will try to discover if the mentioned record is
+            clearly associated with a partner record.
+        """
+        obj_model = self.pool.get(res_model)
+        if obj_model._name == 'res.partner':
+            return res_id
+        elif 'partner_id' in obj_model._columns and obj_model._columns['partner_id']._obj == 'res.partner':
+            bro = obj_model.browse(self, cr, uid, res_id, context=context)
+            return bro.partner_id.id
+        elif 'address_id' in obj_model._columns and obj_model._columns['address_id']._obj == 'res.partner.address':
+            bro = obj_model.browse(self, cr, uid, res_id, context=context)
+            return bro.address_id.partner_id.id
+        return False
+
     def unlink(self, cr, uid, ids, context={}):
         stor = self.pool.get('document.storage')
         unres = []

addons/document/document_data.xml

@@ -9,7 +9,7 @@
     </record>
     
     <record model="document.storage" id="storage_default">
-        <field name="name">Default DB storage</field>
+        <field name="name">Database storage</field>
         <field name="type">db</field>
         <field name="user_id" ref="base.user_admin"/>
     </record>
@@ -17,7 +17,7 @@
     <record model="document.directory" id="dir_root">
         <field name="name">Documents</field>
         <field name="user_id" ref="base.user_admin"/>
-	<field name="storage_id" ref="storage_default"/>
+        <field name="storage_id" ref="storage_default"/>
         <field name="ressource_id">0</field>
     </record>
 
@@ -25,7 +25,6 @@
         <field name="name">My Folder</field>
         <field name="parent_id" ref="dir_root"/>
         <field name="user_id" ref="base.user_admin"/>
-	<field name="storage_id" ref="storage_default"/>
         <field name="ressource_id">0</field>
  
     </record>
@@ -35,7 +34,6 @@
         <field name="parent_id" ref="dir_root"/>
         <field name="type">ressource</field>
         <field name="ressource_tree">1</field>
-	<field name="storage_id" ref="storage_default"/>
         <field name="ressource_id">0</field>
 
         <field name="ressource_type_id" search="[('model','=','res.partner.category')]" />
@@ -57,7 +55,6 @@
         <field name="name">Personal Folders</field>
         <field name="parent_id" ref="dir_root"/>
         <field name="type">ressource</field>
-	<field name="storage_id" ref="storage_default"/>
         <field name="ressource_type_id" ref="base.model_res_users" />
         <field name="ressource_id">0</field>
 

addons/document/document_directory.py

@@ -39,19 +39,23 @@ class document_directory(osv.osv):
         'write_uid':  fields.many2one('res.users', 'Last Modification User', readonly=True),
         'create_date': fields.datetime('Date Created', readonly=True),
         'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
-        'file_type': fields.char('Content Type', size=32),
         'domain': fields.char('Domain', size=128, help="Use a domain if you want to apply an automatic filter on visible resources."),
         'user_id': fields.many2one('res.users', 'Owner'),
         'storage_id': fields.many2one('document.storage', 'Storage'),
         'group_ids': fields.many2many('res.groups', 'document_directory_group_rel', 'item_id', 'group_id', 'Groups'),
-        'parent_id': fields.many2one('document.directory', 'Parent Item'),
+        'parent_id': fields.many2one('document.directory', 'Parent Directory', select=1),
         'child_ids': fields.one2many('document.directory', 'parent_id', 'Children'),
         'file_ids': fields.one2many('ir.attachment', 'parent_id', 'Files'),
         'content_ids': fields.one2many('document.directory.content', 'directory_id', 'Virtual Files'),
-        'type': fields.selection([('directory','Static Directory'),('ressource','Other Resources')], 'Type', required=True),
-        'ressource_type_id': fields.many2one('ir.model', 'Directories Mapped to Objects',
-            help="Select an object here and Open ERP will create a mapping for each of these " \
-                 "objects, using the given domain, when browsing through FTP."),
+        'type': fields.selection([ 
+            ('directory','Static Directory'),
+            ('ressource','Folders per resource'),
+            ],
+            'Type', required=True, select=1,
+            help="Defines directory's behaviour."),
+        
+        'ressource_type_id': fields.many2one('ir.model', 'Resource model',
+            help="Select an object here and there will be one folder per record of that resource."),
         'resource_field': fields.many2one('ir.model.fields', 'Name field', help='Field to be used as name on resource directories. If empty, the "name" will be used.'),
         'ressource_parent_type_id': fields.many2one('ir.model', 'Parent Model',
             help="If you put an object here, this directory template will appear bellow all of these objects. " \
@@ -60,7 +64,7 @@ class document_directory(osv.osv):
         'ressource_tree': fields.boolean('Tree Structure',
             help="Check this if you want to use the same tree structure as the object selected in the system."),
         'dctx_ids': fields.one2many('document.directory.dctx', 'dir_id', 'Context fields'),
-        'company_id': fields.many2one('res.company', 'Company'),        
+        'company_id': fields.many2one('res.company', 'Company'),
     }
 
 
@@ -102,7 +106,8 @@ class document_directory(osv.osv):
     }
     _sql_constraints = [
         ('dirname_uniq', 'unique (name,parent_id,ressource_id,ressource_parent_type_id)', 'The directory name must be unique !'),
-        ('no_selfparent', 'check(parent_id <> id)', 'Directory cannot be parent of itself!')
+        ('no_selfparent', 'check(parent_id <> id)', 'Directory cannot be parent of itself!'),
+        ('dir_parented', 'check(parent_id IS NOT NULL OR storage_id IS NOT NULL)', 'Directory must have a parent or a storage')
     ]
     def name_get(self, cr, uid, ids, context={}):
         res = []
@@ -190,17 +195,8 @@ class document_directory(osv.osv):
         """
         if not context:
                 context = {}
-        lang = context.get('lang',False)
-        if not lang:
-            user = self.pool.get('res.users').browse(cr, uid, uid)
-            lang = user.context_lang 
-            context['lang'] = lang
             
-        try: #just instrumentation
-                return nodes.get_node_context(cr, uid, context).get_uri(cr, uri)
-        except Exception,e:
-                print "exception: ",e
-                raise
+        return nodes.get_node_context(cr, uid, context).get_uri(cr, uri)
 
 
     def _locate_child(self, cr, uid, root_id, uri,nparent, ncontext):
@@ -274,6 +270,8 @@ class document_directory(osv.osv):
             raise osv.except_osv(_('ValidateError'), _('Directory name contains special characters!'))
         return super(document_directory,self).create(cr, uid, vals, context)
 
+    # TODO def unlink(...
+
 document_directory()
 
 class document_directory_dctx(osv.osv):

addons/document/document_storage.py

@@ -24,7 +24,10 @@ from osv import osv, fields
 import os
 import tools
 import base64
+import errno
 import logging
+from StringIO import StringIO
+import psycopg2
 
 from tools.misc import ustr
 from tools.translate import _
@@ -33,7 +36,9 @@ from osv.orm import except_orm
 
 import random
 import string
+import pooler
 import netsvc
+import nodes
 from content_index import cntIndex
 
 DMS_ROOT_PATH = tools.config.get('document_path', os.path.join(tools.config.get('root_path'), 'filestore'))
@@ -84,6 +89,127 @@ def create_directory(path):
     os.makedirs(path)
     return dir_name
 
+class nodefd_file(nodes.node_descriptor):
+    """ A descriptor to a real file
+
+    Inheriting directly from file doesn't work, since file exports
+    some read-only attributes (like 'name') that we don't like.
+    """
+    def __init__(self, parent, path, mode):
+        nodes.node_descriptor.__init__(self, parent)
+        self.__file = open(path, mode)
+        
+        for attr in ('closed', 'read', 'write', 'seek', 'tell'):
+            setattr(self,attr, getattr(self.__file, attr))
+
+    def close(self):
+        # TODO: locking in init, close()
+        self.__file.close()
+
+    
+class nodefd_db(StringIO, nodes.node_descriptor):
+    """ A descriptor to db data
+    """
+    def __init__(self, parent, ira_browse, mode):
+        nodes.node_descriptor.__init__(self, parent)
+        if mode.endswith('b'):
+            mode = mode[:-1]
+        
+        if mode in ('r', 'r+'):
+            cr = ira_browse._cr # reuse the cursor of the browse object, just now
+            cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s',(ira_browse.id,))
+            data = cr.fetchone()[0]
+            StringIO.__init__(self, data)
+        elif mode in ('w', 'w+'):
+            StringIO.__init__(self, None)
+            # at write, we start at 0 (= overwrite), but have the original
+            # data available, in case of a seek()
+        elif mode == 'a':
+            StringIO.__init__(self, None)
+        else:
+            logging.getLogger('document.storage').error("Incorrect mode %s specified", mode)
+            raise IOError(errno.EINVAL, "Invalid file mode")
+        self.mode = mode
+
+    def close(self):
+        # we now open a *separate* cursor, to update the data.
+        # FIXME: this may be improved, for concurrency handling
+        par = self._get_parent()
+        uid = par.context.uid
+        cr = pooler.get_db(par.context.dbname).cursor()
+        try:
+            if self.mode in ('w', 'w+', 'r+'):
+                data = self.getvalue()
+                out = psycopg2.Binary(data)
+                cr.execute("UPDATE ir_attachment SET db_datas = %s, file_size=%s WHERE id = %s",
+                    (out, len(data), par.file_id))
+            elif self.mode == 'a':
+                data = self.getvalue()
+                out = psycopg2.Binary(data)
+                cr.execute("UPDATE ir_attachment " \
+                    "SET db_datas = COALESCE(db_datas,'') || %s, " \
+                    "    file_size = COALESCE(file_size, 0) + %s " \
+                    " WHERE id = %s",
+                    (out, len(data), par.file_id))
+            cr.commit()
+        except Exception, e:
+            logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id)
+            raise
+        finally:
+            cr.close()
+        StringIO.close(self)
+
+class nodefd_db64(StringIO, nodes.node_descriptor):
+    """ A descriptor to db data, base64 (the old way)
+    
+        It stores the data in base64 encoding at the db. Not optimal, but
+        the transparent compression of Postgres will save the day.
+    """
+    def __init__(self, parent, ira_browse, mode):
+        nodes.node_descriptor.__init__(self, parent)
+        if mode.endswith('b'):
+            mode = mode[:-1]
+        
+        if mode in ('r', 'r+'):
+            StringIO.__init__(self, base64.decodestring(ira_browse.db_datas))
+        elif mode in ('w', 'w+'):
+            StringIO.__init__(self, None)
+            # at write, we start at 0 (= overwrite), but have the original
+            # data available, in case of a seek()
+        elif mode == 'a':
+            StringIO.__init__(self, None)
+        else:
+            logging.getLogger('document.storage').error("Incorrect mode %s specified", mode)
+            raise IOError(errno.EINVAL, "Invalid file mode")
+        self.mode = mode
+
+    def close(self):
+        # we now open a *separate* cursor, to update the data.
+        # FIXME: this may be improved, for concurrency handling
+        par = self._get_parent()
+        uid = par.context.uid
+        cr = pooler.get_db(par.context.dbname).cursor()
+        try:
+            if self.mode in ('w', 'w+', 'r+'):
+                out = self.getvalue()
+                cr.execute('UPDATE ir_attachment SET db_datas = %s::bytea, file_size=%s WHERE id = %s',
+                    (base64.encodestring(out), len(out), par.file_id))
+            elif self.mode == 'a':
+                out = self.getvalue()
+                # Yes, we're obviously using the wrong representation for storing our
+                # data as base64-in-bytea
+                cr.execute("UPDATE ir_attachment " \
+                    "SET db_datas = encode( (COALESCE(decode(encode(db_datas,'escape'),'base64'),'') || decode(%s, 'base64')),'base64')::bytea , " \
+                    "    file_size = COALESCE(file_size, 0) + %s " \
+                    " WHERE id = %s",
+                    (base64.encodestring(out), len(out), par.file_id))
+            cr.commit()
+        except Exception, e:
+            logging.getLogger('document.storage').exception('Cannot update db file #%d for close:', par.file_id)
+            raise
+        finally:
+            cr.close()
+        StringIO.close(self)
 
 class document_storage(osv.osv):
     """ The primary object for data storage.
@@ -110,7 +236,7 @@ class document_storage(osv.osv):
         'group_ids': fields.many2many('res.groups', 'document_storage_group_rel', 'item_id', 'group_id', 'Groups'),
         'dir_ids': fields.one2many('document.directory', 'parent_id', 'Directories'),
         'type': fields.selection([('db', 'Database'), ('filestore', 'Internal File storage'),
-            ('realstore', 'External file storage'), ('virtual', 'Virtual storage')], 'Type', required=True),
+                ('realstore','External file storage'),], 'Type', required=True),
         'path': fields.char('Path', size=250, select=1, help="For file storage, the root path of the storage"),
         'online': fields.boolean('Online', help="If not checked, media is currently offline and its contents not available", required=True),
         'readonly': fields.boolean('Read Only', help="If set, media is for reading only"),
@@ -147,6 +273,51 @@ class document_storage(osv.osv):
             ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
         return self.__get_data_3(cr, uid, boo, ira, context)
 
+    def get_file(self, cr, uid, id, file_node, mode, context=None):
+        """ Return a file-like object for the contents of some node
+        """
+        if context is None:
+            context = {}
+        boo = self.browse(cr, uid, id, context)
+        if not boo.online:
+            raise RuntimeError('media offline')
+        
+        ira = self.pool.get('ir.attachment').browse(cr, uid, file_node.file_id, context=context)
+        if boo.type == 'filestore':
+            if not ira.store_fname:
+                # On a migrated db, some files may have the wrong storage type
+                # try to fix their directory.
+                if ira.file_size:
+                    self._doclog.warning( "ir.attachment #%d does not have a filename, but is at filestore, fix it!" % ira.id)
+                raise IOError(errno.ENOENT, 'No file can be located')
+            fpath = os.path.join(boo.path, ira.store_fname)
+            return nodefd_file(file_node, path=fpath, mode=mode)
+
+        elif boo.type == 'db':
+            # TODO: we need a better api for large files
+            return nodefd_db(file_node, ira_browse=ira, mode=mode)
+
+        elif boo.type == 'db64':
+            return nodefd_db64(file_node, ira_browse=ira, mode=mode)
+
+        elif boo.type == 'realstore':
+            if not ira.store_fname:
+                # On a migrated db, some files may have the wrong storage type
+                # try to fix their directory.
+                if ira.file_size:
+                    self._doclog.warning("ir.attachment #%d does not have a filename, trying the name." %ira.id)
+                sfname = ira.name
+            fpath = os.path.join(boo.path,ira.store_fname or ira.name)
+            if not os.path.exists(fpath):
+                raise IOError("File not found: %s" % fpath)
+            return nodefd_file(file_node, path=fpath, mode=mode)
+
+        elif boo.type == 'virtual':
+            raise ValueError('Virtual storage does not support static files')
+        
+        else:
+            raise TypeError("No %s storage" % boo.type)
+
     def __get_data_3(self, cr, uid, boo, ira, context):
         if not boo.online:
             raise RuntimeError('media offline')
@@ -159,13 +330,21 @@ class document_storage(osv.osv):
                 return None
             fpath = os.path.join(boo.path, ira.store_fname)
             return file(fpath, 'rb').read()
-        elif boo.type == 'db':
+        elif boo.type == 'db64':
             # TODO: we need a better api for large files
             if ira.db_datas:
                 out = base64.decodestring(ira.db_datas)
             else:
                 out = ''
             return out
+        elif boo.type == 'db':
+            # We do an explicit query, to avoid type transformations.
+            cr.execute('SELECT db_datas FROM ir_attachment WHERE id = %s', (ira.id,))
+            res = cr.fetchone()
+            if res:
+                return res[0]
+            else:
+                return ''
         elif boo.type == 'realstore':
             if not ira.store_fname:
                 # On a migrated db, some files may have the wrong storage type
@@ -180,6 +359,10 @@ class document_storage(osv.osv):
                 return None
             else:
                 raise IOError("File not found: %s" % fpath)
+
+        elif boo.type == 'virtual':
+            raise ValueError('Virtual storage does not support static files')
+
         else:
             raise TypeError("No %s storage" % boo.type)
 
@@ -228,7 +411,13 @@ class document_storage(osv.osv):
                 raise except_orm(_('Error!'), str(e))
         elif boo.type == 'db':
             filesize = len(data)
-            # will that work for huge data? TODO
+            # will that work for huge data?
+            out = psycopg2.Binary(data)
+            cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s',
+                (out, file_node.file_id))
+        elif boo.type == 'db64':
+            filesize = len(data)
+            # will that work for huge data?
             out = base64.encodestring(data)
             cr.execute('UPDATE ir_attachment SET db_datas = %s WHERE id = %s',
                 (out, file_node.file_id))
@@ -260,6 +449,10 @@ class document_storage(osv.osv):
             except Exception,e :
                 self._doclog.warning("Couldn't save data:", exc_info=True)
                 raise except_orm(_('Error!'), str(e))
+
+        elif boo.type == 'virtual':
+            raise ValueError('Virtual storage does not support static files')
+
         else:
             raise TypeError("No %s storage" % boo.type)
 
@@ -276,16 +469,21 @@ class document_storage(osv.osv):
                 self._doclog.debug('Cannot index file:', exc_info=True)
                 pass
 
+            try:
+                icont_u = ustr(icont)
+            except UnicodeError:
+                icont_u = ''
+
             # a hack: /assume/ that the calling write operation will not try
             # to write the fname and size, and update them in the db concurrently.
             # We cannot use a write() here, because we are already in one.
             cr.execute('UPDATE ir_attachment SET store_fname = %s, file_size = %s, index_content = %s, file_type = %s WHERE id = %s',
-                (store_fname, filesize, ustr(icont), mime, file_node.file_id))
+                (store_fname, filesize, icont_u, mime, file_node.file_id))
             file_node.content_length = filesize
             file_node.content_type = mime
             return True
         except Exception, e :
-            self._doclog.warning( "Couldn't save data:", exc_info=True)
+            self._doclog.warning("Couldn't save data:", exc_info=True)
             # should we really rollback once we have written the actual data?
             # at the db case (only), that rollback would be safe
             raise except_orm(_('Error at doc write!'), str(e))
@@ -303,7 +501,7 @@ class document_storage(osv.osv):
                 return None
             path = storage_bo.path
             return (storage_bo.id, 'file', os.path.join(path, fname))
-        elif storage_bo.type == 'db':
+        elif storage_bo.type in ('db', 'db64'):
             return None
         elif storage_bo.type == 'realstore':
             fname = fil_bo.store_fname
@@ -312,7 +510,7 @@ class document_storage(osv.osv):
             path = storage_bo.path
             return ( storage_bo.id, 'file', os.path.join(path, fname))
         else:
-            raise TypeError("No %s storage" % boo.type)
+            raise TypeError("No %s storage" % storage_bo.type)
 
     def do_unlink(self, cr, uid, unres):
         for id, ktype, fname in unres:
@@ -326,6 +524,92 @@ class document_storage(osv.osv):
 
         return True
 
+    def simple_rename(self, cr, uid, file_node, new_name, context=None):
+        """ A preparation for a file rename.
+            It will not affect the database, but merely check and perhaps
+            rename the realstore file.
+            
+            @return the dict of values that can safely be be stored in the db.
+        """
+        sbro = self.browse(cr, uid, file_node.storage_id, context=context)
+        assert sbro, "The file #%d didn't provide storage" % file_node.file_id
+        
+        if sbro.type in ('filestore', 'db', 'db64'):
+            # nothing to do for a rename, allow to change the db field
+            return { 'name': new_name, 'datas_fname': new_name }
+        elif sbro.type == 'realstore':
+            fname = fil_bo.store_fname
+            if not fname:
+                return ValueError("Tried to rename a non-stored file")
+            path = storage_bo.path
+            oldpath = os.path.join(path, fname)
+            
+            for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'):
+                if ch in new_name:
+                    raise ValueError("Invalid char %s in name %s" %(ch, new_name))
+                
+            file_node.fix_ppath(cr, ira)
+            npath = file_node.full_path() or []
+            dpath = [path,]
+            dpath.extend(npath[:-1])
+            dpath.append(new_name)
+            newpath = os.path.join(*dpath)
+            # print "old, new paths:", oldpath, newpath
+            os.rename(oldpath, newpath)
+            return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name }
+        else:
+            raise TypeError("No %s storage" % boo.type)
+
+    def simple_move(self, cr, uid, file_node, ndir_bro, context=None):
+        """ A preparation for a file move.
+            It will not affect the database, but merely check and perhaps
+            move the realstore file.
+            
+            @param ndir_bro a browse object of document.directory, where this
+                    file should move to.
+            @return the dict of values that can safely be be stored in the db.
+        """
+        sbro = self.browse(cr, uid, file_node.storage_id, context=context)
+        assert sbro, "The file #%d didn't provide storage" % file_node.file_id
+
+        par = ndir_bro
+        psto = None
+        while par:
+            if par.storage_id:
+                psto = par.storage_id.id
+                break
+            par = par.parent_id
+        if file_node.storage_id != psto:
+            self._doclog.debug('Cannot move file %r from %r to %r', file_node, file_node.parent, ndir_bro.name)
+            raise NotImplementedError('Cannot move files between storage media')
+
+        if sbro.type in ('filestore', 'db', 'db64'):
+            # nothing to do for a rename, allow to change the db field
+            return { 'parent_id': ndir_bro.id }
+        elif sbro.type == 'realstore':
+            raise NotImplementedError("Cannot move in realstore, yet") # TODO
+            fname = fil_bo.store_fname
+            if not fname:
+                return ValueError("Tried to rename a non-stored file")
+            path = storage_bo.path
+            oldpath = os.path.join(path, fname)
+            
+            for ch in ('*', '|', "\\", '/', ':', '"', '<', '>', '?', '..'):
+                if ch in new_name:
+                    raise ValueError("Invalid char %s in name %s" %(ch, new_name))
+                
+            file_node.fix_ppath(cr, ira)
+            npath = file_node.full_path() or []
+            dpath = [path,]
+            dpath.extend(npath[:-1])
+            dpath.append(new_name)
+            newpath = os.path.join(*dpath)
+            # print "old, new paths:", oldpath, newpath
+            os.rename(oldpath, newpath)
+            return { 'name': new_name, 'datas_fname': new_name, 'store_fname': new_name }
+        else:
+            raise TypeError("No %s storage" % boo.type)
+
 
 document_storage()
 

addons/document/document_view.xml

@@ -16,7 +16,7 @@
                     <field name="online"/>
                     <field name="readonly"/>
                 </group>
-                <group colspan="2" attrs="{'invisible':[('type','=','db')]}">
+                <group colspan="2" attrs="{'invisible':[('type','in',['db', 'db64'])]}">
                     <field name="path"/>
                 </group>
             </form>
@@ -61,6 +61,7 @@
         name="Storage Media"
         action="action_document_storage_form"
         id="menu_document_storage_media"
+        groups="base.group_extended"
         parent="menu_document_management_configuration"/>
 
     <record model="ir.ui.view" id="view_document_directory_form">
@@ -68,25 +69,32 @@
         <field name="model">document.directory</field>
         <field name="type">form</field>
         <field name="arch" type="xml">
-            <form string="Directories">
+            <form string="Directories" col="6">
                 <field name="name" select="1" colspan="4"/>
-                <field name="user_id"/>
                 <field name="parent_id"/>
-                <field name="storage_id" />
+                <field name="storage_id" widget="selection" />
+                <field name="user_id"/>
                 <field name="company_id" groups="base.group_multi_company" widget="selection"/>
                 <notebook colspan="4">
                 <page string="Definition">
                     <separator string="Directory Type" colspan="4"/>
                     <field name="type"/>
-                    <field name="ressource_type_id"  on_change="onchange_content_id(ressource_type_id)" attrs="{'required': [('type','=','ressource')], 'readonly': [('type','=','static')]}"/>
-                    <newline/>
-                    <field name="domain" attrs="{'required': [('type','=','ressource')], 'readonly': [('type','=','static')]}"/>
-                    <field name="ressource_tree"/>
-                    <field name="resource_field"     domain="[('model_id','=',ressource_type_id), ('ttype', 'in', ('char', 'selection', 'date', 'datetime'))]"/>
-                    <field name="ressource_id" select="2" readonly="1"/>
-                    <field name="ressource_parent_type_id"/>
+                    <group colspan="4" col="4" attrs="{'invisible': [('type','!=','ressource')]}">
+                        <field name="ressource_type_id"  on_change="onchange_content_id(ressource_type_id)" 
+                            attrs="{'required': [('type','=','ressource')] }"/>
+                        <newline/>
+                        <field name="resource_field"     domain="[('model_id','=',ressource_type_id), ('ttype', 'in', ('char', 'selection', 'date', 'datetime'))]"/>
+                        <field name="ressource_tree"/>
+                        <newline/>
+                        <field name="domain" attrs="{'required': [('type','=','ressource')], 'readonly': [('type','=','static')]}"/>
+                    </group>
+                    <group colspan="4" col="4">
+                        <field name="ressource_parent_type_id"/>
+                        <field name="ressource_id" select="2" readonly="1"/>
+                    </group>
+                    
                 </page>
-                <page string="Generated Files">
+                <page string="Generated Files" groups="base.group_extended">
                     <label colspan="4" string="For each entry here, virtual files will appear in this folder." />
                     <field name="content_ids" nolabel="1" colspan="4" attrs="{'readonly': [('ressource_type_id','=',False)]}">
                         <form string="Contents">
@@ -108,6 +116,7 @@
                     </field>
                 </page>
                 <page string="Dynamic context" groups="base.group_extended">
+                    <label colspan="4" string="Define words in the context, for all child directories and files" />
                     <field name="dctx_ids" nolabel="1" colspan="4">
                         <tree string="Fields" editable="bottom">
                             <field name="field"/>
@@ -120,6 +129,8 @@
                     </field>
                 </page>
                 <page string="Security">
+                    <label colspan="4" string="Only members of these groups will have access to this directory and its files." />
+                    <label colspan="4" string="These groups, however, do NOT apply to children directories, which must define their own groups." />
                     <field name="group_ids" colspan="4" nolabel="1"/>
                 </page>
                 </notebook>
@@ -213,44 +224,46 @@
         <field name="arch" type="xml">
             <form string="Documents">
                 <group colspan="4" col="6">
-                    <field name="name" select="1" />
-                    <field name="type"/>
+                    <field name="name" select="1" colspan="4" />
                     <field name="parent_id"/>
+                    <newline/>
+                    <field name="user_id"/>
                     <field name="company_id" groups="base.group_multi_company" widget="selection"/>
                 </group>
                 <notebook colspan="4">
                 <page string="Attachment">
                     <group col="2" colspan="2">
                         <separator string="Data" colspan="2"/>
+                        <field name="type"/>
+                        <newline />
                         <group col="2" colspan="2" attrs="{'invisible':[('type','=','url')]}">
-                        <field name="datas" filename="datas_fname"/>
-                        <field name="datas_fname" select="1"/>
+                            <field name="datas" filename="datas_fname"/>
+                            <field name="datas_fname" select="1"/>
                         </group>
                         <group col="2" colspan="2" attrs="{'invisible':[('type','=','binary')]}">
-                        <field name="url" widget="url"/>
+                            <field name="url" widget="url"/>
                         </group>
                     </group>
                     <group col="2" colspan="2">
-                        <separator string="Relation" colspan="2"/>
-                         <field name="res_name" readonly="1"/>
-                         <field name="partner_id"/>
-                          <field name="user_id"/>
-                    </group>
-                    <group col="4" colspan="4">
-                        <separator string="History" colspan="4"/>
-                        <group col="4" colspan="4">
-                        <field name="create_uid"/>
-                        <field name="create_date"/>
-                        </group>
-                        <group col="4" colspan="4">
-                        <field name="write_uid"/>
-                        <field name="write_date"/>
+                    <field name="res_id" invisible="True"/>
+                        <group col="2" colspan="2" attrs="{'invisible': [('res_id','=',0)]}">
+                            <separator string="Attached To" colspan="2"/>
+                            <field name="res_model" readonly="True" invisible="True"/>
+                            <field name="res_name" readonly="1"/>
                         </group>
+                        <separator string="Related to" colspan="2"/>
+                            <field name="partner_id"/>
+                    </group>
+                    <group col="2" colspan="2" groups="base.group_extended">
+                        <separator string="Created" colspan="2"/>
+                        <field name="create_uid" readonly="1"/>
+                        <field name="create_date" readonly="1"/>
+                    </group>
+                    <group col="2" colspan="2" groups="base.group_extended">
+                        <separator string="Modified" colspan="2"/>
+                        <field name="write_uid" readonly="1"/>
+                        <field name="write_date" readonly="1"/>
                     </group>
-
-                </page>
-                <page string="Security">
-                    <field name="group_ids" colspan="4" nolabel="1"/>
                 </page>
                 <page string="Indexed Content" groups="base.group_extended">
                     <field name="index_content" colspan="4" nolabel="1"/>
@@ -269,11 +282,13 @@
             <field name="type">search</field>
             <field name="arch" type="xml">
              <search string="IR Attachment">
-             <filter icon="terp-go-month" string="Recent"
-                    help="less 1 month modified/created attachments"
+             <filter icon="terp-go-month" string="Recent Month"
+                    help="Attachment modified/created last 30 days"
                     domain="[('create_date','&lt;=', time.strftime('%%Y-%%m-%%d')),('create_date','&gt;',(datetime.date.today()-datetime.timedelta(days=30)).strftime('%%Y-%%m-%%d'))]"
                 />
                  <separator orientation="vertical"/>
+                 <field name="name"/>
+                 <field name="parent_id" />
                  <field name="user_id">
                     <filter icon="terp-personal"
                         domain="[('user_id','=', False)]"
@@ -284,9 +299,9 @@
                 <newline/>
                 <group expand="0" string="Group By...">
                     <filter string="Partner" icon="terp-personal" domain="[]"
-                       context="{'group_by':'partner_id'}" />
+                       context="{'group_by':'partner_id'}"  groups="base.group_extended"/>
                     <filter string="Directory" icon="terp-folder-green" domain="[]" context="{'group_by':'parent_id'}"/>
-                    <filter string="Type"  domain="[]" context="{'group_by':'type'}"/>
+                    <filter string="Type"  domain="[]" context="{'group_by':'type'}"  groups="base.group_extended"/>
                     <filter string="Owner" icon="terp-personal" domain="[]" context="{'group_by':'user_id'}"/>
                     <filter string="Company" icon="terp-personal" domain="[]" context="{'group_by':'company_id'}" groups="base.group_multi_company"/>
                 </group>
@@ -301,11 +316,12 @@
         <field name="arch" type="xml">
             <tree colors="blue:type in ('url',)">
                 <field name="name"/>
-                <field name="type"/>
-                <field name="datas_fname"/>
+                <field name="parent_id" />
                 <field name="user_id"/>
                 <field name="create_date"/>
                 <field name="write_date"/>
+                <field name="partner_id" groups="base.group_extended" />
+                <field name="type"  groups="base.group_extended"/>
             </tree>
 
         </field>

addons/document/report/document_report_view.xml

@@ -33,22 +33,22 @@
             </field>
         </record>
 
-		<record id="view_report_document_user_search" model="ir.ui.view">
+        <record id="view_report_document_user_search" model="ir.ui.view">
             <field name="name">report.document.user.search</field>
             <field name="model">report.document.user</field>
             <field name="type">search</field>
             <field name="arch" type="xml">
                 <search string="All users files">
-                	<group col="12" colspan="4">
+                        <group col="12" colspan="4">
                         <separator orientation="vertical"/>
-						<filter icon="terp-go-year" string="This Year" domain="[('name','=',time.localtime()[0])]" help="All Months Files"/>
-						<filter icon="terp-go-month" string="This Month" domain="[('month','=',time.localtime()[1])]" help="This Months Files"/>
-						<separator orientation="vertical"/>
-						<field name="name" select="1"/>
-						<field name="month" select="1"/>
-	                    <field name="user" select="1"/>
-	                    <field name="directory" select="1"/>
-	           		</group>
+                            <filter icon="terp-go-year" string="This Year" domain="[('name','=',time.localtime()[0])]" help="All Months Files"/>
+                            <filter icon="terp-go-month" string="This Month" domain="[('month','=',time.localtime()[1])]" help="This Months Files"/>
+                            <separator orientation="vertical"/>
+                            <field name="name" select="1"/>
+                            <field name="month" select="1"/>
+                            <field name="user" select="1"/>
+                            <field name="directory" select="1"/>
+                        </group>
                </search>
             </field>
         </record>
@@ -61,7 +61,7 @@
             <field name="res_model">report.document.user</field>
             <field name="view_type">form</field>
             <field name="view_mode">tree</field>
-            <field name="context">{"search_default_user":uid}</field>
+            <field name="context">{'search_default_user': 'user_id'}</field>
             <field name="search_view_id" ref="view_report_document_user_search"/>
         </record>
 

addons/document/test/document_test2.yml

@@ -0,0 +1,73 @@
+- 
+    In order to test the document management
+-
+    I make sure the default installation has some storage and root directory.
+-
+    !assert {model: document.storage, id: storage_default }:
+      - id != False
+-
+    !assert {model: document.directory, id: dir_root}:
+      - storage_id != False
+-
+    I create a "Testing" folder where all the test data will go.
+-
+    !record {model: document.directory, id: dir_tests }:
+        name: 'Testing'
+        parent_id: dir_root
+-
+    I create an attachment into the root folder (w. empty fields, test that
+    defaults work)
+-
+    !record {model: ir.attachment, id: file_test1 }:
+        name: Test file
+-
+    I delete the attachment from the root folder
+-
+    !delete {model: ir.attachment, id: file_test1, search: }
+-
+    I create an attachment into the Testing folder.
+-
+    !record {model: ir.attachment, id: file_test2 }:
+        name: Test file 2
+        parent_id: dir_tests
+-
+    I update the attachment with data, namely "abcd"
+-
+    !record {model: ir.attachment, id: file_test2 }:
+        datas: "abcd"
+-
+    I test that the datas of the attachment are correct
+-
+    !assert {model: ir.attachment, id: file_test2 }:
+        - datas == "abcd\n"
+        - file_size == 5
+        - file_type == 'text/plain'
+-
+    I rename the attachment.
+-
+    !record {model: ir.attachment, id: file_test2 }:
+        name: Test renamed 2
+-
+    I search the testing folder for attachments.
+-
+    !python {model: ir.attachment}: |
+        ids = self.search(cr, uid, [('parent_id.name','=', 'Testing'), ('name','=','Test renamed 2')])
+        assert ids == [ ref("file_test2") ], ids
+-
+    I create an attachment to a 3rd resource, eg. a res.country
+-
+    !record {model: ir.attachment, id: attach_3rd }:
+        name: 'Res country attachment.txt'
+        parent_id: dir_tests
+        datas: 'defg'
+        res_model: res.country
+        res_id: !eval ref("base.za")
+-
+    I search for the res.country attachment
+-
+    !python {model: ir.attachment}: |
+        ids = self.search(cr, uid, [('res_model', '=', 'res.country'), ('res_id', '=', ref("base.za"))])
+        assert ids == [ ref("attach_3rd")], ids
+-
+    !delete {model: ir.attachment, id: attach_3rd, search: }
+-