pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[MERGE] ir_attachement filestorage and deduplication

Browse Source 
can be set with ir.config.parameters key ir_attachement.location

bzr revid: al@openerp.com-20121220025613-sxerd2t6sv7teqn8
This commit is contained in:
Antony Lesuisse
2012-12-20 03:56:13 +01:00
parent a24164ced0 0ed151708a
commit eb6e47e711
8 changed files with 275 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openerp/addons/base/__openerp__.py
-5
View File
@@ -93,16 +93,11 @@ The kernel of OpenERP, needed for all installation.
'res/res_partner_image_demo.xml',
],
'test': [
'test/base_test.xml',
'test/base_test.yml',
'test/test_context.xml',
'test/bug_lp541545.xml',
'test/test_osv_expression.yml',
'test/test_ir_rule.yml', # <-- These tests modify/add/delete ir_rules.
# Commented because this takes some time.
# This must be (un)commented with the corresponding import statement
# in test/__init__.py.
# 'test/test_ir_cron.yml', # <-- These tests perform a roolback.
],
'installable': True,
'auto_install': True,
openerp/addons/base/ir/ir_attachment.py
+163 -56
View File
@@ -18,11 +18,164 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import itertools
import hashlib
import itertools
import os
import re
from openerp import tools
from openerp.osv import fields,osv
class ir_attachment(osv.osv):
"""Attachments are used to link binary files or url to any openerp document.
External attachment storage
---------------------------
The 'data' function field (_data_get,data_set) is implemented using
_file_read, _file_write and _file_delete which can be overridden to
implement other storage engines, shuch methods should check for other
location pseudo uri (example: hdfs://hadoppserver)
The default implementation is the file:dirname location that stores files
on the local filesystem using name based on their sha1 hash
"""
def _name_get_resname(self, cr, uid, ids, object, method, context):
data = {}
for attachment in self.browse(cr, uid, ids, context=context):
model_object = attachment.res_model
res_id = attachment.res_id
if model_object and res_id:
model_pool = self.pool.get(model_object)
res = model_pool.name_get(cr,uid,[res_id],context)
res_name = res and res[0][1] or False
if res_name:
field = self._columns.get('res_name',False)
if field and len(res_name) > field.size:
res_name = res_name[:field.size-3] + '...'
data[attachment.id] = res_name
else:
data[attachment.id] = False
return data
# 'data' field implementation
def _full_path(self, cr, uid, location, path):
# location = 'file:filestore'
assert location.startswith('file:'), "Unhandled filestore location %s" % location
location = location[5:]
# sanitize location name and path
location = re.sub('[.]','',location)
location = location.strip('/\\')
path = re.sub('[.]','',path)
path = path.strip('/\\')
return os.path.join(tools.config['root_path'], location, cr.dbname, path)
def _file_read(self, cr, uid, location, fname, bin_size=False):
full_path = self._full_path(cr, uid, location, fname)
r = ''
try:
if bin_size:
r = os.path.getsize(full_path)
else:
r = open(full_path).read().encode('base64')
except IOError:
_logger.error("_read_file reading %s",full_path)
return r
def _file_write(self, cr, uid, location, value):
bin_value = value.decode('base64')
fname = hashlib.sha1(bin_value).hexdigest()
# scatter files across 1024 dirs
# we use '/' in the db (even on windows)
fname = fname[:3] + '/' + fname
full_path = self._full_path(cr, uid, location, fname)
try:
dirname = os.path.dirname(full_path)
if not os.path.isdir(dirname):
os.makedirs(dirname)
open(full_path,'wb').write(bin_value)
except IOError:
_logger.error("_file_write writing %s",full_path)
return fname
def _file_delete(self, cr, uid, location, fname):
count = self.search(cr, 1, [('store_fname','=',fname)], count=True)
if count <= 1:
full_path = self._full_path(cr, uid, location, fname)
try:
os.unlink(full_path)
except IOError:
# Harmless and needed for race conditions
_logger.error("_file_delete could not unlink %s",full_path)
def _data_get(self, cr, uid, ids, name, arg, context=None):
if context is None:
context = {}
result = {}
location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location')
bin_size = context.get('bin_size')
for attach in self.browse(cr, uid, ids, context=context):
if location and attach.store_fname:
result[attach.id] = self._file_read(cr, uid, location, attach.store_fname, bin_size)
else:
result[attach.id] = attach.db_datas
return result
def _data_set(self, cr, uid, id, name, value, arg, context=None):
# We dont handle setting data to null
if not value:
return True
if context is None:
context = {}
location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location')
file_size = len(value.decode('base64'))
if location:
attach = self.browse(cr, uid, id, context=context)
if attach.store_fname:
self._file_delete(cr, uid, location, attach.store_fname)
fname = self._file_write(cr, uid, location, value)
super(ir_attachment, self).write(cr, uid, [id], {'store_fname': fname, 'file_size': file_size}, context=context)
else:
super(ir_attachment, self).write(cr, uid, [id], {'db_datas': value, 'file_size': file_size}, context=context)
return True
_name = 'ir.attachment'
_columns = {
'name': fields.char('Attachment Name',size=256, required=True),
'datas_fname': fields.char('File Name',size=256),
'description': fields.text('Description'),
'res_name': fields.function(_name_get_resname, type='char', size=128, string='Resource Name', store=True),
'res_model': fields.char('Resource Model',size=64, readonly=True, help="The database object this attachment will be attached to"),
'res_id': fields.integer('Resource ID', readonly=True, help="The record id this is attached to"),
'create_date': fields.datetime('Date Created', readonly=True),
'create_uid': fields.many2one('res.users', 'Owner', readonly=True),
'company_id': fields.many2one('res.company', 'Company', change_default=True),
'type': fields.selection( [ ('url','URL'), ('binary','Binary'), ],
'Type', help="Binary File or URL", required=True, change_default=True),
'url': fields.char('Url', size=1024),
# al: We keep shitty field names for backward compatibility with document
'datas': fields.function(_data_get, fnct_inv=_data_set, string='File Content', type="binary", nodrop=True),
'store_fname': fields.char('Stored Filename', size=256),
'db_datas': fields.binary('Database Data'),
'file_size': fields.integer('File Size'),
}
_defaults = {
'type': 'binary',
'file_size': 0,
'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c),
}
def _auto_init(self, cr, context=None):
super(ir_attachment, self)._auto_init(cr, context)
cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_attachment_res_idx',))
if not cr.fetchone():
cr.execute('CREATE INDEX ir_attachment_res_idx ON ir_attachment (res_model, res_id)')
cr.commit()
def check(self, cr, uid, ids, mode, context=None, values=None):
"""Restricts the access to an ir.attachment, according to referred model
In the 'document' module, it is overriden to relax this hard rule, since
@@ -108,6 +261,8 @@ class ir_attachment(osv.osv):
def write(self, cr, uid, ids, vals, context=None):
self.check(cr, uid, ids, 'write', context=context, values=vals)
if 'file_size' in vals:
del vals['file_size']
return super(ir_attachment, self).write(cr, uid, ids, vals, context)
def copy(self, cr, uid, id, default=None, context=None):
@@ -116,70 +271,22 @@ class ir_attachment(osv.osv):
def unlink(self, cr, uid, ids, context=None):
self.check(cr, uid, ids, 'unlink', context=context)
location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location')
if location:
for attach in self.browse(cr, uid, ids, context=context):
if attach.store_fname:
self._file_delete(cr, uid, location, attach.store_fname)
return super(ir_attachment, self).unlink(cr, uid, ids, context)
def create(self, cr, uid, values, context=None):
self.check(cr, uid, [], mode='create', context=context, values=values)
if 'file_size' in values:
del values['file_size']
return super(ir_attachment, self).create(cr, uid, values, context)
def action_get(self, cr, uid, context=None):
return self.pool.get('ir.actions.act_window').for_xml_id(
cr, uid, 'base', 'action_attachment', context=context)
def _name_get_resname(self, cr, uid, ids, object, method, context):
data = {}
for attachment in self.browse(cr, uid, ids, context=context):
model_object = attachment.res_model
res_id = attachment.res_id
if model_object and res_id:
model_pool = self.pool.get(model_object)
res = model_pool.name_get(cr,uid,[res_id],context)
res_name = res and res[0][1] or False
if res_name:
field = self._columns.get('res_name',False)
if field and len(res_name) > field.size:
res_name = res_name[:field.size-3] + '...'
data[attachment.id] = res_name
else:
data[attachment.id] = False
return data
_name = 'ir.attachment'
_columns = {
'name': fields.char('Attachment Name',size=256, required=True),
'datas': fields.binary('Data'),
'datas_fname': fields.char('File Name',size=256),
'description': fields.text('Description'),
'res_name': fields.function(_name_get_resname, type='char', size=128,
string='Resource Name', store=True),
'res_model': fields.char('Resource Object',size=64, readonly=True,
help="The database object this attachment will be attached to"),
'res_id': fields.integer('Resource ID', readonly=True,
help="The record id this is attached to"),
'url': fields.char('Url', size=512, oldname="link"),
'type': fields.selection(
[ ('url','URL'), ('binary','Binary'), ],
'Type', help="Binary File or external URL", required=True, change_default=True),
'create_date': fields.datetime('Date Created', readonly=True),
'create_uid': fields.many2one('res.users', 'Owner', readonly=True),
'company_id': fields.many2one('res.company', 'Company', change_default=True),
}
_defaults = {
'type': 'binary',
'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c),
}
def _auto_init(self, cr, context=None):
super(ir_attachment, self)._auto_init(cr, context)
cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_attachment_res_idx',))
if not cr.fetchone():
cr.execute('CREATE INDEX ir_attachment_res_idx ON ir_attachment (res_model, res_id)')
cr.commit()
ir_attachment()
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
openerp/addons/base/test/base_test.xml
-14
View File
@@ -1,14 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<openerp>
<data>
<!--
Assert:
-->
<assert model="res.request" id="res_partner_asustek_request" string="Reference field request returns a browsable object">
<test expr="ref_doc1.name">ASUSTeK</test>
</assert>
</data>
</openerp>
openerp/addons/base/test/test_ir_cron.py
-116
View File
@@ -1,116 +0,0 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# Copyright (C) 2011-TODAY OpenERP S.A. <http://www.openerp.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import time
from datetime import datetime
from dateutil.relativedelta import relativedelta
import openerp
JOB = {
'function': u'_0_seconds',
'interval_type': u'minutes',
'user_id': 1,
'name': u'test',
'args': False,
'numbercall': 1,
'nextcall': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'priority': 5,
'doall': True,
'active': True,
'interval_number': 1,
'model': u'ir.cron'
}
class test_ir_cron(openerp.osv.osv.osv):
""" Add a few handy methods to test cron jobs scheduling. """
_inherit = "ir.cron"
def _0_seconds(a, b, c):
print ">>> _0_seconds"
def _20_seconds(self, cr, uid):
print ">>> in _20_seconds"
time.sleep(20)
print ">>> out _20_seconds"
def _80_seconds(self, cr, uid):
print ">>> in _80_seconds"
time.sleep(80)
print ">>> out _80_seconds"
def test_0(self, cr, uid):
now = datetime.now()
t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S')
t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S')
t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S')
self.create(cr, uid, dict(JOB, name='test_0 _20_seconds A', function='_20_seconds', nextcall=t1))
self.create(cr, uid, dict(JOB, name='test_0 _20_seconds B', function='_20_seconds', nextcall=t2))
self.create(cr, uid, dict(JOB, name='test_0 _20_seconds C', function='_20_seconds', nextcall=t3))
def test_1(self, cr, uid):
now = datetime.now()
t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S')
self.create(cr, uid, dict(JOB, name='test_1 _20_seconds * 3', function='_20_seconds', nextcall=t1, numbercall=3))
def test_2(self, cr, uid):
now = datetime.now()
t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S')
self.create(cr, uid, dict(JOB, name='test_2 _80_seconds * 2', function='_80_seconds', nextcall=t1, numbercall=2))
def test_3(self, cr, uid):
now = datetime.now()
t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S')
t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S')
t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S')
self.create(cr, uid, dict(JOB, name='test_3 _80_seconds A', function='_80_seconds', nextcall=t1))
self.create(cr, uid, dict(JOB, name='test_3 _20_seconds B', function='_20_seconds', nextcall=t2))
self.create(cr, uid, dict(JOB, name='test_3 _20_seconds C', function='_20_seconds', nextcall=t3))
# This test assumes 4 cron threads.
def test_00(self, cr, uid):
self.test_00_set = set()
now = datetime.now()
t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S')
t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S')
t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S')
self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_A', function='_20_seconds_A', nextcall=t1))
self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_B', function='_20_seconds_B', nextcall=t2))
self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_C', function='_20_seconds_C', nextcall=t3))
def _expect(self, cr, uid, to_add, to_sleep, to_expect_in, to_expect_out):
assert self.test_00_set == to_expect_in
self.test_00_set.add(to_add)
time.sleep(to_sleep)
self.test_00_set.discard(to_add)
assert self.test_00_set == to_expect_out
def _20_seconds_A(self, cr, uid):
self._expect(cr, uid, 'A', 20, set(), set(['B', 'C']))
def _20_seconds_B(self, cr, uid):
self._expect(cr, uid, 'B', 20, set('A'), set('C'))
def _20_seconds_C(self, cr, uid):
self._expect(cr, uid, 'C', 20, set(['A', 'B']), set())
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
openerp/addons/base/test/test_ir_cron.yml
-61
View File
@@ -1,61 +0,0 @@
-
Test the cron jobs scheduling.
-
Disable the existing cron jobs if any during the tests.
-
!python {model: ir.cron }: |
# For this test to work, as it involves multiple database cursors,
# we have to commit changes. But YAML tests must be rollbacked, so
# the final database state is left untouched. So we have to be a bit
# ugly here: use our own cursor, commit, and clean after ourselves.
# We also pass around some ids using setattr/delattr, and we have to
# rollback the previous tests otherwise we won't be able to touch the
# db.
# Well, this should probably be a standalone, or regular unit test,
# instead of using the YAML infrastructure.
cr.rollback()
our_cr = self.pool.db.cursor()
try:
ids = self.search(our_cr, uid, [], {})
setattr(self, 'saved_ids', ids)
self.write(our_cr, uid, ids, {'active': False}, {})
our_cr.commit()
finally:
our_cr.close()
-
Three concurrent jobs started with a slight time gap. Assume 4 cron threads.
This will take about 2 minutes.
-
!python {model: ir.cron }: |
# Pretend initialization is already done. We the use a try/finally
# to reset _init correctly.
self.pool._init = False
our_cr = self.pool.db.cursor()
try:
self.test_00(our_cr, uid) # this will commit using the passed cursor
import openerp.cron
openerp.cron._thread_slots = 4
# Wake up this db as soon as the master cron thread starts.
openerp.cron.schedule_wakeup(1, self.pool.db.dbname)
# Pretend to be the master thread, for 4 iterations.
openerp.cron.runner_body()
openerp.cron.runner_body()
openerp.cron.runner_body()
openerp.cron.runner_body()
finally:
self.pool._init = True
our_cr.close()
-
Clean after ourselves.
-
!python {model: ir.cron }: |
our_cr = self.pool.db.cursor()
try:
ids = [x for x in self.search(our_cr, uid, ['|', ('active', '=', True), ('active', '=', False)], {}) if x not in self.saved_ids]
self.unlink(our_cr, uid, ids, {})
ids = self.saved_ids
delattr(self, 'saved_ids')
self.write(our_cr, uid, ids, {'active': True}, {})
our_cr.commit()
finally:
our_cr.close()
openerp/addons/base/tests/__init__.py
+8 -2
View File
@@ -1,9 +1,15 @@
import test_base, test_expression, test_search, test_ir_values, test_menu
import test_base
import test_expression
import test_ir_attachment
import test_ir_values
import test_menu
import test_search
checks = [
test_base,
test_expression,
test_search,
test_ir_attachment,
test_ir_values,
test_menu,
test_search,
]
openerp/addons/base/tests/test_ir_attachment.py
+89
View File
@@ -0,0 +1,89 @@
import hashlib
import os
import unittest2
import openerp
import openerp.tests.common
class test_ir_attachment(openerp.tests.common.TransactionCase):
def test_00_attachment_flow(self):
registry, cr, uid = self.registry, self.cr, self.uid
root_path = openerp.tools.config['root_path']
ira = registry('ir.attachment')
# Blob1
blob1 = 'blob1'
blob1_b64 = blob1.encode('base64')
blob1_hash = hashlib.sha1(blob1).hexdigest()
blob1_fname = blob1_hash[:3] + '/' + blob1_hash
# Blob2
blob2 = 'blob2'
blob2_b64 = blob2.encode('base64')
blob2_hash = hashlib.sha1(blob2).hexdigest()
blob2_fname = blob2_hash[:3] + '/' + blob2_hash
# 'ir_attachment.location' is undefined test database storage
a1 = ira.create(cr, uid, {'name': 'a1', 'datas': blob1_b64})
a1_read = ira.read(cr, uid, [a1], ['datas'])
self.assertEqual(a1_read[0]['datas'], blob1_b64)
cr.execute("select id,db_datas from ir_attachment where id = %s", (a1,) )
a1_db_datas = str(cr.fetchall()[0][1])
self.assertEqual(a1_db_datas, blob1_b64)
# define a location for filestore
registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', 'file:///filestore')
# Test file storage
a2 = ira.create(cr, uid, {'name': 'a2', 'datas': blob1_b64})
a2_read = ira.read(cr, uid, [a2], ['datas'])
self.assertEqual(a2_read[0]['datas'], blob1_b64)
cr.execute("select id,store_fname from ir_attachment where id = %s", (a2,) )
a2_store_fname = cr.fetchall()[0][1]
self.assertEqual(a2_store_fname, blob1_fname)
a2_fn = os.path.join(root_path, 'filestore', cr.dbname, blob1_hash[:3], blob1_hash)
fc = file(a2_fn).read()
self.assertEqual(fc, blob1)
# create a3 with same blob
a3 = ira.create(cr, uid, {'name': 'a3', 'datas': blob1_b64})
a3_read = ira.read(cr, uid, [a3], ['datas'])
self.assertEqual(a3_read[0]['datas'], blob1_b64)
cr.execute("select id,store_fname from ir_attachment where id = %s", (a3,) )
a3_store_fname = cr.fetchall()[0][1]
self.assertEqual(a3_store_fname, a2_store_fname)
# create a4 blob2
a4 = ira.create(cr, uid, {'name': 'a4', 'datas': blob2_b64})
a4_read = ira.read(cr, uid, [a4], ['datas'])
self.assertEqual(a4_read[0]['datas'], blob2_b64)
a4_fn = os.path.join(root_path, 'filestore', cr.dbname, blob2_hash[:3], blob2_hash)
self.assertTrue(os.path.isfile(a4_fn))
# delete a3 but file stays
ira.unlink(cr, uid, [a3])
self.assertTrue(os.path.isfile(a2_fn))
# delete a2 it is unlinked
ira.unlink(cr, uid, [a2])
self.assertFalse(os.path.isfile(a2_fn))
# update a4 blob2 by blob1
ira.write(cr, uid, [a4], {'datas': blob1_b64})
a4_read = ira.read(cr, uid, [a4], ['datas'])
self.assertEqual(a4_read[0]['datas'], blob1_b64)
# file of a4 disapear and a2 reappear
self.assertFalse(os.path.isfile(a4_fn))
self.assertTrue(os.path.isfile(a2_fn))
# everybody applause
openerp/tests/__init__.py
+15 -7
View File
@@ -8,12 +8,20 @@ Tests can be explicitely added to the `fast_suite` or `checks` lists or not.
See the :ref:`test-framework` section in the :ref:`features` list.
"""
from . import test_acl
from . import test_expression, test_mail, test_ir_sequence, test_orm, \
test_fields, test_basecase, \
test_view_validation, test_uninstall, test_misc, test_db_cursor, \
test_osv, test_translate
from . import test_ir_filters
import test_acl
import test_basecase
import test_db_cursor
import test_expression
import test_fields
import test_ir_filters
import test_ir_sequence
import test_mail
import test_misc
import test_orm
import test_osv
import test_translate
import test_uninstall
import test_view_validation
fast_suite = [
test_ir_sequence,
@@ -33,5 +41,5 @@ checks = [
test_osv,
test_translate,
]
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: