diff --git a/openerp/addons/base/__openerp__.py b/openerp/addons/base/__openerp__.py index b4473434cdf..d400b28a729 100644 --- a/openerp/addons/base/__openerp__.py +++ b/openerp/addons/base/__openerp__.py @@ -93,16 +93,11 @@ The kernel of OpenERP, needed for all installation. 'res/res_partner_image_demo.xml', ], 'test': [ - 'test/base_test.xml', 'test/base_test.yml', 'test/test_context.xml', 'test/bug_lp541545.xml', 'test/test_osv_expression.yml', 'test/test_ir_rule.yml', # <-- These tests modify/add/delete ir_rules. - # Commented because this takes some time. - # This must be (un)commented with the corresponding import statement - # in test/__init__.py. - # 'test/test_ir_cron.yml', # <-- These tests perform a roolback. ], 'installable': True, 'auto_install': True, diff --git a/openerp/addons/base/ir/ir_attachment.py b/openerp/addons/base/ir/ir_attachment.py index 1e84dadd15c..5f54b4aca8e 100644 --- a/openerp/addons/base/ir/ir_attachment.py +++ b/openerp/addons/base/ir/ir_attachment.py @@ -18,11 +18,164 @@ # along with this program. If not, see . # ############################################################################## -import itertools +import hashlib +import itertools +import os +import re + +from openerp import tools from openerp.osv import fields,osv class ir_attachment(osv.osv): + """Attachments are used to link binary files or url to any openerp document. + + External attachment storage + --------------------------- + + The 'data' function field (_data_get,data_set) is implemented using + _file_read, _file_write and _file_delete which can be overridden to + implement other storage engines, shuch methods should check for other + location pseudo uri (example: hdfs://hadoppserver) + + The default implementation is the file:dirname location that stores files + on the local filesystem using name based on their sha1 hash + """ + def _name_get_resname(self, cr, uid, ids, object, method, context): + data = {} + for attachment in self.browse(cr, uid, ids, context=context): + model_object = attachment.res_model + res_id = attachment.res_id + if model_object and res_id: + model_pool = self.pool.get(model_object) + res = model_pool.name_get(cr,uid,[res_id],context) + res_name = res and res[0][1] or False + if res_name: + field = self._columns.get('res_name',False) + if field and len(res_name) > field.size: + res_name = res_name[:field.size-3] + '...' + data[attachment.id] = res_name + else: + data[attachment.id] = False + return data + + # 'data' field implementation + def _full_path(self, cr, uid, location, path): + # location = 'file:filestore' + assert location.startswith('file:'), "Unhandled filestore location %s" % location + location = location[5:] + + # sanitize location name and path + location = re.sub('[.]','',location) + location = location.strip('/\\') + + path = re.sub('[.]','',path) + path = path.strip('/\\') + return os.path.join(tools.config['root_path'], location, cr.dbname, path) + + def _file_read(self, cr, uid, location, fname, bin_size=False): + full_path = self._full_path(cr, uid, location, fname) + r = '' + try: + if bin_size: + r = os.path.getsize(full_path) + else: + r = open(full_path).read().encode('base64') + except IOError: + _logger.error("_read_file reading %s",full_path) + return r + + def _file_write(self, cr, uid, location, value): + bin_value = value.decode('base64') + fname = hashlib.sha1(bin_value).hexdigest() + # scatter files across 1024 dirs + # we use '/' in the db (even on windows) + fname = fname[:3] + '/' + fname + full_path = self._full_path(cr, uid, location, fname) + try: + dirname = os.path.dirname(full_path) + if not os.path.isdir(dirname): + os.makedirs(dirname) + open(full_path,'wb').write(bin_value) + except IOError: + _logger.error("_file_write writing %s",full_path) + return fname + + def _file_delete(self, cr, uid, location, fname): + count = self.search(cr, 1, [('store_fname','=',fname)], count=True) + if count <= 1: + full_path = self._full_path(cr, uid, location, fname) + try: + os.unlink(full_path) + except IOError: + # Harmless and needed for race conditions + _logger.error("_file_delete could not unlink %s",full_path) + + def _data_get(self, cr, uid, ids, name, arg, context=None): + if context is None: + context = {} + result = {} + location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location') + bin_size = context.get('bin_size') + for attach in self.browse(cr, uid, ids, context=context): + if location and attach.store_fname: + result[attach.id] = self._file_read(cr, uid, location, attach.store_fname, bin_size) + else: + result[attach.id] = attach.db_datas + return result + + def _data_set(self, cr, uid, id, name, value, arg, context=None): + # We dont handle setting data to null + if not value: + return True + if context is None: + context = {} + location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location') + file_size = len(value.decode('base64')) + if location: + attach = self.browse(cr, uid, id, context=context) + if attach.store_fname: + self._file_delete(cr, uid, location, attach.store_fname) + fname = self._file_write(cr, uid, location, value) + super(ir_attachment, self).write(cr, uid, [id], {'store_fname': fname, 'file_size': file_size}, context=context) + else: + super(ir_attachment, self).write(cr, uid, [id], {'db_datas': value, 'file_size': file_size}, context=context) + return True + + _name = 'ir.attachment' + _columns = { + 'name': fields.char('Attachment Name',size=256, required=True), + 'datas_fname': fields.char('File Name',size=256), + 'description': fields.text('Description'), + 'res_name': fields.function(_name_get_resname, type='char', size=128, string='Resource Name', store=True), + 'res_model': fields.char('Resource Model',size=64, readonly=True, help="The database object this attachment will be attached to"), + 'res_id': fields.integer('Resource ID', readonly=True, help="The record id this is attached to"), + 'create_date': fields.datetime('Date Created', readonly=True), + 'create_uid': fields.many2one('res.users', 'Owner', readonly=True), + 'company_id': fields.many2one('res.company', 'Company', change_default=True), + 'type': fields.selection( [ ('url','URL'), ('binary','Binary'), ], + 'Type', help="Binary File or URL", required=True, change_default=True), + 'url': fields.char('Url', size=1024), + # al: We keep shitty field names for backward compatibility with document + 'datas': fields.function(_data_get, fnct_inv=_data_set, string='File Content', type="binary", nodrop=True), + 'store_fname': fields.char('Stored Filename', size=256), + 'db_datas': fields.binary('Database Data'), + 'file_size': fields.integer('File Size'), + } + + _defaults = { + 'type': 'binary', + 'file_size': 0, + 'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c), + } + + def _auto_init(self, cr, context=None): + super(ir_attachment, self)._auto_init(cr, context) + cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_attachment_res_idx',)) + if not cr.fetchone(): + cr.execute('CREATE INDEX ir_attachment_res_idx ON ir_attachment (res_model, res_id)') + cr.commit() + def check(self, cr, uid, ids, mode, context=None, values=None): """Restricts the access to an ir.attachment, according to referred model In the 'document' module, it is overriden to relax this hard rule, since @@ -108,6 +261,8 @@ class ir_attachment(osv.osv): def write(self, cr, uid, ids, vals, context=None): self.check(cr, uid, ids, 'write', context=context, values=vals) + if 'file_size' in vals: + del vals['file_size'] return super(ir_attachment, self).write(cr, uid, ids, vals, context) def copy(self, cr, uid, id, default=None, context=None): @@ -116,70 +271,22 @@ class ir_attachment(osv.osv): def unlink(self, cr, uid, ids, context=None): self.check(cr, uid, ids, 'unlink', context=context) + location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location') + if location: + for attach in self.browse(cr, uid, ids, context=context): + if attach.store_fname: + self._file_delete(cr, uid, location, attach.store_fname) return super(ir_attachment, self).unlink(cr, uid, ids, context) def create(self, cr, uid, values, context=None): self.check(cr, uid, [], mode='create', context=context, values=values) + if 'file_size' in values: + del values['file_size'] return super(ir_attachment, self).create(cr, uid, values, context) def action_get(self, cr, uid, context=None): return self.pool.get('ir.actions.act_window').for_xml_id( cr, uid, 'base', 'action_attachment', context=context) - def _name_get_resname(self, cr, uid, ids, object, method, context): - data = {} - for attachment in self.browse(cr, uid, ids, context=context): - model_object = attachment.res_model - res_id = attachment.res_id - if model_object and res_id: - model_pool = self.pool.get(model_object) - res = model_pool.name_get(cr,uid,[res_id],context) - res_name = res and res[0][1] or False - if res_name: - field = self._columns.get('res_name',False) - if field and len(res_name) > field.size: - res_name = res_name[:field.size-3] + '...' - data[attachment.id] = res_name - else: - data[attachment.id] = False - return data - - _name = 'ir.attachment' - _columns = { - 'name': fields.char('Attachment Name',size=256, required=True), - 'datas': fields.binary('Data'), - 'datas_fname': fields.char('File Name',size=256), - 'description': fields.text('Description'), - 'res_name': fields.function(_name_get_resname, type='char', size=128, - string='Resource Name', store=True), - 'res_model': fields.char('Resource Object',size=64, readonly=True, - help="The database object this attachment will be attached to"), - 'res_id': fields.integer('Resource ID', readonly=True, - help="The record id this is attached to"), - 'url': fields.char('Url', size=512, oldname="link"), - 'type': fields.selection( - [ ('url','URL'), ('binary','Binary'), ], - 'Type', help="Binary File or external URL", required=True, change_default=True), - - 'create_date': fields.datetime('Date Created', readonly=True), - 'create_uid': fields.many2one('res.users', 'Owner', readonly=True), - 'company_id': fields.many2one('res.company', 'Company', change_default=True), - } - - _defaults = { - 'type': 'binary', - 'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'ir.attachment', context=c), - } - - def _auto_init(self, cr, context=None): - super(ir_attachment, self)._auto_init(cr, context) - cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_attachment_res_idx',)) - if not cr.fetchone(): - cr.execute('CREATE INDEX ir_attachment_res_idx ON ir_attachment (res_model, res_id)') - cr.commit() - -ir_attachment() - - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/addons/base/test/base_test.xml b/openerp/addons/base/test/base_test.xml deleted file mode 100644 index 27bcabfc8f9..00000000000 --- a/openerp/addons/base/test/base_test.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - ASUSTeK - - - - \ No newline at end of file diff --git a/openerp/addons/base/test/test_ir_cron.py b/openerp/addons/base/test/test_ir_cron.py deleted file mode 100644 index 6ef79ac3a78..00000000000 --- a/openerp/addons/base/test/test_ir_cron.py +++ /dev/null @@ -1,116 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# OpenERP, Open Source Management Solution -# Copyright (C) 2011-TODAY OpenERP S.A. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -import time -from datetime import datetime -from dateutil.relativedelta import relativedelta - -import openerp - -JOB = { - 'function': u'_0_seconds', - 'interval_type': u'minutes', - 'user_id': 1, - 'name': u'test', - 'args': False, - 'numbercall': 1, - 'nextcall': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - 'priority': 5, - 'doall': True, - 'active': True, - 'interval_number': 1, - 'model': u'ir.cron' -} - -class test_ir_cron(openerp.osv.osv.osv): - """ Add a few handy methods to test cron jobs scheduling. """ - _inherit = "ir.cron" - - def _0_seconds(a, b, c): - print ">>> _0_seconds" - - def _20_seconds(self, cr, uid): - print ">>> in _20_seconds" - time.sleep(20) - print ">>> out _20_seconds" - - def _80_seconds(self, cr, uid): - print ">>> in _80_seconds" - time.sleep(80) - print ">>> out _80_seconds" - - def test_0(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') - t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_0 _20_seconds A', function='_20_seconds', nextcall=t1)) - self.create(cr, uid, dict(JOB, name='test_0 _20_seconds B', function='_20_seconds', nextcall=t2)) - self.create(cr, uid, dict(JOB, name='test_0 _20_seconds C', function='_20_seconds', nextcall=t3)) - - def test_1(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_1 _20_seconds * 3', function='_20_seconds', nextcall=t1, numbercall=3)) - - def test_2(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_2 _80_seconds * 2', function='_80_seconds', nextcall=t1, numbercall=2)) - - def test_3(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') - t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_3 _80_seconds A', function='_80_seconds', nextcall=t1)) - self.create(cr, uid, dict(JOB, name='test_3 _20_seconds B', function='_20_seconds', nextcall=t2)) - self.create(cr, uid, dict(JOB, name='test_3 _20_seconds C', function='_20_seconds', nextcall=t3)) - - # This test assumes 4 cron threads. - def test_00(self, cr, uid): - self.test_00_set = set() - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') - t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_A', function='_20_seconds_A', nextcall=t1)) - self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_B', function='_20_seconds_B', nextcall=t2)) - self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_C', function='_20_seconds_C', nextcall=t3)) - - def _expect(self, cr, uid, to_add, to_sleep, to_expect_in, to_expect_out): - assert self.test_00_set == to_expect_in - self.test_00_set.add(to_add) - time.sleep(to_sleep) - self.test_00_set.discard(to_add) - assert self.test_00_set == to_expect_out - - def _20_seconds_A(self, cr, uid): - self._expect(cr, uid, 'A', 20, set(), set(['B', 'C'])) - - def _20_seconds_B(self, cr, uid): - self._expect(cr, uid, 'B', 20, set('A'), set('C')) - - def _20_seconds_C(self, cr, uid): - self._expect(cr, uid, 'C', 20, set(['A', 'B']), set()) - -# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: - diff --git a/openerp/addons/base/test/test_ir_cron.yml b/openerp/addons/base/test/test_ir_cron.yml deleted file mode 100644 index 16d648e1c30..00000000000 --- a/openerp/addons/base/test/test_ir_cron.yml +++ /dev/null @@ -1,61 +0,0 @@ -- - Test the cron jobs scheduling. -- - Disable the existing cron jobs if any during the tests. -- - !python {model: ir.cron }: | - # For this test to work, as it involves multiple database cursors, - # we have to commit changes. But YAML tests must be rollbacked, so - # the final database state is left untouched. So we have to be a bit - # ugly here: use our own cursor, commit, and clean after ourselves. - # We also pass around some ids using setattr/delattr, and we have to - # rollback the previous tests otherwise we won't be able to touch the - # db. - # Well, this should probably be a standalone, or regular unit test, - # instead of using the YAML infrastructure. - cr.rollback() - our_cr = self.pool.db.cursor() - try: - ids = self.search(our_cr, uid, [], {}) - setattr(self, 'saved_ids', ids) - self.write(our_cr, uid, ids, {'active': False}, {}) - our_cr.commit() - finally: - our_cr.close() -- - Three concurrent jobs started with a slight time gap. Assume 4 cron threads. - This will take about 2 minutes. -- - !python {model: ir.cron }: | - # Pretend initialization is already done. We the use a try/finally - # to reset _init correctly. - self.pool._init = False - our_cr = self.pool.db.cursor() - try: - self.test_00(our_cr, uid) # this will commit using the passed cursor - import openerp.cron - openerp.cron._thread_slots = 4 - # Wake up this db as soon as the master cron thread starts. - openerp.cron.schedule_wakeup(1, self.pool.db.dbname) - # Pretend to be the master thread, for 4 iterations. - openerp.cron.runner_body() - openerp.cron.runner_body() - openerp.cron.runner_body() - openerp.cron.runner_body() - finally: - self.pool._init = True - our_cr.close() -- - Clean after ourselves. -- - !python {model: ir.cron }: | - our_cr = self.pool.db.cursor() - try: - ids = [x for x in self.search(our_cr, uid, ['|', ('active', '=', True), ('active', '=', False)], {}) if x not in self.saved_ids] - self.unlink(our_cr, uid, ids, {}) - ids = self.saved_ids - delattr(self, 'saved_ids') - self.write(our_cr, uid, ids, {'active': True}, {}) - our_cr.commit() - finally: - our_cr.close() diff --git a/openerp/addons/base/tests/__init__.py b/openerp/addons/base/tests/__init__.py index 70f9a932d01..d891bc06fda 100644 --- a/openerp/addons/base/tests/__init__.py +++ b/openerp/addons/base/tests/__init__.py @@ -1,9 +1,15 @@ -import test_base, test_expression, test_search, test_ir_values, test_menu +import test_base +import test_expression +import test_ir_attachment +import test_ir_values +import test_menu +import test_search checks = [ test_base, test_expression, - test_search, + test_ir_attachment, test_ir_values, test_menu, + test_search, ] diff --git a/openerp/addons/base/tests/test_ir_attachment.py b/openerp/addons/base/tests/test_ir_attachment.py new file mode 100644 index 00000000000..4d96ec42b28 --- /dev/null +++ b/openerp/addons/base/tests/test_ir_attachment.py @@ -0,0 +1,89 @@ +import hashlib +import os + +import unittest2 + +import openerp +import openerp.tests.common + +class test_ir_attachment(openerp.tests.common.TransactionCase): + + def test_00_attachment_flow(self): + registry, cr, uid = self.registry, self.cr, self.uid + root_path = openerp.tools.config['root_path'] + ira = registry('ir.attachment') + + # Blob1 + blob1 = 'blob1' + blob1_b64 = blob1.encode('base64') + blob1_hash = hashlib.sha1(blob1).hexdigest() + blob1_fname = blob1_hash[:3] + '/' + blob1_hash + + # Blob2 + blob2 = 'blob2' + blob2_b64 = blob2.encode('base64') + blob2_hash = hashlib.sha1(blob2).hexdigest() + blob2_fname = blob2_hash[:3] + '/' + blob2_hash + + # 'ir_attachment.location' is undefined test database storage + a1 = ira.create(cr, uid, {'name': 'a1', 'datas': blob1_b64}) + a1_read = ira.read(cr, uid, [a1], ['datas']) + self.assertEqual(a1_read[0]['datas'], blob1_b64) + + cr.execute("select id,db_datas from ir_attachment where id = %s", (a1,) ) + a1_db_datas = str(cr.fetchall()[0][1]) + self.assertEqual(a1_db_datas, blob1_b64) + + # define a location for filestore + registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', 'file:///filestore') + + # Test file storage + a2 = ira.create(cr, uid, {'name': 'a2', 'datas': blob1_b64}) + a2_read = ira.read(cr, uid, [a2], ['datas']) + self.assertEqual(a2_read[0]['datas'], blob1_b64) + + cr.execute("select id,store_fname from ir_attachment where id = %s", (a2,) ) + a2_store_fname = cr.fetchall()[0][1] + self.assertEqual(a2_store_fname, blob1_fname) + + a2_fn = os.path.join(root_path, 'filestore', cr.dbname, blob1_hash[:3], blob1_hash) + fc = file(a2_fn).read() + self.assertEqual(fc, blob1) + + # create a3 with same blob + a3 = ira.create(cr, uid, {'name': 'a3', 'datas': blob1_b64}) + a3_read = ira.read(cr, uid, [a3], ['datas']) + self.assertEqual(a3_read[0]['datas'], blob1_b64) + + cr.execute("select id,store_fname from ir_attachment where id = %s", (a3,) ) + a3_store_fname = cr.fetchall()[0][1] + self.assertEqual(a3_store_fname, a2_store_fname) + + # create a4 blob2 + a4 = ira.create(cr, uid, {'name': 'a4', 'datas': blob2_b64}) + a4_read = ira.read(cr, uid, [a4], ['datas']) + self.assertEqual(a4_read[0]['datas'], blob2_b64) + + a4_fn = os.path.join(root_path, 'filestore', cr.dbname, blob2_hash[:3], blob2_hash) + self.assertTrue(os.path.isfile(a4_fn)) + + # delete a3 but file stays + ira.unlink(cr, uid, [a3]) + self.assertTrue(os.path.isfile(a2_fn)) + + # delete a2 it is unlinked + ira.unlink(cr, uid, [a2]) + self.assertFalse(os.path.isfile(a2_fn)) + + # update a4 blob2 by blob1 + ira.write(cr, uid, [a4], {'datas': blob1_b64}) + a4_read = ira.read(cr, uid, [a4], ['datas']) + self.assertEqual(a4_read[0]['datas'], blob1_b64) + + # file of a4 disapear and a2 reappear + self.assertFalse(os.path.isfile(a4_fn)) + self.assertTrue(os.path.isfile(a2_fn)) + + # everybody applause + + diff --git a/openerp/tests/__init__.py b/openerp/tests/__init__.py index 2606f421669..7f82c414fdd 100644 --- a/openerp/tests/__init__.py +++ b/openerp/tests/__init__.py @@ -8,12 +8,20 @@ Tests can be explicitely added to the `fast_suite` or `checks` lists or not. See the :ref:`test-framework` section in the :ref:`features` list. """ -from . import test_acl -from . import test_expression, test_mail, test_ir_sequence, test_orm, \ - test_fields, test_basecase, \ - test_view_validation, test_uninstall, test_misc, test_db_cursor, \ - test_osv, test_translate -from . import test_ir_filters +import test_acl +import test_basecase +import test_db_cursor +import test_expression +import test_fields +import test_ir_filters +import test_ir_sequence +import test_mail +import test_misc +import test_orm +import test_osv +import test_translate +import test_uninstall +import test_view_validation fast_suite = [ test_ir_sequence, @@ -33,5 +41,5 @@ checks = [ test_osv, test_translate, ] - + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: