diff --git a/openerp/addons/base/test/base_test.yml b/openerp/addons/base/test/base_test.yml index 336b3be2a82..5a17ca108c3 100644 --- a/openerp/addons/base/test/base_test.yml +++ b/openerp/addons/base/test/base_test.yml @@ -52,7 +52,6 @@ # but other exceptions probably indicate that open() was executed! assert False, "safe_eval should not allow calling open() builtin" - - "ORM test: verify that parent_store computation are going right" - @@ -145,3 +144,45 @@ !python {model: res.partner.category}: | self.pool._init = True +- + "OSV Memory: Verify that osv_memory properly handles large data allocation" +- + 1. No "count-based" auto-vaccuum when max_count is disabled +- + !python {model: base.language.export}: | + # setup special limits for the test, these will be reset at next pool reload anyway + self._max_count = None + num_recs = 250 + for i in xrange(num_recs): + self.create(cr, uid, {'format':'po'}) + assert (len(self.datas) >= num_recs), "OSV Memory must not auto-vaccum records from the current transaction if max_count is not set" +- + 2. Auto-vaccuum should be enabled when max_count is set +- + !python {model: base.language.export}: | + # setup special limits for the test, these will be reset at next pool reload anyway + self._max_count = 100 + num_recs = 219 + for i in xrange(num_recs): + self.create(cr, uid, {'name': i, 'format':'po'}) + assert (self._max_count <= len(self.datas) < self._max_count + self._check_time), "OSV Memory must auto-expire records when max_count is reached" + for k,v in self.datas.iteritems(): + assert (int(v['name']) >= (num_recs - (self._max_count + self._check_time))), "OSV Memory must auto-expire records based on age" +- + 3. Auto-vaccuum should be based on age only when max_count is not set +- + !python {model: base.language.export}: | + # setup special limits for the test, these will be reset at next pool reload anyway + self._max_count = None + self._max_hours = 0.01 #36 seconds + num_recs = 200 + for i in xrange(num_recs): + self.create(cr, uid, {'format':'po'}) + assert (len(self.datas) >= num_recs), "OSV Memory must not auto-expire records from the current transaction" + + # expire all records + for k,v in self.datas.iteritems(): + v['internal.date_access'] = 0 + self.vaccum(cr, 1, force=True) + + assert (len(self.datas) == 0), "OSV Memory must expire old records after vaccuum" diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 758ab84aaea..e23140d28bd 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -1884,8 +1884,8 @@ class orm_memory(orm_template): _protected = ['read', 'write', 'create', 'default_get', 'perm_read', 'unlink', 'fields_get', 'fields_view_get', 'search', 'name_get', 'distinct_field_get', 'name_search', 'copy', 'import_data', 'search_count', 'exists'] _inherit_fields = {} - _max_count = 200 - _max_hours = 1 + _max_count = config.get('osv_memory_count_limit') + _max_hours = config.get('osv_memory_age_limit') _check_time = 20 def __init__(self, cr): @@ -1899,21 +1899,32 @@ class orm_memory(orm_template): if uid != 1 and self.datas[object_id]['internal.create_uid'] != uid: raise except_orm(_('AccessError'), '%s access is only allowed on your own records for osv_memory objects except for the super-user' % mode.capitalize()) - def vaccum(self, cr, uid): + def vaccum(self, cr, uid, force=False): + """Run the vaccuum cleaning system, expiring and removing old records from the + virtual osv_memory tables if the "max count" or "max age" conditions are enabled + and have been reached. This method can be called very often (e.g. everytime a record + is created), but will only actually trigger the cleanup process once out of + "_check_time" times (by default once out of 20 calls).""" self.check_id += 1 - if self.check_id % self._check_time: + if (not force) and (self.check_id % self._check_time): return True tounlink = [] - max = time.time() - self._max_hours * 60 * 60 - for id in self.datas: - if self.datas[id]['internal.date_access'] < max: - tounlink.append(id) - self.unlink(cr, 1, tounlink) - if len(self.datas) > self._max_count: - sorted = map(lambda x: (x[1]['internal.date_access'], x[0]), self.datas.items()) - sorted.sort() - ids = map(lambda x: x[1], sorted[:len(self.datas)-self._max_count]) - self.unlink(cr, uid, ids) + + # Age-based expiration + if self._max_hours: + max = time.time() - self._max_hours * 60 * 60 + for k,v in self.datas.iteritems(): + if v['internal.date_access'] < max: + tounlink.append(k) + self.unlink(cr, 1, tounlink) + + # Count-based expiration + if self._max_count and len(self.datas) > self._max_count: + # sort by access time to remove only the first/oldest ones in LRU fashion + records = self.datas.items() + records.sort(key=lambda x:x[1]['internal.date_access']) + self.unlink(cr, 1, [x[0] for x in records[:len(self.datas)-self._max_count]]) + return True def read(self, cr, user, ids, fields_to_read=None, context=None, load='_classic_read'): diff --git a/openerp/tools/config.py b/openerp/tools/config.py index e92fe98be90..ba12bc74f9a 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -95,6 +95,8 @@ class configmanager(object): 'secure_cert_file': 'server.cert', 'secure_pkey_file': 'server.pkey', 'publisher_warranty_url': 'http://services.openerp.com/publisher-warranty/', + 'osv_memory_count_limit': None, # number of records in each osv_memory virtual table + 'osv_memory_age_limit': 1, # hours } self.blacklist_for_save = set(["publisher_warranty_url", "load_language"]) @@ -235,6 +237,15 @@ class configmanager(object): group.add_option("--stop-after-init", action="store_true", dest="stop_after_init", default=False, help="stop the server after it initializes") group.add_option("-t", "--timezone", dest="timezone", help="specify reference timezone for the server (e.g. Europe/Brussels") + group.add_option("--osv-memory-count-limit", dest="osv_memory_count_limit", default=False, + help="Force a limit on the maximum number of records kept in the virtual " + "osv_memory tables. The default is False, which means no count-based limit.", + type="int") + group.add_option("--osv-memory-age-limit", dest="osv_memory_age_limit", default=1.0, + help="Force a limit on the maximum age of records kept in the virtual " + "osv_memory tables. This is a decimal value expressed in hours, " + "and the default is 1 hour.", + type="float") parser.add_option_group(group) self.parse_config() @@ -305,7 +316,8 @@ class configmanager(object): 'debug_mode', 'smtp_ssl', 'load_language', 'stop_after_init', 'logrotate', 'without_demo', 'netrpc', 'xmlrpc', 'syslog', 'list_db', 'xmlrpcs', - 'test_file', 'test_disable', 'test_commit', 'test_report_directory' + 'test_file', 'test_disable', 'test_commit', 'test_report_directory', + 'osv_memory_count_limit', 'osv_memory_age_limit', ] for arg in keys: