diff --git a/addons/account/models/chart_template.py b/addons/account/models/chart_template.py index 5245c4b97a9..cb7512f43f4 100644 --- a/addons/account/models/chart_template.py +++ b/addons/account/models/chart_template.py @@ -4,6 +4,7 @@ from odoo.exceptions import AccessError from odoo import api, fields, models, _ from odoo import SUPERUSER_ID from odoo.exceptions import UserError +from odoo.tools import pycompat import logging @@ -390,8 +391,35 @@ class AccountChartTemplate(models.Model): # xmlid is the concatenation of company_id and template_xml_id ir_model_data = self.env['ir.model.data'] template_xmlid = ir_model_data.search([('model', '=', template._name), ('res_id', '=', template.id)]) - new_xml_id = str(company.id)+'_'+template_xmlid.name - return ir_model_data._update(model, template_xmlid.module, vals, xml_id=new_xml_id, store=True, noupdate=True, mode='init', res_id=False) + xml_id = "%s.%s_%s" % (template_xmlid.module, company.id, template_xmlid.name) + data = dict(xml_id=xml_id, values=vals, noupdate=True) + record = self.env[model]._load_records([data]) + return record.id + + @api.model + def _load_records(self, data_list, update=False): + # When creating a chart template create, for the liquidity transfer account + # - an account.account.template: this allow to define account.reconcile.model.template objects refering that liquidity transfer + # account although it's not existing in any xml file + # - an entry in ir_model_data: this allow to still use the method create_record_with_xmlid() and don't make any difference between + # regular accounts created and that liquidity transfer account + records = super(AccountChartTemplate, self)._load_records(data_list, update) + account_data_list = [] + for data, record in pycompat.izip(data_list, records): + # Create the transfer account only for leaf chart template in the hierarchy. + if record.parent_id: + continue + if data.get('xml_id'): + account_xml_id = data['xml_id'] + '_liquidity_transfer' + if not self.env.ref(account_xml_id, raise_if_not_found=False): + account_vals = record._prepare_transfer_account_template() + account_data_list.append(dict( + xml_id=account_xml_id, + values=account_vals, + noupdate=data.get('noupdate'), + )) + self.env['account.account.template']._load_records(account_data_list, update) + return records def _get_account_vals(self, company, account_template, code_acc, tax_template_ref): """ This method generates a dictionary of all the values for the account that will be created. @@ -1036,28 +1064,3 @@ class AccountReconcileModelTemplate(models.Model): ], string="Second Amount type",required=True, default='percentage') second_amount = fields.Float(string='Second Amount', digits=0, required=True, default=100.0, help="Fixed amount will count as a debit if it is negative, as a credit if it is positive.") second_tax_id = fields.Many2one('account.tax.template', string='Second Tax', ondelete='restrict', domain=[('type_tax_use', '=', 'purchase')]) - - -class IrModelData(models.Model): - _inherit = 'ir.model.data' - - @api.model - def _update(self, model, module, values, xml_id=False, store=True, noupdate=False, mode='init', res_id=False): - record_id = super(IrModelData, self)._update(model, module, values, xml_id=xml_id, store=store, noupdate=noupdate, mode=mode, res_id=res_id) - # When creating a chart template create, for the liquidity transfer account - # - an account.account.template: this allow to define account.reconcile.model.template objects refering that liquidity transfer - # account although it's not existing in any xml file - # - an entry in ir_model_data: this allow to still use the method create_record_with_xmlid() and don't make any difference between - # regular accounts created and that liquidity transfer account - if model == 'account.chart.template' and xml_id and module: - chart_template = self.env[model].browse(record_id) - - # Create the transfer account only for leaf chart template in the hierarchy. - if chart_template.parent_id: - return record_id - - new_xml_id = xml_id + '_liquidity_transfer' - if not self.search([('model', '=', 'account.account.template'), ('module', '=', module), ('name', '=', new_xml_id)]): - vals = chart_template._prepare_transfer_account_template() - self._update('account.account.template', module, vals, xml_id=new_xml_id, store=True, noupdate=noupdate, mode=mode, res_id=False) - return record_id diff --git a/addons/mrp/data/mrp_data.xml b/addons/mrp/data/mrp_data.xml index 31013ff0bb4..e8341ce3f52 100644 --- a/addons/mrp/data/mrp_data.xml +++ b/addons/mrp/data/mrp_data.xml @@ -83,15 +83,12 @@ 0 - - stock.picking.type - mrp - - picking_type_manufacturing - - - init - + + diff --git a/addons/stock/data/stock_data.xml b/addons/stock/data/stock_data.xml index f7b73d76467..e3487172f7c 100644 --- a/addons/stock/data/stock_data.xml +++ b/addons/stock/data/stock_data.xml @@ -144,75 +144,36 @@ - - stock.location - stock - - stock_location_stock - - - init - - - - stock.location - stock - - stock_location_company - - - init - - - - stock.location - stock - - stock_location_output - - - init - - - - stock.location - stock - - location_pack_zone - - - init - - - - stock.picking.type - stock - - picking_type_internal - - - init - - - - stock.picking.type - stock - - picking_type_in - - - init - - - - stock.picking.type - stock - - picking_type_out - - - init - + + diff --git a/addons/stock/data/stock_demo2.xml b/addons/stock/data/stock_demo2.xml index eb181e16d21..7b5044cd4f7 100644 --- a/addons/stock/data/stock_demo2.xml +++ b/addons/stock/data/stock_demo2.xml @@ -1,35 +1,20 @@ - - stock.picking.type - stock - - chi_picking_type_in - - - init - - - - stock.picking.type - stock - - chi_picking_type_out - - - init - - - - stock.location - stock - - stock_location_shop0 - - - init - + + diff --git a/addons/stock_account/__init__.py b/addons/stock_account/__init__.py index e6ecd5c5e46..f21215b7ba1 100644 --- a/addons/stock_account/__init__.py +++ b/addons/stock_account/__init__.py @@ -84,10 +84,11 @@ def _configure_journals(cr, registry): if not account_id: account_id = env['account.account'].search([('user_type_id', '=', env.ref('account.data_account_type_current_assets').id)], limit=1).id if account_id: + xml_id = 'stock_account.property_stock_valuation_account_id' vals = { 'name': 'property_stock_valuation_account_id', 'fields_id': fields_id, 'value': 'account.account,'+str(account_id), 'company_id': env.ref('base.main_company').id, } - env['ir.model.data']._update('ir.property', 'stock_account', vals, 'property_stock_valuation_account_id') + env['ir.property']._load_records([dict(xml_id=xml_id, values=vals)]) diff --git a/odoo/addons/base/models/ir_fields.py b/odoo/addons/base/models/ir_fields.py index d7521ed6ba1..9a862f02eee 100644 --- a/odoo/addons/base/models/ir_fields.py +++ b/odoo/addons/base/models/ir_fields.py @@ -297,6 +297,10 @@ class IrFieldsConverter(models.AbstractModel): warnings :rtype: (ID|None, unicode, list) """ + # the function 'flush' comes from BaseModel.load(), and forces the + # creation/update of former records (batch creation) + flush = self._context.get('import_flush', lambda arg=None: None) + id = None warnings = [] action = {'type': 'ir.actions.act_window', 'target': 'new', @@ -330,12 +334,11 @@ class IrFieldsConverter(models.AbstractModel): xmlid = value else: xmlid = "%s.%s" % (self._context.get('_import_current_module', ''), value) - try: - id = self.env.ref(xmlid).id - except ValueError: - pass # leave id is None + flush(xmlid) + id = getattr(self.env.ref(xmlid, raise_if_not_found=False), 'id', None) elif subfield is None: field_type = _(u"name") + flush() ids = RelatedModel.name_search(name=value, operator='=') if ids: if len(ids) > 1: diff --git a/odoo/addons/base/models/ir_model.py b/odoo/addons/base/models/ir_model.py index 4cc2a2d7913..ba3a17c09a0 100644 --- a/odoo/addons/base/models/ir_model.py +++ b/odoo/addons/base/models/ir_model.py @@ -1403,139 +1403,104 @@ class IrModelData(models.Model): """ return self.xmlid_to_object("%s.%s" % (module, xml_id), raise_if_not_found=True) - @api.model - def _update_dummy(self, model, module, xml_id=False, store=True): - if xml_id: - try: - # One step to check the ID is defined and the record actually exists - record = self.get_object(module, xml_id) - if record: - self.loads[(module, xml_id)] = (model, record.id) - for parent_model, parent_field in self.env[model]._inherits.items(): - parent = record[parent_field] - parent_xid = '%s_%s' % (xml_id, parent_model.replace('.', '_')) - self.loads[(module, parent_xid)] = (parent_model, parent.id) - return record.id - except Exception: - pass - return False - @api.multi def unlink(self): """ Regular unlink method, but make sure to clear the caches. """ self.clear_caches() return super(IrModelData, self).unlink() + def _lookup_xmlids(self, xml_ids, model): + """ Look up the given XML ids of the given model. """ + if not xml_ids: + return [] + + # group xml_ids by prefix + bymodule = defaultdict(set) + for xml_id in xml_ids: + prefix, suffix = xml_id.split('.', 1) + bymodule[prefix].add(suffix) + + # query xml_ids by prefix + result = [] + cr = self.env.cr + for prefix, suffixes in bymodule.items(): + query = """ + SELECT d.id, d.module, d.name, d.model, d.res_id, d.noupdate, r.id + FROM ir_model_data d LEFT JOIN "{}" r on d.res_id=r.id + WHERE d.module=%s AND d.name IN %s + """.format(model._table) + for subsuffixes in cr.split_for_in_conditions(suffixes): + cr.execute(query, (prefix, subsuffixes)) + result.extend(cr.fetchall()) + + return result + + def _generate_xmlids(self, xml_id, model): + """ Return all the XML ids to create for the given model. """ + yield xml_id + for parent_model in model._inherits: + yield '%s_%s' % (xml_id, parent_model.replace('.', '_')) + @api.model - def _update(self, model, module, values, xml_id=False, store=True, noupdate=False, mode='init', res_id=False): - # records created during module install should not display the messages of OpenChatter - self = self.with_context(install_mode=True) - current_module = module + def _update_xmlids(self, data_list, update=False): + """ Create or update the given XML ids. - if xml_id and ('.' in xml_id): - assert len(xml_id.split('.')) == 2, _("'%s' contains too many dots. XML ids should not contain dots ! These are used to refer to other modules data, as in module.reference_id") % xml_id - module, xml_id = xml_id.split('.') + :param data_list: list of dicts with keys `xml_id` (XMLID to + assign), `noupdate` (flag on XMLID), `record` (target record). + :param update: should be ``True`` when upgrading a module + """ + if not data_list: + return - action = self.browse() - record = self.env[model].browse(res_id) - - if xml_id: - self._cr.execute("""SELECT imd.id, imd.res_id, md.id, imd.model, imd.noupdate - FROM ir_model_data imd LEFT JOIN %s md ON (imd.res_id = md.id) - WHERE imd.module=%%s AND imd.name=%%s""" % record._table, - (module, xml_id)) - results = self._cr.fetchall() - for imd_id, imd_res_id, real_id, imd_model, imd_noupdate in results: - # In update mode, do not update a record if it's ir.model.data is flagged as noupdate - if mode == 'update' and imd_noupdate: - return imd_res_id - if not real_id: - self.clear_caches() - self._cr.execute('DELETE FROM ir_model_data WHERE id=%s', (imd_id,)) - record = record.browse() - else: - assert model == imd_model, "External ID conflict, %s already refers to a `%s` record,"\ - " you can't define a `%s` record with this ID." % (xml_id, imd_model, model) - action = self.browse(imd_id) - record = record.browse(imd_res_id) - - if action and record: - record.write(values) - action.sudo().write({'date_update': fields.Datetime.now()}) - - elif record: - record.write(values) - if xml_id: - for parent_model, parent_field in record._inherits.items(): - self.sudo().create({ - 'name': xml_id + '_' + parent_model.replace('.', '_'), - 'model': parent_model, - 'module': module, - 'res_id': record[parent_field].id, - 'noupdate': noupdate, - }) - self.sudo().create({ - 'name': xml_id, - 'model': model, - 'module': module, - 'res_id': record.id, - 'noupdate': noupdate, - }) - - elif mode == 'init' or (mode == 'update' and xml_id): - existing_parents = set() # {parent_model, ...} - if xml_id: - for parent_model, parent_field in record._inherits.items(): - xid = self.sudo().search([ - ('module', '=', module), - ('name', '=', xml_id + '_' + parent_model.replace('.', '_')), - ]) - # XML ID found in the database, try to recover an existing record - if xid: - parent = self.env[xid.model].browse(xid.res_id) - if parent.exists(): - existing_parents.add(xid.model) - values[parent_field] = parent.id - else: - xid.unlink() - - record = record.create(values) - if xml_id: - #To add an external identifiers to all inherits model - inherit_models = [record] - while inherit_models: - current_model = inherit_models.pop() - for parent_model_name, parent_field in current_model._inherits.items(): - inherit_models.append(self.env[parent_model_name]) - if parent_model_name in existing_parents: - continue - self.sudo().create({ - 'name': xml_id + '_' + parent_model_name.replace('.', '_'), - 'model': parent_model_name, - 'module': module, - 'res_id': record[parent_field].id, - 'noupdate': noupdate, - }) - existing_parents.add(parent_model_name) - self.sudo().create({ - 'name': xml_id, - 'model': model, - 'module': module, - 'res_id': record.id, - 'noupdate': noupdate - }) - if current_module and module != current_module: - _logger.warning("Creating the ir.model.data %s in module %s instead of %s.", - xml_id, module, current_module) - - - if xml_id and record: - self.loads[(module, xml_id)] = (model, record.id) + # rows to insert + rowf = "(%s, %s, %s, %s, %s, now() at time zone 'UTC', now() at time zone 'UTC')" + rows = set() + for data in data_list: + prefix, suffix = data['xml_id'].split('.', 1) + record = data['record'] + noupdate = bool(data.get('noupdate')) + rows.add((prefix, suffix, record._name, record.id, noupdate)) + # also create XML ids for parent records for parent_model, parent_field in record._inherits.items(): - parent_xml_id = xml_id + '_' + parent_model.replace('.', '_') - self.loads[(module, parent_xml_id)] = (parent_model, record[parent_field].id) + parent = record[parent_field] + puffix = suffix + '_' + parent_model.replace('.', '_') + rows.add((prefix, puffix, parent._name, parent.id, noupdate)) - return record.id + for sub_rows in self.env.cr.split_for_in_conditions(rows): + # insert rows or update them + query = """ + INSERT INTO ir_model_data (module, name, model, res_id, noupdate, date_init, date_update) + VALUES {rows} + ON CONFLICT (module, name) + DO UPDATE SET date_update=(now() at time zone 'UTC') {where} + """.format( + rows=", ".join([rowf] * len(sub_rows)), + where="WHERE NOT ir_model_data.noupdate" if update else "", + ) + try: + self.env.cr.execute(query, [arg for row in sub_rows for arg in row]) + except Exception: + _logger.error("Failed to insert ir_model_data\n%s", "\n".join(str(row) for row in sub_rows)) + raise + + # update self.loads + for prefix, suffix, res_model, res_id, noupdate in rows: + self.loads[(prefix, suffix)] = (res_model, res_id) + + @api.model + def _load_xmlid(self, xml_id): + """ Simply mark the given XML id as being loaded, and return the + corresponding record. + """ + record = self.xmlid_to_object(xml_id) + if record: + prefix, suffix = xml_id.split('.', 1) + self.loads[(prefix, suffix)] = (record._name, record.id) + for parent_model, parent_field in record._inherits.items(): + parent = record[parent_field] + puffix = suffix + '_' + parent_model.replace('.', '_') + self.loads[(prefix, puffix)] = (parent._name, parent.id) + return record @api.model def _module_data_uninstall(self, modules_to_remove): diff --git a/odoo/addons/base/tests/__init__.py b/odoo/addons/base/tests/__init__.py index 3e6195ed523..0a620a87608 100644 --- a/odoo/addons/base/tests/__init__.py +++ b/odoo/addons/base/tests/__init__.py @@ -10,6 +10,7 @@ from . import test_ir_actions from . import test_ir_attachment from . import test_ir_http from . import test_ir_filters +from . import test_ir_model from . import test_ir_sequence from . import test_ir_sequence_date_range from . import test_ir_default diff --git a/odoo/addons/base/tests/test_ir_model.py b/odoo/addons/base/tests/test_ir_model.py new file mode 100644 index 00000000000..d00d2d1fb55 --- /dev/null +++ b/odoo/addons/base/tests/test_ir_model.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from odoo.tests.common import TransactionCase + + +class TestXMLID(TransactionCase): + def get_data(self, xml_id): + """ Return the 'ir.model.data' record corresponding to ``xml_id``. """ + module, suffix = xml_id.split('.', 1) + domain = [('module', '=', module), ('name', '=', suffix)] + return self.env['ir.model.data'].search(domain) + + def test_create(self): + model = self.env['res.partner.category'] + xml_id = 'test_convert.category_foo' + + # create category (flag 'noupdate' should be False by default) + data = dict(xml_id=xml_id, values={'name': 'Foo'}) + category = model._load_records([data]) + self.assertEqual(category, self.env.ref(xml_id, raise_if_not_found=False)) + self.assertEqual(category.name, 'Foo') + self.assertEqual(self.get_data(xml_id).noupdate, False) + + # update category + data = dict(xml_id=xml_id, values={'name': 'Bar'}) + category1 = model._load_records([data], update=True) + self.assertEqual(category, category1) + self.assertEqual(category.name, 'Bar') + self.assertEqual(self.get_data(xml_id).noupdate, False) + + # update category + data = dict(xml_id=xml_id, values={'name': 'Baz'}, noupdate=True) + category2 = model._load_records([data], update=True) + self.assertEqual(category, category2) + self.assertEqual(category.name, 'Baz') + self.assertEqual(self.get_data(xml_id).noupdate, False) + + def test_create_noupdate(self): + model = self.env['res.partner.category'] + xml_id = 'test_convert.category_foo' + + # create category + data = dict(xml_id=xml_id, values={'name': 'Foo'}, noupdate=True) + category = model._load_records([data]) + self.assertEqual(category, self.env.ref(xml_id, raise_if_not_found=False)) + self.assertEqual(category.name, 'Foo') + self.assertEqual(self.get_data(xml_id).noupdate, True) + + # update category + data = dict(xml_id=xml_id, values={'name': 'Bar'}, noupdate=False) + category1 = model._load_records([data], update=True) + self.assertEqual(category, category1) + self.assertEqual(category.name, 'Foo') + self.assertEqual(self.get_data(xml_id).noupdate, True) + + # update category + data = dict(xml_id=xml_id, values={'name': 'Baz'}, noupdate=True) + category2 = model._load_records([data], update=True) + self.assertEqual(category, category2) + self.assertEqual(category.name, 'Foo') + self.assertEqual(self.get_data(xml_id).noupdate, True) + + def test_create_noupdate_multi(self): + model = self.env['res.partner.category'] + data_list = [ + dict(xml_id='test_convert.category_foo', values={'name': 'Foo'}, noupdate=True), + dict(xml_id='test_convert.category_bar', values={'name': 'Bar'}, noupdate=True), + ] + + # create category + categories = model._load_records(data_list) + foo = self.env.ref('test_convert.category_foo') + bar = self.env.ref('test_convert.category_bar') + self.assertEqual(categories, foo + bar) + self.assertEqual(foo.name, 'Foo') + self.assertEqual(bar.name, 'Bar') + + # check data + self.assertEqual(self.get_data('test_convert.category_foo').noupdate, True) + self.assertEqual(self.get_data('test_convert.category_bar').noupdate, True) + + def test_create_order(self): + model = self.env['res.partner.category'] + data_list = [ + dict(xml_id='test_convert.category_foo', values={'name': 'Foo'}), + dict(xml_id='test_convert.category_bar', values={'name': 'Bar'}, noupdate=True), + dict(xml_id='test_convert.category_baz', values={'name': 'Baz'}), + ] + + # create categories + foo = model._load_records([data_list[0]]) + bar = model._load_records([data_list[1]]) + baz = model._load_records([data_list[2]]) + self.assertEqual(foo.name, 'Foo') + self.assertEqual(bar.name, 'Bar') + self.assertEqual(baz.name, 'Baz') + + # update them, and check the order of result + for data in data_list: + data['values']['name'] += 'X' + cats = model._load_records(data_list, update=True) + self.assertEqual(list(cats), [foo, bar, baz]) + self.assertEqual(foo.name, 'FooX') + self.assertEqual(bar.name, 'Bar') + self.assertEqual(baz.name, 'BazX') + + def test_create_inherits(self): + model = self.env['res.users'] + xml_id = 'test_convert.user_foo' + par_xml_id = xml_id + '_res_partner' + + # create user + user = model._load_records([dict(xml_id=xml_id, values={'name': 'Foo', 'login': 'foo'})]) + self.assertEqual(user, self.env.ref(xml_id, raise_if_not_found=False)) + self.assertEqual(user.partner_id, self.env.ref(par_xml_id, raise_if_not_found=False)) + self.assertEqual(user.name, 'Foo') + self.assertEqual(user.login, 'foo') + + def test_recreate(self): + model = self.env['res.partner.category'] + xml_id = 'test_convert.category_foo' + data = dict(xml_id=xml_id, values={'name': 'Foo'}) + + # create category + category = model._load_records([data]) + self.assertEqual(category, self.env.ref(xml_id, raise_if_not_found=False)) + self.assertEqual(category.name, 'Foo') + + # suppress category + category.unlink() + self.assertFalse(self.env.ref(xml_id, raise_if_not_found=False)) + + # update category, this should recreate it + category = model._load_records([data], update=True) + self.assertEqual(category, self.env.ref(xml_id, raise_if_not_found=False)) + self.assertEqual(category.name, 'Foo') + + def test_create_xmlids(self): + # create users and assign them xml ids + foo, bar = self.env['res.users'].create([ + {'name': 'Foo', 'login': 'foo'}, + {'name': 'Bar', 'login': 'bar'}, + ]) + self.env['ir.model.data']._update_xmlids([ + dict(xml_id='test_convert.foo', record=foo, noupdate=True), + dict(xml_id='test_convert.bar', record=bar, noupdate=True), + ]) + + self.assertEqual(foo, self.env.ref('test_convert.foo', raise_if_not_found=False)) + self.assertEqual(bar, self.env.ref('test_convert.bar', raise_if_not_found=False)) + + self.assertEqual(foo.partner_id, self.env.ref('test_convert.foo_res_partner', raise_if_not_found=False)) + self.assertEqual(bar.partner_id, self.env.ref('test_convert.bar_res_partner', raise_if_not_found=False)) + + self.assertEqual(self.get_data('test_convert.foo').noupdate, True) + self.assertEqual(self.get_data('test_convert.bar').noupdate, True) diff --git a/odoo/addons/base/tests/test_user_has_group.py b/odoo/addons/base/tests/test_user_has_group.py index 9b8911d5348..d7e44410104 100644 --- a/odoo/addons/base/tests/test_user_has_group.py +++ b/odoo/addons/base/tests/test_user_has_group.py @@ -8,25 +8,19 @@ class TestHasGroup(TransactionCase): def setUp(self): super(TestHasGroup, self).setUp() - group0 = self.env['ir.model.data']._update( - 'res.groups', 'test_user_has_group', - {'name': 'group0'}, - xml_id='group0' - ) self.group0 = 'test_user_has_group.group0' - self.env['ir.model.data']._update( - 'res.groups', 'test_user_has_group', - {'name': 'group1'}, - xml_id='group1' - ) self.group1 = 'test_user_has_group.group1' + group0, group1 = self.env['res.groups']._load_records([ + dict(xml_id=self.group0, values={'name': 'group0'}), + dict(xml_id=self.group1, values={'name': 'group1'}), + ]) self.test_user = self.env['res.users'].create({ 'login': 'testuser', 'partner_id': self.env['res.partner'].create({ 'name': "Strawman Test User" }).id, - 'groups_id': [(4, group0, 0)] + 'groups_id': [(4, group0.id, 0)] }) def test_env_uid(self): diff --git a/odoo/models.py b/odoo/models.py index 0fda0e7ff46..d9edbd6e052 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -840,36 +840,84 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): messages = [] ModelData = self.env['ir.model.data'] ModelData.clear_caches() - extracted = self._extract_records(fields, data, log=messages.append) - converted = self._convert_records(extracted, log=messages.append) - for id, xid, record, info in converted: + + # list of (xid, vals, info) for records to be created in batch + batch = [] + batch_xml_ids = set() + + def flush(xml_id=None): + if not batch: + return + if xml_id and xml_id not in batch_xml_ids: + return + + data_list = [ + dict(xml_id=xid, values=vals, info=info, noupdate=noupdate) + for xid, vals, info in batch + ] + batch.clear() + batch_xml_ids.clear() + try: cr.execute('SAVEPOINT model_load_save') except psycopg2.InternalError as e: # broken transaction, exit and hope the source error was # already logged if not any(message['type'] == 'error' for message in messages): - messages.append(dict(info, type='error',message=u"Unknown database error: '%s'" % e)) - break + info = data_list[0]['info'] + messages.append(dict(info, type='error', message=u"Unknown database error: '%s'" % e)) + return + + # try to create in batch try: - ids.append(ModelData._update(self._name, current_module, record, mode=mode, - xml_id=xid, noupdate=noupdate, res_id=id)) + recs = self._load_records(data_list, mode == 'update') + ids.extend(recs.ids) cr.execute('RELEASE SAVEPOINT model_load_save') - except psycopg2.Warning as e: - messages.append(dict(info, type='warning', message=str(e))) - cr.execute('ROLLBACK TO SAVEPOINT model_load_save') - except psycopg2.Error as e: - messages.append(dict(info, type='error', **PGERROR_TO_OE[e.pgcode](self, fg, info, e))) - # Failed to write, log to messages, rollback savepoint (to - # avoid broken transaction) and keep going - cr.execute('ROLLBACK TO SAVEPOINT model_load_save') - except Exception as e: - message = (_(u'Unknown error during import:') + u' %s: %s' % (type(e), e)) - moreinfo = _('Resolve other errors first') - messages.append(dict(info, type='error', message=message, moreinfo=moreinfo)) - # Failed for some reason, perhaps due to invalid data supplied, - # rollback savepoint and keep going + return + except Exception: cr.execute('ROLLBACK TO SAVEPOINT model_load_save') + + # try again, this time record by record + for rec_data in data_list: + try: + cr.execute('SAVEPOINT model_load_save') + rec = self._load_records([rec_data], mode == 'update') + ids.append(rec.id) + cr.execute('RELEASE SAVEPOINT model_load_save') + except psycopg2.Warning as e: + info = rec_data['info'] + messages.append(dict(info, type='warning', message=str(e))) + cr.execute('ROLLBACK TO SAVEPOINT model_load_save') + except psycopg2.Error as e: + info = rec_data['info'] + messages.append(dict(info, type='error', **PGERROR_TO_OE[e.pgcode](self, fg, info, e))) + # Failed to write, log to messages, rollback savepoint (to + # avoid broken transaction) and keep going + cr.execute('ROLLBACK TO SAVEPOINT model_load_save') + except Exception as e: + info = rec_data['info'] + message = (_(u'Unknown error during import:') + u' %s: %s' % (type(e), e)) + moreinfo = _('Resolve other errors first') + messages.append(dict(info, type='error', message=message, moreinfo=moreinfo)) + # Failed for some reason, perhaps due to invalid data supplied, + # rollback savepoint and keep going + cr.execute('ROLLBACK TO SAVEPOINT model_load_save') + + # make 'flush' available to the methods below, in the case where XMLID + # resolution fails, for instance + flush_self = self.with_context(import_flush=flush) + extracted = flush_self._extract_records(fields, data, log=messages.append) + converted = flush_self._convert_records(extracted, log=messages.append) + + for id, xid, record, info in converted: + if xid: + xid = xid if '.' in xid else "%s.%s" % (current_module, xid) + batch_xml_ids.update(ModelData._generate_xmlids(xid, self)) + elif id: + record['id'] = id + batch.append((xid, record, info)) + + flush() if any(message['type'] == 'error' for message in messages): cr.execute('ROLLBACK TO SAVEPOINT model_load') ids = False @@ -3568,6 +3616,93 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): modified_ids = {row[0] for row in cr.fetchall()} self.browse(modified_ids).modified(['parent_path']) + def _load_records(self, data_list, update=False): + """ Create or update records of this model, and assign XMLIDs. + + :param data_list: list of dicts with keys `xml_id` (XMLID to + assign), `noupdate` (flag on XMLID), `values` (field values) + :param update: should be ``True`` when upgrading a module + + :return: the records corresponding to ``data_list`` + """ + original_self = self.browse() + # records created during installation should not display messages + self = self.with_context(install_mode=True) + imd = self.env['ir.model.data'].sudo() + + # The algorithm below partitions 'data_list' into three sets: the ones + # to create, the ones to update, and the others. For each set, we assign + # data['record'] for each data. All those records are then retrieved for + # the result. + + # determine existing xml_ids + xml_ids = [data['xml_id'] for data in data_list if data.get('xml_id')] + existing = { + ("%s.%s" % row[1:3]): row + for row in imd._lookup_xmlids(xml_ids, self) + } + + # determine which records to create and update + to_create = [] # list of data + to_update = [] # list of data + + for data in data_list: + xml_id = data.get('xml_id') + if not xml_id: + vals = data['values'] + if vals.get('id'): + data['record'] = self.browse(vals['id']) + to_update.append(data) + elif not update: + to_create.append(data) + continue + row = existing.get(xml_id) + if not row: + to_create.append(data) + continue + d_id, d_module, d_name, d_model, d_res_id, d_noupdate, r_id = row + record = self.browse(d_res_id) + if update and d_noupdate: + data['record'] = record + elif r_id: + data['record'] = record + to_update.append(data) + else: + imd.browse(d_id).unlink() + to_create.append(data) + + # update existing records + for data in to_update: + data['record'].write(data['values']) + + # determine existing parents for new records + for parent_model, parent_field in self._inherits.items(): + suffix = '_' + parent_model.replace('.', '_') + xml_ids_vals = { + (data['xml_id'] + suffix): data['values'] + for data in to_create + if data.get('xml_id') + } + for row in imd._lookup_xmlids(xml_ids_vals, self.env[parent_model]): + d_id, d_module, d_name, d_model, d_res_id, d_noupdate, r_id = row + if r_id: + xml_id = '%s.%s' % (d_module, d_name) + xml_ids_vals[xml_id][parent_field] = r_id + else: + imd.browse(d_id).unlink() + + # create records + records = self.create([data['values'] for data in to_create]) + for data, record in pycompat.izip(to_create, records): + data['record'] = record + + # create or update XMLIDs + if to_create or to_update: + imd_data_list = [data for data in data_list if data.get('xml_id')] + imd._update_xmlids(imd_data_list, update) + + return original_self.concat(*(data['record'] for data in data_list)) + # TODO: ameliorer avec NULL @api.model def _where_calc(self, domain, active_test=True): diff --git a/odoo/tools/convert.py b/odoo/tools/convert.py index 02cee45c11f..6e276a65cf6 100644 --- a/odoo/tools/convert.py +++ b/odoo/tools/convert.py @@ -241,6 +241,11 @@ class xml_import(object): return self.id_get(node_uid) return self.uid + def make_xml_id(self, xml_id): + if not xml_id or '.' in xml_id: + return xml_id + return "%s.%s" % (self.module, xml_id) + def _test_xml_id(self, xml_id): id = xml_id if '.' in xml_id: @@ -316,17 +321,17 @@ form: module.record_id""" % (xml_id,) pf_id = self.id_get(pf_name) res['paperformat_id'] = pf_id - id = self.env['ir.model.data']._update("ir.actions.report", self.module, res, xml_id, noupdate=self.isnoupdate(data_node), mode=self.mode) - self.idref[xml_id] = int(id) + xid = self.make_xml_id(xml_id) + data = dict(xml_id=xid, values=res, noupdate=self.isnoupdate(data_node)) + report = self.env['ir.actions.report']._load_records([data], self.mode == 'update') + self.idref[xml_id] = report.id if not rec.get('menu') or safe_eval(rec.get('menu','False')): - report = self.env['ir.actions.report'].browse(id) report.create_action() elif self.mode=='update' and safe_eval(rec.get('menu','False'))==False: # Special check for report having attribute menu=False on update - report = self.env['ir.actions.report'].browse(id) report.unlink_action() - return id + return report.id def _tag_function(self, rec, data_node=None, mode=None): if self.isnoupdate(data_node) and self.mode != 'init': @@ -434,8 +439,10 @@ form: module.record_id""" % (xml_id,) if rec.get('key2') in (None, 'client_action_relate'): if not res.get('multi'): res['binding_type'] = 'action_form_only' - id = self.env['ir.model.data']._update('ir.actions.act_window', self.module, res, xml_id, noupdate=self.isnoupdate(data_node), mode=self.mode) - self.idref[xml_id] = int(id) + + xid = self.make_xml_id(xml_id) + data = dict(xml_id=xid, values=res, noupdate=self.isnoupdate(data_node)) + self.env['ir.actions.act_window']._load_records([data], self.mode == 'update') def _tag_menuitem(self, rec, data_node=None, mode=None): rec_id = rec.get("id") @@ -497,12 +504,9 @@ form: module.record_id""" % (xml_id,) if rec.get('web_icon'): values['web_icon'] = rec.get('web_icon') - pid = self.env['ir.model.data']._update('ir.ui.menu', self.module, values, rec_id, noupdate=self.isnoupdate(data_node), mode=self.mode, res_id=res and res[0] or False) - - if rec_id and pid: - self.idref[rec_id] = int(pid) - - return 'ir.ui.menu', pid + xid = self.make_xml_id(rec_id) + data = dict(xml_id=xid, values=values, noupdate=self.isnoupdate(data_node)) + self.env['ir.ui.menu']._load_records([data], self.mode == 'update') def _assert_equals(self, f1, f2, prec=4): return not round(f1 - f2, prec) @@ -579,24 +583,21 @@ form: module.record_id""" % (xml_id,) rec_context['install_filename'] = self.xml_filename self._test_xml_id(rec_id) + xid = self.make_xml_id(rec_id) + # in update mode, the record won't be updated if the data node explicitly # opt-out using @noupdate="1". A second check will be performed in - # ir.model.data#_update() using the record's ir.model.data `noupdate` field. + # model._load_records() using the record's ir.model.data `noupdate` field. if self.isnoupdate(data_node) and self.mode != 'init': # check if the xml record has no id, skip if not rec_id: return None - if '.' in rec_id: - module,rec_id2 = rec_id.split('.') - else: - module = self.module - rec_id2 = rec_id - id = self.env['ir.model.data']._update_dummy(rec_model, module, rec_id2) - if id: + record = self.env['ir.model.data']._load_xmlid(xid) + if record: # if the resource already exists, don't update it but store # its database id (can be useful) - self.idref[rec_id] = int(id) + self.idref[rec_id] = record.id return None elif not self.nodeattr2bool(rec, 'forcecreate', True): # if it doesn't exist and we shouldn't create it, skip it @@ -650,12 +651,13 @@ form: module.record_id""" % (xml_id,) f_val = str2bool(f_val) res[f_name] = f_val - id = self.env(context=rec_context)['ir.model.data']._update(rec_model, self.module, res, rec_id or False, not self.isnoupdate(data_node), noupdate=self.isnoupdate(data_node), mode=self.mode) + data = dict(xml_id=xid, values=res, noupdate=self.isnoupdate(data_node)) + record = model.with_context(rec_context)._load_records([data], self.mode == 'update') if rec_id: - self.idref[rec_id] = int(id) + self.idref[rec_id] = record.id if config.get('import_partial'): self.cr.commit() - return rec_model, id + return rec_model, record.id def _tag_template(self, el, data_node=None, mode=None): # This helper transforms a