[IMP] base: batch XID generation during export

* add UUIDs to XID sections (4 bytes / 8 hex digits) so it's not
  necessary to handle collisions & add a fallback generator
* use COPY for performances over executemany: executemany just
  performs an implicit loop on all statements (in psycopg2)
* ~pure SQL was possible:

      INSERT INTO ir_model_data (module, model, name, res_id)
      SELECT
          '__export__',
          '{model}',
          '{table}_' || A.id || '_' || uuid_generate_v4(),
          A.id
      FROM {table} A
      LEFT JOIN ir_model_data B
             ON A.id = B.res_id AND B.model = %s
      WHERE B.res_id IS NULL;

  but would have required installing pg extensions (problematic
  especially in stable) and performances are about the same as
  the COPY version

task 36343
Fixes #22493

odoo/addons/test_impex/tests/test_export.py

@@ -2,6 +2,8 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
 import itertools
+import unittest
+from cProfile import Profile
 
 from odoo.tests import common
 from odoo.tools import pycompat
@@ -71,6 +73,16 @@ class test_integer_field(CreatorCase):
             self.export(2**31-1),
             [[pycompat.text_type(2**31-1)]])
 
+    @unittest.skip("Only benches/profiles")
+    def test_xid_perfs(self):
+        for i in range(10000):
+            self.make(i)
+        self.model.invalidate_cache()
+        records = self.model.search([])
+
+        p = Profile()
+        p.runcall(records._export_rows, [['id'], ['value']])
+        p.dump_stats('xid_perfs.pstats')
 
 class test_float_field(CreatorCase):
     model_name = 'export.float'
@@ -325,11 +337,9 @@ class test_m2o(CreatorCase):
         record = self.env['export.integer'].create({'value': 42})
         # Expecting the m2o target model name in the external id,
         # not this model's name
-        external_id = u'__export__.export_integer_%d' % record.id
-        self.assertEqual(
-            self.export(record.id, fields=['value/id']),
-            [[external_id]])
-
+        self.assertRegexpMatches(
+            self.export(record.id, fields=['value/id'])[0][0],
+            u'__export__.export_integer_%d_[0-9a-f]{8}' % record.id)
 
 class test_o2m(CreatorCase):
     model_name = 'export.one2many'

odoo/models.py

@@ -27,10 +27,12 @@ import collections
 import dateutil
 import functools
 import itertools
+import io
 import logging
 import operator
 import pytz
 import re
+import uuid
 from collections import defaultdict, MutableMapping, OrderedDict
 from contextlib import closing
 from inspect import getmembers, currentframe
@@ -615,33 +617,67 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
     def _is_an_ordinary_table(self):
         return tools.table_kind(self.env.cr, self._table) == 'r'
 
-    def __export_xml_id(self):
-        """ Return a valid xml_id for the record ``self``. """
+    def __ensure_xml_id(self, skip=False):
+        """ Create missing external ids for records in ``self``, and return an
+            iterator of pairs ``(record, xmlid)`` for the records in ``self``.
+
+        :rtype: Iterable[Model, str | None]
+        """
+        if skip:
+            return ((record, None) for record in self)
+
+        if not self:
+            return iter([])
+
         if not self._is_an_ordinary_table():
             raise Exception(
                 "You can not export the column ID of model %s, because the "
                 "table %s is not an ordinary table."
                 % (self._name, self._table))
-        ir_model_data = self.sudo().env['ir.model.data']
-        data = ir_model_data.search([('model', '=', self._name), ('res_id', '=', self.id)])
-        if data:
-            if data[0].module:
-                return '%s.%s' % (data[0].module, data[0].name)
-            else:
-                return data[0].name
-        else:
-            postfix = 0
-            name = '%s_%s' % (self._table, self.id)
-            while ir_model_data.search([('module', '=', '__export__'), ('name', '=', name)]):
-                postfix += 1
-                name = '%s_%s_%s' % (self._table, self.id, postfix)
-            ir_model_data.create({
-                'model': self._name,
-                'res_id': self.id,
-                'module': '__export__',
-                'name': name,
-            })
-            return '__export__.' + name
+
+        modname = '__export__'
+
+        cr = self.env.cr
+        cr.execute("""
+            SELECT res_id, module, name
+            FROM ir_model_data
+            WHERE model = %s AND res_id in %s
+        """, (self._name, tuple(self.ids)))
+        xids = {
+            res_id: (module, name)
+            for res_id, module, name in cr.fetchall()
+        }
+
+        # create missing xml ids
+        missing = self.filtered(lambda r: r.id not in xids)
+        xids.update(
+            (r.id, (modname, '%s_%s_%s' % (
+                r._table,
+                r.id,
+                uuid.uuid4().hex[:8],
+            )))
+            for r in missing
+        )
+        cr.copy_from(io.StringIO(
+            u'\n'.join(
+                u"%s\t%s\t%s\t%d" % (
+                    modname,
+                    record._name,
+                    xids[record.id][1],
+                    record.id,
+                )
+                for record in missing
+            )),
+            table='ir_model_data',
+            columns=['module', 'model', 'name', 'res_id'],
+        )
+
+        self.invalidate_cache()
+
+        return (
+            (record, '%s.%s' % xids[record.id])
+            for record in self
+        )
 
     @api.multi
     def _export_rows(self, fields):
@@ -659,13 +695,16 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
             from the cache after it's been iterated in full
             """
             for idx in range(0, len(rs), 1000):
-                sub = rs[idx: idx+1000]
+                sub = rs[idx:idx+1000]
                 for rec in sub:
                     yield rec
                 rs.invalidate_cache(ids=sub.ids)
 
+        # both _ensure_xml_id and the splitter want to work on recordsets but
+        # neither returns one, so can't really be composed...
+        xids = dict(self.__ensure_xml_id(skip=['id'] not in fields))
         # memory stable but ends up prefetching 275 fields (???)
-        for idx, record in enumerate(splittor(self)):
+        for record in splittor(self):
             # main line of record, initially empty
             current = [''] * len(fields)
             lines.append(current)
@@ -685,7 +724,9 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
                 if name == '.id':
                     current[i] = str(record.id)
                 elif name == 'id':
-                    current[i] = record.__export_xml_id()
+                    xid = xids.get(record)
+                    assert xid, "no xid was generated for the record %s" % record
+                    current[i] = xid
                 else:
                     field = record._fields[name]
                     value = record[name]
@@ -700,7 +741,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
                         # in import_compat mode, m2m should always be exported as
                         # a comma-separated list of xids in a single cell
                         if import_compatible and field.type == 'many2many' and len(path) > 1 and path[1] == 'id':
-                            xml_ids = [r.__export_xml_id() for r in value]
+                            xml_ids = [xid for _, xid in value.__ensure_xml_id()]
                             current[i] = ','.join(xml_ids) or False
                             continue