[MERGE] base: custom models improvements

Custom model improvements needed for a Studio task:
- allow ordering custom models by something else than their id
- allow a basic `group_expand` functionality for custom models
  allowing 'stage-like' behaviours (e.g. a `read_group` on `x_stage_id`
will return all the stages, including empty ones)
- small fixes detected during development:
  - SQL identifiers escaping
  - correct reset of the registry after `SingleTransactionCase` and `SavepointCase` tests
  - correct removal of a custom class from the registry

Part of Task 2091654
Related to odoo/enterprise#7326

closes odoo/odoo#43981

Signed-off-by: Damien Bouvy (dbo) <dbo@odoo.com>

odoo/addons/base/models/ir_model.py

@@ -4,6 +4,7 @@ import datetime
 import dateutil
 import itertools
 import logging
+import re
 import time
 from ast import literal_eval
 from collections import defaultdict, Mapping
@@ -19,7 +20,7 @@ from odoo.tools.safe_eval import safe_eval
 _logger = logging.getLogger(__name__)
 
 MODULE_UNINSTALL_FLAG = '_force_unlink'
-
+RE_ORDER_FIELDS = re.compile(r'"?(\w+)"?\s*(?:asc|desc)?', flags=re.I)
 
 # base environment for doing a safe_eval
 SAFE_EVAL_BASE = {
@@ -47,8 +48,8 @@ def query_insert(cr, table, rows):
         rows = [rows]
     cols = list(rows[0])
     query = INSERT_QUERY.format(
-        table=table,
-        cols=",".join(cols),
+        table='"{}"'.format(table),
+        cols=",".join(['"{}"'.format(col) for col in cols]),
         rows=",".join("%s" for row in rows),
     )
     params = [tuple(row[col] for col in cols) for row in rows]
@@ -61,9 +62,9 @@ def query_update(cr, table, values, selectors):
     """
     setters = set(values) - set(selectors)
     query = UPDATE_QUERY.format(
-        table=table,
-        assignment=",".join("{0}=%({0})s".format(s) for s in setters),
-        condition=" AND ".join("{0}=%({0})s".format(s) for s in selectors),
+        table='"{}"'.format(table),
+        assignment=",".join('"{0}"=%({0})s'.format(s) for s in setters),
+        condition=" AND ".join('"{0}"=%({0})s'.format(s) for s in selectors),
     )
     cr.execute(query, values)
     return [row[0] for row in cr.fetchall()]
@@ -99,6 +100,8 @@ class IrModel(models.Model):
 
     name = fields.Char(string='Model Description', translate=True, required=True)
     model = fields.Char(default='x_', required=True, index=True)
+    order = fields.Char(string='Order', default='id', required=True,
+                        help='SQL expression for ordering records in the model; e.g. "x_sequence asc, id desc"')
     info = fields.Text(string='Information')
     field_id = fields.One2many('ir.model.fields', 'model_id', string='Fields', required=True, copy=True,
                                default=_default_field_id)
@@ -156,6 +159,24 @@ class IrModel(models.Model):
             if not models.check_object_name(model.model):
                 raise ValidationError(_("The model name can only contain lowercase characters, digits, underscores and dots."))
 
+    @api.constrains('order', 'field_id')
+    def _check_order(self):
+        for model in self:
+            try:
+                model._check_qorder(model.order)  # regex check for the whole clause ('is it valid sql?')
+            except UserError as e:
+                raise ValidationError(str(e))
+            stored_fields = model.field_id.filtered('store').mapped('name')
+            if self.env.get(model.model) is None:
+                # model hasn't been init'd yet, which means that some fields are not yet in its
+                # list of fields but will be right after its creation - these fields can be used
+                # for ordering, so let's add them to the list of stored fields manually
+                stored_fields += models.MAGIC_COLUMNS
+            order_fields = RE_ORDER_FIELDS.findall(model.order)
+            for field in order_fields:
+                if field not in stored_fields:
+                    raise ValidationError(_("Unable to order by %s: fields used for ordering must be present on the model and stored.") % field)
+
     _sql_constraints = [
         ('obj_name_uniq', 'unique (model)', 'Each model must be unique!'),
     ]
@@ -238,7 +259,12 @@ class IrModel(models.Model):
         # writes (4,id,False) even for non dirty items.
         if 'field_id' in vals:
             vals['field_id'] = [op for op in vals['field_id'] if op[0] != 4]
-        return super(IrModel, self).write(vals)
+        res = super(IrModel, self).write(vals)
+        # ordering has been changed, reload registry to reflect update + signaling
+        if 'order' in vals:
+            self.flush()  # setup_models need to fetch the updated values from the db
+            self.pool.setup_models(self._cr)
+        return res
 
     @api.model
     def create(self, vals):
@@ -264,6 +290,7 @@ class IrModel(models.Model):
         return {
             'model': model._name,
             'name': model._description,
+            'order': model._order,
             'info': next(cls.__doc__ for cls in type(model).mro() if cls.__doc__),
             'state': 'manual' if model._custom else 'base',
             'transient': model._transient,
@@ -299,6 +326,7 @@ class IrModel(models.Model):
             _module = False
             _custom = True
             _transient = bool(model_data['transient'])
+            _order = model_data['order']
             __doc__ = model_data['info']
 
         return CustomModel
@@ -308,7 +336,7 @@ class IrModel(models.Model):
         # clean up registry first
         custom_models = [name for name, model_class in self.pool.items() if model_class._custom]
         for name in custom_models:
-            del self.pool.models[name]
+            del self.pool[name]
         # add manual models
         cr = self.env.cr
         cr.execute('SELECT * FROM ir_model WHERE state=%s', ['manual'])
@@ -369,6 +397,12 @@ class IrModelFields(models.Model):
                                             "specified as a Python expression defining a list of triplets. "
                                             "For example: [('color','=','red')]")
     groups = fields.Many2many('res.groups', 'ir_model_fields_group_rel', 'field_id', 'group_id') # CLEANME unimplemented field (empty table)
+    group_expand = fields.Boolean(string="Expand Groups",
+                                  help="If checked, all the records of the target model will be included\n"
+                                        "in a grouped result (e.g. 'Group By' filters, Kanban columns, etc.).\n"
+                                        "Note that it can significantly reduce performance if the target model\n"
+                                        "of the field contains a lot of records; usually used on models with\n"
+                                        "few records (e.g. Stages, Job Positions, Event Types, etc.).")
     selectable = fields.Boolean(default=True)
     modules = fields.Char(compute='_in_modules', string='In Apps', help='List of modules in which the field is defined')
     relation_table = fields.Char(help="Used for custom many2many fields to define a custom relation table name")
@@ -966,6 +1000,7 @@ class IrModelFields(models.Model):
             attrs['comodel_name'] = field_data['relation']
             attrs['ondelete'] = field_data['on_delete']
             attrs['domain'] = safe_eval(field_data['domain'] or '[]')
+            attrs['group_expand'] = '_read_group_expand_full' if field_data['group_expand'] else None
         elif field_data['ttype'] == 'one2many':
             if not self.pool.loaded and not (
                 field_data['relation'] in self.env and (

odoo/addons/base/tests/test_ir_model.py

@@ -3,7 +3,8 @@
 
 from psycopg2 import IntegrityError
 
-from odoo.tests.common import TransactionCase
+from odoo.exceptions import ValidationError
+from odoo.tests.common import TransactionCase, SavepointCase
 from odoo.tools import mute_logger
 
 
@@ -168,3 +169,143 @@ class TestXMLID(TransactionCase):
         }]
         with self.assertRaisesRegex(IntegrityError, 'ir_model_data_name_nospaces'):
             model._load_records(data_list)
+
+
+class TestIrModel(SavepointCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+
+        # The test mode is necessary in this case.  After each test, we call
+        # registry.reset_changes(), which opens a new cursor to retrieve custom
+        # models and fields.  A regular cursor would correspond to the state of
+        # the database before setUpClass(), which is not correct.  Instead, a
+        # test cursor will correspond to the state of the database of cls.cr at
+        # that point, i.e., before the call to setUp().
+        cls.registry.enter_test_mode(cls.cr)
+        cls.addClassCleanup(cls.registry.leave_test_mode)
+
+        # model and records for banana stages
+        cls.env['ir.model'].create({
+            'name': 'Banana Ripeness',
+            'model': 'x_banana_ripeness',
+            'field_id': [
+                (0, 0, {'name': 'x_name', 'ttype': 'char', 'field_description': 'Name'}),
+            ]
+        })
+        # stage values are pairs (id, display_name)
+        cls.ripeness_green = cls.env['x_banana_ripeness'].name_create('Green')
+        cls.ripeness_okay = cls.env['x_banana_ripeness'].name_create('Okay, I guess?')
+        cls.ripeness_gone = cls.env['x_banana_ripeness'].name_create('Walked away on its own')
+
+        # model and records for bananas
+        cls.bananas_model = cls.env['ir.model'].create({
+            'name': 'Bananas',
+            'model': 'x_bananas',
+            'field_id': [
+                (0, 0, {'name': 'x_name', 'ttype': 'char', 'field_description': 'Name'}),
+                (0, 0, {'name': 'x_length', 'ttype': 'float', 'field_description': 'Length'}),
+                (0, 0, {'name': 'x_color', 'ttype': 'integer', 'field_description': 'Color'}),
+                (0, 0, {'name': 'x_ripeness_id', 'ttype': 'many2one',
+                        'field_description': 'Ripeness','relation': 'x_banana_ripeness',
+                        'group_expand': True})
+            ]
+        })
+        # add non-stored field that is not valid in order
+        cls.env['ir.model.fields'].create({
+            'name': 'x_is_yellow',
+            'field_description': 'Is the banana yellow?',
+            'ttype': 'boolean',
+            'model_id': cls.bananas_model.id,
+            'store': False,
+            'depends': 'x_color',
+            'compute': "for banana in self:\n    banana['x_is_yellow'] = banana.x_color == 9"
+        })
+        # default stage is ripeness_green
+        cls.env['ir.default'].set('x_bananas', 'x_ripeness_id', cls.ripeness_green[0])
+        cls.env['x_bananas'].create([{
+            'x_name': 'Banana #1',
+            'x_length': 3.14159,
+            'x_color': 9,
+        }, {
+            'x_name': 'Banana #2',
+            'x_length': 0,
+            'x_color': 6,
+        }, {
+            'x_name': 'Banana #3',
+            'x_length': 10,
+            'x_color': 6,
+        }])
+
+    def setUp(self):
+        # this cleanup is necessary after each test, and must be done last
+        self.addCleanup(self.registry.reset_changes)
+        super().setUp()
+
+    def test_model_order_constraint(self):
+        """Check that the order constraint is properly enforced."""
+        VALID_ORDERS = ['id', 'id desc', 'id asc, x_length', 'x_color, x_length, create_uid']
+        for order in VALID_ORDERS:
+            self.bananas_model.order = order
+
+        INVALID_ORDERS = ['', 'x_wat', 'id esc', 'create_uid,', 'id, x_is_yellow']
+        for order in INVALID_ORDERS:
+            with self.assertRaises(ValidationError), self.cr.savepoint():
+                self.bananas_model.order = order
+
+        # check that the constraint is checked at model creation
+        fields_value = [
+            (0, 0, {'name': 'x_name', 'ttype': 'char', 'field_description': 'Name'}),
+            (0, 0, {'name': 'x_length', 'ttype': 'float', 'field_description': 'Length'}),
+            (0, 0, {'name': 'x_color', 'ttype': 'integer', 'field_description': 'Color'}),
+        ]
+        self.env['ir.model'].create({
+            'name': 'MegaBananas',
+            'model': 'x_mega_bananas',
+            'order': 'x_name asc, id desc',         # valid order
+            'field_id': fields_value,
+        })
+        with self.assertRaises(ValidationError):
+            self.env['ir.model'].create({
+                'name': 'GigaBananas',
+                'model': 'x_giga_bananas',
+                'order': 'x_name asc, x_wat',       # invalid order
+                'field_id': fields_value,
+            })
+
+    def test_model_order_search(self):
+        """Check that custom orders are applied when querying a model."""
+        ORDERS = {
+            'id asc': ['Banana #1', 'Banana #2', 'Banana #3'],
+            'id desc': ['Banana #3', 'Banana #2', 'Banana #1'],
+            'x_color asc, id asc': ['Banana #2', 'Banana #3', 'Banana #1'],
+            'x_color asc, id desc': ['Banana #3', 'Banana #2', 'Banana #1'],
+            'x_length asc, id': ['Banana #2', 'Banana #1', 'Banana #3'],
+        }
+        for order, names in ORDERS.items():
+            self.bananas_model.order = order
+            self.assertEqual(self.env['x_bananas']._order, order)
+
+            bananas = self.env['x_bananas'].search([])
+            self.assertEqual(bananas.mapped('x_name'), names, 'failed to order by %s' % order)
+
+    def test_group_expansion(self):
+        """Check that the basic custom group expansion works."""
+        groups = self.env['x_bananas'].read_group(domain=[],
+                                                  fields=['x_ripeness_id'],
+                                                  groupby=['x_ripeness_id'])
+        expected = [{
+            'x_ripeness_id': self.ripeness_green,
+            'x_ripeness_id_count': 3,
+            '__domain': [('x_ripeness_id', '=', self.ripeness_green[0])],
+        }, {
+            'x_ripeness_id': self.ripeness_okay,
+            'x_ripeness_id_count': 0,
+            '__domain': [('x_ripeness_id', '=', self.ripeness_okay[0])],
+        }, {
+            'x_ripeness_id': self.ripeness_gone,
+            'x_ripeness_id_count': 0,
+            '__domain': [('x_ripeness_id', '=', self.ripeness_gone[0])],
+        }]
+        self.assertEqual(groups, expected, 'should include 2 empty ripeness stages')

odoo/addons/base/views/ir_model_views.xml

@@ -31,6 +31,7 @@
                             <field name="id" invisible="1"/>
                             <field name="name"/>
                             <field name="model" attrs="{'readonly': [('id', '!=', False)]}"/>
+                            <field name="order"/>
                             <field name="transient" attrs="{'readonly': [('id', '!=', False)]}" groups="base.group_no_one"/>
                         </group>
                         <group>
@@ -273,6 +274,9 @@
                                             attrs="{'required': [('ttype','in',['many2one','one2many','many2many'])],
                                                     'readonly': [('ttype','not in',['many2one','one2many','many2many'])],
                                                     'invisible': [('ttype','not in',['many2one','one2many','many2many'])]}"/>
+                                        <field name="group_expand" groups="base.group_no_one"
+                                            attrs="{'readonly': [('ttype','!=','many2one')],
+                                                    'invisible': [('ttype','!=','many2one')]}"/>
                                         <field name="on_delete" groups="base.group_no_one"
                                             attrs="{'readonly': [('ttype','!=','many2one')],
                                                     'invisible': [('ttype','!=','many2one')]}"/>

odoo/models.py

@@ -1754,6 +1754,11 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
         """
         cls.pool._clear_cache()
 
+    @api.model
+    def _read_group_expand_full(self, groups, domain, order):
+        """Extend the group to include all targer records by default."""
+        return groups.search([], order=order)
+
     @api.model
     def _read_group_fill_results(self, domain, groupby, remaining_groupbys,
                                  aggregated_fields, count_field,

odoo/modules/registry.py

@@ -183,6 +183,13 @@ class Registry(Mapping):
         """ Add or replace a model in the registry."""
         self.models[model_name] = model
 
+    def __delitem__(self, model_name):
+        """ Remove a (custom) model from the registry. """
+        del self.models[model_name]
+        # the custom model can inherit from mixins ('mail.thread', ...)
+        for Model in self.models.values():
+            Model._inherit_children.discard(model_name)
+
     def descendants(self, model_names, *kinds):
         """ Return the models corresponding to ``model_names`` and all those
         that inherit/inherits from them.

odoo/tests/common.py

@@ -560,6 +560,7 @@ class SingleTransactionCase(BaseCase):
     def setUpClass(cls):
         super().setUpClass()
         cls.registry = odoo.registry(get_db_name())
+        cls.addClassCleanup(cls.registry.reset_changes)
         cls.addClassCleanup(cls.registry.clear_caches)
 
         cls.cr = cls.registry.cursor()