[IMP] core, base: add tooling to populate database

Some use case like testing performance or upgrade scripts required a database
with prefilled data, covering basic corner cases. A solution can be to
create data a procedural way.

This commit proposes an API to easily populate a model, usually by giving a
list of possible values for each field or by giving a compute method that will
be based on raw values of other fields.

The basic way to define how to populate a new field is to override `_populate_factories`,
a method that returns a sequence of pairs `(field_name, factory)`.

The definition of a field is a "factory", a function that returns a neverending iterator
combining its value(s) with the values of the iterator given in parameter.
Some factory helpers are given in `tools.populate.py`:
- `iterate(vals, weighs)` ensures that one record is created for each value
by iterating on them, then resumes as `random.choice` on those vals following weights
once the first iteration is finished.
- `cartesian(vals, weights)` makes a cartesian product of its own values with the values
of its input iterator, then resumes as a randomized generator.
- `compute(function)` calls the given function with the current values dict and a random object,
and assigns the current field to the returned value.
- ...

Each iterator yields dictionaries of field values, and the factory should add a
value for the current field(s).  The yielded dictionaries also contain a pseudo_field
`"__complete"`, that indicates whether this step is some randomized data
to reach the expected count of records.  A falsy value indicates that the iterator
is still covering mandatory cases.  This indicates whether a cartesian product is
finished, or an `iterate` has consumed all its values.

The order of the factories is quite important, since some computed fields may need
other fields to be defined, and `cartesian` factories should always be at the beginning
to avoid having too many combination.  That is why the factories are given as a list of
pairs instead of a dictionary; this makes it easier to insert elements at any place.

Example:
field A: cartesian([T, F])
field B: cartesian([0, 1])
field C: iterate([a, b, c, d, e])
field D: compute(1-B)

_c is shortcut for __complete
_ is a random value, or result of a random value

```
iter | root  | field A  | field B | field C     | field D   | result
0     {_c:F}  {... A:T}  {... B:0} {...C:a}      {...D:1}    T,0,a,1 complete:False
                         {... B:1} {...C:b}      {...D:0}    T,1,b,0 complete:False
              {... A:F}  {... B:0} {...C:c}      {...D:1}    F,0,c,1 complete:False
                         {... B:1} {...C:d}      {...D:0}    F,1,d,0 complete:False
1     {_c:T}  {... A:_}  {... B:_} {...C:e,_c:F} {...D:_}    _,_,e,_ complete:False
2     {_c:T}  {... A:_}  {... B:_} {...C:_}      {...D:_}    _,_,_,_ complete:True
```

X-original-commit: 4c0182dafa584853ed83a166096f45c33c06a245

odoo/addons/test_populate/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+from . import models

odoo/addons/test_populate/__manifest__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+{
+    'name': 'test-populate',
+    'version': '0.1',
+    'category': 'Tests',
+    'description': """A module to test populate.""",
+    'depends': ['base'],
+    'data': [
+        'ir.model.access.csv',
+    ],
+    'installable': True,
+    'auto_install': False,
+}

odoo/addons/test_populate/ir.model.access.csv

@@ -0,0 +1,5 @@
+"id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink"
+access_test_populate,access_test_populate,model_test_populate,,1,1,1,1
+access_test_populate_category,access_test_populate_category,model_test_populate_category,,1,1,1,1
+access_test_no_populat,access_test_no_populat,model_test_no_populate,,1,1,1,1
+access_test_populate_inherit,access_test_populate_inherit,model_test_populate_inherit,,1,1,1,1

odoo/addons/test_populate/models.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo import models, fields, api
+
+from odoo.tools import populate, pycompat
+
+
+class TestPopulateModel(models.Model):
+    _name = 'test.populate'
+    _description = 'Test Populate'
+
+    name = fields.Char(default='Foo')
+    state = fields.Selection([('a', 'A'), ('b', 'B')], default='a')
+    active = fields.Boolean('Active', default=True)
+    category_id = fields.Many2one('test.populate.category', 'Category')
+    some_ref = fields.Integer('Reference')
+    dependant_field_1 = fields.Char('Dependant 1')
+    dependant_field_2 = fields.Char('Dependant 2')
+
+    _populate_dependencies = ['test.populate.category']
+
+    _populate_sizes = {
+        'small': 20,
+        'medium': 30,
+        'large': 100,
+    }
+
+    def _populate_factories(self):
+
+        # cross dependant field in a sub generator, cartesian product of two fields
+        dependant_factories = [
+            ('dependant_field_1', populate.cartesian(['d1_1', 'd1_2'])),
+            ('dependant_field_2', populate.cartesian(['d2_1', 'd2_2', 'd2_3_{counter}'])),
+        ]
+        def generate_dependant(iterator, *args):
+            dependants_generator = populate.chain_factories(dependant_factories, self._name)
+            for values in dependants_generator:
+                dependant_values = next(iterator)
+                yield {**values, **dependant_values, '__complete': values['__complete'] and dependant_values['__complete']}
+
+        def get_name(values=None, counter=0, **kwargs):
+            active = 'active' if values['active'] else 'inactive'
+            cat = 'filling' if values['__complete'] else 'corner'
+            return '%s_%s_%s' % (active, cat, counter)
+
+        category_ids = self.env.registry.populated_models['test.populate.category']
+
+        return [
+            ('active', populate.cartesian([True, False], [3, 1])),
+            ('state', populate.cartesian([False] + self.env['test.populate']._fields['state'].get_values(self.env))),
+            ('some_ref', populate.iterate([False, 1, 2, 3, 4])),
+            ('_dependant', generate_dependant),
+            ('name', populate.compute(get_name)),
+            ('category_id', populate.randomize([False] + category_ids)),
+        ]
+
+class TestPopulateDependencyModel(models.Model):
+    _name = 'test.populate.category'
+    _description = 'Test Populate Category'
+
+    _populate_sizes = {
+        'small': 3,
+        'medium': 10,
+        'large': 20,
+    }
+    name = fields.Char('Name', required=True, default='Cat1')
+    active = fields.Boolean('Active', default=True)
+
+    def _populate_factories(self):
+        return [
+            ('active', populate.cartesian([True, False], [9, 1])),
+            ('name', populate.cartesian(['Cat1', 'Cat2', 'Cat3'])),
+        ]
+
+class TestNoPopulateModelInherit(models.Model):
+    _name = 'test.populate.inherit'
+    _inherit = 'test.populate'
+
+    _description = 'Test populate inherit'
+
+    additionnal_field = fields.Char(required=True)
+
+    def _populate_factories(self):
+        return super()._populate_factories() + [
+            ('additionnal_field', populate.iterate(['V1', 'V2', 'V3'])),
+        ]
+
+
+class TestNoPopulateModel(models.Model):
+    _name = 'test.no.populate'
+    _description = 'A model with no populate method and a required field, should not crash'
+
+    name = fields.Char(required=True)

odoo/addons/test_populate/tests/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+from . import test_populate
+

odoo/addons/test_populate/tests/test_populate.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+import collections
+import logging
+from odoo.tests import common
+from odoo.cli.populate import Populate
+from odoo.tools import mute_logger, populate
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
+
+
+_logger = logging.getLogger(__name__)
+
+# todo patch cursor commit
+class TestPopulate(common.TransactionCase):
+    def setUp(self):
+        super(TestPopulate, self).setUp()
+        patcher = patch.object(self.cr, 'commit')
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_dependency(self):
+        ordered_models = Populate._get_ordered_models(self.env, ['test.populate'])
+        ordered_models_names = [model._name for model in ordered_models]
+        self.assertEqual(ordered_models_names, ['test.populate.category', 'test.populate'])
+
+    @mute_logger('odoo.cli.populate')
+    def test_no_populate(self):
+        """ Check that model with no populate method are not populated"""
+        model = 'test.no.populate'
+        populated = Populate.populate(self.env, 'small', [model])
+        new = populated[model]
+        self.assertFalse(new)
+
+    @mute_logger('odoo.cli.populate')
+    def test_populate(self):
+        """ Check that model with populate methods are correctly populated"""
+        model = 'test.populate'
+        populated = Populate.populate(self.env, 'small', [model])
+        records = self.check_test_populate_values(populated, model)
+
+        # pseudo random after cartesian with ~ 1/4 False, 3/4 True
+        # seed is model dependant
+        self.assertEqual(records.mapped('active')[6:20], [
+            True, True, True, True, True, True, False, False, True, True, True, True, False, True
+        ])
+        # pseudo random after iterate
+        self.assertEqual(records.mapped('some_ref')[5:20], [
+            1, 0, 2, 4, 4, 3, 4, 1, 2, 2, 2, 4, 4, 1, 2
+        ])
+
+    @mute_logger('odoo.cli.populate')
+    def test_populate_inherit(self):
+        """ Check that model with populate methods are correctly populated"""
+        model = 'test.populate.inherit'
+        populated = Populate.populate(self.env, 'small', [model])
+        records = self.check_test_populate_values(populated, model)  # should be same values as base class
+        # and additionnal_field has own values set
+        # iterate then pseudo random
+        self.assertEqual(records.mapped('additionnal_field')[:20], [
+            'V1', 'V2', 'V3',  # iterate
+            'V3', 'V1', 'V2', 'V1', 'V2', 'V1', 'V2', 'V2', 'V2', 'V1', 'V1', 'V3', 'V1', 'V2', 'V2', 'V3', 'V2'  # pseudorandom
+        ])
+
+    def check_test_populate_values(self, populated, model):
+        new = populated[model]
+        self.assertTrue(new)
+        records = self.env[model].browse(new)
+            # main cartesian product
+        self.assertEqual(records.mapped('active')[:6], [
+            True, True, True,
+            False, False, False,
+        ])
+        self.assertEqual(records.mapped('state')[:6], [
+            False, 'a', 'b',
+            False, 'a', 'b',
+        ])
+
+        # custom name call
+        self.assertEqual(records.mapped('name')[:6], [
+            'active_corner_0', 'active_corner_1', 'active_corner_2',
+            'inactive_corner_3', 'inactive_corner_4', 'inactive_corner_5',
+        ])
+        self.assertIn('filling', records.mapped('name')[6]) # filling when cartesian and iterate are done
+        # iterate then pseudo random
+        self.assertEqual(records.mapped('some_ref')[:5], [
+            0, 1, 2, 3, 4 # iterate
+        ])
+
+        # some custom multi field generator (as cartesian product in this example)
+        self.assertEqual(records.mapped('dependant_field_1')[:6], [
+            'd1_1', 'd1_1', 'd1_1',
+            'd1_2', 'd1_2', 'd1_2'
+        ])
+        self.assertEqual(records.mapped('dependant_field_2')[:6], [
+            'd2_1', 'd2_2', 'd2_3_0',
+            'd2_1', 'd2_2', 'd2_3_1'
+        ])
+        used_category_ids = set(records.mapped('category_id').ids[:20])
+        self.assertEqual(len(used_category_ids), 6) # event if id may change, with given seed, the 6 category are used
+        generated_category_ids = set(populated['test.populate.category'])
+        self.assertFalse(used_category_ids-generated_category_ids) # all category are the generated one
+        self.assertFalse(hasattr(self.env.registry, 'populated_models'), 'populated_models flag has been removed from registry')
+
+        return records
+
+
+@common.tagged('-at_install', 'post_install')
+class TestPopulateValidation(common.TransactionCase):
+    """ check that all fields in _populate_factories exists """
+    def setUp(self):
+        super(TestPopulateValidation, self).setUp()
+        self.env.registry.populated_models = collections.defaultdict(list)
+        self.addCleanup(delattr, self.env.registry, 'populated_models')
+
+    def test_populate_factories(self):
+        for model in self.env.values():
+            factories = model._populate_factories() or []
+            factories_fields = set([field_name for field_name, factory in factories if not field_name.startswith('_')])
+            missing = factories_fields - model._fields.keys()
+            self.assertFalse(missing, 'Fields %s not found in model %s' % (missing, model._name))
+
+
+@common.tagged('-standard', '-at_install', 'post_install', 'missing_populate')
+class TestPopulateMissing(common.TransactionCase):
+    """ check that all fields in _populate_factories exists """
+    def setUp(self):
+        super(TestPopulateMissing, self).setUp()
+        self.env.registry.populated_models = collections.defaultdict(list)
+        self.addCleanup(delattr, self.env.registry, 'populated_models')
+
+    def test_populate_missing_factories(self):
+        no_factory_models = []
+        for model in self.env.values():
+            factories = model._populate_factories()
+            if not factories:
+                if model._transient or model._abstract:
+                    continue
+                ir_model = self.env['ir.model'].search([('model', '=', model._name)])
+                if all(module.startswith('test_') for module in ir_model.modules.split(',')):
+                    continue
+                no_factory_models.append(model._name)
+            else:
+                factories_fields = next(populate.chain_factories(factories, model._name)).keys()
+                def is_electable(field):
+                    return not field.compute \
+                        and field.store \
+                        and field.name not in ('create_uid', 'write_uid', 'write_date', 'create_date', 'id') \
+                        and field.type not in ('many2many', 'one2many')
+                electable_fields = set([key for key, field in model._fields.items() if is_electable(field)])
+                no_factory_fields = set(electable_fields - factories_fields)
+                if no_factory_fields:
+                    _logger.info('Model %s has some undefined field: %s', model._name, no_factory_fields)
+
+        _logger.info('No populate factories defiend for %s', no_factory_models)

odoo/cli/__init__.py

@@ -11,3 +11,4 @@ from . import scaffold
 from . import server
 from . import shell
 from . import start
+from . import populate

odoo/cli/populate.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import time
+import fnmatch
+import logging
+import optparse
+import odoo
+
+from . import Command
+_logger = logging.getLogger(__name__)
+
+
+class Populate(Command):
+
+    def run(self, cmdargs):
+        parser = odoo.tools.config.parser
+        group = optparse.OptionGroup(parser, "Populate Configuration")
+        group.add_option("--size", dest="population_size",
+                        help="Populate database with auto-generated data. Value should be the population size: small, medium or large",
+                        default='small')
+        group.add_option("--models",
+                         dest='populate_models',
+                         help="Comma separated list of model or pattern (fnmatch)")
+        parser.add_option_group(group)
+        opt = odoo.tools.config.parse_config(cmdargs)
+        populate_models = opt.populate_models and set(opt.populate_models.split(','))
+        population_size = opt.population_size
+        with odoo.api.Environment.manage():
+            dbname = odoo.tools.config['db_name']
+            registry = odoo.registry(dbname)
+            with registry.cursor() as cr:
+                env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
+                self.populate(env, population_size, populate_models)
+
+
+    @classmethod
+    def populate(cls, env, size, model_patterns=False):
+        registry = env.registry
+        populated_models = None
+        try:
+            registry.populated_models = {}  # todo master, initialize with already populated models
+            ordered_models = cls._get_ordered_models(env, model_patterns)
+
+            _logger.log(25, 'Populating database')
+            for model in ordered_models:
+                _logger.info('Populating database for model %s', model._name)
+                t0 = time.time()
+                registry.populated_models[model._name] = model._populate(size).ids
+                # todo indicate somewhere that model is populated
+                env.cr.commit()
+                model_time = time.time() - t0
+                if model_time > 1:
+                    _logger.info('Populated database for model %s in %ss', model._name, model_time)
+        except:
+            _logger.exception('Something went wrong populating database')
+        finally:
+            populated_models = registry.populated_models
+            del registry.populated_models
+
+        return populated_models
+
+    @classmethod
+    def _get_ordered_models(cls, env, model_patterns=False):
+        _logger.info('Computing model order')
+        processed = set()
+        ordered_models = []
+        visited = set()
+        def add_model(model):
+            if model not in processed:
+                if model in visited:
+                    raise ValueError('Cyclic dependency detected for %s' % model)
+                visited.add(model)
+                for dep in model._populate_dependencies:
+                    add_model(env[dep])
+                ordered_models.append(model)
+                processed.add(model)
+        for model in env.values():
+            ir_model = env['ir.model'].search([('model', '=', model._name)])
+            if model_patterns and not any(fnmatch.fnmatch(model._name, match) for match in model_patterns):
+                continue
+            if model._transient or model._abstract:
+                continue
+            if not model_patterns and all(module.startswith('test_') for module in ir_model.modules.split(',')):
+                continue
+            add_model(model)
+
+        return ordered_models

odoo/models.py

@@ -61,6 +61,7 @@ from .tools.func import frame_codeinfo
 from .tools.misc import CountingStream, clean_context, DEFAULT_SERVER_DATETIME_FORMAT, DEFAULT_SERVER_DATE_FORMAT, get_lang
 from .tools.translate import _
 from .tools import date_utils
+from .tools import populate
 
 _logger = logging.getLogger(__name__)
 _schema = logging.getLogger(__name__ + '.schema')
@@ -6160,6 +6161,66 @@ Record ids: %(records)s
         """
         return 'placeholder.png'
 
+    def _populate_factories(self):
+        """ Return a list of pairs ``(field_name, factory)``, ``factory`` is a function that creates
+        a generator of values (dict of field values).  Its API suggested by::
+
+            # generator yield values by iterating on base_generator and adding a value
+            # for the given field in the dicts yielded by base_generator
+            generator = factory(base_generator, field_name, model_name)
+            # suggested usage
+            for values in generator:
+                model.create(values)
+        """
+        return []
+
+    @property
+    def _populate_sizes(self):
+        """ Return a dict mapping symbolic sizes (``'small'``, ``'medium'``, ``'large'``) to integers,
+        giving the minimal number of records that method ``_populate`` should create.
+        """
+        return {
+            'small': 10,  # minimal representative set
+            'medium': 100,  # average database load
+            'large': 1000, # maxi database load
+        }
+
+    @property
+    def _populate_dependencies(self):
+        return []
+
+    def _populate(self, size):
+        """ Create records to populate this model.
+        
+        :param size: symbolic size for the number of records: ``'small'``, ``'medium'`` or ``'large'``
+        """
+        batch_size = 1000
+        min_size = self._populate_sizes[size]
+
+        record_count = 0
+        create_values = []
+        complete = False
+        field_generators = self._populate_factories()
+        if not field_generators:
+            return self.browse() # maybe create an automatic generator?
+            
+        records_batches = []
+        generator = populate.chain_factories(field_generators, self._name)
+        while record_count <= min_size or not complete:
+            values = next(generator)
+            complete = values.pop('__complete')
+            create_values.append(values)
+            record_count += 1
+            if len(create_values) >= batch_size:
+                _logger.info('Batch: %s/%s', record_count, min_size)
+                records_batches.append(self.create(create_values))
+                create_values = []
+
+        if create_values:
+            records_batches.append(self.create(create_values))
+        return self.concat(*records_batches)
+
+
 collections.Set.register(BaseModel)
 # not exactly true as BaseModel doesn't have __reversed__, index or count
 collections.Sequence.register(BaseModel)

odoo/tools/config.py

@@ -352,9 +352,10 @@ class configmanager(object):
 
             odoo.tools.config.parse_config(sys.argv[1:])
         """
-        self._parse_config(args)
+        opt = self._parse_config(args)
         odoo.netsvc.init_logger()
         odoo.modules.module.initialize_sys_path()
+        return opt
 
     def _parse_config(self, args=None):
         if args is None:
@@ -529,6 +530,8 @@ class configmanager(object):
             m.strip() for m in self.options['server_wide_modules'].split(',') if m.strip()
         ]
 
+        return opt
+
     def _is_addons_path(self, path):
         from odoo.modules.module import MANIFEST_NAMES
         for f in os.listdir(path):

odoo/tools/populate.py

@@ -0,0 +1,113 @@
+import random
+from odoo.tools import pycompat
+
+
+def Random(seed):
+    """ Return a random number generator object with the given seed. """
+    r = random.Random()
+    r.seed(seed, version=2)
+    return r
+
+
+def format_str(val, counter, values):
+    """ Format the given value (with method ``format``) when it is a string. """
+    if isinstance(val, str):
+        return val.format(counter=counter, values=values)
+    return val
+
+
+def chain_factories(field_factories, model_name): 
+    """ Instanciate a generator by calling all the field factories. """
+    generator = root_factory()
+    for (fname, field_factory) in field_factories:
+        generator = field_factory(generator, fname, model_name)
+    return generator
+
+
+def root_factory():
+    """ Return a generator with empty values dictionaries (except for the flag ``__complete``). """
+    yield {'__complete': False}
+    while True:
+        yield {'__complete': True}
+
+
+def randomize(vals, weights=None, seed=False, formatter=format_str, counter_offset=0):
+    """ Return a factory for an iterator of values dicts with pseudo-randomly
+    chosen values (among ``vals``) for a field.
+    """
+    def generate(iterator, field_name, model_name):
+        r = Random('%s+field+%s' % (model_name, seed or field_name))
+        for counter, values in enumerate(iterator):
+            val = r.choices(vals, weights)[0]
+            values[field_name] = formatter(val, counter + counter_offset, values)
+            yield values
+    return generate
+
+
+def cartesian(vals, weights=None, seed=False, formatter=format_str, then=None):
+    """ Return a factory for an iterator of values dicts that combines all ``vals`` for
+    the field with the other field values in input.
+    """
+    def generate(iterator, field_name, model_name):
+        counter = 0
+        for values in iterator:
+            if values['__complete']:
+                break  # will consume and lose an element, (complete so a filling element). If it is a problem, use peekable instead.
+            for val in vals:
+                yield {**values, field_name: formatter(val, counter, values)}
+            counter += 1
+        factory = then or randomize(vals, weights, seed, formatter, counter)
+        yield from factory(iterator, field_name, model_name)
+    return generate
+
+
+def iterate(vals, weights=None, seed=False, formatter=format_str, then=None):
+    """ Return a factory for an iterator of values dicts that picks a value among ``vals``
+    for each input.  Once all ``vals`` have been used once, resume as a ``randomize``
+    generator.
+    """
+    def generate(iterator, field_name, model_name):
+        counter = 0
+        for val in vals: # iteratable order is important, shortest first
+            values = next(iterator)
+            values[field_name] = formatter(val, counter, values)
+            values['__complete'] = False
+            yield values
+            counter += 1
+        factory = then or randomize(vals, weights, seed, formatter, counter)
+        yield from factory(iterator, field_name, model_name)
+    return generate
+
+
+def constant(val, formatter=format_str):
+    """ Return a factory for an iterator of values dicts that sets the field
+    to the given value in each input dict.
+    """
+    def generate(iterator, field_name, _):
+        for counter, values in enumerate(iterator):
+            values[field_name] = formatter(val, counter, values)
+            yield values
+    return generate
+
+
+def compute(function, seed=None):
+    """ Return a factory for an iterator of values dicts that computes the field value
+    as ``function(values, counter, random)``, where ``values`` is the other field values,
+    ``counter`` is an integer, and ``random`` is a pseudo-random number generator.
+    """
+    def generate(iterator, field_name, model_name):
+        r = Random('%s+field+%s' % (model_name, seed or field_name))
+        for counter, values in enumerate(iterator):
+            val = function(values=values, counter=counter, random=r)
+            values[field_name] = val
+            yield values
+    return generate
+
+
+def randint(a, b, seed=None):
+    """ Return a factory for an iterator of values dicts that sets the field
+    to the random integer between a and b included in each input dict.
+    """
+    def get_rand_int(random=None, **kwargs):
+        random.randint(a, b)
+    return compute(get_rand_int, seed=seed)