Xavier-Do
f545fd274d
Some use case like testing performance or upgrade scripts required a database
with prefilled data, covering basic corner cases. A solution can be to
create data a procedural way.
This commit proposes an API to easily populate a model, usually by giving a
list of possible values for each field or by giving a compute method that will
be based on raw values of other fields.
The basic way to define how to populate a new field is to override `_populate_factories`,
a method that returns a sequence of pairs `(field_name, factory)`.
The definition of a field is a "factory", a function that returns a neverending iterator
combining its value(s) with the values of the iterator given in parameter.
Some factory helpers are given in `tools.populate.py`:
- `iterate(vals, weighs)` ensures that one record is created for each value
by iterating on them, then resumes as `random.choice` on those vals following weights
once the first iteration is finished.
- `cartesian(vals, weights)` makes a cartesian product of its own values with the values
of its input iterator, then resumes as a randomized generator.
- `compute(function)` calls the given function with the current values dict and a random object,
and assigns the current field to the returned value.
- ...
Each iterator yields dictionaries of field values, and the factory should add a
value for the current field(s). The yielded dictionaries also contain a pseudo_field
`"__complete"`, that indicates whether this step is some randomized data
to reach the expected count of records. A falsy value indicates that the iterator
is still covering mandatory cases. This indicates whether a cartesian product is
finished, or an `iterate` has consumed all its values.
The order of the factories is quite important, since some computed fields may need
other fields to be defined, and `cartesian` factories should always be at the beginning
to avoid having too many combination. That is why the factories are given as a list of
pairs instead of a dictionary; this makes it easier to insert elements at any place.
Example:
field A: cartesian([T, F])
field B: cartesian([0, 1])
field C: iterate([a, b, c, d, e])
field D: compute(1-B)
_c is shortcut for __complete
_ is a random value, or result of a random value
```
iter | root | field A | field B | field C | field D | result
0 {_c:F} {... A:T} {... B:0} {...C:a} {...D:1} T,0,a,1 complete:False
{... B:1} {...C:b} {...D:0} T,1,b,0 complete:False
{... A:F} {... B:0} {...C:c} {...D:1} F,0,c,1 complete:False
{... B:1} {...C:d} {...D:0} F,1,d,0 complete:False
1 {_c:T} {... A:_} {... B:_} {...C:e,_c:F} {...D:_} _,_,e,_ complete:False
2 {_c:T} {... A:_} {... B:_} {...C:_} {...D:_} _,_,_,_ complete:True
```
X-original-commit: 4c0182dafa584853ed83a166096f45c33c06a245
89 lines
3.5 KiB
Python
89 lines
3.5 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import time
|
|
import fnmatch
|
|
import logging
|
|
import optparse
|
|
import odoo
|
|
|
|
from . import Command
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Populate(Command):
|
|
|
|
def run(self, cmdargs):
|
|
parser = odoo.tools.config.parser
|
|
group = optparse.OptionGroup(parser, "Populate Configuration")
|
|
group.add_option("--size", dest="population_size",
|
|
help="Populate database with auto-generated data. Value should be the population size: small, medium or large",
|
|
default='small')
|
|
group.add_option("--models",
|
|
dest='populate_models',
|
|
help="Comma separated list of model or pattern (fnmatch)")
|
|
parser.add_option_group(group)
|
|
opt = odoo.tools.config.parse_config(cmdargs)
|
|
populate_models = opt.populate_models and set(opt.populate_models.split(','))
|
|
population_size = opt.population_size
|
|
with odoo.api.Environment.manage():
|
|
dbname = odoo.tools.config['db_name']
|
|
registry = odoo.registry(dbname)
|
|
with registry.cursor() as cr:
|
|
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
|
|
self.populate(env, population_size, populate_models)
|
|
|
|
|
|
@classmethod
|
|
def populate(cls, env, size, model_patterns=False):
|
|
registry = env.registry
|
|
populated_models = None
|
|
try:
|
|
registry.populated_models = {} # todo master, initialize with already populated models
|
|
ordered_models = cls._get_ordered_models(env, model_patterns)
|
|
|
|
_logger.log(25, 'Populating database')
|
|
for model in ordered_models:
|
|
_logger.info('Populating database for model %s', model._name)
|
|
t0 = time.time()
|
|
registry.populated_models[model._name] = model._populate(size).ids
|
|
# todo indicate somewhere that model is populated
|
|
env.cr.commit()
|
|
model_time = time.time() - t0
|
|
if model_time > 1:
|
|
_logger.info('Populated database for model %s in %ss', model._name, model_time)
|
|
except:
|
|
_logger.exception('Something went wrong populating database')
|
|
finally:
|
|
populated_models = registry.populated_models
|
|
del registry.populated_models
|
|
|
|
return populated_models
|
|
|
|
@classmethod
|
|
def _get_ordered_models(cls, env, model_patterns=False):
|
|
_logger.info('Computing model order')
|
|
processed = set()
|
|
ordered_models = []
|
|
visited = set()
|
|
def add_model(model):
|
|
if model not in processed:
|
|
if model in visited:
|
|
raise ValueError('Cyclic dependency detected for %s' % model)
|
|
visited.add(model)
|
|
for dep in model._populate_dependencies:
|
|
add_model(env[dep])
|
|
ordered_models.append(model)
|
|
processed.add(model)
|
|
for model in env.values():
|
|
ir_model = env['ir.model'].search([('model', '=', model._name)])
|
|
if model_patterns and not any(fnmatch.fnmatch(model._name, match) for match in model_patterns):
|
|
continue
|
|
if model._transient or model._abstract:
|
|
continue
|
|
if not model_patterns and all(module.startswith('test_') for module in ir_model.modules.split(',')):
|
|
continue
|
|
add_model(model)
|
|
|
|
return ordered_models
|