pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[REF] core: read_group() now uses _read_group()

Browse Source 
Make the public `read_group` depends of its private method `_read_group`
refactored to match the backend usage.

We try to keep the public API similar for this first part of the
rafactor, but there are still some API change:
- We cannot order by `id` anymore.
- The display_name of many2x group values are not lazy anymore.

Part-of: odoo/odoo#110737
This commit is contained in:
Rémy Voet (ryv)
2023-04-19 21:58:28 +02:00
parent e5b42a0a20
commit 81a892ac8c
4 changed files with 204 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/project/tests/test_project_sharing.py
+2 -2
View File
@@ -283,7 +283,7 @@ class TestProjectSharing(TestProjectSharingCommon):
task_read_group = self.env['project.task'].read_group(
expression.AND([expression.TRUE_DOMAIN, domain]),
['id'],
['id:min'],
[],
)
self.assertEqual(task_read_group[0]['__count'], 1, 'The task should be found with the read_group method containing a truly tuple.')
@@ -291,7 +291,7 @@ class TestProjectSharing(TestProjectSharingCommon):
task_read_group = self.env['project.task'].read_group(
expression.AND([expression.FALSE_DOMAIN, domain]),
['id'],
['id:min'],
[],
)
self.assertFalse(task_read_group[0]['__count'], 'No result should found with the read_group since the domain is falsy.')
odoo/addons/test_performance/tests/test_performance.py
-18
View File
@@ -592,24 +592,6 @@ class TestPerformance(SavepointCaseWithUserDemo):
result = model.read_group([], ['partner_id', 'value'], ['partner_id'])
self.assertEqual(result, expected)
@users('__system__', 'demo')
def test_read_group_without_name_get(self):
model = self.env['test_performance.base']
expected = self.expected_read_group()
# use read_group and check the expected result
with self.assertQueryCount(__system__=1, demo=1):
self.env.invalidate_all()
result = model.read_group([], ['partner_id', 'value'], ['partner_id'])
self.assertEqual(len(result), len(expected))
for res, exp in zip(result, expected):
self.assertEqual(res['__domain'], exp['__domain'])
self.assertEqual(res['partner_id'][0], exp['partner_id'][0])
self.assertEqual(res['partner_id_count'], exp['partner_id_count'])
self.assertEqual(res['value'], exp['value'])
# now serialize to json, which should force evaluation
with self.assertQueryCount(__system__=1, demo=1):
json.dumps(result, default=json_default)
@tagged('bacon_and_eggs')
class TestIrPropertyOptimizations(TransactionCase):
odoo/addons/test_read_group/tests/test_m2m_grouping.py
+18 -14
View File
@@ -74,13 +74,13 @@ class TestM2MGrouping(common.TransactionCase):
'user_ids': (self.users[0].id, "Mario"),
'user_ids_count': 1,
'name': ["Super Mario Bros."],
'__domain': ['&', ('user_ids', '=', self.users[0].id), ('id', '=', self.tasks[0].id)],
'__domain': ['&', ('id', '=', self.tasks[0].id), ('user_ids', '=', self.users[0].id)],
},
{ # task of Luigi
'user_ids': (self.users[1].id, "Luigi"),
'user_ids_count': 1,
'name': ["Super Mario Bros."],
'__domain': ['&', ('user_ids', '=', self.users[1].id), ('id', '=', self.tasks[0].id)],
'__domain': ['&', ('id', '=', self.tasks[0].id), ('user_ids', '=', self.users[1].id)],
},
])
@@ -130,18 +130,23 @@ class TestM2MGrouping(common.TransactionCase):
'domain_force': [('id', '=', self.users[0].id)],
})
# warmup
as_admin = self.tasks.read_group(
domain=[],
fields=['name:array_agg'],
groupby=['user_ids'],
)
# as superuser, ir.rule should not apply
expected = """
SELECT
min("test_read_group_task".id) AS id,
count("test_read_group_task".id) AS "user_ids_count",
array_agg("test_read_group_task"."name") AS "name",
"test_read_group_task__user_ids"."user_id" AS "user_ids"
"test_read_group_task__user_ids"."user_id",
COUNT(*),
ARRAY_AGG("test_read_group_task"."name" ORDER BY "test_read_group_task"."id")
FROM "test_read_group_task"
LEFT JOIN "test_read_group_task_user_rel" AS "test_read_group_task__user_ids"
ON ("test_read_group_task"."id" = "test_read_group_task__user_ids"."task_id")
LEFT JOIN "test_read_group_task_user_rel" AS "test_read_group_task__user_ids" ON ("test_read_group_task"."id" = "test_read_group_task__user_ids"."task_id")
GROUP BY "test_read_group_task__user_ids"."user_id"
ORDER BY "user_ids"
ORDER BY "test_read_group_task__user_ids"."user_id" ASC
"""
with self.assertQueries([expected]):
as_admin = self.tasks.read_group(
@@ -178,10 +183,9 @@ class TestM2MGrouping(common.TransactionCase):
expected = """
SELECT
min("test_read_group_task".id) AS id,
count("test_read_group_task".id) AS "user_ids_count",
array_agg("test_read_group_task"."name") AS "name",
"test_read_group_task__user_ids"."user_id" AS "user_ids"
"test_read_group_task__user_ids"."user_id",
COUNT(*),
ARRAY_AGG("test_read_group_task"."name" ORDER BY "test_read_group_task"."id")
FROM "test_read_group_task"
LEFT JOIN "test_read_group_task_user_rel" AS "test_read_group_task__user_ids"
ON (
@@ -193,7 +197,7 @@ class TestM2MGrouping(common.TransactionCase):
)
)
GROUP BY "test_read_group_task__user_ids"."user_id"
ORDER BY "user_ids"
ORDER BY "test_read_group_task__user_ids"."user_id" ASC
"""
with self.assertQueries([expected]):
as_demo = tasks.read_group(
odoo/models.py
+184 -394
View File
@@ -2140,13 +2140,12 @@ class BaseModel(metaclass=MetaModel):
return groups.search([], order=order)
@api.model
def _read_group_fill_results(self, domain, groupby, remaining_groupbys,
aggregated_fields, count_field,
read_group_result, read_group_order=None):
def _read_group_fill_results(self, domain, groupby, annoted_aggregates, read_group_result, read_group_order=None):
"""Helper method for filling in empty groups for all possible values of
the field being grouped by"""
field = self._fields[groupby]
if not field.group_expand:
field_name, __ = groupby.split(':') if ':' in groupby else (groupby, 'month')
field = self._fields[field_name]
if not field or not field.group_expand:
return read_group_result
# field.group_expand is a callable or the name of a method, that returns
@@ -2164,14 +2163,13 @@ class BaseModel(metaclass=MetaModel):
if field.relational:
# groups is a recordset; determine order on groups's model
groups = self.env[field.comodel_name].browse([value[0] for value in values])
groups = self.env[field.comodel_name].browse([value.id for value in values])
order = groups._order
if read_group_order == groupby + ' desc':
order = tools.reverse_order(order)
groups = group_expand(self, groups, domain, order)
groups = groups.sudo()
values = lazy_name_get(groups)
value2key = lambda value: value and value[0]
values = groups.sudo()
value2key = lambda value: value and value.id
else:
# groups is a list of values
@@ -2183,27 +2181,28 @@ class BaseModel(metaclass=MetaModel):
# Merge the current results (list of dicts) with all groups. Determine
# the global order of results groups, which is supposed to be in the
# same order as read_group_result (in the case of a many2one field).
result = OrderedDict((value2key(value), {}) for value in values)
# fill in results from read_group_result
read_group_result_as_dict = {}
for line in read_group_result:
key = value2key(line[groupby])
if not result.get(key):
result[key] = line
else:
result[key][count_field] = line[count_field]
read_group_result_as_dict[value2key(line[groupby])] = line
# fill in missing results from all groups
empty_item = {
name: self._read_group_empty_value(spec)
for name, spec in annoted_aggregates.items()
}
result = {}
# fill result with the values order
for value in values:
key = value2key(value)
if not result[key]:
line = dict.fromkeys(aggregated_fields, False)
line[groupby] = value
line[groupby + '_count'] = 0
line['__domain'] = [(groupby, '=', key)] + domain
if remaining_groupbys:
line['__context'] = {'group_by': remaining_groupbys}
result[key] = line
if key in read_group_result_as_dict:
result[key] = read_group_result_as_dict.pop(key)
else:
result[key] = dict(empty_item, **{groupby: value})
for line in read_group_result_as_dict.values():
key = value2key(line[groupby])
result[key] = line
# add folding information if present
if field.relational and groups._fold_name in groups._fields:
@@ -2215,7 +2214,7 @@ class BaseModel(metaclass=MetaModel):
return list(result.values())
@api.model
def _read_group_fill_temporal(self, data, groupby, aggregated_fields, annotated_groupbys,
def _read_group_fill_temporal(self, data, groupby, annoted_aggregates,
fill_from=False, fill_to=False, min_groups=False):
"""Helper method for filling date/datetime 'holes' in a result set.
@@ -2289,8 +2288,8 @@ class BaseModel(metaclass=MetaModel):
Aug Sep Oct Nov
:param list data: the data containing groups
:param list groupby: name of the first group by
:param list aggregated_fields: list of aggregated fields in the query
:param list groupby: list of fields being grouped on
:param list annoted_aggregates: dict of "<key_name>:<aggregate specification>"
:param str fill_from: (inclusive) string representation of a
date/datetime, start bound of the fill_temporal range
formats: date -> %Y-%m-%d, datetime -> %Y-%m-%d %H:%M:%S
@@ -2302,27 +2301,35 @@ class BaseModel(metaclass=MetaModel):
:rtype: list
:return: list
"""
first_a_gby = annotated_groupbys[0]
if first_a_gby['type'] not in ('date', 'datetime'):
# TODO: remove min_groups
first_group = groupby[0]
field_name = first_group.split(':')[0]
field = self._fields[field_name]
if field.type not in ('date', 'datetime'):
return data
interval = first_a_gby['interval']
granularity = first_a_gby['granularity']
tz = pytz.timezone(self._context['tz']) if first_a_gby["tz_convert"] else False
groupby_name = groupby[0]
granularity = first_group.split(':')[1] if ':' in first_group else 'month'
interval = READ_GROUP_TIME_GRANULARITY[granularity]
tz = False
if field.type == 'datetime' and self._context.get('tz') in pytz.all_timezones_set:
tz = pytz.timezone(self._context['tz'])
# TODO: refactor remaing lines here
# existing non null datetimes
existing = [d[groupby_name] for d in data if d[groupby_name]] or [None]
existing = [d[first_group] for d in data if d[first_group]] or [None]
# assumption: existing data is sorted by field 'groupby_name'
existing_from, existing_to = existing[0], existing[-1]
if fill_from:
fill_from = date_utils.start_of(odoo.fields.Datetime.to_datetime(fill_from), granularity)
fill_from = odoo.fields.Datetime.to_datetime(fill_from) if isinstance(fill_from, datetime.datetime) else odoo.fields.Date.to_date(fill_from)
fill_from = date_utils.start_of(fill_from, granularity)
if tz:
fill_from = tz.localize(fill_from)
elif existing_from:
fill_from = existing_from
if fill_to:
fill_to = date_utils.start_of(odoo.fields.Datetime.to_datetime(fill_to), granularity)
fill_to = odoo.fields.Datetime.to_datetime(fill_to) if isinstance(fill_to, datetime.datetime) else odoo.fields.Date.to_date(fill_to)
fill_to = date_utils.start_of(fill_to, granularity)
if tz:
fill_to = tz.localize(fill_to)
elif existing_to:
@@ -2348,17 +2355,20 @@ class BaseModel(metaclass=MetaModel):
else:
existing = sorted(set().union(existing, required_dates))
empty_item = {'id': False, (groupby_name.split(':')[0] + '_count'): 0}
empty_item.update({key: False for key in aggregated_fields})
empty_item.update({key: False for key in [group['groupby'] for group in annotated_groupbys[1:]]})
empty_item = {
name: self._read_group_empty_value(spec)
for name, spec in annoted_aggregates.items()
}
for group in groupby[1:]:
empty_item[group] = self._read_group_empty_value(group)
grouped_data = collections.defaultdict(list)
for d in data:
grouped_data[d[groupby_name]].append(d)
grouped_data[d[first_group]].append(d)
result = []
for dt in existing:
result.extend(grouped_data[dt] or [dict(empty_item, **{groupby_name: dt})])
result.extend(grouped_data[dt] or [dict(empty_item, **{first_group: dt})])
if False in grouped_data:
result.extend(grouped_data[False])
@@ -2366,149 +2376,7 @@ class BaseModel(metaclass=MetaModel):
return result
@api.model
def _read_group_prepare(self, orderby, aggregated_fields, annotated_groupbys, query):
"""
Prepares the GROUP BY and ORDER BY terms for the read_group method. Adds the missing JOIN clause
to the query if order should be computed against m2o field.
:param orderby: the orderby definition in the form "%(field)s %(order)s"
:param aggregated_fields: list of aggregated fields in the query
:param annotated_groupbys: list of dictionaries returned by
:meth:`_read_group_process_groupby`
These dictionaries contain the qualified name of each groupby
(fully qualified SQL name for the corresponding field),
and the (non raw) field name.
:param Query query: the query under construction
:return: (groupby_terms, orderby_terms)
"""
orderby_terms = []
groupby_terms = [gb['qualified_field'] for gb in annotated_groupbys]
if not orderby:
return groupby_terms, orderby_terms
self._check_qorder(orderby)
# when a field is grouped as 'foo:bar', both orderby='foo' and
# orderby='foo:bar' generate the clause 'ORDER BY "foo:bar"'
groupby_fields = {
gb[key]: gb['groupby']
for gb in annotated_groupbys
for key in ('field', 'groupby')
}
for order_part in orderby.split(','):
order_split = order_part.split() # potentially ["field:group_func", "desc"]
order_field = order_split[0]
is_many2one_id = order_field.endswith(".id")
if is_many2one_id:
order_field = order_field[:-3]
if order_field == 'id' or order_field in groupby_fields:
order_field_name = order_field.split(':')[0]
if self._fields[order_field_name].type == 'many2one' and not is_many2one_id:
order_clause = self._generate_order_by(order_part, query)
order_clause = order_clause.replace('ORDER BY ', '')
if order_clause:
orderby_terms.append(order_clause)
groupby_terms += [order_term.split()[0] for order_term in order_clause.split(',')]
else:
order_split[0] = '"%s"' % groupby_fields.get(order_field, order_field)
orderby_terms.append(' '.join(order_split))
elif order_field in aggregated_fields:
order_split[0] = '"%s"' % order_field
orderby_terms.append(' '.join(order_split))
elif order_field not in self._fields:
raise ValueError("Invalid field %r on model %r" % (order_field, self._name))
elif order_field == 'sequence':
pass
else:
# Cannot order by a field that will not appear in the results (needs to be grouped or aggregated)
_logger.warning('%s: read_group order by `%s` ignored, cannot sort on empty columns (not grouped/aggregated)',
self._name, order_part)
return groupby_terms, orderby_terms
@api.model
def _read_group_process_groupby(self, gb, query):
"""
Helper method to collect important information about groupbys: raw
field name, type, time information, qualified name, ...
"""
split = gb.split(':')
field = self._fields.get(split[0])
if not field:
raise ValueError("Invalid field %r on model %r" % (split[0], self._name))
field_type = field.type
gb_function = split[1] if len(split) == 2 else 'month'
temporal = field_type in ('date', 'datetime')
tz_convert = field_type == 'datetime' and self._context.get('tz') in pytz.all_timezones
qualified_field = self._inherits_join_calc(self._table, split[0], query)
if temporal:
display_formats = {
# Careful with week/year formats:
# - yyyy (lower) must always be used, *except* for week+year formats
# - YYYY (upper) must always be used for week+year format
# e.g. 2006-01-01 is W52 2005 in some locales (de_DE),
# and W1 2006 for others
#
# Mixing both formats, e.g. 'MMM YYYY' would yield wrong results,
# such as 2006-01-01 being formatted as "January 2005" in some locales.
# Cfr: http://babel.pocoo.org/en/latest/dates.html#date-fields
'hour': 'hh:00 dd MMM',
'day': 'dd MMM yyyy', # yyyy = normal year
'week': "'W'w YYYY", # w YYYY = ISO week-year
'month': 'MMMM yyyy',
'quarter': 'QQQ yyyy',
'year': 'yyyy',
}
time_intervals = {
'hour': dateutil.relativedelta.relativedelta(hours=1),
'day': dateutil.relativedelta.relativedelta(days=1),
'week': datetime.timedelta(days=7),
'month': dateutil.relativedelta.relativedelta(months=1),
'quarter': dateutil.relativedelta.relativedelta(months=3),
'year': dateutil.relativedelta.relativedelta(years=1)
}
if tz_convert:
qualified_field = "timezone('%s', timezone('UTC',%s))" % (self._context.get('tz', 'UTC'), qualified_field)
if gb_function == 'week':
# first_week_day: 0=Monday, 1=Tuesday, ...
first_week_day = int(get_lang(self.env).week_start) - 1
days_offset = first_week_day and 7 - first_week_day
qualified_field = f"date_trunc('{gb_function}', {qualified_field}::timestamp - INTERVAL '-{days_offset} DAY') + INTERVAL '-{days_offset} DAY'"
else:
qualified_field = f"date_trunc('{gb_function}', {qualified_field}::timestamp)"
if field_type == 'boolean':
qualified_field = "coalesce(%s,false)" % qualified_field
return {
'field': split[0],
'groupby': gb,
'type': field_type,
'display_format': display_formats[gb_function] if temporal else None,
'interval': time_intervals[gb_function] if temporal else None,
'granularity': gb_function if temporal else None,
'tz_convert': tz_convert,
'qualified_field': qualified_field,
}
@api.model
def _read_group_prepare_data(self, key, value, groupby_dict):
"""
Helper method to sanitize the data received by read_group. The None
values are converted to False, and the date/datetime are formatted,
and corrected according to the timezones.
"""
value = False if value is None else value
gb = groupby_dict.get(key)
if gb and gb['type'] in ('date', 'datetime') and value:
if isinstance(value, str):
dt_format = DEFAULT_SERVER_DATETIME_FORMAT if gb['type'] == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
value = datetime.datetime.strptime(value, dt_format)
if gb['tz_convert']:
value = pytz.timezone(self._context['tz']).localize(value)
return value
@api.model
def _read_group_format_result(self, data, annotated_groupbys, groupby, domain):
def _read_group_format_result(self, rows_dict, lazy_groupby):
"""
Helper method to format the data contained in the dictionary data by
adding the domain corresponding to its values, the groupbys in the
@@ -2520,68 +2388,67 @@ class BaseModel(metaclass=MetaModel):
:param domain: original domain for read_group
"""
sections = []
for gb in annotated_groupbys:
ftype = gb['type']
value = data[gb['groupby']]
for group in lazy_groupby:
field_name = group.split(':')[0]
field = self._fields[field_name]
# full domain for this groupby spec
d = None
if value:
if ftype in ['many2one', 'many2many']:
value = value[0]
elif ftype in ('date', 'datetime'):
locale = get_lang(self.env).code
fmt = DEFAULT_SERVER_DATETIME_FORMAT if ftype == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
tzinfo = None
range_start = value
range_end = value + gb['interval']
# value from postgres is in local tz (so range is
# considered in local tz e.g. "day" is [00:00, 00:00[
# local rather than UTC which could be [11:00, 11:00]
# local) but domain and raw value should be in UTC
if gb['tz_convert']:
tzinfo = range_start.tzinfo
range_start = range_start.astimezone(pytz.utc)
# take into account possible hour change between start and end
range_end = tzinfo.localize(range_end.replace(tzinfo=None))
range_end = range_end.astimezone(pytz.utc)
if field.type in ('many2one', 'many2many') or field_name == 'id':
ids = [row[group].id for row in rows_dict if row[group]]
m2x_records = self.env[field.comodel_name].browse(ids)
name_get_dict = dict(m2x_records.sudo().name_get())
range_start = range_start.strftime(fmt)
range_end = range_end.strftime(fmt)
if ftype == 'datetime':
label = babel.dates.format_datetime(
value, format=gb['display_format'],
tzinfo=tzinfo, locale=locale
)
elif field.type in ('date', 'datetime'):
locale = get_lang(self.env).code
fmt = DEFAULT_SERVER_DATETIME_FORMAT if field.type == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
granularity = group.split(':')[1] if ':' in group else 'month'
interval = READ_GROUP_TIME_GRANULARITY[granularity]
for row in rows_dict:
value = row[group]
if field.type in ('many2one', 'many2many'):
value = value.id
row[group] = (value, name_get_dict[value]) if value else value
additional_domain = [(field_name, '=', value)]
if field.type in ('date', 'datetime'):
if value:
range_start = value
range_end = value + interval
if field.type == 'datetime':
tzinfo = None
if self._context.get('tz') in pytz.all_timezones_set:
tzinfo = pytz.timezone(self._context['tz'])
range_start = tzinfo.localize(range_start).astimezone(pytz.utc)
# take into account possible hour change between start and end
range_end = tzinfo.localize(range_end).astimezone(pytz.utc)
label = babel.dates.format_datetime(
range_start, format=READ_GROUP_DISPLAY_FORMAT[granularity],
tzinfo=tzinfo, locale=locale
)
else:
label = babel.dates.format_date(
value, format=READ_GROUP_DISPLAY_FORMAT[granularity],
locale=locale
)
range_start = range_start.strftime(fmt)
range_end = range_end.strftime(fmt)
row[group] = label # TODO should put raw data
row.setdefault('__range', {})[group] = {'from': range_start, 'to': range_end}
additional_domain = [
'&',
(field_name, '>=', range_start),
(field_name, '<', range_end),
]
else:
label = babel.dates.format_date(
value, format=gb['display_format'],
locale=locale
)
data[gb['groupby']] = ('%s/%s' % (range_start, range_end), label)
data.setdefault('__range', {})[gb['groupby']] = {'from': range_start, 'to': range_end}
d = [
'&',
(gb['field'], '>=', range_start),
(gb['field'], '<', range_end),
]
elif ftype in ('date', 'datetime'):
# Set the __range of the group containing records with an unset
# date/datetime field value to False.
data.setdefault('__range', {})[gb['groupby']] = False
if d is None:
d = [(gb['field'], '=', value)]
sections.append(d)
sections.append(domain)
data['__domain'] = expression.AND(sections)
if len(groupby) - len(annotated_groupbys) >= 1:
data['__context'] = { 'group_by': groupby[len(annotated_groupbys):]}
del data['id']
return data
# Set the __range of the group containing records with an unset
# date/datetime field value to False.
row.setdefault('__range', {})[group] = False
row['__domain'] = expression.AND([row['__domain'], additional_domain])
@api.model
def read_group(self, domain, fields, groupby, offset=0, limit=None, orderby=False, lazy=True):
@@ -2621,178 +2488,101 @@ class BaseModel(metaclass=MetaModel):
:rtype: [{'field_name_1': value, ...}, ...]
:raise AccessError: if user is not allowed to access requested information
"""
result = self._read_group_raw(domain, fields, groupby, offset=offset, limit=limit, orderby=orderby, lazy=lazy)
groupby = [groupby] if isinstance(groupby, str) else groupby[:1] if lazy else OrderedSet(groupby)
groupby_dates = [
groupby_description for groupby_description in groupby
if self._fields[groupby_description.split(':')[0]].type in ('date', 'datetime') # e.g. 'date:month'
]
if not groupby_dates:
return result
groupby = [groupby] if isinstance(groupby, str) else groupby
lazy_groupby = groupby[:1] if lazy else groupby
# iterate on all results and replace the "full" date/datetime value (<=> group[df])
# which is a tuple (range, label) by just the formatted label, in-place.
for group in result:
for groupby_date in groupby_dates:
# could group on a date(time) field which is empty in some
# records, in which case as with m2o the _raw value will be
# `False` instead of a (value, label) pair. In that case,
# leave the `False` value alone
if group.get(groupby_date):
group[groupby_date] = group[groupby_date][1]
return result
@api.model
def _read_group_raw(self, domain, fields, groupby, offset=0, limit=None, orderby=False, lazy=True):
self.check_access_rights('read')
query = self._where_calc(domain)
fields = fields or [f.name for f in self._fields.values() if f.store]
groupby = [groupby] if isinstance(groupby, str) else list(OrderedSet(groupby))
groupby_list = groupby[:1] if lazy else groupby
annotated_groupbys = [self._read_group_process_groupby(gb, query) for gb in groupby_list]
groupby_fields = [g['field'] for g in annotated_groupbys]
order = orderby or ','.join([g for g in groupby_list])
groupby_dict = {gb['groupby']: gb for gb in annotated_groupbys}
self._apply_ir_rules(query, 'read')
for gb in groupby_fields:
if gb not in self._fields:
raise UserError(_("Unknown field %r in 'groupby'", gb))
if not self._fields[gb].base_field.groupable:
raise UserError(_(
"Field %s is not a stored field, only stored fields (regular or "
"many2many) are valid for the 'groupby' parameter", self._fields[gb],
))
aggregated_fields = []
select_terms = []
fnames = [] # list of fields to flush
for fspec in fields:
if fspec == 'sequence':
continue
if fspec == '__count':
# the web client sometimes adds this pseudo-field in the list
continue
match = regex_field_agg.match(fspec)
if not match:
raise UserError(_("Invalid field specification %r.", fspec))
name, func, fname = match.groups()
if func:
# we have either 'name:func' or 'name:func(fname)'
fname = fname or name
field = self._fields.get(fname)
if not field:
raise ValueError("Invalid field %r on model %r" % (fname, self._name))
if not (field.base_field.store and field.base_field.column_type):
raise UserError(_("Cannot aggregate field %r.", fname))
if func not in READ_GROUP_AGGREGATE:
raise UserError(_("Invalid aggregation function %r.", func))
# Compatibility layer with _read_group, it should be remove in the second part of the refactoring
# - Modify `groupby` default value 'month' into specifique groupby specification
# - Modify `fields` into aggregates specification of _read_group
# - Modify the order to be compatible with the _read_group specification
annoted_groupby = {} # Key as the name in the result, value as the explicit groupby specification
for group_spec in lazy_groupby:
field_name, granularity = parse_read_group_spec(group_spec)
if field_name not in self._fields:
raise ValueError(f"Invalid field {field_name!r} on model {self._name!r}")
field = self._fields[field_name]
if field.type in ('date', 'datetime'):
annoted_groupby[group_spec] = f"{field_name}:{granularity or 'month'}"
else:
# we have 'name', retrieve the aggregator on the field
field = self._fields.get(name)
if not field:
raise ValueError("Invalid field %r on model %r" % (name, self._name))
if not (field.base_field.store and
field.base_field.column_type and field.group_operator):
continue
func, fname = field.group_operator, name
annoted_groupby[group_spec] = group_spec
fnames.append(fname)
if fname in groupby_fields:
continue
if name in aggregated_fields:
raise UserError(_("Output name %r is used twice.", name))
aggregated_fields.append(name)
expr = self._inherits_join_calc(self._table, fname, query)
if func.lower() == 'count_distinct':
term = 'COUNT(DISTINCT %s) AS "%s"' % (expr, name)
else:
term = '%s(%s) AS "%s"' % (func, expr, name)
select_terms.append(term)
for gb in annotated_groupbys:
select_terms.append('%s as "%s" ' % (gb['qualified_field'], gb['groupby']))
self._flush_search(domain, fields=fnames + groupby_fields, order=order)
groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, annotated_groupbys, query)
from_clause, where_clause, where_clause_params = query.get_sql()
if lazy and (len(groupby_fields) >= 2 or not self._context.get('group_by_no_leaf')):
count_field = groupby_fields[0] if len(groupby_fields) >= 1 else '_'
else:
count_field = '_'
count_field += '_count'
prefix_terms = lambda prefix, terms: (prefix + " " + ",".join(terms)) if terms else ''
prefix_term = lambda prefix, term: ('%s %s' % (prefix, term)) if term else ''
query = """
SELECT min("%(table)s".id) AS id, count("%(table)s".id) AS "%(count_field)s" %(extra_fields)s
FROM %(from)s
%(where)s
%(groupby)s
%(orderby)s
%(limit)s
%(offset)s
""" % {
'table': self._table,
'count_field': count_field,
'extra_fields': prefix_terms(',', select_terms),
'from': from_clause,
'where': prefix_term('WHERE', where_clause),
'groupby': prefix_terms('GROUP BY', groupby_terms),
'orderby': prefix_terms('ORDER BY', orderby_terms),
'limit': prefix_term('LIMIT', int(limit) if limit else None),
'offset': prefix_term('OFFSET', int(offset) if limit else None),
annoted_aggregates = { # Key as the name in the result, value as the explicit aggregate specification
f"{lazy_groupby[0].split(':')[0]}_count" if lazy and len(lazy_groupby) == 1 else '__count': '__count',
}
self._cr.execute(query, where_clause_params)
fetched_data = self._cr.dictfetchall()
for field_spec in fields:
if field_spec == '__count':
continue
match = regex_field_agg.match(field_spec)
if not match:
raise ValueError(f"Invalid field specification {field_spec!r}.")
name, func, fname = match.groups()
if not groupby_fields:
return fetched_data
if fname: # Manage this kind of specification : "field_min:min(field)"
annoted_aggregates[name] = f"{fname}:{func}"
continue
if func: # Manage this kind of specification : "field:min"
annoted_aggregates[name] = f"{name}:{func}"
continue
self._read_group_resolve_many2x_fields(fetched_data, annotated_groupbys)
if name not in self._fields:
raise ValueError(f"Invalid field {field_name!r} on model {self._name!r}")
field = self._fields[name]
if field.base_field.store and field.base_field.column_type and field.group_operator and field_spec not in annoted_groupby:
annoted_aggregates[name] = f"{name}:{field.group_operator}"
data = [{k: self._read_group_prepare_data(k, v, groupby_dict) for k, v in r.items()} for r in fetched_data]
if orderby:
new_terms = []
for order_term in orderby.split(','):
order_term = order_term.strip()
for key_name, annoted in itertools.chain(reversed(annoted_groupby.items()), annoted_aggregates.items()):
key_name = key_name.split(':')[0]
if order_term.startswith(f'{key_name} ') or key_name == order_term:
order_term = order_term.replace(key_name, annoted)
break
new_terms.append(order_term)
orderby = ','.join(new_terms)
else:
orderby = ','.join(annoted_groupby.values())
rows = self._read_group(domain, annoted_groupby.values(), annoted_aggregates.values(), offset=offset, limit=limit, order=orderby)
rows_dict = [
dict(zip(itertools.chain(annoted_groupby, annoted_aggregates), row))
for row in rows
]
fill_temporal = self.env.context.get('fill_temporal')
if (data and fill_temporal) or isinstance(fill_temporal, dict):
if lazy_groupby and (rows_dict and fill_temporal) or isinstance(fill_temporal, dict):
# fill_temporal = {} is equivalent to fill_temporal = True
# if fill_temporal is a dictionary and there is no data, there is a chance that we
# want to display empty columns anyway, so we should apply the fill_temporal logic
if not isinstance(fill_temporal, dict):
fill_temporal = {}
data = self._read_group_fill_temporal(data, groupby, aggregated_fields,
annotated_groupbys, **fill_temporal)
# TODO Shouldn't be possible with a limit
rows_dict = self._read_group_fill_temporal(
rows_dict, lazy_groupby,
annoted_aggregates, **fill_temporal,
)
result = [self._read_group_format_result(d, annotated_groupbys, groupby, domain) for d in data]
if lazy:
if lazy_groupby and lazy:
# Right now, read_group only fill results in lazy mode (by default).
# If you need to have the empty groups in 'eager' mode, then the
# method _read_group_fill_results need to be completely reimplemented
# in a sane way
result = self._read_group_fill_results(
domain, groupby_fields[0], groupby[len(annotated_groupbys):],
aggregated_fields, count_field, result, read_group_order=order,
# TODO Shouldn't be possible with a limit or the limit should be in account
rows_dict = self._read_group_fill_results(
domain, lazy_groupby[0],
annoted_aggregates, rows_dict, read_group_order=orderby,
)
return result
def _read_group_resolve_many2x_fields(self, data, fields):
many2xfields = {field['field'] for field in fields if field['type'] in ['many2one', 'many2many']}
for field in many2xfields:
ids_set = {d[field] for d in data if d[field]}
m2x_records = self.env[self._fields[field].comodel_name].browse(ids_set)
data_dict = dict(lazy_name_get(m2x_records.sudo()))
for d in data:
d[field] = (d[field], data_dict[d[field]]) if d[field] else False
for row in rows_dict:
row['__domain'] = domain
if len(lazy_groupby) < len(groupby):
row['__context'] = {'group_by': groupby[len(lazy_groupby):]}
self._read_group_format_result(rows_dict, lazy_groupby)
return rows_dict
def _inherits_join_add(self, current_model, parent_model_name, query):
"""