diff --git a/addons/mail/models/mail_activity_mixin.py b/addons/mail/models/mail_activity_mixin.py index aeb1f543ba5..5c599b70ecf 100644 --- a/addons/mail/models/mail_activity_mixin.py +++ b/addons/mail/models/mail_activity_mixin.py @@ -8,6 +8,7 @@ import pytz from odoo import api, fields, models from odoo.osv import expression +from odoo.tools import SQL _logger = logging.getLogger(__name__) @@ -263,26 +264,30 @@ class MailActivityMixin(models.AbstractModel): self.env['res.partner'].flush_model(['tz']) tz = 'UTC' - if 'tz' in self.env.context and self.env.context.get('tz') in pytz.all_timezones_set: + if self.env.context.get('tz') in pytz.all_timezones_set: tz = self.env.context['tz'] - join_query = f""" - (SELECT res_id, - CASE - WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, '{tz}'))::date) > 0 THEN 'planned' - WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, '{tz}'))::date) < 0 THEN 'overdue' - WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, '{tz}'))::date) = 0 THEN 'today' - ELSE null - END AS activity_state - FROM mail_activity - JOIN res_users ON (res_users.id = mail_activity.user_id) - JOIN res_partner ON (res_partner.id = res_users.partner_id) - WHERE res_model = '{self._name}' - GROUP BY res_id) - """ - alias = query.join(self._table, "id", join_query, "res_id", "last_activity_state") + sql_join = SQL( + """ + (SELECT res_id, + CASE + WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %(tz)s))::date) > 0 THEN 'planned' + WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %(tz)s))::date) < 0 THEN 'overdue' + WHEN min(date_deadline - (now() AT TIME ZONE COALESCE(res_partner.tz, %(tz)s))::date) = 0 THEN 'today' + ELSE null + END AS activity_state + FROM mail_activity + JOIN res_users ON (res_users.id = mail_activity.user_id) + JOIN res_partner ON (res_partner.id = res_users.partner_id) + WHERE res_model = %(res_model)s + GROUP BY res_id) + """, + res_model=self._name, + tz=tz, + ) + alias = query.join(self._table, "id", sql_join, "res_id", "last_activity_state") - return f'"{alias}"."activity_state"', ['activity_state'] + return SQL.identifier(alias, 'activity_state'), ['activity_state'] def toggle_active(self): """ Before archiving the record we should also remove its ongoing diff --git a/addons/project/report/project_task_burndown_chart_report.py b/addons/project/report/project_task_burndown_chart_report.py index de35f6b98fc..4d5594e5d46 100644 --- a/addons/project/report/project_task_burndown_chart_report.py +++ b/addons/project/report/project_task_burndown_chart_report.py @@ -81,6 +81,7 @@ class ReportProjectTaskBurndownChart(models.AbstractModel): simple_date_groupby_sql, __ = self._read_group_groupby(f"date:{interval}", main_query) # Removing unexistant table name from the expression + simple_date_groupby_sql = self.env.cr.mogrify(simple_date_groupby_sql).decode() simple_date_groupby_sql = simple_date_groupby_sql.replace('"project_task_burndown_chart_report".', '') burndown_chart_query = """ @@ -232,7 +233,7 @@ class ReportProjectTaskBurndownChart(models.AbstractModel): def _read_group_select(self, aggregate_spec, query): if aggregate_spec == '__count': - return f'SUM("{self._table}"."__count")', [] + return SQL("SUM(%s)", SQL.identifier(self._table, '__count')), [] return super()._read_group_select(aggregate_spec, query) def _read_group(self, domain, groupby=(), aggregates=(), having=(), offset=0, limit=None, order=None): diff --git a/addons/purchase_stock/report/vendor_delay_report.py b/addons/purchase_stock/report/vendor_delay_report.py index aac602768c9..ea43727cecd 100644 --- a/addons/purchase_stock/report/vendor_delay_report.py +++ b/addons/purchase_stock/report/vendor_delay_report.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -from odoo import api, fields, models, tools -from odoo.exceptions import UserError -from odoo.osv.expression import expression +from odoo import fields, models, tools +from odoo.tools import SQL class VendorDelayReport(models.Model): @@ -57,6 +56,6 @@ GROUP BY m.id def _read_group_select(self, aggregate_spec, query): if aggregate_spec == 'on_time_rate:sum': # Make a weigthed average instead of simple average for these fields - sql_expression = 'SUM(qty_on_time) / SUM(qty_total) * 100' - return sql_expression, ['on_time_rate', 'qty_on_time', 'qty_total'] + sql_expr = SQL('SUM(qty_on_time) / SUM(qty_total) * 100') + return sql_expr, ['on_time_rate', 'qty_on_time', 'qty_total'] return super()._read_group_select(aggregate_spec, query) diff --git a/addons/stock/models/stock_quant.py b/addons/stock/models/stock_quant.py index 5432317de7d..aa62e6b1e74 100644 --- a/addons/stock/models/stock_quant.py +++ b/addons/stock/models/stock_quant.py @@ -11,7 +11,7 @@ from psycopg2 import Error from odoo import _, api, fields, models, SUPERUSER_ID from odoo.exceptions import UserError, ValidationError from odoo.osv import expression -from odoo.tools import check_barcode_encoding, groupby +from odoo.tools import check_barcode_encoding, groupby, SQL from odoo.tools.float_utils import float_compare, float_is_zero _logger = logging.getLogger(__name__) @@ -300,11 +300,12 @@ class StockQuant(models.Model): def _read_group_select(self, aggregate_spec, query): if aggregate_spec == 'inventory_quantity:sum' and self.env.context.get('inventory_report_mode'): - return "NULL", [] + return SQL("NULL"), [] if aggregate_spec == 'available_quantity:sum': - quantity_expr, quantity_fnames = self._read_group_select('quantity:sum', query) - reserved_quantity_expr, reserved_quantity_fnames = self._read_group_select('reserved_quantity:sum', query) - return f'{quantity_expr} - {reserved_quantity_expr}', quantity_fnames + reserved_quantity_fnames + sql_quantity, quantity_fnames = self._read_group_select('quantity:sum', query) + sql_reserved_quantity, reserved_quantity_fnames = self._read_group_select('reserved_quantity:sum', query) + sql_expr = SQL("%s - %s", sql_quantity, sql_reserved_quantity) + return sql_expr, quantity_fnames + reserved_quantity_fnames if aggregate_spec == 'inventory_quantity_auto_apply:sum': return self._read_group_select('quantity:sum', query) return super()._read_group_select(aggregate_spec, query) diff --git a/odoo/models.py b/odoo/models.py index f747955df0a..6aeb445a6a7 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -352,17 +352,17 @@ READ_GROUP_TIME_GRANULARITY = { # valid SQL aggregation functions READ_GROUP_AGGREGATE = { - 'sum': lambda table, expr: f'SUM({expr})', - 'avg': lambda table, expr: f'AVG({expr})', - 'max': lambda table, expr: f'MAX({expr})', - 'min': lambda table, expr: f'MIN({expr})', - 'bool_and': lambda table, expr: f'BOOL_AND({expr})', - 'bool_or': lambda table, expr: f'BOOL_OR({expr})', - 'array_agg': lambda table, expr: f'ARRAY_AGG({expr} ORDER BY "{table}"."id")', + 'sum': lambda table, expr: SQL('SUM(%s)', expr), + 'avg': lambda table, expr: SQL('AVG(%s)', expr), + 'max': lambda table, expr: SQL('MAX(%s)', expr), + 'min': lambda table, expr: SQL('MIN(%s)', expr), + 'bool_and': lambda table, expr: SQL('BOOL_AND(%s)', expr), + 'bool_or': lambda table, expr: SQL('BOOL_OR(%s)', expr), + 'array_agg': lambda table, expr: SQL('ARRAY_AGG(%s ORDER BY %s)', expr, SQL.identifier(table, 'id')), # 'recordset' aggregates will be post-processed to become recordsets - 'recordset': lambda table, expr: f'ARRAY_AGG({expr} ORDER BY "{table}"."id")', - 'count': lambda table, expr: f'COUNT({expr})', - 'count_distinct': lambda table, expr: f'COUNT(DISTINCT {expr})', + 'recordset': lambda table, expr: SQL('ARRAY_AGG(%s ORDER BY %s)', expr, SQL.identifier(table, 'id')), + 'count': lambda table, expr: SQL('COUNT(%s)', expr), + 'count_distinct': lambda table, expr: SQL('COUNT(DISTINCT %s)', expr), } READ_GROUP_DISPLAY_FORMAT = { @@ -1860,28 +1860,25 @@ class BaseModel(metaclass=MetaModel): query = self._search(domain) fnames_to_flush = OrderedSet() - groupby_terms_map = {} # {spec: } + + groupby_terms: dict[str, SQL] = {} for spec in groupby: - sql_expression, fnames_used = self._read_group_groupby(spec, query) - groupby_terms_map[spec] = SQL(sql_expression) + groupby_terms[spec], fnames_used = self._read_group_groupby(spec, query) fnames_to_flush.update(fnames_used) - select_terms = [] # [,] + select_terms: list[SQL] = [] for spec in aggregates: - sql_expression, fnames_used = self._read_group_select(spec, query) - select_terms.append(SQL(sql_expression)) + sql_expr, fnames_used = self._read_group_select(spec, query) + select_terms.append(sql_expr) fnames_to_flush.update(fnames_used) - having_expression, having_params, fnames_used = self._read_group_having(having, query) - having_term = SQL(having_expression, *having_params) + sql_having, fnames_used = self._read_group_having(having, query) fnames_to_flush.update(fnames_used) - orderby_terms, extra_groupby_terms, fnames_used = self._read_group_orderby(order, groupby_terms_map, query) - orderby_terms = [SQL(term) for term in orderby_terms] - extra_groupby_terms = [SQL(term) for term in extra_groupby_terms] + sql_order, sql_extra_groupby, fnames_used = self._read_group_orderby(order, groupby_terms, query) fnames_to_flush.update(fnames_used) - groupby_terms = list(groupby_terms_map.values()) + groupby_terms = list(groupby_terms.values()) query_parts = [ SQL("SELECT %s", SQL(", ").join(groupby_terms + select_terms)), @@ -1890,11 +1887,13 @@ class BaseModel(metaclass=MetaModel): if query.where_clause: query_parts.append(SQL("WHERE %s", query.where_clause)) if groupby_terms: - query_parts.append(SQL("GROUP BY %s", SQL(", ").join(groupby_terms + extra_groupby_terms))) - if having_term: - query_parts.append(SQL("HAVING %s", having_term)) - if orderby_terms: - query_parts.append(SQL("ORDER BY %s", SQL(", ").join(orderby_terms))) + if sql_extra_groupby: + groupby_terms.append(sql_extra_groupby) + query_parts.append(SQL("GROUP BY %s", SQL(", ").join(groupby_terms))) + if sql_having: + query_parts.append(SQL("HAVING %s", sql_having)) + if sql_order: + query_parts.append(SQL("ORDER BY %s", sql_order)) if limit: query_parts.append(SQL("LIMIT %s", limit)) if offset: @@ -1927,13 +1926,12 @@ class BaseModel(metaclass=MetaModel): # return [(a1, b1, c1), (a2, b2, c2), ...] return list(zip(*column_result)) - @api.model - def _read_group_select(self, aggregate_spec, query): + def _read_group_select(self, aggregate_spec: str, query: Query) -> tuple[SQL, list[str]]: """ Return a pair (, []) corresponding to the given aggregation. """ if aggregate_spec == '__count': - return 'COUNT(*)', [] + return SQL("COUNT(*)"), [] fname, property_name, func = parse_read_group_spec(aggregate_spec) @@ -1950,13 +1948,11 @@ class BaseModel(metaclass=MetaModel): if func == 'recordset' and not (field.relational or fname == 'id'): raise ValueError(f"Aggregate method {func!r} can be only used on relational field (or id) (for {aggregate_spec!r}).") - field_expression = self._inherits_join_calc(self._table, access_fname, query) + sql_field = self._field_to_sql(self._table, access_fname, query) + sql_expr = READ_GROUP_AGGREGATE[func](self._table, sql_field) + return sql_expr, [fname] - expression = READ_GROUP_AGGREGATE[func](self._table, field_expression) - return expression, [fname] - - @api.model - def _read_group_groupby(self, groupby_spec, query): + def _read_group_groupby(self, groupby_spec: str, query: Query) -> tuple[SQL, list[str]]: """ Return a pair (, []) corresponding to the given groupby element. """ @@ -1967,10 +1963,8 @@ class BaseModel(metaclass=MetaModel): field = self._fields[fname] if property_name: - check_property_field_value_name(property_name) if field.type != "properties": raise ValueError(f"Property set on a non properties field: {property_name!r}") - access_fname = f"{fname}.{property_name}" else: access_fname = fname @@ -1978,13 +1972,9 @@ class BaseModel(metaclass=MetaModel): if granularity and field.type not in ('datetime', 'date', 'properties'): raise ValueError(f"Granularity set on a no-datetime field or property: {groupby_spec!r}") - if not field.store and not field.inherited: # TODO: should be manage by _inherits_join_calc - raise ValueError(f"Groupby on no-store field: {groupby_spec!r}") - - field_expression = self._inherits_join_calc(self._table, access_fname, query) + sql_expr = self._field_to_sql(self._table, access_fname, query) if field.type == 'datetime' and self.env.context.get('tz') in pytz.all_timezones_set: - # TODO: `self.env.context.get('tz')` should be passed as args - field_expression = f"timezone('{self.env.context.get('tz')}', timezone('UTC', {field_expression}))" + sql_expr = SQL("timezone(%s, timezone('UTC', %s))", self.env.context['tz'], sql_expr) if field.type in ('datetime', 'date') or (field.type == 'properties' and granularity): if not granularity: @@ -1992,67 +1982,66 @@ class BaseModel(metaclass=MetaModel): if granularity not in READ_GROUP_TIME_GRANULARITY: raise ValueError(f"Granularity specification isn't correct: {granularity!r}") - # TODO: `granularity` should be passed as args if granularity == 'week': # first_week_day: 0=Monday, 1=Tuesday, ... first_week_day = int(get_lang(self.env).week_start) - 1 days_offset = first_week_day and 7 - first_week_day - field_expression = f"(date_trunc('week', {field_expression}::timestamp - INTERVAL '-{days_offset} DAY') + INTERVAL '-{days_offset} DAY')" + interval = f"-{days_offset} DAY" + sql_expr = SQL( + "(date_trunc('week', %s::timestamp - INTERVAL %s) + INTERVAL %s)", + sql_expr, interval, interval, + ) else: - field_expression = f"date_trunc('{granularity}', {field_expression}::timestamp)" + sql_expr = SQL("date_trunc(%s, %s::timestamp)", granularity, sql_expr) + if field.type == 'date': - field_expression += '::date' + sql_expr = SQL("%s::date", sql_expr) + elif field.type == 'boolean': - field_expression = f"COALESCE({field_expression}, FALSE)" + sql_expr = SQL("COALESCE(%s, FALSE)", sql_expr) - return field_expression, [fname] + return sql_expr, [fname] - @api.model - def _read_group_having(self, having_domain, query): - """ Return (, [], []) - corresponding to the having domain. + def _read_group_having(self, having_domain: list, query: Query) -> tuple[SQL, list[str]]: + """ Return a pair (, []) corresponding + to the having domain. """ if not having_domain: - return "", [], [] + return SQL(), [] - # stack of [(sql_expr, params)] - stack = [] + stack: list[SQL] = [] fnames_used = [] SUPPORTED = ('in', 'not in', '<', '>', '<=', '>=', '=', '!=') for item in reversed(having_domain): if item == '!': - expr, params = stack.pop() - stack.append((f'(NOT {expr})', params)) - elif item in ('&', '|'): - expr1, params1 = stack.pop() - expr2, params2 = stack.pop() - operator = 'AND' if item == '&' else 'OR' - stack.append((f'({expr1} {operator} {expr2})', params1 + params2)) + stack.append(SQL("(NOT %s)", stack.pop())) + elif item == '&': + stack.append(SQL("(%s AND %s)", stack.pop(), stack.pop())) + elif item == '|': + stack.append(SQL("(%s OR %s)", stack.pop(), stack.pop())) elif isinstance(item, (list, tuple)) and len(item) == 3: left, operator, right = item if operator not in SUPPORTED: raise ValueError(f"Invalid having clause {item!r}: supported comparators are {SUPPORTED}") - expr, fnames = self._read_group_select(left, query) - stack.append((f'{expr} {operator} %s', [right])) + sql_left, fnames = self._read_group_select(left, query) + sql_operator = expression.SQL_OPERATORS[operator] + stack.append(SQL("%s %s %s", sql_left, sql_operator, right)) fnames_used.extend(fnames) else: raise ValueError(f"Invalid having clause {item!r}: it should be a domain-like clause") while len(stack) > 1: - expr1, params1 = stack.pop() - expr2, params2 = stack.pop() - stack.append((f'({expr1} AND {expr2})', params1 + params2)) + stack.append(SQL("(%s AND %s)", stack.pop(), stack.pop())) - expr, params = stack[0] - return expr, params, fnames_used + return stack[0], fnames_used - @api.model - def _read_group_orderby(self, order, groupby_terms, query): - """ Return ([], [], []) + def _read_group_orderby(self, order: str, groupby_terms: dict[str, SQL], + query: Query) -> tuple[SQL, SQL, list[str]]: + """ Return (, , []) corresponding to the given order and groupby terms. - :param order: The order to use in SQL format - :param groupby_terms: The group by terms ({spec: sql_expression}) + :param order: the order specification + :param groupby_terms: the group by terms mapping ({spec: sql_expression}) :param query: The query we are building """ if order: @@ -2061,40 +2050,50 @@ class BaseModel(metaclass=MetaModel): order = ','.join(groupby_terms) traverse_many2one = False + if not order: + return SQL(), SQL(), [] + orderby_terms = [] extra_groupby_terms = [] fnames_used = [] - if not order: - return orderby_terms, extra_groupby_terms, fnames_used for order_part in order.split(','): order_match = regex_order.match(order_part) if not order_match: raise ValueError(f"Invalid order {order!r} for _read_group()") - order_term = order_match['term'] - order_direction = (order_match['direction'] or 'ASC').upper() - order_nulls = (order_match['nulls'] or '').upper() + term = order_match['term'] + direction = (order_match['direction'] or 'ASC').upper() + nulls = (order_match['nulls'] or '').upper() - if order_term not in groupby_terms: + sql_direction = SQL(direction) if direction in ('ASC', 'DESC') else SQL() + sql_nulls = SQL(nulls) if nulls in ('NULLS FIRST', 'NULLS LAST') else SQL() + + if term not in groupby_terms: try: - sql_expression, fnames_used_select = self._read_group_select(order_term, query) - fnames_used.extend(fnames_used_select) - orderby_terms.append(f'{sql_expression} {order_direction} {order_nulls}') - continue + sql_expr, fnames = self._read_group_select(term, query) except ValueError as e: raise ValueError(f"Order term {order_part!r} is not a valid aggregate nor valid groupby") from e + orderby_terms.append(SQL("%s %s %s", sql_expr, sql_direction, sql_nulls)) + fnames_used.extend(fnames) + continue + + field = self._fields.get(term) + if traverse_many2one and field and field.type == 'many2one': + # this generates an extra clause to add in the group by + sql_order = self._order_to_sql(f'{term} {direction} {nulls}', query) + orderby_terms.append(sql_order) + sql_order_str = self.env.cr.mogrify(sql_order).decode() + extra_groupby_terms.extend( + SQL(order.strip().split()[0]) + for order in sql_order_str.split(",") + if order.strip() + ) - if traverse_many2one and order_term in self and self._fields[order_term].type == 'many2one': - # It will generated extra clause to add in the group by. - extra_order = self._generate_order_by(f'{order_term} {order_direction} {order_nulls}', query) - extra_order = extra_order.replace(' ORDER BY ', '') - orderby_terms.extend(order.strip() for order in extra_order.split(",") if order.strip()) - extra_groupby_terms.extend(order.strip().split()[0] for order in extra_order.split(",") if order.strip()) else: - sql_expression = groupby_terms[order_term] - orderby_terms.append(SQL(f'%s {order_direction} {order_nulls}', sql_expression)) + sql_expr = groupby_terms[term] + orderby_terms.append(SQL("%s %s %s", sql_expr, sql_direction, sql_nulls)) - return orderby_terms, extra_groupby_terms, fnames_used + return SQL(", ").join(orderby_terms), SQL(", ").join(extra_groupby_terms), fnames_used @api.model def _read_group_check_field_access_rights(self, field_names):