From d29a622740f6c34d25c52add5367bfdf58bbaf49 Mon Sep 17 00:00:00 2001 From: "Victor Piryns (pivi)" Date: Fri, 15 Mar 2024 17:38:37 +0100 Subject: [PATCH] [PERF] account: speed up dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Description: Add supporting indexes that are used in the queries generated when openning the accounting dashboard. A query is tweaked to hit those indexes and avoid `JOIN` where possible. Benchmark: Hot loading the default accounting dashboard, default filters and 1 company selected on a staging database with millions of accounting related entries. | | Before | After | |---------|----------|----------| | Timings | 7.32 sec | 1.6 sec | Reference: task-3805835 closes odoo/odoo#157892 Signed-off-by: William André (wan) --- .../account/models/account_bank_statement.py | 8 +++ .../models/account_bank_statement_line.py | 10 ++++ .../models/account_journal_dashboard.py | 57 ++++++++++++------- addons/account/models/account_move.py | 8 +++ addons/account/models/account_move_line.py | 12 +++- 5 files changed, 75 insertions(+), 20 deletions(-) diff --git a/addons/account/models/account_bank_statement.py b/addons/account/models/account_bank_statement.py index cc8b30b7110..a01be31655c 100644 --- a/addons/account/models/account_bank_statement.py +++ b/addons/account/models/account_bank_statement.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from odoo import api, fields, models, _, Command from odoo.exceptions import UserError +from odoo.tools import create_index from odoo.tools.misc import formatLang class AccountBankStatement(models.Model): @@ -102,6 +103,13 @@ class AccountBankStatement(models.Model): string="Attachments", ) + def init(self): + super().init() + create_index(self.env.cr, + indexname='account_bank_statement_journal_id_date_desc_id_desc_idx', + tablename='account_bank_statement', + expressions=['journal_id', 'date DESC', 'id DESC']) + # ------------------------------------------------------------------------- # COMPUTE METHODS # ------------------------------------------------------------------------- diff --git a/addons/account/models/account_bank_statement_line.py b/addons/account/models/account_bank_statement_line.py index 6fb3c307951..54a98db850d 100644 --- a/addons/account/models/account_bank_statement_line.py +++ b/addons/account/models/account_bank_statement_line.py @@ -6,6 +6,8 @@ from odoo.addons.base.models.res_bank import sanitize_account_number from xmlrpc.client import MAXINT +from odoo.tools import create_index + class AccountBankStatementLine(models.Model): _name = "account.bank.statement.line" @@ -152,6 +154,14 @@ class AccountBankStatementLine(models.Model): # Technical field to store details about the bank statement line transaction_details = fields.Json(readonly=True) + def init(self): + super().init() + create_index(self.env.cr, + indexname='account_bank_statement_line_internal_index_move_id_amount_idx', + tablename='account_bank_statement_line', + expressions=['internal_index', 'move_id', 'amount'], + where='statement_id IS NULL') + # ------------------------------------------------------------------------- # COMPUTE METHODS # ------------------------------------------------------------------------- diff --git a/addons/account/models/account_journal_dashboard.py b/addons/account/models/account_journal_dashboard.py index 88f27ef6441..2ffcddcd719 100644 --- a/addons/account/models/account_journal_dashboard.py +++ b/addons/account/models/account_journal_dashboard.py @@ -9,7 +9,7 @@ from odoo import models, api, _, fields from odoo.exceptions import UserError from odoo.osv import expression from odoo.release import version -from odoo.tools import DEFAULT_SERVER_DATE_FORMAT as DF +from odoo.tools import DEFAULT_SERVER_DATE_FORMAT as DF, SQL from odoo.tools.misc import formatLang, format_date as odoo_format_date, get_lang @@ -123,23 +123,26 @@ class account_journal(models.Model): journal.json_activity_data = json.dumps({'activities': activities[journal.id]}) def _query_has_sequence_holes(self): - self.env['res.company'].flush_model(['fiscalyear_lock_date']) self.env['account.move'].flush_model(['journal_id', 'date', 'sequence_prefix', 'sequence_number', 'state']) - self.env.cr.execute(""" - SELECT move.journal_id, - move.sequence_prefix - FROM account_move move - JOIN res_company company ON company.id = move.company_id - WHERE move.journal_id = ANY(%(journal_ids)s) - AND move.company_id = ANY(%(company_ids)s) - AND (move.state = 'posted' OR (move.state = 'draft' AND move.name != '/')) - AND (company.fiscalyear_lock_date IS NULL OR move.date > company.fiscalyear_lock_date) - GROUP BY move.journal_id, move.sequence_prefix - HAVING COUNT(*) != MAX(move.sequence_number) - MIN(move.sequence_number) + 1 - """, { - 'journal_ids': self.ids, - 'company_ids': self.env.companies.ids, - }) + queries = [] + for company in self.env.companies: + queries.append(SQL( + """ + SELECT move.journal_id, + move.sequence_prefix + FROM account_move move + WHERE move.journal_id = ANY(%(journal_ids)s) + AND move.company_id = %(company_id)s + AND (move.state = 'posted' OR (move.state = 'draft' AND move.name != '/')) + AND %(fiscalyear_lock_date_clause)s + GROUP BY move.journal_id, move.sequence_prefix + HAVING COUNT(*) != MAX(move.sequence_number) - MIN(move.sequence_number) + 1 + """, + journal_ids=self.ids, + company_id=company.id, + fiscalyear_lock_date_clause=SQL('move.date > %s', lock_date) if (lock_date := company.fiscalyear_lock_date) else SQL('TRUE') + )) + self.env.cr.execute(SQL(' UNION ALL '.join(['%s'] * len(queries)), *queries)) return self.env.cr.fetchall() def _compute_has_sequence_holes(self): @@ -508,7 +511,23 @@ class account_journal(models.Model): ) } - sale_purchase_journals._fill_dashboard_data_count(dashboard_data, 'account.move', 'entries_count', []) + self.env.cr.execute(SQL(""" + SELECT id, moves_exists + FROM account_journal journal + LEFT JOIN LATERAL ( + SELECT EXISTS(SELECT 1 + FROM account_move move + WHERE move.journal_id = journal.id + AND move.company_id = ANY (%(companies_ids)s) AND + move.journal_id = ANY (%(journal_ids)s)) AS moves_exists + ) moves ON TRUE + WHERE journal.id = ANY (%(journal_ids)s); + """, + journal_ids=sale_purchase_journals.ids, + companies_ids=self.env.companies.ids, + )) + is_sample_data_by_journal_id = {row[0]: not row[1] for row in self.env.cr.fetchall()} + for journal in sale_purchase_journals: # User may have read access on the journal but not on the company currency = journal.currency_id or self.env['res.currency'].browse(journal.company_id.sudo().currency_id.id) @@ -527,7 +546,7 @@ class account_journal(models.Model): 'sum_waiting': currency.format(sum_waiting), 'sum_late': currency.format(sum_late), 'has_sequence_holes': journal.has_sequence_holes, - 'is_sample_data': dashboard_data[journal.id]['entries_count'], + 'is_sample_data': is_sample_data_by_journal_id[journal.id], }) def _fill_general_dashboard_data(self, dashboard_data): diff --git a/addons/account/models/account_move.py b/addons/account/models/account_move.py index e6eb5850f61..95ec93b9f8b 100644 --- a/addons/account/models/account_move.py +++ b/addons/account/models/account_move.py @@ -30,6 +30,7 @@ from odoo.tools import ( groupby, index_exists, is_html_empty, + create_index, ) _logger = logging.getLogger(__name__) @@ -624,6 +625,13 @@ class AccountMove(models.Model): ON account_move (journal_id, sequence_prefix desc, (sequence_number+1) desc) """) + def init(self): + super().init() + create_index(self.env.cr, + indexname='account_move_journal_id_company_id_idx', + tablename='account_move', + expressions=['journal_id', 'company_id', 'date']) + # ------------------------------------------------------------------------- # COMPUTE METHODS # ------------------------------------------------------------------------- diff --git a/addons/account/models/account_move_line.py b/addons/account/models/account_move_line.py index 3f3a68443c6..abe7ccf94d5 100644 --- a/addons/account/models/account_move_line.py +++ b/addons/account/models/account_move_line.py @@ -88,7 +88,7 @@ class AccountMoveLine(models.Model): string='Account', compute='_compute_account_id', store=True, readonly=False, precompute=True, inverse='_inverse_account_id', - index=True, + index=False, # covered by account_move_line_account_id_date_idx defined in init() auto_join=True, ondelete="cascade", domain="[('deprecated', '=', False), ('account_type', '!=', 'off_balance')]", @@ -1432,6 +1432,16 @@ class AccountMoveLine(models.Model): # Match exactly how the ORM converts domains to ensure the query planner uses it create_index(self._cr, 'account_move_line__unreconciled_index', 'account_move_line', ['account_id', 'partner_id'], where="(reconciled IS NULL OR reconciled = false OR reconciled IS NOT true) AND parent_state = 'posted'") + create_index(self.env.cr, + indexname='account_move_line_journal_id_neg_amnt_residual_idx', + tablename='account_move_line', + expressions=['journal_id'], + where="amount_residual < 0 AND parent_state = 'posted'") + # covers the standard index on account_id + create_index(self.env.cr, + indexname='account_move_line_account_id_date_idx', + tablename='account_move_line', + expressions=['account_id', 'date']) super().init() def default_get(self, fields_list):