From 012d6e898116dc58e8617ea3d02ef5c9293eb7ac Mon Sep 17 00:00:00 2001 From: william-andre Date: Wed, 22 Feb 2023 09:41:41 +0000 Subject: [PATCH] [IMP] account: dashboard performances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As the number of `account.move`, `account.move.line`, `account.bank.statement.line` and `account.journal` is growing, the accounting dashboard, which is the entry point of the app, gets slower and slower. There are multiple issues being addressed in this commit: * The data for each journal is computed journal by journal. This means that the number of queries run increases linearly with the number of journals. While the boilerplate around running multiple queries is negligible compared to the running time of the queries in this case, some queries take as much time to run for one journal or for many. To improve this, all the queries are now batched. This has been done by refactoring the code; all these functions are now called on as many records as needed[^1]: - `_get_journal_bank_account_balance` - `_get_journal_outstanding_payments_account_balance` - `_get_last_bank_statement` - `get_line_graph_datas` - `get_bar_graph_datas` - `get_journal_dashboard_datas` * The gap detection and the entries' count are computed fields (`has_sequence_holes` and `entries_count` respectively). We don't need to display/compute these fields for all types of journals, but since they were mentioned by using a `` node in the view, they were computed for all journals displayed. Instead of using the `` node, we are now setting the value in the `kanban_dashboard` field. * Documents in foreign currencies on journals in foreign currencies need to get the rate in order to be aggregated in the journal's currency. There are 3 cases: - Document in journal's currency - Company's currency is the same as the journal's - Document, company and journal have 3 different currencies Before this commit, the second case will still fetch the daily rate for the document in order to do the conversion, but we actually already know the conversion; it is stored on the document. Benchmark ========= On a `populate` database with: - 4 `res.company` (with accounting enabled) - 45 `account.journal` - 19k `account.move` - 140k `account.move.line` - 4k `account.bank.statement.line` - 4 `account.bank.statement` | Query count | Query time | Remaining time --------------------------- | ----------- | ---------- | -------------- Before fix | 279 | 0.333s | 0.375s After fix (without update) | 40 | 0.120s | 0.170s After fix (with update[^2]) | 38 | 0.100s | 0.170s Note that the currency conversion was disabled because the populate database doesn't represent a realistic dataset regarding this. Disabling it only improves the numbers before the fix. Note ==== A lot of the time remaining comes from the aggregation of draft/unpaid invoices with the correct rate done in python instead of in SQL. This commit doesn't change the behavior but this could be rethought from a function point of view. ________________________________________________________________________ [^1]: the old functions have been kept for compatibility, new ones are suffixed by `_batched` and made private if it wasn't the case. [^2]: some optimization require the views and indexes to be updated closes odoo/odoo#117964 X-original-commit: 0a386932d2cbcb44cf36c1830422e52b285ae990 Related: odoo/enterprise#39461 Signed-off-by: Quentin De Paoli Signed-off-by: William André (wan) --- .../account/models/account_bank_statement.py | 1 + addons/account/models/account_journal.py | 4 +- .../models/account_journal_dashboard.py | 796 +++++++++++------- .../tests/test_account_journal_dashboard.py | 2 +- .../views/account_journal_dashboard_view.xml | 6 +- .../models/account_journal.py | 13 +- 6 files changed, 487 insertions(+), 335 deletions(-) diff --git a/addons/account/models/account_bank_statement.py b/addons/account/models/account_bank_statement.py index 0b3119aae31..761a53ee998 100644 --- a/addons/account/models/account_bank_statement.py +++ b/addons/account/models/account_bank_statement.py @@ -25,6 +25,7 @@ class AccountBankStatement(models.Model): date = fields.Date( compute='_compute_date_index', store=True, + index=True, ) # The internal index of the first line of a statement, it is used for sorting the statements diff --git a/addons/account/models/account_journal.py b/addons/account/models/account_journal.py index 2aeeaff36a5..d4b87105dce 100644 --- a/addons/account/models/account_journal.py +++ b/addons/account/models/account_journal.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from odoo import api, Command, fields, models, _ -from odoo.osv import expression from odoo.exceptions import UserError, ValidationError from odoo.addons.base.models.res_bank import sanitize_account_number from odoo.tools import remove_accents @@ -812,6 +811,7 @@ class AccountJournal(models.Model): # REPORTING METHODS # ------------------------------------------------------------------------- + # TODO move to `account_reports` in master (simple read_group) def _get_journal_bank_account_balance(self, domain=None): ''' Get the bank balance of the current journal by filtering the journal items using the journal's accounts. @@ -872,6 +872,7 @@ class AccountJournal(models.Model): account_ids.add(line.payment_account_id.id or self.company_id.account_journal_payment_credit_account_id.id) return self.env['account.account'].browse(account_ids) + # TODO remove in master def _get_journal_outstanding_payments_account_balance(self, domain=None, date=None): ''' Get the outstanding payments balance of the current journal by filtering the journal items using the journal's accounts. @@ -938,6 +939,7 @@ class AccountJournal(models.Model): total_balance += balance return total_balance, nb_lines + # TODO move to `account_reports` in master def _get_last_bank_statement(self, domain=None): ''' Retrieve the last bank statement created using this journal. :param domain: An additional domain to be applied on the account.bank.statement model. diff --git a/addons/account/models/account_journal_dashboard.py b/addons/account/models/account_journal_dashboard.py index 7ba5a422527..253a6c1a21e 100644 --- a/addons/account/models/account_journal_dashboard.py +++ b/addons/account/models/account_journal_dashboard.py @@ -1,81 +1,102 @@ -import json -from datetime import datetime, timedelta - +import ast from babel.dates import format_datetime, format_date +from collections import defaultdict +from datetime import datetime, timedelta +import json +import random + from odoo import models, api, _, fields from odoo.exceptions import UserError from odoo.osv import expression from odoo.release import version from odoo.tools import DEFAULT_SERVER_DATE_FORMAT as DF from odoo.tools.misc import formatLang, format_date as odoo_format_date, get_lang -import random -import ast + +def group_by_journal(vals_list): + res = defaultdict(list) + for vals in vals_list: + res[vals['journal_id']].append(vals) + return res class account_journal(models.Model): _inherit = "account.journal" - def _kanban_dashboard(self): - for journal in self: - journal.kanban_dashboard = json.dumps(journal.get_journal_dashboard_datas()) - - def _kanban_dashboard_graph(self): - for journal in self: - if (journal.type in ['sale', 'purchase']): - journal.kanban_dashboard_graph = json.dumps(journal.get_bar_graph_datas()) - elif (journal.type in ['cash', 'bank']): - journal.kanban_dashboard_graph = json.dumps(journal.get_line_graph_datas()) - else: - journal.kanban_dashboard_graph = False - - def _get_json_activity_data(self): - for journal in self: - activities = [] - # search activity on move on the journal - sql_query = ''' - SELECT act.id, - act.res_id, - act.res_model, - act.summary, - act_type.name as act_type_name, - act_type.category as activity_category, - act.date_deadline, - m.date, - m.ref, - CASE WHEN act.date_deadline < CURRENT_DATE THEN 'late' ELSE 'future' END as status - FROM account_move m - LEFT JOIN mail_activity act ON act.res_id = m.id - LEFT JOIN mail_activity_type act_type ON act.activity_type_id = act_type.id - WHERE act.res_model = 'account.move' - AND m.journal_id = %s - AND (act_type.category != 'tax_report' OR (act_type.category = 'tax_report' AND act.date_deadline <= CURRENT_DATE)) - ''' - self.env.cr.execute(sql_query, (journal.id,)) - for activity in self.env.cr.dictfetchall(): - act = { - 'id': activity.get('id'), - 'res_id': activity.get('res_id'), - 'res_model': activity.get('res_model'), - 'status': activity.get('status'), - 'name': (activity.get('summary') or activity.get('act_type_name')), - 'activity_category': activity.get('activity_category'), - 'date': odoo_format_date(self.env, activity.get('date_deadline')) - } - if activity.get('activity_category') == 'tax_report' and activity.get('res_model') == 'account.move': - act['name'] = activity.get('ref') - - activities.append(act) - journal.json_activity_data = json.dumps({'activities': activities}) - kanban_dashboard = fields.Text(compute='_kanban_dashboard') kanban_dashboard_graph = fields.Text(compute='_kanban_dashboard_graph') json_activity_data = fields.Text(compute='_get_json_activity_data') show_on_dashboard = fields.Boolean(string='Show journal on dashboard', help="Whether this journal should be displayed on the dashboard or not", default=True) color = fields.Integer("Color Index", default=0) + current_statement_balance = fields.Monetary(compute='_compute_current_statement_balance') # technical field used to avoid computing the value multiple times + has_statement_lines = fields.Boolean(compute='_compute_current_statement_balance') # technical field used to avoid computing the value multiple times entries_count = fields.Integer(compute='_compute_entries_count') has_sequence_holes = fields.Boolean(compute='_compute_has_sequence_holes') + def _compute_current_statement_balance(self): + query_result = self._get_journal_dashboard_bank_running_balance() + for journal in self: + journal.has_statement_lines, journal.current_statement_balance = query_result.get(journal.id) + + def _kanban_dashboard(self): + dashboard_data = self._get_journal_dashboard_data_batched() + for journal in self: + journal.kanban_dashboard = json.dumps(dashboard_data[journal.id]) + + @api.depends('current_statement_balance') + def _kanban_dashboard_graph(self): + bank_cash_journals = self.filtered(lambda journal: journal.type in ('bank', 'cash')) + bank_cash_graph_datas = bank_cash_journals._get_bank_cash_graph_data() + for journal in bank_cash_journals: + journal.kanban_dashboard_graph = json.dumps(bank_cash_graph_datas[journal.id]) + + sale_purchase_journals = self.filtered(lambda journal: journal.type in ('sale', 'purchase')) + sale_purchase_graph_datas = sale_purchase_journals._get_sale_purchase_graph_data() + for journal in sale_purchase_journals: + journal.kanban_dashboard_graph = json.dumps(sale_purchase_graph_datas[journal.id]) + + (self - bank_cash_journals - sale_purchase_journals).kanban_dashboard_graph = False + + def _get_json_activity_data(self): + today = fields.Date.context_today(self) + activities = defaultdict(list) + # search activity on move on the journal + sql_query = """ + SELECT activity.id, + activity.res_id, + activity.res_model, + activity.summary, + CASE WHEN activity.date_deadline < %(today)s THEN 'late' ELSE 'future' END as status, + act_type.name as act_type_name, + act_type.category as activity_category, + activity.date_deadline, + move.date, + move.ref, + move.journal_id + FROM account_move move + JOIN mail_activity activity ON activity.res_id = move.id AND activity.res_model = 'account.move' + LEFT JOIN mail_activity_type act_type ON activity.activity_type_id = act_type.id + WHERE move.journal_id = ANY(%(ids)s) + AND (act_type.category != 'tax_report' OR (act_type.category = 'tax_report' AND activity.date_deadline <= %(today)s)) + """ + self.env.cr.execute(sql_query, {'ids': self.ids, 'today': today}) + for activity in self.env.cr.dictfetchall(): + act = { + 'id': activity['id'], + 'res_id': activity['res_id'], + 'res_model': activity['res_model'], + 'status': activity['status'], + 'name': activity['summary'] or activity['act_type_name'], + 'activity_category': activity['activity_category'], + 'date': odoo_format_date(self.env, activity['date_deadline']) + } + if activity['activity_category'] == 'tax_report' and activity['res_model'] == 'account.move': + act['name'] = activity['ref'] + + activities[activity['journal_id']].append(act) + for journal in self: + journal.json_activity_data = json.dumps({'activities': activities[journal.id]}) + def _query_has_sequence_holes(self): self.env.cr.execute(""" SELECT move.journal_id, @@ -84,7 +105,7 @@ class account_journal(models.Model): JOIN res_company company ON company.id = move.company_id WHERE move.journal_id = ANY(%(journal_ids)s) AND move.state = 'posted' - AND (company.fiscalyear_lock_date IS NULL OR move.date >= company.fiscalyear_lock_date) + AND (company.fiscalyear_lock_date IS NULL OR move.date >= company.fiscalyear_lock_date) GROUP BY move.journal_id, move.sequence_prefix HAVING COUNT(*) != MAX(move.sequence_number) - MIN(move.sequence_number) + 1 """, { @@ -117,324 +138,457 @@ class account_journal(models.Model): elif self.type == 'bank': return ['', _('Bank: Balance')] - # Below method is used to get data of bank and cash statemens + # TODO remove in master def get_line_graph_datas(self): - """Computes the data used to display the graph for bank and cash journals in the accounting dashboard""" - currency = self.currency_id or self.company_id.currency_id + self.ensure_one() + return self._get_bank_cash_graph_data()[self.id] - def build_graph_data(date, amount): + def _get_bank_cash_graph_data(self): + """Computes the data used to display the graph for bank and cash journals in the accounting dashboard""" + def build_graph_data(date, amount, currency): #display date in locale format name = format_date(date, 'd LLLL Y', locale=locale) short_name = format_date(date, 'd MMM', locale=locale) - return {'x':short_name,'y': amount, 'name':name} + return {'x': short_name, 'y': currency.round(amount), 'name': name} - self.ensure_one() - data = [] today = datetime.today() last_month = today + timedelta(days=-30) locale = get_lang(self.env).code - #starting point of the graph is the last statement - last_stmt = self._get_last_bank_statement(domain=[('move_id.state', '=', 'posted')]) + query = """ + SELECT move.journal_id, + move.date, + SUM(st_line.amount) AS amount + FROM account_bank_statement_line st_line + JOIN account_move move ON move.id = st_line.move_id + WHERE move.journal_id = ANY(%s) + AND move.date > %s + AND move.date <= %s + GROUP BY move.date, move.journal_id + ORDER BY move.date DESC + """ + self.env.cr.execute(query, (self.ids, last_month, today)) + query_result = group_by_journal(self.env.cr.dictfetchall()) - last_balance = last_stmt and last_stmt.balance_end_real or 0 - data.append(build_graph_data(today, last_balance)) + result = {} + for journal in self: + graph_title, graph_key = journal._graph_title_and_key() + currency = journal.currency_id or journal.company_id.currency_id + journal_result = query_result[journal.id] - #then we subtract the total amount of bank statement lines per day to get the previous points - #(graph is drawn backward) - date = today - amount = last_balance - query = ''' - SELECT move.date, sum(st_line.amount) as amount - FROM account_bank_statement_line st_line - JOIN account_move move ON move.id = st_line.move_id - WHERE move.journal_id = %s - AND move.date > %s - AND move.date <= %s - GROUP BY move.date - ORDER BY move.date desc - ''' - self.env.cr.execute(query, (self.id, last_month, today)) - query_result = self.env.cr.dictfetchall() - for val in query_result: - date = val['date'] - if date != today.strftime(DF): # make sure the last point in the graph is today - data[:0] = [build_graph_data(date, amount)] - amount = currency.round(amount - val['amount']) + color = '#875A7B' if 'e' in version else '#7c7bad' + is_sample_data = not journal.has_statement_lines - # make sure the graph starts 1 month ago - if date.strftime(DF) != last_month.strftime(DF): - data[:0] = [build_graph_data(last_month, amount)] - - [graph_title, graph_key] = self._graph_title_and_key() - color = '#875A7B' if 'e' in version else '#7c7bad' - - is_sample_data = not last_stmt and len(query_result) == 0 - if is_sample_data: data = [] - for i in range(30, 0, -5): - current_date = today + timedelta(days=-i) - data.append(build_graph_data(current_date, random.randint(-5, 15))) + if is_sample_data: + for i in range(30, 0, -5): + current_date = today + timedelta(days=-i) + data.append(build_graph_data(current_date, random.randint(-5, 15), currency)) + else: + last_balance = journal.current_statement_balance + data.append(build_graph_data(today, last_balance, currency)) + date = today + amount = last_balance + #then we subtract the total amount of bank statement lines per day to get the previous points + #(graph is drawn backward) + for val in journal_result: + date = val['date'] + if date != today.strftime(DF): # make sure the last point in the graph is today + data[:0] = [build_graph_data(date, amount, currency)] + amount -= val['amount'] - return [{'values': data, 'title': graph_title, 'key': graph_key, 'area': True, 'color': color, 'is_sample_data': is_sample_data}] + # make sure the graph starts 1 month ago + if date.strftime(DF) != last_month.strftime(DF): + data[:0] = [build_graph_data(last_month, amount, currency)] + result[journal.id] = [{'values': data, 'title': graph_title, 'key': graph_key, 'area': True, 'color': color, 'is_sample_data': is_sample_data}] + return result + + # TODO remove in master def get_bar_graph_datas(self): - data = [] + self.ensure_one() + return self._get_sale_purchase_graph_data()[self.id] + + def _get_sale_purchase_graph_data(self): today = fields.Date.today() - data.append({'label': _('Due'), 'value':0.0, 'type': 'past'}) day_of_week = int(format_datetime(today, 'e', locale=get_lang(self.env).code)) first_day_of_week = today + timedelta(days=-day_of_week+1) - for i in range(-1,4): - if i==0: - label = _('This Week') - elif i==3: - label = _('Not Due') - else: - start_week = first_day_of_week + timedelta(days=i*7) - end_week = start_week + timedelta(days=6) - if start_week.month == end_week.month: - label = str(start_week.day) + '-' + str(end_week.day) + ' ' + format_date(end_week, 'MMM', locale=get_lang(self.env).code) - else: - label = format_date(start_week, 'd MMM', locale=get_lang(self.env).code) + '-' + format_date(end_week, 'd MMM', locale=get_lang(self.env).code) - data.append({'label':label,'value':0.0, 'type': 'past' if i<0 else 'future'}) + format_month = lambda d: format_date(d, 'MMM', locale=get_lang(self.env).code) - # Build SQL query to find amount aggregated by week - (select_sql_clause, query_args) = self._get_bar_graph_select_query() - query = '' - start_date = (first_day_of_week + timedelta(days=-7)) - weeks = [] - for i in range(0,6): - if i == 0: - query += "("+select_sql_clause+" and invoice_date_due < '"+start_date.strftime(DF)+"')" - weeks.append((start_date.min, start_date)) - elif i == 5: - query += " UNION ALL ("+select_sql_clause+" and invoice_date_due >= '"+start_date.strftime(DF)+"')" - weeks.append((start_date, start_date.max)) - else: - next_date = start_date + timedelta(days=7) - query += " UNION ALL ("+select_sql_clause+" and invoice_date_due >= '"+start_date.strftime(DF)+"' and invoice_date_due < '"+next_date.strftime(DF)+"')" - weeks.append((start_date, next_date)) - start_date = next_date - # Ensure results returned by postgres match the order of data list - self.env.cr.execute(query, query_args) - query_results = self.env.cr.dictfetchall() - is_sample_data = True - for index in range(0, len(query_results)): - if query_results[index].get('aggr_date') != None: - is_sample_data = False - aggr_date = query_results[index]['aggr_date'] - week_index = next(i for i in range(0, len(weeks)) if weeks[i][0] <= aggr_date < weeks[i][1]) - data[week_index]['value'] = query_results[index].get('total') - - [graph_title, graph_key] = self._graph_title_and_key() - - if is_sample_data: - for index in range(0, len(query_results)): - data[index]['type'] = 'o_sample_data' - # we use unrealistic values for the sample data - data[index]['value'] = random.randint(0, 20) - graph_key = _('Sample data') - - return [{'values': data, 'title': graph_title, 'key': graph_key, 'is_sample_data': is_sample_data}] - - def _get_bar_graph_select_query(self): - """ - Returns a tuple containing the base SELECT SQL query used to gather - the bar graph's data as its first element, and the arguments dictionary - for it as its second. - """ - sign = '' if self.type == 'sale' else '-' - return (''' - SELECT - ''' + sign + ''' + SUM(move.amount_residual_signed) AS total, - MIN(invoice_date_due) AS aggr_date - FROM account_move move - WHERE move.journal_id = %(journal_id)s - AND move.state = 'posted' - AND move.payment_state in ('not_paid', 'partial') - AND move.move_type IN %(invoice_types)s - ''', { + self.env.cr.execute(""" + SELECT move.journal_id, + COALESCE(SUM(move.amount_residual_signed) FILTER (WHERE invoice_date_due < %(start_week1)s), 0) AS total_before, + COALESCE(SUM(move.amount_residual_signed) FILTER (WHERE invoice_date_due >= %(start_week1)s AND invoice_date_due < %(start_week2)s), 0) AS total_week1, + COALESCE(SUM(move.amount_residual_signed) FILTER (WHERE invoice_date_due >= %(start_week2)s AND invoice_date_due < %(start_week3)s), 0) AS total_week2, + COALESCE(SUM(move.amount_residual_signed) FILTER (WHERE invoice_date_due >= %(start_week3)s AND invoice_date_due < %(start_week4)s), 0) AS total_week3, + COALESCE(SUM(move.amount_residual_signed) FILTER (WHERE invoice_date_due >= %(start_week4)s AND invoice_date_due < %(start_week5)s), 0) AS total_week4, + COALESCE(SUM(move.amount_residual_signed) FILTER (WHERE invoice_date_due >= %(start_week5)s), 0) AS total_after + FROM account_move move + WHERE move.journal_id = ANY(%(journal_ids)s) + AND move.state = 'posted' + AND move.payment_state in ('not_paid', 'partial') + AND move.move_type IN %(invoice_types)s + GROUP BY move.journal_id + """, { 'invoice_types': tuple(self.env['account.move'].get_invoice_types(True)), - 'journal_id': self.id + 'journal_ids': self.ids, + 'start_week1': first_day_of_week + timedelta(days=-7), + 'start_week2': first_day_of_week + timedelta(days=0), + 'start_week3': first_day_of_week + timedelta(days=7), + 'start_week4': first_day_of_week + timedelta(days=14), + 'start_week5': first_day_of_week + timedelta(days=21), }) + query_results = {r['journal_id']: r for r in self.env.cr.dictfetchall()} + result = {} + for journal in self: + currency = journal.currency_id or journal.company_id.currency_id + graph_title, graph_key = journal._graph_title_and_key() + sign = 1 if journal.type == 'sale' else -1 + journal_data = query_results.get(journal.id) + data = [] + data.append({'label': _('Due'), 'type': 'past'}) + for i in range(-1, 3): + if i == 0: + label = _('This Week') + else: + start_week = first_day_of_week + timedelta(days=i*7) + end_week = start_week + timedelta(days=6) + if start_week.month == end_week.month: + label = f"{start_week.day} - {end_week.day} {format_month(end_week)}" + else: + label = f"{start_week.day} {format_month(start_week)} - {end_week.day} {format_month(end_week)}" + data.append({'label': label, 'type': 'past' if i < 0 else 'future'}) + data.append({'label': _('Not Due'), 'type': 'future'}) - def get_journal_dashboard_datas(self): - currency = self.currency_id or self.company_id.currency_id - number_to_reconcile = number_to_check = last_balance = 0 - has_at_least_one_statement = False - bank_account_balance = nb_lines_bank_account_balance = 0 - outstanding_pay_account_balance = nb_lines_outstanding_pay_account_balance = 0 - title = '' - number_draft = number_waiting = number_late = to_check_balance = 0 - sum_draft = sum_waiting = sum_late = 0.0 - if self.type in ('bank', 'cash'): - last_statement = self._get_last_bank_statement( - domain=[('move_id.state', '=', 'posted')]) - last_balance = last_statement.balance_end - has_at_least_one_statement = bool(last_statement) - bank_account_balance, nb_lines_bank_account_balance = self._get_journal_bank_account_balance( - domain=[('parent_state', '=', 'posted')]) - outstanding_pay_account_balance, nb_lines_outstanding_pay_account_balance = self._get_journal_outstanding_payments_account_balance( - domain=[('parent_state', '=', 'posted')]) - - if self.default_account_id: - self._cr.execute(''' - SELECT COUNT(st_line.id) - FROM account_bank_statement_line st_line - JOIN account_move st_line_move ON st_line_move.id = st_line.move_id - JOIN account_move_line aml ON aml.move_id = st_line_move.id - WHERE st_line_move.journal_id IN %s - AND NOT st_line.is_reconciled - AND st_line_move.to_check IS NOT TRUE - AND st_line_move.state = 'posted' - AND aml.account_id = %s - ''', [tuple(self.ids), self.default_account_id.id]) - number_to_reconcile = self.env.cr.fetchone()[0] + is_sample_data = not journal_data + if not is_sample_data: + data[0]['value'] = currency.round(sign * journal_data['total_before']) + data[1]['value'] = currency.round(sign * journal_data['total_week1']) + data[2]['value'] = currency.round(sign * journal_data['total_week2']) + data[3]['value'] = currency.round(sign * journal_data['total_week3']) + data[4]['value'] = currency.round(sign * journal_data['total_week4']) + data[5]['value'] = currency.round(sign * journal_data['total_after']) else: - number_to_reconcile = 0 + for index in range(6): + data[index]['type'] = 'o_sample_data' + # we use unrealistic values for the sample data + data[index]['value'] = random.randint(0, 20) + graph_key = _('Sample data') - to_check_ids = self.to_check_ids() - number_to_check = len(to_check_ids) - to_check_balance = sum([r.amount for r in to_check_ids]) - #TODO need to check if all invoices are in the same currency than the journal!!!! - elif self.type in ['sale', 'purchase']: - title = _('Bills to pay') if self.type == 'purchase' else _('Invoices owed to you') - self.env['account.move'].flush_model() + result[journal.id] = [{'values': data, 'title': graph_title, 'key': graph_key, 'is_sample_data': is_sample_data}] + return result - (query, query_args) = self._get_open_bills_to_pay_query() - self.env.cr.execute(query, query_args) - query_results_to_pay = self.env.cr.dictfetchall() + # TODO remove in master + def get_journal_dashboard_datas(self): + return self._get_journal_dashboard_data_batched()[self.id] - (query, query_args) = self._get_draft_bills_query() - self.env.cr.execute(query, query_args) - query_results_drafts = self.env.cr.dictfetchall() + def _get_journal_dashboard_data_batched(self): + self.env['account.move'].flush_model() + self.env['account.move.line'].flush_model() + dashboard_data = {} # container that will be filled by functions below + for journal in self: + dashboard_data[journal.id] = { + 'currency_id': journal.currency_id.id or journal.company_id.currency_id.id, + 'company_count': len(self.env.companies), + } + self._fill_bank_cash_dashboard_data(dashboard_data) + self._fill_sale_purchase_dashboard_data(dashboard_data) + self._fill_general_dashboard_data(dashboard_data) + return dashboard_data - (query, query_args) = self._get_late_bills_query() - self.env.cr.execute(query, query_args) - late_query_results = self.env.cr.dictfetchall() + def _fill_dashboard_data_count(self, dashboard_data, model, name, domain): + """Populate the dashboard data with the result of a count. - curr_cache = {} - (number_waiting, sum_waiting) = self._count_results_and_sum_amounts(query_results_to_pay, currency, curr_cache=curr_cache) - (number_draft, sum_draft) = self._count_results_and_sum_amounts(query_results_drafts, currency, curr_cache=curr_cache) - (number_late, sum_late) = self._count_results_and_sum_amounts(late_query_results, currency, curr_cache=curr_cache) - read = self.env['account.move'].read_group([('journal_id', '=', self.id), ('to_check', '=', True)], ['amount_total_signed'], 'journal_id', lazy=False) - if read: - number_to_check = read[0]['__count'] - to_check_balance = read[0]['amount_total_signed'] - elif self.type == 'general': - read = self.env['account.move'].read_group([('journal_id', '=', self.id), ('to_check', '=', True)], ['amount_total_signed'], 'journal_id', lazy=False) - if read: - number_to_check = read[0]['__count'] - to_check_balance = read[0]['amount_total_signed'] + :param dashboard_data: a mapping between a journal ids and the data needed to display their + dashboard kanban card. + :type dashboard_data: dict[int, dict] + :param model: the model on which to perform the count + :type model: str + :param name: the name of the variable to inject in the dashboard's data + :type name: str + :param domain: the domain of records to count + :type domain: list[tuple] + """ + res = { + r['journal_id'][0]: r['journal_id_count'] + for r in self.env[model]._read_group( + domain=[('journal_id', 'in', self.ids)] + domain, + fields=['journal_id'], + groupby=['journal_id'], + ) + } + for journal in self: + dashboard_data[journal.id][name] = res.get(journal.id, 0) - is_sample_data = self.kanban_dashboard_graph and any(data.get('is_sample_data', False) for data in json.loads(self.kanban_dashboard_graph)) + def _fill_bank_cash_dashboard_data(self, dashboard_data): + """Populate all bank and cash journal's data dict with relevant information for the kanban card.""" + bank_cash_journals = self.filtered(lambda journal: journal.type in ('bank', 'cash')) + if not bank_cash_journals: + return - return { - 'number_to_check': number_to_check, - 'to_check_balance': formatLang(self.env, to_check_balance, currency_obj=currency), - 'number_to_reconcile': number_to_reconcile, - 'account_balance': formatLang(self.env, currency.round(bank_account_balance), currency_obj=currency), - 'has_at_least_one_statement': has_at_least_one_statement, - 'nb_lines_bank_account_balance': nb_lines_bank_account_balance, - 'outstanding_pay_account_balance': formatLang(self.env, currency.round(outstanding_pay_account_balance), currency_obj=currency), - 'nb_lines_outstanding_pay_account_balance': nb_lines_outstanding_pay_account_balance, - 'last_balance': formatLang(self.env, currency.round(last_balance) + 0.0, currency_obj=currency), - 'number_draft': number_draft, - 'number_waiting': number_waiting, - 'number_late': number_late, - 'sum_draft': formatLang(self.env, currency.round(sum_draft) + 0.0, currency_obj=currency), - 'sum_waiting': formatLang(self.env, currency.round(sum_waiting) + 0.0, currency_obj=currency), - 'sum_late': formatLang(self.env, currency.round(sum_late) + 0.0, currency_obj=currency), - 'currency_id': currency.id, - 'bank_statements_source': self.bank_statements_source, - 'title': title, - 'is_sample_data': is_sample_data, - 'company_count': len(self.env.companies) + # Number to reconcile + self._cr.execute(""" + SELECT st_line_move.journal_id, + COUNT(st_line.id) + FROM account_bank_statement_line st_line + JOIN account_move st_line_move ON st_line_move.id = st_line.move_id + WHERE st_line_move.journal_id IN %s + AND NOT st_line.is_reconciled + AND st_line_move.to_check IS NOT TRUE + AND st_line_move.state = 'posted' + GROUP BY st_line_move.journal_id + """, [tuple(bank_cash_journals.ids)]) + number_to_reconcile = { + journal_id: count + for journal_id, count in self.env.cr.fetchall() } + # Last statement + self.env.cr.execute(""" + SELECT journal.id, statement.id + FROM account_journal journal + LEFT JOIN LATERAL ( + SELECT id + FROM account_bank_statement + WHERE journal_id = journal.id + ORDER BY date DESC + LIMIT 1 + ) statement ON TRUE + WHERE journal.id = ANY(%s) + """, [self.ids]) + last_statements = {journal_id: statement_id for journal_id, statement_id in self.env.cr.fetchall()} + self.env['account.bank.statement'].browse(i for i in last_statements.values() if i).mapped('balance_end_real') # prefetch + + outstanding_pay_account_balances = bank_cash_journals._get_journal_dashboard_outstanding_payments() + + # To check + to_check = { + res['journal_id'][0]: (res['amount'], res['journal_id_count']) + for res in self.env['account.bank.statement.line'].read_group( + domain=[ + ('journal_id', 'in', bank_cash_journals.ids), + ('move_id.to_check', '=', True), + ('move_id.state', '=', 'posted'), + ], + fields=['amount'], + groupby=['journal_id'], + ) + } + + for journal in bank_cash_journals: + last_statement = self.env['account.bank.statement'].browse(last_statements.get(journal.id)) + currency = journal.currency_id or journal.company_id.currency_id + has_outstanding, outstanding_pay_account_balance = outstanding_pay_account_balances[journal.id] + to_check_balance, number_to_check = to_check.get(journal.id, (0, 0)) + + dashboard_data[journal.id].update({ + 'number_to_check': number_to_check, + 'to_check_balance': currency.format(to_check_balance), + 'number_to_reconcile': number_to_reconcile.get(journal.id, 0), + 'account_balance': currency.format(journal.current_statement_balance), + 'has_at_least_one_statement': bool(last_statement), + 'nb_lines_bank_account_balance': bool(last_statement), + 'outstanding_pay_account_balance': currency.format(outstanding_pay_account_balance), + 'nb_lines_outstanding_pay_account_balance': has_outstanding, + 'last_balance': currency.format(last_statement.balance_end_real), + 'bank_statements_source': journal.bank_statements_source, + 'is_sample_data': journal.has_statement_lines, + }) + + def _fill_sale_purchase_dashboard_data(self, dashboard_data): + """Populate all sale and purchase journal's data dict with relevant information for the kanban card.""" + sale_purchase_journals = self.filtered(lambda journal: journal.type in ('sale', 'purchase')) + if not sale_purchase_journals: + return + field_list = [ + "account_move.journal_id", + "(CASE WHEN account_move.move_type IN ('out_refund', 'in_refund') THEN -1 ELSE 1 END) * account_move.amount_residual AS amount_total", + "account_move.amount_residual_signed AS amount_total_company", + "account_move.currency_id AS currency", + "account_move.move_type", + "account_move.invoice_date", + "account_move.company_id", + ] + query, params = sale_purchase_journals._get_open_bills_to_pay_query().select(*field_list) + self.env.cr.execute(query, params) + query_results_to_pay = group_by_journal(self.env.cr.dictfetchall()) + + query, params = sale_purchase_journals._get_draft_bills_query().select(*field_list) + self.env.cr.execute(query, params) + query_results_drafts = group_by_journal(self.env.cr.dictfetchall()) + + query, params = sale_purchase_journals._get_late_bills_query().select(*field_list) + self.env.cr.execute(query, params) + late_query_results = group_by_journal(self.env.cr.dictfetchall()) + + to_check_vals = { + vals['journal_id']: vals + for vals in self.env['account.move'].read_group( + domain=[('journal_id', 'in', sale_purchase_journals.ids), ('to_check', '=', True)], + fields=['amount_total_signed'], + groupby='journal_id', + ) + } + + curr_cache = {} + sale_purchase_journals._fill_dashboard_data_count(dashboard_data, 'account.move', 'entries_count', []) + for journal in sale_purchase_journals: + currency = journal.currency_id or journal.company_id.currency_id + (number_waiting, sum_waiting) = self._count_results_and_sum_amounts(query_results_to_pay[journal.id], currency, curr_cache=curr_cache) + (number_draft, sum_draft) = self._count_results_and_sum_amounts(query_results_drafts[journal.id], currency, curr_cache=curr_cache) + (number_late, sum_late) = self._count_results_and_sum_amounts(late_query_results[journal.id], currency, curr_cache=curr_cache) + to_check = to_check_vals.get(journal.id, {}) + dashboard_data[journal.id].update({ + 'number_to_check': to_check.get('__count', 0), + 'to_check_balance': to_check.get('amount_total_signed', 0), + 'title': _('Bills to pay') if journal.type == 'purchase' else _('Invoices owed to you'), + 'number_draft': number_draft, + 'number_waiting': number_waiting, + 'number_late': number_late, + 'sum_draft': currency.format(sum_draft), + 'sum_waiting': currency.format(sum_waiting), + 'sum_late': currency.format(sum_late), + 'has_sequence_holes': journal.has_sequence_holes, + 'is_sample_data': dashboard_data[journal.id]['entries_count'], + }) + + def _fill_general_dashboard_data(self, dashboard_data): + """Populate all miscelaneous journal's data dict with relevant information for the kanban card.""" + general_journals = self.filtered(lambda journal: journal.type == 'general') + if not general_journals: + return + to_check_vals = { + vals['journal_id']: vals + for vals in self.env['account.move'].read_group( + domain=[('journal_id', 'in', general_journals.ids), ('to_check', '=', True)], + fields=['amount_total_signed'], + groupby='journal_id', + lazy=False, + ) + } + for journal in general_journals: + vals = to_check_vals.get('journal_id', {}) + dashboard_data[journal.id].update({ + 'number_to_check': vals.get('__count', 0), + 'to_check_balance': vals.get('amount_total_signed', 0), + }) + def _get_open_bills_to_pay_query(self): - """ - Returns a tuple containing the SQL query used to gather the open bills - data as its first element, and the arguments dictionary to use to run - it as its second. - """ - return (''' - SELECT - (CASE WHEN move.move_type IN ('out_refund', 'in_refund') THEN -1 ELSE 1 END) * move.amount_residual AS amount_total, - move.currency_id AS currency, - move.move_type, - move.invoice_date, - move.company_id - FROM account_move move - WHERE move.journal_id = %(journal_id)s - AND move.state = 'posted' - AND move.payment_state in ('not_paid', 'partial') - AND move.move_type IN ('out_invoice', 'out_refund', 'in_invoice', 'in_refund', 'out_receipt', 'in_receipt'); - ''', {'journal_id': self.id}) + return self.env['account.move']._where_calc([ + ('journal_id', 'in', self.ids), + ('state', '=', 'posted'), + ('payment_state', 'in', ('not_paid', 'partial')), + ('move_type', 'in', self.env['account.move'].get_invoice_types(include_receipts=True)), + ]) def _get_draft_bills_query(self): - """ - Returns a tuple containing as its first element the SQL query used to - gather the bills in draft state data, and the arguments - dictionary to use to run it as its second. - """ - return (''' - SELECT - (CASE WHEN move.move_type IN ('out_refund', 'in_refund') THEN -1 ELSE 1 END) * move.amount_total AS amount_total, - move.currency_id AS currency, - move.move_type, - move.invoice_date, - move.company_id - FROM account_move move - WHERE move.journal_id = %(journal_id)s - AND move.state = 'draft' - AND move.payment_state in ('not_paid', 'partial') - AND move.move_type IN ('out_invoice', 'out_refund', 'in_invoice', 'in_refund', 'out_receipt', 'in_receipt'); - ''', {'journal_id': self.id}) + return self.env['account.move']._where_calc([ + ('journal_id', 'in', self.ids), + ('state', '=', 'draft'), + ('payment_state', 'in', ('not_paid', 'partial')), + ('move_type', 'in', self.env['account.move'].get_invoice_types(include_receipts=True)), + ]) def _get_late_bills_query(self): - return """ - SELECT - (CASE WHEN move_type IN ('out_refund', 'in_refund') THEN -1 ELSE 1 END) * amount_residual AS amount_total, - currency_id AS currency, - move_type, - invoice_date, - company_id - FROM account_move move - WHERE journal_id = %(journal_id)s - AND invoice_date_due < %(today)s - AND state = 'posted' - AND payment_state in ('not_paid', 'partial') - AND move_type IN ('out_invoice', 'out_refund', 'in_invoice', 'in_refund', 'out_receipt', 'in_receipt'); - """, {'journal_id': self.id, 'today': fields.Date.context_today(self)} + return self.env['account.move']._where_calc([ + ('journal_id', 'in', self.ids), + ('invoice_date_due', '<', fields.Date.context_today(self)), + ('state', '=', 'posted'), + ('payment_state', 'in', ('not_paid', 'partial')), + ('move_type', 'in', self.env['account.move'].get_invoice_types(include_receipts=True)), + ]) def _count_results_and_sum_amounts(self, results_dict, target_currency, curr_cache=None): """ Loops on a query result to count the total number of invoices and sum their amount_total field (expressed in the given target currency). amount_total must be signed! """ - rslt_count = 0 - rslt_sum = 0.0 # Create a cache with currency rates to avoid unnecessary SQL requests. Do not copy # curr_cache on purpose, so the dictionary is modified and can be re-used for subsequent # calls of the method. curr_cache = {} if curr_cache is None else curr_cache + total_amount = 0 for result in results_dict: - cur = self.env['res.currency'].browse(result.get('currency')) + document_currency = self.env['res.currency'].browse(result.get('currency')) company = self.env['res.company'].browse(result.get('company_id')) or self.env.company - rslt_count += 1 date = result.get('invoice_date') or fields.Date.context_today(self) - amount = result.get('amount_total', 0) or 0 - if cur != target_currency: - key = (cur, target_currency, company, date) - # Using setdefault will call _get_conversion_rate, so we explicitly check the - # existence of the key in the cache instead. + if document_currency == target_currency: + total_amount += result.get('amount_total') or 0 + elif company.currency_id == target_currency: + total_amount += result.get('amount_total_company') or 0 + else: + key = (document_currency, target_currency, company, date) if key not in curr_cache: curr_cache[key] = self.env['res.currency']._get_conversion_rate(*key) - amount *= curr_cache[key] - rslt_sum += target_currency.round(amount) - return (rslt_count, rslt_sum) + total_amount += (result.get('amount_total') or 0) * curr_cache[key] + return (len(results_dict), target_currency.round(total_amount)) + + def _get_journal_dashboard_bank_running_balance(self): + # In order to not recompute everything from the start, we take the last + # bank statement and only sum starting from there. + self._cr.execute(""" + SELECT journal.id AS journal_id, + statement.id AS statement_id, + COALESCE(statement.balance_end_real, 0) AS balance_end_real, + without_statement.amount AS unlinked_amount, + without_statement.count AS unlinked_count + FROM account_journal journal + LEFT JOIN LATERAL ( -- select latest statement based on the date + SELECT id, + first_line_index, + balance_end_real + FROM account_bank_statement + WHERE journal_id = journal.id + ORDER BY date DESC + LIMIT 1 + ) statement ON TRUE + LEFT JOIN LATERAL ( -- sum all the lines not linked to a statement with a higher index than the last line of the statement + SELECT COALESCE(SUM(stl.amount), 0.0) AS amount, + COUNT(*) + FROM account_bank_statement_line stl + JOIN account_move move ON move.id = stl.move_id + WHERE stl.statement_id IS NULL + AND move.state != 'cancel' + AND move.journal_id = journal.id + AND stl.internal_index >= statement.first_line_index + ) without_statement ON TRUE + WHERE journal.id = ANY(%s) + """, [(self.ids)]) + query_res = {res['journal_id']: res for res in self.env.cr.dictfetchall()} + result = {} + for journal in self: + journal_vals = query_res[journal.id] + result[journal.id] = ( + bool(journal_vals['statement_id'] or journal_vals['unlinked_count']), + journal_vals['balance_end_real'] + journal_vals['unlinked_amount'], + ) + return result + + def _get_journal_dashboard_outstanding_payments(self): + self.env.cr.execute(""" + SELECT move.journal_id AS journal_id, + move.company_id AS company_id, + move.currency_id AS currency, + SUM(CASE + WHEN payment.payment_type = 'outbound' THEN -payment.amount + ELSE payment.amount + END) AS amount_total, + SUM(amount_company_currency_signed) AS amount_total_company + FROM account_payment payment + JOIN account_move move ON move.payment_id = payment.id + WHERE payment.is_matched IS NOT TRUE + AND move.state = 'posted' + AND move.journal_id = ANY(%s) + GROUP BY move.company_id, move.journal_id, move.currency_id + """, [self.ids]) + query_result = group_by_journal(self.env.cr.dictfetchall()) + result = {} + curr_cache = {} + for journal in self: + currency = journal.currency_id or journal.company_id.currency_id + result[journal.id] = self._count_results_and_sum_amounts(query_result[journal.id], currency, curr_cache) + return result def _get_move_action_context(self): ctx = self._context.copy() diff --git a/addons/account/tests/test_account_journal_dashboard.py b/addons/account/tests/test_account_journal_dashboard.py index c64b7899865..138389ac51b 100644 --- a/addons/account/tests/test_account_journal_dashboard.py +++ b/addons/account/tests/test_account_journal_dashboard.py @@ -55,7 +55,7 @@ class TestAccountJournalDashboard(AccountTestInvoicingCommon): dashboard_data = journal.get_journal_dashboard_datas() self.assertEqual(dashboard_data['number_draft'], 1) - self.assertIn('-13.30', dashboard_data['sum_draft']) + self.assertIn('-\N{ZERO WIDTH NO-BREAK SPACE}13.30', dashboard_data['sum_draft']) self.assertEqual(dashboard_data['number_waiting'], 1) self.assertIn('81.72', dashboard_data['sum_waiting']) diff --git a/addons/account/views/account_journal_dashboard_view.xml b/addons/account/views/account_journal_dashboard_view.xml index c6014deea8a..9c5a21aa945 100644 --- a/addons/account/views/account_journal_dashboard_view.xml +++ b/addons/account/views/account_journal_dashboard_view.xml @@ -53,8 +53,7 @@ - - + Gaps in the sequence @@ -298,8 +297,7 @@ - - + diff --git a/addons/account_check_printing/models/account_journal.py b/addons/account_check_printing/models/account_journal.py index fb4e04906b4..aa5a0797409 100644 --- a/addons/account_check_printing/models/account_journal.py +++ b/addons/account_check_printing/models/account_journal.py @@ -74,17 +74,14 @@ class AccountJournal(models.Model): 'company_id': journal.company_id.id, }) - def get_journal_dashboard_datas(self): - domain_checks_to_print = [ - ('journal_id', '=', self.id), + def _get_journal_dashboard_data_batched(self): + dashboard_data = super()._get_journal_dashboard_data_batched() + self._fill_dashboard_data_count(dashboard_data, 'account.payment', 'num_checks_to_print', [ ('payment_method_line_id.code', '=', 'check_printing'), ('state', '=', 'posted'), ('is_move_sent','=', False), - ] - return dict( - super(AccountJournal, self).get_journal_dashboard_datas(), - num_checks_to_print=self.env['account.payment'].search_count(domain_checks_to_print), - ) + ]) + return dashboard_data def action_checks_to_print(self): payment_method_line = self.outbound_payment_method_line_ids.filtered(lambda l: l.code == 'check_printing')