[FIX] account: optimize the bank reconciliation widget + add of a button to auto-reconcile instead of auto-reconciling directly when opening the widget.

* This new button is a behavior change, but this is needed for big databases with a lot of statement lines to process. Indeed, in that case we prefer to load the widget as soon as possible and thus let the user process the lines manually or click on that button (but at that time he knows why it takes time).
* Opimization of various method in order to speed reconciliation and view opening : added indexes, direct use of sql queries...and also removed the code that was looping on open debit/credit to propose a sum of entries to match the statement line's amount, because it was a little greedy and wouldn't give accurate results most of the time.
This commit is contained in:
Cédric Snauwaert
2016-08-17 14:23:08 +02:00
committed by qdp-odoo
parent 8787476b50
commit 5b4404f628
5 changed files with 221 additions and 162 deletions
+150 -124
View File
@@ -288,58 +288,54 @@ class AccountBankStatement(models.Model):
"""
statements = self
bsl_obj = self.env['account.bank.statement.line']
# NB : The field account_id can be used at the statement line creation/import to avoid the reconciliation process on it later on,
# this is why we filter out statements lines where account_id is set
st_lines_filter = [('journal_entry_ids', '=', False), ('account_id', '=', False)]
sql_query = """SELECT stl.id
FROM account_bank_statement_line stl
WHERE account_id IS NULL AND not exists (select 1 from account_move m where m.statement_line_id = stl.id)
AND company_id = %s
"""
params = (self.env.user.company_id.id,)
if statements:
st_lines_filter += [('statement_id', 'in', statements.ids)]
sql_query += ' AND stl.statement_id IN %s'
params += (tuple(statements.ids),)
sql_query += ' ORDER BY stl.id'
self.env.cr.execute(sql_query, params)
st_lines_left = self.env['account.bank.statement.line'].browse([line.get('id') for line in self.env.cr.dictfetchall()])
# Try to automatically reconcile statement lines
automatic_reconciliation_entries = []
st_lines_left = self.env['account.bank.statement.line']
for st_line in bsl_obj.search(st_lines_filter):
res = st_line.auto_reconcile()
if not res:
st_lines_left = (st_lines_left | st_line)
else:
automatic_reconciliation_entries.append(res.ids)
# Try to set statement line's partner
for st_line in st_lines_left:
if st_line.name and not st_line.partner_id:
additional_domain = [('ref', '=', st_line.name)]
match_recs = st_line.get_move_lines_for_reconciliation(limit=1, additional_domain=additional_domain, overlook_partner=True)
if match_recs and match_recs[0].partner_id:
st_line.write({'partner_id': match_recs[0].partner_id.id})
# Collect various informations for the reconciliation widget
notifications = []
num_auto_reconciled = len(automatic_reconciliation_entries)
if num_auto_reconciled > 0:
auto_reconciled_message = num_auto_reconciled > 1 \
and _("%d transactions were automatically reconciled.") % num_auto_reconciled \
or _("1 transaction was automatically reconciled.")
notifications += [{
'type': 'info',
'message': auto_reconciled_message,
'details': {
'name': _("Automatically reconciled items"),
'model': 'account.move',
'ids': automatic_reconciliation_entries
}
}]
lines = []
for el in statements:
lines.extend(el.line_ids.ids)
lines = list(set(lines))
#try to assign partner to bank_statement_line
stl_to_assign_partner = [stl.id for stl in st_lines_left if not stl.partner_id]
refs = list(set([st.name for st in st_lines_left if not stl.partner_id]))
if st_lines_left and stl_to_assign_partner and refs:
sql_query = """SELECT aml.partner_id, aml.ref, stl.id
FROM account_move_line aml
JOIN account_account acc ON acc.id = aml.account_id
JOIN account_bank_statement_line stl ON aml.ref = stl.name
WHERE (aml.company_id = %s
AND aml.partner_id IS NOT NULL)
AND (
(aml.statement_id IS NULL AND aml.account_id IN %s)
OR
(acc.internal_type IN ('payable', 'receivable') AND aml.reconciled = false)
)
AND aml.ref IN %s
"""
params = (self.env.user.company_id.id, (st_lines_left[0].journal_id.default_credit_account_id.id, st_lines_left[0].journal_id.default_debit_account_id.id), tuple(refs))
if statements:
sql_query += 'AND stl.id IN %s'
params += (tuple(stl_to_assign_partner),)
self.env.cr.execute(sql_query, params)
results = self.env.cr.dictfetchall()
st_line = self.env['account.bank.statement.line']
for line in results:
st_line.browse(line.get('id')).write({'partner_id': line.get('partner_id')})
return {
'st_lines_ids': st_lines_left.ids,
'notifications': notifications,
'notifications': [],
'statement_name': len(statements) == 1 and statements[0].name or False,
'num_already_reconciled_lines': statements and bsl_obj.search_count([('journal_entry_ids', '!=', False), ('id', 'in', lines)]) or 0,
'num_already_reconciled_lines': 0,
}
@api.multi
@@ -448,6 +444,40 @@ class AccountBankStatementLine(models.Model):
# Reconciliation interface methods
####################################################
@api.multi
def reconciliation_widget_auto_reconcile(self, num_already_reconciled_lines):
automatic_reconciliation_entries = self.env['account.bank.statement.line']
unreconciled = self.env['account.bank.statement.line']
for stl in self:
res = stl.auto_reconcile()
if res:
automatic_reconciliation_entries += stl
else:
unreconciled += stl
# Collect various informations for the reconciliation widget
notifications = []
num_auto_reconciled = len(automatic_reconciliation_entries)
if num_auto_reconciled > 0:
auto_reconciled_message = num_auto_reconciled > 1 \
and _("%d transactions were automatically reconciled.") % num_auto_reconciled \
or _("1 transaction was automatically reconciled.")
notifications += [{
'type': 'info',
'message': auto_reconciled_message,
'details': {
'name': _("Automatically reconciled items"),
'model': 'account.move',
'ids': automatic_reconciliation_entries.ids
}
}]
return {
'st_lines_ids': unreconciled.ids,
'notifications': notifications,
'statement_name': False,
'num_already_reconciled_lines': num_auto_reconciled + num_already_reconciled_lines,
}
@api.multi
def get_data_for_reconciliation_widget(self, excluded_ids=None):
""" Returns the data required to display a reconciliation widget, for each statement line in self """
@@ -560,86 +590,66 @@ class AccountBankStatementLine(models.Model):
return self.env['account.move.line'].search(domain, offset=offset, limit=limit, order="date_maturity asc, id asc")
def _get_domain_maker_move_line_amount(self):
""" Returns a function that can create the appropriate domain to search on move.line amount based on statement.line currency/amount """
company_currency = self.journal_id.company_id.currency_id
st_line_currency = self.currency_id or self.journal_id.currency_id
currency = (st_line_currency and st_line_currency != company_currency) and st_line_currency.id or False
field = currency and 'amount_residual_currency' or 'amount_residual'
precision = st_line_currency and st_line_currency.decimal_places or company_currency.decimal_places
def ret(comparator, amount, p=precision, f=field, c=currency):
if comparator == '<':
if amount < 0:
domain = [(f, '<', 0), (f, '>', amount)]
else:
domain = [(f, '>', 0), (f, '<', amount)]
elif comparator == '=':
if f == 'amount_residual':
liquidity_field = amount > 0 and 'debit' or 'credit'
domain = [
'|', (f, '=', float_round(amount, precision_digits=p)),
'&', ('account_id.internal_type', '=', 'liquidity'),
(liquidity_field, '=', amount),
]
else:
domain = [
'|', (f, '=', float_round(amount, precision_digits=p)),
'&', ('account_id.internal_type', '=', 'liquidity'),
('amount_currency', '=', amount),
]
else:
raise UserError(_("Programmation error : domain_maker_move_line_amount requires comparator '=' or '<'"))
domain += [('currency_id', '=', c)]
return domain
return ret
def _get_common_sql_query(self, overlook_partner = False, excluded_ids = None, split = False):
acc_type = "acc.internal_type IN ('payable', 'receivable')" if (self.partner_id or overlook_partner) else "acc.reconcile = true"
select_clause = "SELECT aml.id "
from_clause = "FROM account_move_line aml JOIN account_account acc ON acc.id = aml.account_id "
where_clause = """WHERE aml.company_id = %(company_id)s
AND (
(aml.statement_id IS NULL AND aml.account_id IN %(account_payable_receivable)s)
OR
("""+acc_type+""" AND aml.reconciled = false)
)"""
where_clause = where_clause + ' AND aml.partner_id = %(partner_id)s' if self.partner_id else where_clause
where_clause = where_clause + ' AND aml.id NOT IN %(excluded_ids)s' if excluded_ids else where_clause
if split:
return select_clause, from_clause, where_clause
return select_clause + from_clause + where_clause
def get_reconciliation_proposition(self, excluded_ids=None):
""" Returns move lines that constitute the best guess to reconcile a statement line
Note: it only looks for move lines in the same currency as the statement line.
"""
self.ensure_one()
if not excluded_ids:
excluded_ids = []
amount = self.amount_currency or self.amount
company_currency = self.journal_id.company_id.currency_id
st_line_currency = self.currency_id or self.journal_id.currency_id
currency = (st_line_currency and st_line_currency != company_currency) and st_line_currency.id or False
precision = st_line_currency and st_line_currency.decimal_places or company_currency.decimal_places
params = {'company_id': self.env.user.company_id.id,
'account_payable_receivable': (self.journal_id.default_credit_account_id.id, self.journal_id.default_debit_account_id.id),
'amount': float_round(amount, precision_digits=precision),
'partner_id': self.partner_id.id,
'excluded_ids': tuple(excluded_ids),
'ref': self.name,
}
# Look for structured communication match
if self.name:
overlook_partner = not self.partner_id # If the transaction has no partner, look for match in payable and receivable account anyway
domain = [('ref', '=', self.name)]
match_recs = self.get_move_lines_for_reconciliation(excluded_ids=excluded_ids, limit=2, additional_domain=domain, overlook_partner=overlook_partner)
if match_recs and len(match_recs) == 1:
return match_recs
elif len(match_recs) == 0:
move = self.env['account.move'].search([('name', '=', self.name)], limit=1)
if move:
domain = [('move_id', '=', move.id)]
match_recs = self.get_move_lines_for_reconciliation(excluded_ids=excluded_ids, limit=2, additional_domain=domain, overlook_partner=overlook_partner)
if match_recs and len(match_recs) == 1:
return match_recs
# How to compare statement line amount and move lines amount
amount_domain_maker = self._get_domain_maker_move_line_amount()
amount = self.amount_currency or self.amount
add_to_select = ", CASE WHEN aml.ref = %(ref)s THEN 1 ELSE 2 END as temp_field_order "
add_to_from = " JOIN account_move m ON m.id = aml.move_id "
select_clause, from_clause, where_clause = self._get_common_sql_query(overlook_partner=True, excluded_ids=excluded_ids, split=True)
sql_query = select_clause + add_to_select + from_clause + add_to_from + where_clause
sql_query += " AND (aml.ref= %(ref)s or m.name = %(ref)s) \
ORDER BY temp_field_order, date_maturity asc, aml.id asc"
self.env.cr.execute(sql_query, params)
results = self.env.cr.fetchone()
if results:
return self.env['account.move.line'].browse(results[0])
# Look for a single move line with the same amount
match_recs = self.get_move_lines_for_reconciliation(excluded_ids=excluded_ids, limit=1, additional_domain=amount_domain_maker('=', amount))
if match_recs:
return match_recs
field = currency and 'amount_residual_currency' or 'amount_residual'
liquidity_field = currency and 'amount_currency' or amount > 0 and 'debit' or 'credit'
sql_query = self._get_common_sql_query(excluded_ids=excluded_ids) + \
" AND ("+field+" = %(amount)s OR (acc.internal_type = 'liquidity' AND "+liquidity_field+" = %(amount)s)) \
ORDER BY date_maturity asc, aml.id asc LIMIT 1"
self.env.cr.execute(sql_query, params)
results = self.env.cr.fetchone()
if results:
return self.env['account.move.line'].browse(results[0])
if not self.partner_id:
return self.env['account.move.line']
# Select move lines until their total amount is greater than the statement line amount
domain = [('reconciled', '=', False)]
domain += [('account_id.user_type_id.type', '=', amount > 0 and 'receivable' or 'payable')] # Make sure we can't mix receivable and payable
domain += amount_domain_maker('<', amount) # Will also enforce > 0
mv_lines = self.get_move_lines_for_reconciliation(excluded_ids=excluded_ids, limit=5, additional_domain=domain)
st_line_currency = self.currency_id or self.journal_id.currency_id or self.journal_id.company_id.currency_id
ret = self.env['account.move.line']
total = 0
for line in mv_lines:
total += line.currency_id and line.amount_residual_currency or line.amount_residual
if float_compare(total, abs(amount), precision_digits=st_line_currency.rounding) != -1:
break
ret = (ret | line)
return ret
return self.env['account.move.line']
def _get_move_lines_for_auto_reconcile(self):
""" Returns the move lines that the method auto_reconcile can use to try to reconcile the statement line """
@@ -653,28 +663,44 @@ class AccountBankStatementLine(models.Model):
self.ensure_one()
match_recs = self.env['account.move.line']
# How to compare statement line amount and move lines amount
amount_domain_maker = self._get_domain_maker_move_line_amount()
equal_amount_domain = amount_domain_maker('=', self.amount_currency or self.amount)
amount = self.amount_currency or self.amount
company_currency = self.journal_id.company_id.currency_id
st_line_currency = self.currency_id or self.journal_id.currency_id
currency = (st_line_currency and st_line_currency != company_currency) and st_line_currency.id or False
precision = st_line_currency and st_line_currency.decimal_places or company_currency.decimal_places
params = {'company_id': self.env.user.company_id.id,
'account_payable_receivable': (self.journal_id.default_credit_account_id.id, self.journal_id.default_debit_account_id.id),
'amount': float_round(amount, precision_digits=precision),
'partner_id': self.partner_id.id,
'ref': self.name,
}
field = currency and 'amount_residual_currency' or 'amount_residual'
liquidity_field = currency and 'amount_currency' or amount > 0 and 'debit' or 'credit'
# Look for structured communication match
if self.name:
overlook_partner = not self.partner_id # If the transaction has no partner, look for match in payable and receivable account anyway
domain = equal_amount_domain + [('ref', '=', self.name)]
match_recs = self.get_move_lines_for_reconciliation(limit=2, additional_domain=domain, overlook_partner=overlook_partner)
if match_recs and len(match_recs) != 1:
sql_query = self._get_common_sql_query() + \
" AND aml.ref = %(ref)s AND ("+field+" = %(amount)s OR (acc.internal_type = 'liquidity' AND "+liquidity_field+" = %(amount)s)) \
ORDER BY date_maturity asc, aml.id asc"
self.env.cr.execute(sql_query, params)
match_recs = self.env.cr.dictfetchall()
if len(match_recs) > 1:
return False
# Look for a single move line with the same partner, the same amount
if not match_recs:
if self.partner_id:
match_recs = self.get_move_lines_for_reconciliation(limit=2, additional_domain=equal_amount_domain)
if match_recs and len(match_recs) != 1:
sql_query = self._get_common_sql_query() + \
" AND ("+field+" = %(amount)s OR (acc.internal_type = 'liquidity' AND "+liquidity_field+" = %(amount)s)) \
ORDER BY date_maturity asc, aml.id asc"
self.env.cr.execute(sql_query, params)
match_recs = self.env.cr.dictfetchall()
if len(match_recs) > 1:
return False
if not match_recs:
return False
match_recs = self.env['account.move.line'].browse([aml.get('id') for aml in match_recs])
# Now reconcile
counterpart_aml_dicts = []
payment_aml_rec = self.env['account.move.line']
+13 -3
View File
@@ -91,7 +91,7 @@ class AccountMove(models.Model):
company_id = fields.Many2one('res.company', related='journal_id.company_id', string='Company', store=True, readonly=True,
default=lambda self: self.env.user.company_id)
matched_percentage = fields.Float('Percentage Matched', compute='_compute_matched_percentage', digits=0, store=True, readonly=True, help="Technical field used in cash basis method")
statement_line_id = fields.Many2one('account.bank.statement.line', string='Bank statement line reconciled with this entry', copy=False, readonly=True)
statement_line_id = fields.Many2one('account.bank.statement.line', index=True, string='Bank statement line reconciled with this entry', copy=False, readonly=True)
# Dummy Account field to search on account.move by account_id
dummy_account_id = fields.Many2one('account.account', related='line_ids.account_id', string='Account', store=False)
@@ -232,6 +232,16 @@ class AccountMoveLine(models.Model):
_description = "Journal Item"
_order = "date desc, id desc"
def init(self, cr):
""" change index on partner_id to a multi-column index on (partner_id, ref), the new index will behave in the
same way when we search on partner_id, with the addition of being optimal when having a query that will
search on partner_id and ref at the same time (which is the case when we open the bank reconciliation widget)
"""
cr.execute('DROP INDEX IF EXISTS account_move_line_partner_id_index')
cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('account_move_line_partner_id_ref_idx',))
if not cr.fetchone():
cr.execute('CREATE INDEX account_move_line_partner_id_ref_idx ON account_move_line (partner_id, ref)')
@api.depends('debit', 'credit', 'amount_currency', 'currency_id', 'matched_debit_ids', 'matched_credit_ids', 'matched_debit_ids.amount', 'matched_credit_ids.amount', 'account_id.currency_id', 'move_id.state')
def _amount_residual(self):
""" Computes the residual amount of a move line from a reconciliable account in the company currency and the line's currency.
@@ -365,7 +375,7 @@ class AccountMoveLine(models.Model):
move_id = fields.Many2one('account.move', string='Journal Entry', ondelete="cascade",
help="The move of this entry line.", index=True, required=True, auto_join=True)
narration = fields.Text(related='move_id.narration', string='Internal Note')
ref = fields.Char(related='move_id.ref', string='Partner Reference', store=True, copy=False)
ref = fields.Char(related='move_id.ref', string='Partner Reference', store=True, copy=False, index=True)
payment_id = fields.Many2one('account.payment', string="Originator Payment", help="Payment that created this entry")
statement_id = fields.Many2one('account.bank.statement', string='Statement',
help="The bank statement used for bank reconciliation", index=True, copy=False)
@@ -390,7 +400,7 @@ class AccountMoveLine(models.Model):
# TODO: put the invoice link and partner_id on the account_move
invoice_id = fields.Many2one('account.invoice', oldname="invoice")
partner_id = fields.Many2one('res.partner', string='Partner', index=True, ondelete='restrict')
partner_id = fields.Many2one('res.partner', string='Partner', ondelete='restrict')
user_type_id = fields.Many2one('account.account.type', related='account_id.user_type_id', index=True, store=True, oldname="user_type")
_sql_constraints = [
@@ -1127,6 +1127,7 @@ var bankStatementReconciliation = abstractReconciliation.extend({
this.reconciliation_menu_id = false; // Used to update the needaction badge
// The same move line cannot be selected for multiple reconciliations
this.excluded_move_lines_ids = {};
this.widget_childrens = [];
},
serverPreprocessResultHandler: function(data) {
@@ -1178,44 +1179,64 @@ var bankStatementReconciliation = abstractReconciliation.extend({
single_statement: self.single_statement,
total_lines: self.num_already_reconciled_lines+self.lines.length
}));
self.updateProgressbar();
var reconciliations_to_show = self.lines.slice(0, self.num_reconciliations_fetched_in_batch);
self.last_displayed_reconciliation_index = reconciliations_to_show.length;
self.$(".reconciliation_lines_container").css("opacity", 0);
// If everything is reconciled, show end message
if (self.lines.length === 0) {
if (self.notifications)
self.displayNotifications(self.notifications, 0);
self.displayDoneMessage(true);
return;
}
// Display the reconciliations
return self.model_bank_statement_line
.call("get_data_for_reconciliation_widget", [reconciliations_to_show])
.then(function (data) {
var child_promises = [];
var datum = data.shift();
if (datum !== undefined)
child_promises.push(self.displayReconciliation(datum.st_line.id, 'match', false, true, datum.st_line, datum.reconciliation_proposition));
while ((datum = data.shift()) !== undefined)
child_promises.push(self.displayReconciliation(datum.st_line.id, 'inactive', false, true, datum.st_line, datum.reconciliation_proposition));
// When reconciliations are instanciated, make an entrance
$.when.apply($, child_promises).then(function(){
self.$(".reconciliation_lines_container").animate({opacity: 1}, self.aestetic_animation_speed, function() {
if (self.notifications) {
self.displayNotifications(self.notifications);
self.updateShowMoreButton();
}
self.$el.find('.js_automatic_reconciliation').click(function() {
// Let odoo try to reconcile entries for the user
self.model_bank_statement_line
.call("reconciliation_widget_auto_reconcile", [self.lines || undefined, self.num_already_reconciled_lines])
.then(function(data){ self.serverPreprocessResultHandler(data); })
.then(function(){ self.$('.js_automatic_reconciliation').hide();
return self.display_reconciliation_propositions();
});
});
});
});
return self.display_reconciliation_propositions();
});
});
},
display_reconciliation_propositions: function() {
var self = this;
self.updateProgressbar();
var reconciliations_to_show = self.lines.slice(0, self.num_reconciliations_fetched_in_batch);
self.last_displayed_reconciliation_index = reconciliations_to_show.length;
self.$(".reconciliation_lines_container").css("opacity", 0);
// Delete previous bankStatementReconciliationLine
$.each(self.widget_childrens, function(index, child) {
child.destroy();
});
// If everything is reconciled, show end message
if (self.lines.length === 0) {
self.$(".reconciliation_lines_container").hide();
if (self.notifications)
self.displayNotifications(self.notifications, 0);
self.displayDoneMessage(true);
return;
}
// Display the reconciliations
return self.model_bank_statement_line
.call("get_data_for_reconciliation_widget", [reconciliations_to_show])
.then(function (data) {
var child_promises = [];
var datum = data.shift();
if (datum !== undefined)
child_promises.push(self.displayReconciliation(datum.st_line.id, 'match', false, true, datum.st_line, datum.reconciliation_proposition));
while ((datum = data.shift()) !== undefined)
child_promises.push(self.displayReconciliation(datum.st_line.id, 'inactive', false, true, datum.st_line, datum.reconciliation_proposition));
// When reconciliations are instanciated, make an entrance
$.when.apply($, child_promises).then(function(){
self.$(".reconciliation_lines_container").animate({opacity: 1}, self.aestetic_animation_speed, function() {
if (self.notifications) {
self.displayNotifications(self.notifications);
self.updateShowMoreButton();
}
});
});
});
},
statementNameClickHandler: function() {
if (! this.single_statement) return;
this.$(".statement_name span").hide();
@@ -1320,6 +1341,7 @@ var bankStatementReconciliation = abstractReconciliation.extend({
reconciliation_proposition: initial_data_provided ? reconciliation_proposition : undefined,
};
var widget = new self.children_widget(self, context);
this.widget_childrens.push(widget);
return widget.appendTo(self.$(".reconciliation_lines_container"));
},
@@ -12,6 +12,7 @@
<div class="progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" t-att-aria-valuemax="total_lines" style="width: 0%;">
</div>
</div>
<button class="btn btn-default js_automatic_reconciliation pull-right" title="Let odoo try to reconcile entries for the user">Automatic reconciliation</button>
<div class="oe_clear o_clear"></div>
<div class="notification_area"></div>
<div class="interface_options oe_form o_form"></div>
@@ -79,8 +79,8 @@ class AccountPartialReconcile(models.Model):
#in that case, mark the reference on the partial reconciliations and the entries
self.env['account.full.reconcile'].with_context(check_move_validity=False).create({
'partial_reconcile_ids': [(6, 0, partial_rec_ids)],
'reconciled_line_ids': [(6, 0, aml_ids)]})
'partial_reconcile_ids': [(4, p_id) for p_id in partial_rec_ids],
'reconciled_line_ids': [(4, a_id) for a_id in aml_ids]})
return res
@api.multi