[REF] account: reconciliation models: improve performance
When opening the reconciliation widget and calling the matching rules, instead of calling all the rules in a single huge SQL query and then iterating on all the statement lines to match them with the results, we now first group the lines per applicable model and each time call a distinct (but way smaller) query for each group, stopping trying other rules for a statement line when one returns candidates. More queries are performed, but they are way more simple.
This commit is contained in:
oco-odoo
@@ -5,7 +5,9 @@ from odoo.tools import float_compare, float_is_zero
|
||||
from odoo.exceptions import UserError
|
||||
import re
|
||||
from math import copysign
|
||||
|
||||
import itertools
|
||||
from collections import defaultdict
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
class AccountReconcileModelLine(models.Model):
|
||||
_name = 'account.reconcile.model.line'
|
||||
@@ -102,7 +104,7 @@ class AccountReconcileModel(models.Model):
|
||||
rule_type = fields.Selection(selection=[
|
||||
('writeoff_button', 'Manually create a write-off on clicked button.'),
|
||||
('writeoff_suggestion', 'Suggest counterpart values.'),
|
||||
('invoice_matching', 'Match existing invoices/bills.')
|
||||
('invoice_matching', 'Match existing invoices/bills.'),
|
||||
], string='Type', default='writeoff_button', required=True)
|
||||
auto_reconcile = fields.Boolean(string='Auto-validate',
|
||||
help='Validate the statement line automatically (reconciliation based on your rule).')
|
||||
@@ -187,7 +189,7 @@ class AccountReconcileModel(models.Model):
|
||||
action.update({
|
||||
'context': {},
|
||||
'domain': [('id', 'in', self._cr.fetchone()[0])],
|
||||
'help': """<p class="o_view_nocontent_empty_folder">{}</p>""".format(_('No move from this reconciliation model')),
|
||||
'help': """<p class="o_view_nocontent_empty_folder">{}</p>""".format(_('This reconciliation model has created no entry so far')),
|
||||
})
|
||||
return action
|
||||
|
||||
@@ -354,177 +356,216 @@ class AccountReconcileModel(models.Model):
|
||||
# RECONCILIATION CRITERIA
|
||||
####################################################
|
||||
|
||||
def _apply_conditions(self, query, params):
|
||||
self.ensure_one()
|
||||
rule = self
|
||||
# Filter on journals.
|
||||
if rule.match_journal_ids:
|
||||
query += ' AND st_line_move.journal_id IN %s'
|
||||
params += [tuple(rule.match_journal_ids.ids)]
|
||||
def _apply_rules(self, st_lines, excluded_ids=None, partner_map=None):
|
||||
''' Apply criteria to get candidates for all reconciliation models.
|
||||
|
||||
# Filter on amount nature.
|
||||
if rule.match_nature == 'amount_received':
|
||||
query += ' AND st_line.amount >= 0.0'
|
||||
elif rule.match_nature == 'amount_paid':
|
||||
query += ' AND st_line.amount <= 0.0'
|
||||
This function is called in enterprise by the reconciliation widget to match
|
||||
the statement lines with the available candidates (using the reconciliation models).
|
||||
|
||||
# Filter on amount.
|
||||
if rule.match_amount:
|
||||
query += ' AND ROUND(ABS(st_line.amount), jnl_precision.dp) '
|
||||
if rule.match_amount == 'lower':
|
||||
query += '< %s'
|
||||
params += [self.match_amount_max]
|
||||
elif rule.match_amount == 'greater':
|
||||
query += '> %s'
|
||||
params += [self.match_amount_min]
|
||||
else:
|
||||
# if self.match_amount == 'between'
|
||||
query += 'BETWEEN %s AND %s'
|
||||
params += [rule.match_amount_min, rule.match_amount_max]
|
||||
|
||||
# Filter on label, note and transaction_type
|
||||
for table, field, column in [('st_line', 'label', 'payment_ref'), ('st_line_move', 'note', 'narration'), ('st_line', 'transaction_type', 'transaction_type')]:
|
||||
if rule['match_' + field] == 'contains':
|
||||
query += ' AND {}.{} ILIKE %s'.format(table, column)
|
||||
params += ['%%%s%%' % rule['match_' + field + '_param']]
|
||||
elif rule['match_' + field] == 'not_contains':
|
||||
query += ' AND {}.{} NOT ILIKE %s'.format(table, column)
|
||||
params += ['%%%s%%' % rule['match_' + field + '_param']]
|
||||
elif rule['match_' + field] == 'match_regex':
|
||||
query += ' AND {}.{} ~* %s'.format(table, column)
|
||||
params += [rule['match_' + field + '_param']]
|
||||
|
||||
# Filter on partners.
|
||||
if rule.match_partner:
|
||||
query += ' AND line_partner.partner_id != 0'
|
||||
|
||||
if rule.match_partner_ids:
|
||||
query += ' AND line_partner.partner_id IN %s'
|
||||
params += [tuple(rule.match_partner_ids.ids)]
|
||||
|
||||
if rule.match_partner_category_ids:
|
||||
query += '''
|
||||
AND line_partner.partner_id IN (
|
||||
SELECT DISTINCT categ.partner_id FROM res_partner_res_partner_category_rel categ WHERE categ.category_id IN %s
|
||||
)
|
||||
'''
|
||||
params += [tuple(rule.match_partner_category_ids.ids)]
|
||||
|
||||
return query, params
|
||||
|
||||
def _get_with_tables(self, st_lines, partner_map=None):
|
||||
with_tables = '''
|
||||
WITH jnl_precision AS (
|
||||
SELECT
|
||||
j.id AS journal_id, currency.decimal_places AS dp
|
||||
FROM account_journal j
|
||||
LEFT JOIN res_company c ON j.company_id = c.id
|
||||
LEFT JOIN res_currency currency ON COALESCE(j.currency_id, c.currency_id) = currency.id
|
||||
WHERE j.type IN ('bank', 'cash')
|
||||
)'''
|
||||
# Compute partners values table.
|
||||
# This is required since some statement line's partners could be shadowed in the reconciliation widget.
|
||||
partners_list = []
|
||||
for line in st_lines:
|
||||
partner_id = partner_map and partner_map.get(line.id) or line.partner_id.id or 0
|
||||
partners_list.append('(%d, %d)' % (line.id, partner_id))
|
||||
partners_table = 'SELECT * FROM (VALUES %s) AS line_partner (line_id, partner_id)' % ','.join(partners_list)
|
||||
with_tables += ', partners_table AS (' + partners_table + ')'
|
||||
return with_tables
|
||||
|
||||
def _get_invoice_matching_query(self, st_lines, excluded_ids=None, partner_map=None):
|
||||
''' Get the query applying all rules trying to match existing entries with the given statement lines.
|
||||
:param st_lines: Account.bank.statement.lines recordset.
|
||||
:param excluded_ids: Account.move.lines to exclude.
|
||||
:param partner_map: Dict mapping each line with new partner eventually.
|
||||
:return: A dict mapping each statement line id with:
|
||||
* aml_ids: A list of account.move.line ids.
|
||||
* model: An account.reconcile.model record (optional).
|
||||
* status: 'reconciled' if the lines has been already reconciled, 'write_off' if the write-off must be
|
||||
applied on the statement line.
|
||||
'''
|
||||
# This functions uses SQL to compute its results. We need to flush before doing anything more.
|
||||
for model_name in ('account.bank.statement', 'account.bank.statement.line', 'account.move', 'account.move.line', 'res.company', 'account.journal', 'account.account'):
|
||||
self.env[model_name].flush(self.env[model_name]._fields)
|
||||
|
||||
results = {line.id: {'aml_ids': []} for line in st_lines}
|
||||
|
||||
available_models = self.filtered(lambda m: m.rule_type != 'writeoff_button').sorted(key=lambda m: (m.sequence, m.id))
|
||||
aml_ids_to_exclude = set() # Keep track of already processed amls.
|
||||
reconciled_amls_ids = set() # Keep track of already reconciled amls.
|
||||
|
||||
# First associate with each rec models all the statement lines for which it is applicable
|
||||
lines_with_partner_per_model = defaultdict(lambda: [])
|
||||
for st_line in st_lines:
|
||||
partner = (partner_map and partner_map.get(st_line.id) and self.env['res.partner'].browse(partner_map[st_line.id])) or st_line.partner_id
|
||||
|
||||
for rec_model in available_models:
|
||||
if rec_model._is_applicable_for(st_line, partner):
|
||||
lines_with_partner_per_model[rec_model].append((st_line, partner))
|
||||
|
||||
# Execute only one SQL query for each model (for performance)
|
||||
matched_lines = self.env['account.bank.statement.line']
|
||||
for rec_model in available_models:
|
||||
|
||||
# We filter the lines for this model, in case a previous one has already found something for them
|
||||
filtered_st_lines_with_partner = [x for x in lines_with_partner_per_model[rec_model] if x[0] not in matched_lines]
|
||||
|
||||
if not filtered_st_lines_with_partner:
|
||||
# No unreconciled statement line for this model
|
||||
continue
|
||||
|
||||
all_model_candidates = rec_model._get_candidates(filtered_st_lines_with_partner, excluded_ids)
|
||||
|
||||
for st_line, partner in filtered_st_lines_with_partner:
|
||||
candidates = all_model_candidates[st_line.id]
|
||||
if candidates:
|
||||
model_rslt, new_reconciled_aml_ids, new_treated_aml_ids = rec_model._get_rule_result(st_line, candidates, aml_ids_to_exclude, reconciled_amls_ids, partner)
|
||||
|
||||
if model_rslt:
|
||||
results[st_line.id] = model_rslt
|
||||
reconciled_amls_ids |= new_reconciled_aml_ids
|
||||
aml_ids_to_exclude |= new_treated_aml_ids
|
||||
matched_lines += st_line
|
||||
|
||||
return results
|
||||
|
||||
def _is_applicable_for(self, st_line, partner):
|
||||
""" Returns true iff this reconciliation model can be used to search for matches
|
||||
for the provided statement line and partner.
|
||||
"""
|
||||
self.ensure_one()
|
||||
|
||||
# Filter on journals, amount nature, amount and partners
|
||||
# All the conditions defined in this block are non-match conditions.
|
||||
if ((self.match_journal_ids and st_line.move_id.journal_id not in self.match_journal_ids)
|
||||
or (self.match_nature == 'amount_received' and st_line.amount < 0)
|
||||
or (self.match_nature == 'amount_paid' and st_line.amount > 0)
|
||||
or (self.match_amount == 'lower' and abs(st_line.amount) >= self.match_amount_max)
|
||||
or (self.match_amount == 'greater' and abs(st_line.amount) <= self.match_amount_min)
|
||||
or (self.match_amount == 'between' and (abs(st_line.amount) > self.match_amount_max or abs(st_line.amount) < self.match_amount_min))
|
||||
or (self.match_partner and not partner)
|
||||
or (self.match_partner and self.match_partner_ids and partner not in self.match_partner_ids)
|
||||
or (self.match_partner and self.match_partner_category_ids and partner.category_id not in self.match_partner_category_ids)
|
||||
):
|
||||
return False
|
||||
|
||||
# Filter on label, note and transaction_type
|
||||
for record, rule_field, record_field in [(st_line, 'label', 'payment_ref'), (st_line.move_id, 'note', 'narration'), (st_line, 'transaction_type', 'transaction_type')]:
|
||||
rule_term = (self['match_' + rule_field + '_param'] or '').lower()
|
||||
record_term = (record[record_field] or '').lower()
|
||||
|
||||
# This defines non-match conditions
|
||||
if ((self['match_' + rule_field] == 'contains' and rule_term not in record_term)
|
||||
or (self['match_' + rule_field] == 'not_contains' and rule_term in record_term)
|
||||
or (self['match_' + rule_field] == 'match_regex' and not re.match(rule_term, record_term))
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _get_candidates(self, st_lines_with_partner, excluded_ids):
|
||||
""" Returns the match candidates for this rule, with respect to the provided parameters.
|
||||
|
||||
:param st_lines_with_partner: A list of tuples (statement_line, partner),
|
||||
associating each statement line to treate with
|
||||
the corresponding partner, given by the partner map
|
||||
:param excluded_ids: a set containing the ids of the amls to ignore during the search
|
||||
(because they already been matched by another rule)
|
||||
"""
|
||||
self.ensure_one()
|
||||
|
||||
treatment_map = {
|
||||
'invoice_matching': lambda x: x._get_invoice_matching_query(st_lines_with_partner, excluded_ids),
|
||||
'writeoff_suggestion': lambda x: x._get_writeoff_suggestion_query(st_lines_with_partner, excluded_ids),
|
||||
}
|
||||
|
||||
query_generator = treatment_map[self.rule_type]
|
||||
query, params = query_generator(self)
|
||||
self._cr.execute(query, params)
|
||||
|
||||
rslt = defaultdict(lambda: [])
|
||||
for candidate_dict in self._cr.dictfetchall():
|
||||
rslt[candidate_dict['id']].append(candidate_dict)
|
||||
|
||||
return rslt
|
||||
|
||||
def _get_invoice_matching_query(self, st_lines_with_partner, excluded_ids):
|
||||
''' Returns the query applying the current invoice_matching reconciliation
|
||||
model to the provided statement lines.
|
||||
|
||||
:param st_lines_with_partner: A list of tuples (statement_line, partner),
|
||||
associating each statement line to treate with
|
||||
the corresponding partner, given by the partner map
|
||||
:param excluded_ids: Account.move.lines to exclude.
|
||||
:return: (query, params)
|
||||
'''
|
||||
if any(m.rule_type != 'invoice_matching' for m in self):
|
||||
self.ensure_one()
|
||||
if self.rule_type != 'invoice_matching':
|
||||
raise UserError(_('Programmation Error: Can\'t call _get_invoice_matching_query() for different rules than \'invoice_matching\''))
|
||||
|
||||
queries = []
|
||||
all_params = []
|
||||
for rule in self:
|
||||
# N.B: 'communication_flag' is there to distinguish invoice matching through the number/reference
|
||||
# (higher priority) from invoice matching using the partner (lower priority).
|
||||
query = r'''
|
||||
SELECT
|
||||
%s AS sequence,
|
||||
%s AS model_id,
|
||||
st_line.id AS id,
|
||||
aml.id AS aml_id,
|
||||
aml.currency_id AS aml_currency_id,
|
||||
aml.date_maturity AS aml_date_maturity,
|
||||
aml.amount_residual AS aml_amount_residual,
|
||||
aml.amount_residual_currency AS aml_amount_residual_currency,
|
||||
aml.balance AS aml_balance,
|
||||
aml.amount_currency AS aml_amount_currency,
|
||||
account.internal_type AS account_internal_type,
|
||||
# N.B: 'communication_flag' is there to distinguish invoice matching through the number/reference
|
||||
# (higher priority) from invoice matching using the partner (lower priority).
|
||||
query = r'''
|
||||
SELECT
|
||||
%(sequence)s AS sequence,
|
||||
%(model_id)s AS model_id,
|
||||
st_line.id AS id,
|
||||
aml.id AS aml_id,
|
||||
aml.currency_id AS aml_currency_id,
|
||||
aml.date_maturity AS aml_date_maturity,
|
||||
aml.amount_residual AS aml_amount_residual,
|
||||
aml.amount_residual_currency AS aml_amount_residual_currency,
|
||||
aml.balance AS aml_balance,
|
||||
aml.amount_currency AS aml_amount_currency,
|
||||
account.internal_type AS account_internal_type,
|
||||
|
||||
-- Determine a matching or not with the statement line communication using the aml.name, move.name or move.ref.
|
||||
(
|
||||
aml.name IS NOT NULL
|
||||
AND
|
||||
substring(REGEXP_REPLACE(aml.name, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*') != ''
|
||||
AND
|
||||
regexp_split_to_array(substring(REGEXP_REPLACE(aml.name, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'),'\s+')
|
||||
&& regexp_split_to_array(substring(REGEXP_REPLACE(st_line.payment_ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'), '\s+')
|
||||
)
|
||||
OR
|
||||
regexp_split_to_array(substring(REGEXP_REPLACE(move.name, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'),'\s+')
|
||||
&& regexp_split_to_array(substring(REGEXP_REPLACE(st_line.payment_ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'), '\s+')
|
||||
OR
|
||||
(
|
||||
move.ref IS NOT NULL
|
||||
AND
|
||||
substring(REGEXP_REPLACE(move.ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*') != ''
|
||||
AND
|
||||
regexp_split_to_array(substring(REGEXP_REPLACE(move.ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'),'\s+')
|
||||
&& regexp_split_to_array(substring(REGEXP_REPLACE(st_line.payment_ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'), '\s+')
|
||||
) AS communication_flag,
|
||||
-- Determine a matching or not with the statement line communication using the move.payment_reference.
|
||||
(
|
||||
move.payment_reference IS NOT NULL
|
||||
AND
|
||||
regexp_replace(move.payment_reference, '\s+', '', 'g') = regexp_replace(st_line.payment_ref, '\s+', '', 'g')
|
||||
) AS payment_reference_flag
|
||||
FROM account_bank_statement_line st_line
|
||||
JOIN account_move st_line_move ON st_line_move.id = st_line.move_id
|
||||
JOIN account_journal journal ON journal.id = st_line_move.journal_id
|
||||
LEFT JOIN jnl_precision ON jnl_precision.journal_id = journal.id
|
||||
JOIN res_company company ON company.id = st_line_move.company_id
|
||||
LEFT JOIN partners_table line_partner ON line_partner.line_id = st_line.id
|
||||
, account_move_line aml
|
||||
LEFT JOIN account_move move ON move.id = aml.move_id AND move.state = 'posted'
|
||||
LEFT JOIN account_account account ON account.id = aml.account_id
|
||||
WHERE st_line.id IN %s
|
||||
AND aml.company_id = st_line_move.company_id
|
||||
AND move.state = 'posted'
|
||||
AND (
|
||||
-- the field match_partner of the rule might enforce the second part of
|
||||
-- the OR condition, later in _apply_conditions()
|
||||
line_partner.partner_id = 0
|
||||
OR
|
||||
aml.partner_id = line_partner.partner_id
|
||||
)
|
||||
AND CASE WHEN st_line.amount > 0.0
|
||||
THEN aml.balance > 0
|
||||
ELSE aml.balance < 0
|
||||
END
|
||||
|
||||
-- if there is a partner, propose all aml of the partner, otherwise propose only the ones
|
||||
-- matching the statement line communication
|
||||
-- Determine a matching or not with the statement line communication using the aml.name, move.name or move.ref.
|
||||
(
|
||||
aml.name IS NOT NULL
|
||||
AND
|
||||
(
|
||||
substring(REGEXP_REPLACE(aml.name, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*') != ''
|
||||
AND
|
||||
regexp_split_to_array(substring(REGEXP_REPLACE(aml.name, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'),'\s+')
|
||||
&& regexp_split_to_array(substring(REGEXP_REPLACE(st_line.payment_ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'), '\s+')
|
||||
)
|
||||
OR
|
||||
regexp_split_to_array(substring(REGEXP_REPLACE(move.name, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'),'\s+')
|
||||
&& regexp_split_to_array(substring(REGEXP_REPLACE(st_line.payment_ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'), '\s+')
|
||||
OR
|
||||
(
|
||||
move.ref IS NOT NULL
|
||||
AND
|
||||
substring(REGEXP_REPLACE(move.ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*') != ''
|
||||
AND
|
||||
regexp_split_to_array(substring(REGEXP_REPLACE(move.ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'),'\s+')
|
||||
&& regexp_split_to_array(substring(REGEXP_REPLACE(st_line.payment_ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*'), '\s+')
|
||||
) AS communication_flag,
|
||||
-- Determine a matching or not with the statement line communication using the move.payment_reference.
|
||||
(
|
||||
move.payment_reference IS NOT NULL
|
||||
AND
|
||||
regexp_replace(move.payment_reference, '\s+', '', 'g') = regexp_replace(st_line.payment_ref, '\s+', '', 'g')
|
||||
) AS payment_reference_flag
|
||||
FROM account_bank_statement_line st_line
|
||||
JOIN account_move st_line_move ON st_line_move.id = st_line.move_id
|
||||
JOIN account_journal journal ON journal.id = st_line_move.journal_id
|
||||
JOIN res_company company ON company.id = st_line_move.company_id
|
||||
, account_move_line aml
|
||||
LEFT JOIN account_move move ON move.id = aml.move_id AND move.state = 'posted'
|
||||
LEFT JOIN account_account account ON account.id = aml.account_id
|
||||
LEFT JOIN res_partner aml_partner ON aml.partner_id = aml_partner.id
|
||||
WHERE
|
||||
aml.company_id = st_line_move.company_id
|
||||
AND move.state = 'posted'
|
||||
AND account.reconcile IS TRUE
|
||||
AND aml.reconciled IS FALSE
|
||||
'''
|
||||
|
||||
# Add conditions to handle each of the statement lines we want to match
|
||||
st_lines_queries = []
|
||||
for st_line, partner in st_lines_with_partner:
|
||||
# In case we don't have any partner for this line, we try assigning one with the rule mapping
|
||||
if st_line.amount > 0:
|
||||
st_line_subquery = r"aml.balance > 0"
|
||||
else:
|
||||
st_line_subquery = r"aml.balance < 0"
|
||||
|
||||
if self.match_same_currency:
|
||||
st_line_subquery += r" AND COALESCE(aml.currency_id, company.currency_id) = %s" % (st_line.foreign_currency_id.id or st_line.move_id.currency_id.id)
|
||||
|
||||
if partner:
|
||||
st_line_subquery += r" AND aml.partner_id = %s" % partner.id
|
||||
else:
|
||||
st_line_subquery += r"""
|
||||
AND
|
||||
(
|
||||
line_partner.partner_id != 0
|
||||
AND
|
||||
aml.partner_id = line_partner.partner_id
|
||||
)
|
||||
OR
|
||||
(
|
||||
line_partner.partner_id = 0
|
||||
AND
|
||||
substring(REGEXP_REPLACE(st_line.payment_ref, '[^0-9|^\s]', '', 'g'), '\S(?:.*\S)*') != ''
|
||||
AND
|
||||
(
|
||||
@@ -556,65 +597,119 @@ class AccountReconcileModel(models.Model):
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
AND account.reconcile IS TRUE
|
||||
AND aml.reconciled IS FALSE
|
||||
'''
|
||||
# Filter on the same currency.
|
||||
if rule.match_same_currency:
|
||||
query += '''
|
||||
AND COALESCE(st_line.foreign_currency_id, st_line_move.currency_id) = COALESCE(aml.currency_id, company.currency_id)
|
||||
'''
|
||||
"""
|
||||
|
||||
st_lines_queries.append(r"st_line.id = %s AND (%s)" % (st_line.id, st_line_subquery))
|
||||
|
||||
query += r" AND (%s) " % " OR ".join(st_lines_queries)
|
||||
|
||||
params = {
|
||||
'sequence': self.sequence,
|
||||
'model_id': self.id,
|
||||
}
|
||||
|
||||
# Filter out excluded account.move.line.
|
||||
if excluded_ids:
|
||||
query += 'AND aml.id NOT IN %(excluded_aml_ids)s'
|
||||
params['excluded_aml_ids'] = tuple(excluded_ids)
|
||||
|
||||
params = [rule.sequence, rule.id, tuple(st_lines.ids)]
|
||||
# Filter out excluded account.move.line.
|
||||
if excluded_ids:
|
||||
query += 'AND aml.id NOT IN %s'
|
||||
params += [tuple(excluded_ids)]
|
||||
query, params = rule._apply_conditions(query, params)
|
||||
queries.append(query)
|
||||
all_params += params
|
||||
full_query = self._get_with_tables(st_lines, partner_map=partner_map)
|
||||
full_query += ' UNION ALL '.join(queries)
|
||||
# Oldest due dates come first.
|
||||
full_query += ' ORDER BY aml_date_maturity, aml_id'
|
||||
return full_query, all_params
|
||||
query += ' ORDER BY aml_date_maturity, aml_id'
|
||||
|
||||
def _get_writeoff_suggestion_query(self, st_lines, excluded_ids=None, partner_map=None):
|
||||
''' Get the query applying all reconciliation rules.
|
||||
:param st_lines: Account.bank.statement.lines recordset.
|
||||
return query, params
|
||||
|
||||
def _get_writeoff_suggestion_query(self, st_lines_with_partner, excluded_ids=None):
|
||||
''' Returns the query applying the current writeoff_suggestion reconciliation
|
||||
model to the provided statement lines.
|
||||
|
||||
:param st_lines_with_partner: A list of tuples (statement_line, partner),
|
||||
associating each statement line to treate with
|
||||
the corresponding partner, given by the partner map
|
||||
:param excluded_ids: Account.move.lines to exclude.
|
||||
:param partner_map: Dict mapping each line with new partner eventually.
|
||||
:return: (query, params)
|
||||
'''
|
||||
if any(m.rule_type != 'writeoff_suggestion' for m in self):
|
||||
raise UserError(_('Programmation Error: Can\'t call _get_wo_suggestion_query() for different rules than \'writeoff_suggestion\''))
|
||||
self.ensure_one()
|
||||
|
||||
queries = []
|
||||
all_params = []
|
||||
for rule in self:
|
||||
query = '''
|
||||
SELECT
|
||||
%s AS sequence,
|
||||
%s AS model_id,
|
||||
st_line.id AS id
|
||||
FROM account_bank_statement_line st_line
|
||||
JOIN account_move st_line_move ON st_line_move.id = st_line.move_id
|
||||
LEFT JOIN account_journal journal ON journal.id = st_line_move.journal_id
|
||||
LEFT JOIN jnl_precision ON jnl_precision.journal_id = journal.id
|
||||
LEFT JOIN res_company company ON company.id = st_line_move.company_id
|
||||
LEFT JOIN partners_table line_partner ON line_partner.line_id = st_line.id
|
||||
WHERE st_line.id IN %s
|
||||
'''
|
||||
params = [rule.sequence, rule.id, tuple(st_lines.ids)]
|
||||
if self.rule_type != 'writeoff_suggestion':
|
||||
raise UserError(_("Programmation Error: Can't call _get_writeoff_suggestion_query() for different rules than 'writeoff_suggestion'"))
|
||||
|
||||
query, params = rule._apply_conditions(query, params)
|
||||
queries.append(query)
|
||||
all_params += params
|
||||
query = '''
|
||||
SELECT
|
||||
%(sequence)s AS sequence,
|
||||
%(model_id)s AS model_id,
|
||||
st_line.id AS id
|
||||
FROM account_bank_statement_line st_line
|
||||
JOIN account_move st_line_move ON st_line_move.id = st_line.move_id
|
||||
LEFT JOIN account_journal journal ON journal.id = st_line_move.journal_id
|
||||
LEFT JOIN res_company company ON company.id = st_line_move.company_id
|
||||
WHERE st_line.id IN %(st_line_ids)s
|
||||
'''
|
||||
params = {
|
||||
'sequence': self.sequence,
|
||||
'model_id': self.id,
|
||||
'st_line_ids': tuple(st_line.id for (st_line, partner) in st_lines_with_partner),
|
||||
}
|
||||
|
||||
full_query = self._get_with_tables(st_lines, partner_map=partner_map)
|
||||
full_query += ' UNION ALL '.join(queries)
|
||||
return full_query, all_params
|
||||
return query, params
|
||||
|
||||
def _get_rule_result(self, st_line, candidates, aml_ids_to_exclude, reconciled_amls_ids, partner_map):
|
||||
""" Get the result of a rule from the list of available candidates, depending on the
|
||||
other reconciliations performed by previous rules.
|
||||
"""
|
||||
self.ensure_one()
|
||||
|
||||
if self.rule_type == 'invoice_matching':
|
||||
return self._get_invoice_matching_rule_result(st_line, candidates, aml_ids_to_exclude, reconciled_amls_ids, partner_map)
|
||||
elif self.rule_type == 'writeoff_suggestion':
|
||||
return self._get_writeoff_suggestion_rule_result(st_line, partner_map), set(), set()
|
||||
else:
|
||||
return None, set(), set()
|
||||
|
||||
def _get_invoice_matching_rule_result(self, st_line, candidates, aml_ids_to_exclude, reconciled_amls_ids, partner):
|
||||
new_reconciled_aml_ids = set()
|
||||
new_treated_aml_ids = set()
|
||||
candidates, priorities = self._filter_candidates(candidates, aml_ids_to_exclude, reconciled_amls_ids)
|
||||
|
||||
# Special case: the amounts are the same, submit the line directly.
|
||||
st_line_currency = st_line.foreign_currency_id or st_line.currency_id
|
||||
candidate_currencies = set(candidate['aml_currency_id'] or st_line.company_id.currency_id.id for candidate in candidates)
|
||||
if candidate_currencies == {st_line_currency.id}:
|
||||
for candidate in candidates:
|
||||
residual_amount = candidate['aml_currency_id'] and candidate['aml_amount_residual_currency'] or candidate['aml_amount_residual']
|
||||
if st_line_currency.is_zero(residual_amount + st_line.amount_residual):
|
||||
candidates, priorities = self._filter_candidates([candidate], aml_ids_to_exclude, reconciled_amls_ids)
|
||||
break
|
||||
|
||||
# We check the amount criteria of the reconciliation model, and select the
|
||||
# candidates if they pass the verification. Candidates from the first priority
|
||||
# level (even already selected) bypass this check, and are selected anyway.
|
||||
if priorities & {1,2} or self._check_rule_propositions(st_line, candidates):
|
||||
rslt = {
|
||||
'model': self,
|
||||
'aml_ids': [candidate['aml_id'] for candidate in candidates],
|
||||
}
|
||||
new_treated_aml_ids = set(rslt['aml_ids'])
|
||||
|
||||
# Create write-off lines.
|
||||
lines_vals_list = self._prepare_reconciliation(st_line, aml_ids=rslt['aml_ids'], partner=partner)
|
||||
|
||||
# A write-off must be applied if there are some 'new' lines to propose.
|
||||
if not lines_vals_list or any(not line_vals.get('id') for line_vals in lines_vals_list):
|
||||
rslt['status'] = 'write_off'
|
||||
|
||||
# Process auto-reconciliation. We only do that for the first two priorities, if they are not matched elsewhere.
|
||||
if lines_vals_list and priorities & {1, 3} and self.auto_reconcile:
|
||||
if not st_line.partner_id and partner:
|
||||
st_line.partner_id = partner
|
||||
|
||||
st_line.reconcile(lines_vals_list)
|
||||
rslt['status'] = 'reconciled'
|
||||
rslt['reconciled_lines'] = st_line.line_ids
|
||||
new_reconciled_aml_ids = new_treated_aml_ids
|
||||
else:
|
||||
rslt = None
|
||||
|
||||
return rslt, new_reconciled_aml_ids, new_treated_aml_ids
|
||||
|
||||
def _check_rule_propositions(self, statement_line, candidates):
|
||||
''' Check restrictions that can't be handled for each move.line separately.
|
||||
@@ -658,172 +753,82 @@ class AccountReconcileModel(models.Model):
|
||||
return False
|
||||
return amount_percentage >= self.match_total_amount_param
|
||||
|
||||
def _apply_rules(self, st_lines, excluded_ids=None, partner_map=None):
|
||||
''' Apply criteria to get candidates for all reconciliation models.
|
||||
:param st_lines: Account.bank.statement.lines recordset.
|
||||
:param excluded_ids: Account.move.lines to exclude.
|
||||
:param partner_map: Dict mapping each line with new partner eventually.
|
||||
:return: A dict mapping each statement line id with:
|
||||
* aml_ids: A list of account.move.line ids.
|
||||
* model: An account.reconcile.model record (optional).
|
||||
* status: 'reconciled' if the lines has been already reconciled, 'write_off' if the write-off must be
|
||||
applied on the statement line.
|
||||
'''
|
||||
available_models = self.filtered(lambda m: m.rule_type != 'writeoff_button')
|
||||
def _filter_candidates(self, candidates, aml_ids_to_exclude, reconciled_amls_ids):
|
||||
""" Sorts reconciliation candidates by priority and filters them so that only
|
||||
the most prioritary are kept.
|
||||
"""
|
||||
candidates_by_priority = self._sort_reconciliation_candidates_by_priority(candidates, aml_ids_to_exclude, reconciled_amls_ids)
|
||||
max_priority = min(candidates_by_priority.keys())
|
||||
|
||||
results = dict((r.id, {'aml_ids': []}) for r in st_lines)
|
||||
filtered_candidates = candidates_by_priority[max_priority]
|
||||
filtered_priorities = {max_priority,}
|
||||
|
||||
if not available_models:
|
||||
return results
|
||||
if max_priority in (1, 3, 5):
|
||||
# We also keep the already proposed values of the same priority level
|
||||
proposed_priority = max_priority + 1
|
||||
filtered_candidates += candidates_by_priority[proposed_priority]
|
||||
if candidates_by_priority[proposed_priority]:
|
||||
filtered_priorities.add(proposed_priority)
|
||||
|
||||
ordered_models = available_models.sorted(key=lambda m: (m.sequence, m.id))
|
||||
return filtered_candidates, filtered_priorities
|
||||
|
||||
grouped_candidates = {}
|
||||
def _sort_reconciliation_candidates_by_priority(self, candidates, already_proposed_aml_ids, already_reconciled_aml_ids):
|
||||
""" Sorts the provided candidates and returns a mapping of candidates by
|
||||
priority (1 being the highest).
|
||||
|
||||
# Type == 'invoice_matching'.
|
||||
# Map each (st_line.id, model_id) with matching amls.
|
||||
invoices_models = ordered_models.filtered(lambda m: m.rule_type == 'invoice_matching')
|
||||
self.env['account.move'].flush(['state'])
|
||||
self.env['account.move.line'].flush(['balance', 'reconciled'])
|
||||
self.env['account.bank.statement.line'].flush(['company_id'])
|
||||
if invoices_models:
|
||||
query, params = invoices_models._get_invoice_matching_query(st_lines, excluded_ids=excluded_ids, partner_map=partner_map)
|
||||
self._cr.execute(query, params)
|
||||
query_res = self._cr.dictfetchall()
|
||||
The priorities are defined as follows:
|
||||
|
||||
for res in query_res:
|
||||
grouped_candidates.setdefault(res['id'], {})
|
||||
grouped_candidates[res['id']].setdefault(res['model_id'], [])
|
||||
grouped_candidates[res['id']][res['model_id']].append(res)
|
||||
1: payment_reference_flag is true, so the move's payment_reference
|
||||
field matches the statement line's.
|
||||
|
||||
# Type == 'writeoff_suggestion'.
|
||||
# Map each (st_line.id, model_id) with a flag indicating the st_line matches the criteria.
|
||||
write_off_models = ordered_models.filtered(lambda m: m.rule_type == 'writeoff_suggestion')
|
||||
if write_off_models:
|
||||
query, params = write_off_models._get_writeoff_suggestion_query(st_lines, excluded_ids=excluded_ids, partner_map=partner_map)
|
||||
self._cr.execute(query, params)
|
||||
query_res = self._cr.dictfetchall()
|
||||
2: Same as 1, but the candidates have already been proposed for a previous statement line
|
||||
|
||||
for res in query_res:
|
||||
grouped_candidates.setdefault(res['id'], {})
|
||||
grouped_candidates[res['id']].setdefault(res['model_id'], True)
|
||||
3: communication_flag is true, so either the move's ref, move's name or
|
||||
aml's name match the statement line's payment reference.
|
||||
|
||||
# Keep track of already processed amls.
|
||||
amls_ids_to_exclude = set()
|
||||
4: Same as 3, but the candidates have already been proposed for a previous statement line
|
||||
|
||||
# Keep track of already reconciled amls.
|
||||
reconciled_amls_ids = set()
|
||||
5: candidates proposed by the query, but no match with the statement
|
||||
line's payment ref could be found.
|
||||
|
||||
# Iterate all and create results.
|
||||
for line in st_lines:
|
||||
line_currency = line.foreign_currency_id or line.currency_id
|
||||
6: Same as 5, but the candidates have already been proposed for a previous statement line
|
||||
"""
|
||||
candidates_by_priority = defaultdict(lambda: [])
|
||||
|
||||
# Search for applicable rule.
|
||||
# /!\ BREAK are very important here to avoid applying multiple rules on the same line.
|
||||
for model in ordered_models:
|
||||
# No result found.
|
||||
if not grouped_candidates.get(line.id) or not grouped_candidates[line.id].get(model.id):
|
||||
continue
|
||||
for candidate in filter(lambda x: x['aml_id'] not in already_reconciled_aml_ids, candidates):
|
||||
|
||||
if model.rule_type == 'invoice_matching':
|
||||
candidates = grouped_candidates[line.id][model.id]
|
||||
if candidate['payment_reference_flag']:
|
||||
priority = 1
|
||||
elif candidate['communication_flag']:
|
||||
priority = 3
|
||||
else:
|
||||
priority = 5
|
||||
|
||||
# If some invoices match on the communication, suggest them.
|
||||
# Otherwise, suggest all invoices having the same partner.
|
||||
# N.B: The only way to match a line without a partner is through the communication.
|
||||
first_batch_candidates = []
|
||||
first_batch_candidates_proposed = []
|
||||
second_batch_candidates = []
|
||||
second_batch_candidates_proposed = []
|
||||
third_batch_candidates = []
|
||||
third_batch_candidates_proposed = []
|
||||
for c in candidates:
|
||||
# Don't take into account already reconciled lines.
|
||||
if c['aml_id'] in reconciled_amls_ids:
|
||||
continue
|
||||
if candidate['aml_id'] in already_proposed_aml_ids:
|
||||
# So, priorities 2, 4 and 6 are created here
|
||||
priority += 1
|
||||
|
||||
# Dispatch candidates between lines matching invoices with the communication or only the partner.
|
||||
elif c['payment_reference_flag']:
|
||||
if c['aml_id'] in amls_ids_to_exclude:
|
||||
first_batch_candidates_proposed.append(c)
|
||||
else:
|
||||
first_batch_candidates.append(c)
|
||||
elif c['communication_flag']:
|
||||
if c['aml_id'] in amls_ids_to_exclude:
|
||||
second_batch_candidates_proposed.append(c)
|
||||
else:
|
||||
second_batch_candidates.append(c)
|
||||
elif not first_batch_candidates:
|
||||
if c['aml_id'] in amls_ids_to_exclude:
|
||||
third_batch_candidates_proposed.append(c)
|
||||
else:
|
||||
third_batch_candidates.append(c)
|
||||
available_candidates = (first_batch_candidates + first_batch_candidates_proposed
|
||||
or second_batch_candidates + second_batch_candidates_proposed
|
||||
or third_batch_candidates + third_batch_candidates_proposed)
|
||||
candidates_by_priority[priority].append(candidate)
|
||||
|
||||
# Special case: the amount are the same, submit the line directly.
|
||||
candidate_currencies = set(candidate['aml_currency_id'] or line.company_currency_id.id for candidate in available_candidates)
|
||||
if candidate_currencies == {line_currency.id}:
|
||||
for c in available_candidates:
|
||||
residual_amount = c['aml_currency_id'] and c['aml_amount_residual_currency'] or c['aml_amount_residual']
|
||||
return candidates_by_priority
|
||||
|
||||
if float_is_zero(residual_amount + line.amount_residual, precision_rounding=line_currency.rounding):
|
||||
available_candidates = [c]
|
||||
break
|
||||
def _get_writeoff_suggestion_rule_result(self, st_line, partner):
|
||||
rslt = {
|
||||
'model': self,
|
||||
'status': 'write_off',
|
||||
'aml_ids': [],
|
||||
}
|
||||
|
||||
# Needed to handle check on total residual amounts.
|
||||
if first_batch_candidates or first_batch_candidates_proposed or model._check_rule_propositions(line, available_candidates):
|
||||
results[line.id]['model'] = model
|
||||
# Create write-off lines.
|
||||
lines_vals_list = self._prepare_reconciliation(st_line, partner=partner)
|
||||
|
||||
# Add candidates to the result.
|
||||
for candidate in available_candidates:
|
||||
results[line.id]['aml_ids'].append(candidate['aml_id'])
|
||||
amls_ids_to_exclude.add(candidate['aml_id'])
|
||||
# Process auto-reconciliation.
|
||||
if lines_vals_list and self.auto_reconcile:
|
||||
if not st_line.partner_id and partner:
|
||||
st_line.partner_id = partner
|
||||
|
||||
# Create write-off lines.
|
||||
move_lines = self.env['account.move.line'].browse(results[line.id]['aml_ids'])
|
||||
partner = partner_map and partner_map.get(line.id) and self.env['res.partner'].browse(partner_map[line.id])
|
||||
lines_vals_list = model._prepare_reconciliation(line, aml_ids=results[line.id]['aml_ids'], partner=partner)
|
||||
st_line.reconcile(lines_vals_list)
|
||||
rslt['status'] = 'reconciled'
|
||||
rslt['reconciled_lines'] = st_line.line_ids
|
||||
|
||||
# A write-off must be applied since there is some 'new' lines to propose.
|
||||
if not lines_vals_list or any(not line_vals.get('id') for line_vals in lines_vals_list):
|
||||
results[line.id]['status'] = 'write_off'
|
||||
|
||||
# Process auto-reconciliation.
|
||||
if lines_vals_list and (first_batch_candidates or second_batch_candidates) and model.auto_reconcile:
|
||||
if not line.partner_id and partner:
|
||||
line.partner_id = partner
|
||||
line.reconcile(lines_vals_list)
|
||||
|
||||
results[line.id]['status'] = 'reconciled'
|
||||
results[line.id]['reconciled_lines'] = line.line_ids
|
||||
|
||||
# The reconciled move lines are no longer candidates for another rule.
|
||||
reconciled_amls_ids.update(move_lines.ids)
|
||||
|
||||
# Break models loop.
|
||||
break
|
||||
|
||||
elif model.rule_type == 'writeoff_suggestion' and grouped_candidates[line.id][model.id]:
|
||||
results[line.id]['model'] = model
|
||||
results[line.id]['status'] = 'write_off'
|
||||
|
||||
# Create write-off lines.
|
||||
partner = partner_map and partner_map.get(line.id) and self.env['res.partner'].browse(partner_map[line.id])
|
||||
lines_vals_list = model._prepare_reconciliation(line, partner=partner)
|
||||
|
||||
if not lines_vals_list:
|
||||
break
|
||||
|
||||
# Process auto-reconciliation.
|
||||
if model.auto_reconcile:
|
||||
if not line.partner_id and partner:
|
||||
line.partner_id = partner
|
||||
line.reconcile(lines_vals_list)
|
||||
results[line.id]['status'] = 'reconciled'
|
||||
results[line.id]['reconciled_lines'] = line.line_ids
|
||||
|
||||
# Break models loop.
|
||||
break
|
||||
return results
|
||||
return rslt
|
||||
|
||||
@@ -125,7 +125,7 @@ class TestReconciliationMatchingRules(AccountTestInvoicingCommon):
|
||||
if statements is None:
|
||||
statements = self.bank_st + self.cash_st
|
||||
statement_lines = statements.mapped('line_ids').sorted()
|
||||
matching_values = rules._apply_rules(statement_lines)
|
||||
matching_values = rules._apply_rules(statement_lines, None)
|
||||
for st_line_id, values in matching_values.items():
|
||||
values.pop('reconciled_lines', None)
|
||||
self.assertDictEqual(values, expected_values[st_line_id])
|
||||
@@ -434,4 +434,27 @@ class TestReconciliationMatchingRules(AccountTestInvoicingCommon):
|
||||
self.bank_line_2.unlink()
|
||||
self.bank_line_1.write({'partner_id': partner.id, 'foreign_currency_id': currency_statement.id, 'amount_currency': 100, 'payment_ref': 'test'})
|
||||
self.env['account.reconcile.model'].flush()
|
||||
self._check_statement_matching(self.rule_1, {self.bank_line_1.id: {'aml_ids': []}}, statements=self.bank_st)
|
||||
self._check_statement_matching(self.rule_1, {self.bank_line_1.id: {'aml_ids': []}}, statements=self.bank_st)
|
||||
|
||||
def test_invoice_matching_rule_no_partner(self):
|
||||
""" Tests that a statement line without any partner can be matched to the
|
||||
right invoice if they have the same payment reference.
|
||||
"""
|
||||
self.invoice_line_1.move_id.write({'payment_reference': 'Tournicoti66'})
|
||||
|
||||
self.bank_line_1.write({
|
||||
'payment_ref': 'Tournicoti66',
|
||||
'partner_id': None,
|
||||
})
|
||||
|
||||
self.rule_1.write({
|
||||
'line_ids': [(5, 0, 0)],
|
||||
'match_partner': False,
|
||||
'match_label': 'contains',
|
||||
'match_label_param': 'Tournicoti', # So that we only match what we want to test
|
||||
})
|
||||
|
||||
self._check_statement_matching(self.rule_1, {
|
||||
self.bank_line_1.id: {'aml_ids': [self.invoice_line_1.id], 'model': self.rule_1},
|
||||
self.bank_line_2.id: {'aml_ids': []},
|
||||
}, self.bank_st)
|
||||
|
||||
@@ -174,6 +174,7 @@
|
||||
</tree>
|
||||
</field>
|
||||
</group>
|
||||
|
||||
</sheet>
|
||||
</form>
|
||||
</field>
|
||||
|
||||
Reference in New Issue
Block a user