[IMP] account: Improve perf of reconciliation

Allow to perform multiple reconciliation at once in order to:
- ...batch the creation of records as much as possible.
Creating an account.partial.reconcile force the orm to search for amls in order to invalidate the reconciliation fields like amount_residual/amount_residual_currency.
- ...reduce the number of flush inside the orm and batch the compute.
Each call to reconcile is looking for the payment's state of invoices before/after the reconciliation.
This step is costly because this will flush all the reconciliation data and then call the computes to get the fresh value of payment_state.
- ...modify more easily the way the lines are matched together.
Before this commit, the lines were split into two batches sorted by some criteria including the currency: debit and credit.
Then, we were matching them together sequentially.
Now, we make the same things except we do that first for each batch of amls sharing the same currency in order to reduce the number of cross-currencies reconciliation.
- ...allow the orm to prefetch all the partials in the reconciliation chain all at once.
The full reconcile needs to be creating on the full reconciliation graph starting on the current amls so we need to travel the matched_debit_ids/matched_credit_ids in order to find all the involved amls.
Fetching all this data at once is also reducing the number of queries made by the orm.

Let's take an example:
Suppose 10 amls: a1, a2, ..., a10
Suppose 10 amls: b1, b2, ..., b10
You want to reconcile respectively a1 with b1, ... , a10 with b10.

Before this commit, each reconciliation was done as follow:
- Check the payment_state of invoice (on 2 amls)
- Create a partial reconcile (single record)
- Create a full reconcile (single record)
- Compute the reconciliation data to compute payment_state (on 2 amls)
All of that, 10 times sequentially.

With the new '_reconcile_plan' method, we are able to give a list of recordset [a1 + b1, ..., a10 + b10]:
- Check the payment_state of invoice (on 20 amls)
- Create a partial reconcile (10 records)
- Create a full reconcile (10 records)
- Compute the reconciliation data to compute payment_state (on 20 amls)

For a reconciliation using 2000 records (a1, ..., a1000 & b1, ..., b1000), the time to reconcile it was about +-34 seconds. Now, it's about +- 3 seconds.

closes odoo/odoo#113680

Related: odoo/enterprise#37543
Signed-off-by: Brice Bartoletti (bib) <bib@odoo.com>

addons/account/models/account_partial_reconcile.py

@@ -520,10 +520,17 @@ class AccountPartialReconcile(models.Model):
 
                 moves_to_create.append(move_vals)
 
-        moves = self.env['account.move'].create(moves_to_create)
+        moves = self.env['account.move']\
+            .with_context(
+                skip_invoice_sync=True,
+                skip_invoice_line_sync=True,
+                skip_account_move_synchronization=True,
+            )\
+            .create(moves_to_create)
         moves._post(soft=False)
 
         # Reconcile the tax lines being on a reconcile tax basis transfer account.
+        reconciliation_plan = []
         for lines, move_index, sequence in to_reconcile_after:
 
             # In expenses, all move lines are created manually without any grouping on tax lines.
@@ -538,6 +545,8 @@ class AccountPartialReconcile(models.Model):
             if counterpart_line.reconciled:
                 continue
 
-            (lines + counterpart_line).reconcile()
+            reconciliation_plan.append((counterpart_line + lines))
+
+        self.env['account.move.line']._reconcile_plan(reconciliation_plan)
 
         return moves

addons/account/tests/test_account_move_date_algorithm.py

@@ -151,10 +151,9 @@ class TestAccountMoveDateAlgorithm(AccountTestInvoicingCommon):
         (invoice + refund).action_post()
         self._set_lock_date('2017-01-31')
 
-        res = (invoice + refund).line_ids\
-            .filtered(lambda x: x.account_id.account_type == 'asset_receivable')\
-            .reconcile()
-        exchange_move = res['partials'].exchange_move_id
+        amls = (invoice + refund).line_ids.filtered(lambda x: x.account_id.account_type == 'asset_receivable')
+        amls.reconcile()
+        exchange_move = amls.matched_debit_ids.exchange_move_id
 
         self.assertRecordValues(exchange_move, [{
             'date': fields.Date.from_string('2017-02-01'),
@@ -167,10 +166,9 @@ class TestAccountMoveDateAlgorithm(AccountTestInvoicingCommon):
         refund = self._create_invoice('out_refund', '2017-01-01', currency_id=self.currency_data['currency'].id)
         (invoice + refund).action_post()
 
-        res = (invoice + refund).line_ids\
-            .filtered(lambda x: x.account_id.account_type == 'asset_receivable')\
-            .reconcile()
-        exchange_move = res['partials'].exchange_move_id
+        amls = (invoice + refund).line_ids.filtered(lambda x: x.account_id.account_type == 'asset_receivable')
+        amls.reconcile()
+        exchange_move = amls.matched_debit_ids.exchange_move_id
 
         self._set_lock_date('2017-01-31')
         (invoice + refund).line_ids.remove_move_reconcile()

addons/account/tests/test_account_move_entry.py

@@ -898,12 +898,11 @@ class TestAccountMove(AccountTestInvoicingCommon):
         ])
         moves.action_post()
 
-        res = moves.line_ids\
+        moves.line_ids\
             .filtered(lambda x: x.account_id == self.company_data['default_account_receivable'])\
             .reconcile()
 
-        self.assertTrue(res.get('partials'))
-        exchange_diff = res['partials'].exchange_move_id
+        exchange_diff = moves.line_ids.matched_debit_ids.exchange_move_id
         self.assertTrue(exchange_diff)
         with self.assertRaises(UserError), self.cr.savepoint():
             exchange_diff.button_draft()

addons/account_edi/models/account_move.py

@@ -471,16 +471,26 @@ class AccountMoveLine(models.Model):
         }
         return res
 
-    def reconcile(self):
-        # OVERRIDE
+    def _reconcile_pre_hook(self):
+        # EXTENDS 'account'
         # In some countries, the payments must be sent to the government under some condition. One of them could be
         # there is at least one reconciled invoice to the payment. Then, we need to update the state of the edi
         # documents during the reconciliation.
+        results = super()._reconcile_pre_hook()
         all_lines = self + self.matched_debit_ids.debit_move_id + self.matched_credit_ids.credit_move_id
-        payments = all_lines.move_id.filtered(lambda move: move.payment_id or move.statement_line_id)
+        results['edi_payments'] = all_lines.move_id\
+            .filtered(lambda move: move.payment_id or move.statement_line_id)
+        results['edi_invoices_per_payment_before'] = {
+            pay: pay._get_reconciled_invoices()
+            for pay in results['edi_payments']
+        }
+        return results
 
-        invoices_per_payment_before = {pay: pay._get_reconciled_invoices() for pay in payments}
-        res = super().reconcile()
+    def _reconcile_post_hook(self, data):
+        # EXTENDS 'account'
+        super()._reconcile_post_hook(data)
+        payments = data['edi_payments']
+        invoices_per_payment_before = data['edi_invoices_per_payment_before']
         invoices_per_payment_after = {pay: pay._get_reconciled_invoices() for pay in payments}
 
         changed_payments = self.env['account.move']
@@ -491,8 +501,6 @@ class AccountMoveLine(models.Model):
                 changed_payments |= payment
         changed_payments._update_payments_edi_documents()
 
-        return res
-
     def remove_move_reconcile(self):
         # OVERRIDE
         # When a payment has been sent to the government, it usually contains some information about reconciled

addons/purchase_stock/tests/test_stockvaluation.py

@@ -1104,11 +1104,11 @@ class TestStockValuationWithCOA(AccountTestInvoicingCommon):
             {'balance': 15.0,   'amount_currency': 30.0,    'account_id': self.stock_input_account.id},
             {'balance': -15.0,  'amount_currency': -30.0,   'account_id': self.company_data['default_account_payable'].id},
         ])
-        self.assertRecordValues(inv.line_ids.full_reconcile_id.reconciled_line_ids, [
+        self.assertRecordValues(inv.line_ids.full_reconcile_id.reconciled_line_ids.sorted('balance'), [
             # pylint: disable=C0326
             {'balance': -42.86, 'amount_currency': -30.0,   'account_id': self.stock_input_account.id},
-            {'balance': 27.86,  'amount_currency': 0.0,     'account_id': self.stock_input_account.id},
             {'balance': 15.0,   'amount_currency': 30.0,    'account_id': self.stock_input_account.id},
+            {'balance': 27.86,  'amount_currency': 0.0,     'account_id': self.stock_input_account.id},
         ])
 
     def test_average_realtime_with_two_delivery_anglo_saxon_valuation_multicurrency_different_dates(self):