[IMP] stock: optimize move reservation

This commit improves the performance of reservation by removing
the `get_available_quantity` and `update_reserved_quantity` inside
`_action_assign` which each does a search on quants with the same domain,
both of which are done per move to assign.

Instead the quants needed by the moves to reserve are fetched first and
are then filtered in memory for each move to reserve from.

A new function is provided `get_quants_by_product` which should be used
when reserving moves to fetch the relevant quants once and then call
`_update_reserved_quantity` on them which will filter them in memory.
This avoids searching for quants for every move.

The changes in `point_of_sale` and `product_expiry` are all
due to the the method signature change and change nothing in the flow.

Performance stats:

These stats are from the Odoo profiler on the `_action_assign` called
on a specific number of moves. For each number of moves the duration was
taken multiple times and the average was taken. The 10 moves durations
were quite random since the numbers were very small and could be considered
an outlier.

Before this commit:

| No. of Moves  | SQL Queries | Duration (s) |
| ------------- | ------------- | ------------|
| 10  | 70  | 0.044440283 |
| 100  | 633  | 0.398992664 |
| 1,000  | 4,518  | 6.264924025 |
| 10,000  | 46,954  | 79.76809310 |

After this commit:

| No. of Moves  | SQL Queries | Duration (s) |
| ------------- | ------------- | ------------|
| 10  | 62  | 0.062840558 |
| 100  | 540  | 0.323033646 |
| 1,000  | 3,893  | 4.062329989 |
| 10,000  | 39,259  | 49.92596793 |

Besides 10 moves duration, there's a general performance
improvement of 20% - 35%

closes odoo/odoo#116803

Related: odoo/enterprise#40633
Signed-off-by: Tiffany Chang <tic@odoo.com>

addons/point_of_sale/models/stock_picking.py

@@ -313,7 +313,5 @@ class StockMove(models.Model):
                         if existing_lots:
                             existing_lot = existing_lots.filtered_domain([('product_id', '=', line.product_id.id), ('name', '=', lot.lot_name)])
                             if existing_lot:
-                                available_quantity = move._get_available_quantity(move.location_id, lot_id=existing_lot, strict=True)
-                                if not float_is_zero(available_quantity, precision_rounding=line.product_id.uom_id.rounding):
-                                    move._update_reserved_quantity(qty, min(qty, available_quantity), move.location_id, existing_lot)
-                                    continue
+                                move._update_reserved_quantity(qty, move.location_id, lot_id=existing_lot)
+                                continue

addons/product_expiry/models/stock_move.py

@@ -72,7 +72,7 @@ class StockMove(models.Model):
                     break
         return options
 
-    def _update_reserved_quantity(self, need, available_quantity, location_id, lot_id=None, package_id=None, owner_id=None, strict=True):
+    def _update_reserved_quantity(self, need, location_id, quant_ids=None, lot_id=None, package_id=None, owner_id=None, strict=True):
         if self.product_id.use_expiration_date:
-            return super(StockMove, self.with_context(with_expiration=self.date))._update_reserved_quantity(need, available_quantity, location_id, lot_id, package_id, owner_id, strict)
-        return super()._update_reserved_quantity(need, available_quantity, location_id, lot_id, package_id, owner_id, strict)
+            return super(StockMove, self.with_context(with_expiration=self.date))._update_reserved_quantity(need, location_id, quant_ids, lot_id, package_id, owner_id, strict)
+        return super()._update_reserved_quantity(need, location_id, quant_ids, lot_id, package_id, owner_id, strict)

addons/stock/models/stock_move.py

@@ -1509,11 +1509,15 @@ Please change the quantity done or the rounding precision of your unit of measur
             )
         return vals
 
-    def _update_reserved_quantity(self, need, available_quantity, location_id, lot_id=None, package_id=None, owner_id=None, strict=True):
-        """ Create or update move lines.
+    def _update_reserved_quantity(self, need, location_id, quant_ids=None, lot_id=None, package_id=None, owner_id=None, strict=True):
+        """ Create or update move lines and reserves quantity from quants
+            Expects the need (qty to reserve) and location_id to reserve from.
+            `quant_ids` can be passed as an optimization since no search on the database
+            is performed and reservation is done on the passed quants set
         """
         self.ensure_one()
-
+        if not quant_ids:
+            quant_ids = self.env['stock.quant']
         if not lot_id:
             lot_id = self.env['stock.lot']
         if not package_id:
@@ -1521,39 +1525,15 @@ Please change the quantity done or the rounding precision of your unit of measur
         if not owner_id:
             owner_id = self.env['res.partner']
 
-        # do full packaging reservation when it's needed
-        if self.product_packaging_id and self.product_id.product_tmpl_id.categ_id.packaging_reserve_method == "full":
-            available_quantity = self.product_packaging_id._check_qty(available_quantity, self.product_id.uom_id, "DOWN")
+        quants = quant_ids._get_reserve_quantity(
+            self.product_id, location_id, need, product_packaging_id=self.product_packaging_id,
+            uom_id=self.product_uom, lot_id=lot_id, package_id=package_id, owner_id=owner_id, strict=strict)
 
-        taken_quantity = min(available_quantity, need)
-
-        # `taken_quantity` is in the quants unit of measure. There's a possibility that the move's
-        # unit of measure won't be respected if we blindly reserve this quantity, a common usecase
-        # is if the move's unit of measure's rounding does not allow fractional reservation. We chose
-        # to convert `taken_quantity` to the move's unit of measure with a down rounding method and
-        # then get it back in the quants unit of measure with an half-up rounding_method. This
-        # way, we'll never reserve more than allowed. We do not apply this logic if
-        # `available_quantity` is brought by a chained move line. In this case, `_prepare_move_line_vals`
-        # will take care of changing the UOM to the UOM of the product.
-        if not strict and self.product_id.uom_id != self.product_uom:
-            taken_quantity_move_uom = self.product_id.uom_id._compute_quantity(taken_quantity, self.product_uom, rounding_method='DOWN')
-            taken_quantity = self.product_uom._compute_quantity(taken_quantity_move_uom, self.product_id.uom_id, rounding_method='HALF-UP')
-
-        quants = []
+        taken_quantity = 0
         rounding = self.env['decimal.precision'].precision_get('Product Unit of Measure')
-
-        if self.product_id.tracking == 'serial':
-            if float_compare(taken_quantity, int(taken_quantity), precision_digits=rounding) != 0:
-                taken_quantity = 0
-
-        if not float_is_zero(taken_quantity, precision_rounding=self.product_id.uom_id.rounding):
-            quants = self.env['stock.quant']._get_reserve_quantity(
-                self.product_id, location_id, taken_quantity, lot_id=lot_id,
-                package_id=package_id, owner_id=owner_id, strict=strict
-            )
-
         # Find a candidate move line to update or create a new one.
         for reserved_quant, quantity in quants:
+            taken_quantity += quantity
             to_update = next((line for line in self.move_line_ids if line._reservation_is_updatable(quantity, reserved_quant)), False)
             if to_update:
                 uom_quantity = self.product_id.uom_id._compute_quantity(quantity, to_update.product_uom_id, rounding_method='HALF-UP')
@@ -1649,6 +1629,9 @@ Please change the quantity done or the rounding precision of your unit of measur
         moves_to_assign = self
         if not force_qty:
             moves_to_assign = self.filtered(lambda m: m.state in ['confirmed', 'waiting', 'partially_available'])
+        moves_to_reserve = moves_to_assign.filtered(lambda m: not m._should_bypass_reservation())
+        quants_by_product = self.env['stock.quant']._get_quants_by_products_locations(moves_to_reserve.product_id, moves_to_reserve.location_id)
+
         for move in moves_to_assign:
             rounding = roundings[move]
             if not force_qty:
@@ -1657,6 +1640,7 @@ Please change the quantity done or the rounding precision of your unit of measur
                 missing_reserved_uom_quantity = force_qty
             missing_reserved_uom_quantity -= reserved_availability[move]
             missing_reserved_quantity = move.product_uom._compute_quantity(missing_reserved_uom_quantity, move.product_id.uom_id, rounding_method='HALF-UP')
+            quants = quants_by_product[move.product_id.id]
             if move._should_bypass_reservation():
                 # create the move line(s) but do not impact quants
                 if move.move_orig_ids:
@@ -1707,10 +1691,7 @@ Please change the quantity done or the rounding precision of your unit of measur
                         continue
                     # Reserve new quants and create move lines accordingly.
                     forced_package_id = move.package_level_id.package_id or None
-                    available_quantity = move._get_available_quantity(move.location_id, package_id=forced_package_id)
-                    if available_quantity <= 0:
-                        continue
-                    taken_quantity = move._update_reserved_quantity(need, available_quantity, move.location_id, package_id=forced_package_id, strict=False)
+                    taken_quantity = move._update_reserved_quantity(need, move.location_id, quant_ids=quants, package_id=forced_package_id, strict=False)
                     if float_is_zero(taken_quantity, precision_rounding=rounding):
                         continue
                     moves_to_redirect.add(move.id)
@@ -1738,10 +1719,8 @@ Please change the quantity done or the rounding precision of your unit of measur
                         # partially `quantity`. When this happens, we chose to reserve the maximum
                         # still available. This situation could not happen on MTS move, because in
                         # this case `quantity` is directly the quantity on the quants themselves.
-                        available_quantity = move._get_available_quantity(location_id, lot_id=lot_id, package_id=package_id, owner_id=owner_id, strict=True)
-                        if float_is_zero(available_quantity, precision_rounding=rounding):
-                            continue
-                        taken_quantity = move._update_reserved_quantity(need, min(quantity, available_quantity), location_id, lot_id, package_id, owner_id)
+
+                        taken_quantity = move._update_reserved_quantity(min(quantity, need), location_id, quants, lot_id, package_id, owner_id)
                         if float_is_zero(taken_quantity, precision_rounding=rounding):
                             continue
                         moves_to_redirect.add(move.id)

addons/stock/models/stock_move_line.py

@@ -534,10 +534,12 @@ class StockMoveLine(models.Model):
 
     def unlink(self):
         precision = self.env['decimal.precision'].precision_get('Product Unit of Measure')
+        quants_by_product = self.env['stock.quant']._get_quants_by_products_locations(self.product_id, self.location_id)
         for ml in self:
             # Unlinking a move line should unreserve.
             if not float_is_zero(ml.reserved_qty, precision_digits=precision) and ml.move_id and not ml.move_id._should_bypass_reservation(ml.location_id):
-                self.env['stock.quant']._update_reserved_quantity(ml.product_id, ml.location_id, -ml.reserved_qty, lot_id=ml.lot_id, package_id=ml.package_id, owner_id=ml.owner_id, strict=True)
+                quants = quants_by_product[ml.product_id.id]
+                quants._update_reserved_quantity(ml.product_id, ml.location_id, -ml.reserved_qty, lot_id=ml.lot_id, package_id=ml.package_id, owner_id=ml.owner_id, strict=True)
         moves = self.mapped('move_id')
         res = super(StockMoveLine, self).unlink()
         if moves:

addons/stock/models/stock_quant.py

@@ -700,7 +700,19 @@ class StockQuant(models.Model):
             return domain, 'in_date ASC, id'
         raise UserError(_('Removal strategy %s not implemented.') % (removal_strategy,))
 
+    def _get_removal_strategy_sort_key(self, removal_strategy):
+        key = lambda q: (q.in_date, q.id)
+        reverse = False
+        if removal_strategy == 'lifo':
+            reverse = True
+        elif removal_strategy == 'closest':
+            key = lambda q: (q.location_id, -q.id)
+        return key, reverse
+
     def _gather(self, product_id, location_id, lot_id=None, package_id=None, owner_id=None, strict=False, qty=0):
+        """ if records in self, the records are filtered based on the wanted characteristics passed to this function
+            if not, a search is done with all the characteristics passed.
+        """
         removal_strategy = self._get_removal_strategy(product_id, location_id)
         domain = [('product_id', '=', product_id.id)]
         if not strict:
@@ -720,14 +732,19 @@ class StockQuant(models.Model):
             domain = expression.AND([[('expiration_date', '>=', self.env.context['with_expiration'])], domain])
 
         domain, order = self._get_removal_strategy_domain_order(domain, removal_strategy, qty)
-        return self.search(domain, order=order).sorted(lambda q: not q.lot_id)
+        if self:
+            sort_key = self._get_removal_strategy_sort_key(removal_strategy)
+            res = self.filtered_domain(domain).sorted(key=sort_key[0], reverse=sort_key[1])
+        else:
+            res = self.search(domain, order=order)
+        return res.sorted(lambda q: not q.lot_id)
 
-    @api.model
     def _get_available_quantity(self, product_id, location_id, lot_id=None, package_id=None, owner_id=None, strict=False, allow_negative=False):
         """ Return the available quantity, i.e. the sum of `quantity` minus the sum of
         `reserved_quantity`, for the set of quants sharing the combination of `product_id,
         location_id` if `strict` is set to False or sharing the *exact same characteristics*
         otherwise.
+        The set of quants to filter from can be in `self`, if not a search will be done
         This method is called in the following usecases:
             - when a stock move checks its availability
             - when a stock move actually assign
@@ -762,19 +779,46 @@ class StockQuant(models.Model):
             else:
                 return sum([available_quantity for available_quantity in availaible_quantities.values() if float_compare(available_quantity, 0, precision_rounding=rounding) > 0])
 
-    def _get_reserve_quantity(self, product_id, location_id, quantity, lot_id=None, package_id=None, owner_id=None, strict=False):
+    def _get_reserve_quantity(self, product_id, location_id, quantity, product_packaging_id=None, uom_id=None, lot_id=None, package_id=None, owner_id=None, strict=False):
         """ Get the quantity available to reserve for the set of quants
         sharing the combination of `product_id, location_id` if `strict` is set to False or sharing
-        the *exact same characteristics* otherwise. Typically, this method is called before the
-        `stock.move.line` creation to know the reserved_qty that could be use. It's also call by
-        `_update_reserve_quantity` to find the quant to reserve.
+        the *exact same characteristics* otherwise. If no quants are in self, `_gather` will do a search to fetch the quants
+        Typically, this method is called before the `stock.move.line` creation to know the reserved_qty that could be use.
+        It's also called by `_update_reserve_quantity` to find the quant to reserve.
 
         :return: a list of tuples (quant, quantity_reserved) showing on which quant the reservation
             could be done and how much the system is able to reserve on it
         """
         self = self.sudo()
         rounding = product_id.uom_id.rounding
-        quants = self or self._gather(product_id, location_id, lot_id=lot_id, package_id=package_id, owner_id=owner_id, strict=strict, qty=quantity)
+
+        quants = self._gather(product_id, location_id, lot_id=lot_id, package_id=package_id, owner_id=owner_id, strict=strict, qty=quantity)
+
+        # avoid quants with negative qty to not lower available_qty
+        available_quantity = quants._get_available_quantity(product_id, location_id, lot_id, package_id, owner_id, strict)
+
+        # do full packaging reservation when it's needed
+        if product_packaging_id and product_id.product_tmpl_id.categ_id.packaging_reserve_method == "full":
+            available_quantity = product_packaging_id._check_qty(available_quantity, product_id.uom_id, "DOWN")
+
+        quantity = min(quantity, available_quantity)
+
+        # `quantity` is in the quants unit of measure. There's a possibility that the move's
+        # unit of measure won't be respected if we blindly reserve this quantity, a common usecase
+        # is if the move's unit of measure's rounding does not allow fractional reservation. We chose
+        # to convert `quantity` to the move's unit of measure with a down rounding method and
+        # then get it back in the quants unit of measure with an half-up rounding_method. This
+        # way, we'll never reserve more than allowed. We do not apply this logic if
+        # `available_quantity` is brought by a chained move line. In this case, `_prepare_move_line_vals`
+        # will take care of changing the UOM to the UOM of the product.
+        if not strict and uom_id and product_id.uom_id != uom_id:
+            quantity_move_uom = product_id.uom_id._compute_quantity(quantity, uom_id, rounding_method='DOWN')
+            quantity = uom_id._compute_quantity(quantity_move_uom, product_id.uom_id, rounding_method='HALF-UP')
+
+        if self.product_id.tracking == 'serial':
+            if float_compare(quantity, int(quantity), precision_digits=rounding) != 0:
+                quantity = 0
+
         reserved_quants = []
 
         if float_compare(quantity, 0, precision_rounding=rounding) > 0:
@@ -906,6 +950,18 @@ class StockQuant(models.Model):
         self.write({'inventory_quantity': 0, 'user_id': False})
         self.write({'inventory_diff_quantity': 0})
 
+    @api.model
+    def _get_quants_by_products_locations(self, product_ids, location_ids):
+        quants_by_product = defaultdict(lambda: self.env['stock.quant'])
+        if product_ids and location_ids:
+            needed_quants = self.env['stock.quant'].read_group([('product_id', 'in', product_ids.ids),
+                                                                ('location_id', 'child_of', location_ids.ids)],
+                                                            ['ids:array_agg(id)', 'product_id'],
+                                                            ['product_id'])
+            for quant in needed_quants:
+                quants_by_product[quant['product_id'][0]] = self.env['stock.quant'].browse(quant['ids'])
+        return quants_by_product
+
     @api.model
     def _update_available_quantity(self, product_id, location_id, quantity, lot_id=None, package_id=None, owner_id=None, in_date=None):
         """ Increase or decrease `reserved_quantity` of a set of quants for a given set of