[REF] *mrp*: batch _bom_find

Some usage of `_bom_find` are performance bottleneck (one request by
product). By example, when the mrp is installed, search products
with fields compute by `_compute_quantities` (e.g. 'Negative forecasted
quantity'). it is due to the override of `_compute_quantities`
in mrp which will make (in the worst case) a `_bom_find` for each
product in the DB.
To avoid this situation the `_bom_find` method become batched
which can handle several products in once. The signature of the method
has changed and uniformize in all module.

Example performance Gain:
------------------------
In a DB with 7000 products (type 'product'), 500 locations, 1800 BoM,
9000 Stock moves, etc. Search in the tree view with filter "Negative
forecasted quantity":
Before: 10879 (nb SQL request) 12.67 +- 0.11 sec (Total RPC Time)
After: 159 (nb SQL request) 1.82 +- 0.03 sec (Total RPC Time)

task-2439019

addons/mrp/models/mrp_bom.py

@@ -7,6 +7,7 @@ from odoo.osv.expression import AND, NEGATIVE_TERM_OPERATORS
 from odoo.tools import float_round
 
 from itertools import groupby
+from collections import defaultdict
 
 
 class MrpBom(models.Model):
@@ -15,7 +16,7 @@ class MrpBom(models.Model):
     _description = 'Bill of Material'
     _inherit = ['mail.thread']
     _rec_name = 'product_tmpl_id'
-    _order = "sequence"
+    _order = "sequence, id"
     _check_company_auto = True
 
     def _get_default_product_uom_id(self):
@@ -168,34 +169,47 @@ class MrpBom(models.Model):
         return self._search(AND([domain, args]), limit=limit, access_rights_uid=name_get_uid)
 
     @api.model
-    def _bom_find_domain(self, product_tmpl=None, product=None, picking_type=None, company_id=False, bom_type=False):
-        if product:
-            if not product_tmpl:
-                product_tmpl = product.product_tmpl_id
-            domain = ['|', ('product_id', '=', product.id), '&', ('product_id', '=', False), ('product_tmpl_id', '=', product_tmpl.id)]
-        elif product_tmpl:
-            domain = [('product_tmpl_id', '=', product_tmpl.id)]
-        else:
-            # neither product nor template, makes no sense to search
-            raise UserError(_('You should provide either a product or a product template to search a BoM'))
-        if picking_type:
-            domain += ['|', ('picking_type_id', '=', picking_type.id), ('picking_type_id', '=', False)]
+    def _bom_find_domain(self, products, picking_type=None, company_id=False, bom_type=False):
+        domain = ['|', ('product_id', 'in', products.ids), '&', ('product_id', '=', False), ('product_tmpl_id', 'in', products.product_tmpl_id.ids)]
         if company_id or self.env.context.get('company_id'):
-            domain = domain + ['|', ('company_id', '=', False), ('company_id', '=', company_id or self.env.context.get('company_id'))]
+            domain = AND([domain, ['|', ('company_id', '=', False), ('company_id', '=', company_id or self.env.context.get('company_id'))]])
+        if picking_type:
+            domain = AND([domain, ['|', ('picking_type_id', '=', picking_type.id), ('picking_type_id', '=', False)]])
         if bom_type:
-            domain += [('type', '=', bom_type)]
-        # order to prioritize bom with product_id over the one without
+            domain = AND([domain, [('type', '=', bom_type)]])
         return domain
 
     @api.model
-    def _bom_find(self, product_tmpl=None, product=None, picking_type=None, company_id=False, bom_type=False):
-        """ Finds BoM for particular product, picking and company """
-        if product and product.type == 'service' or product_tmpl and product_tmpl.type == 'service':
-            return self.env['mrp.bom']
-        domain = self._bom_find_domain(product_tmpl=product_tmpl, product=product, picking_type=picking_type, company_id=company_id, bom_type=bom_type)
-        if domain is False:
-            return self.env['mrp.bom']
-        return self.search(domain, order='sequence, product_id', limit=1)
+    def _bom_find(self, products, picking_type=None, company_id=False, bom_type=False):
+        """ Find the first BoM for each products
+
+        :param products: `product.product` recordset
+        :return: One bom (or empty recordset `mrp.bom` if none find) by product (`product.product` record)
+        :rtype: defaultdict(`lambda: self.env['mrp.bom']`)
+        """
+        bom_by_product = defaultdict(lambda: self.env['mrp.bom'])
+        products = products.filtered(lambda p: p.type != 'service')
+        if not products:
+            return bom_by_product
+        domain = self._bom_find_domain(products, picking_type=picking_type, company_id=company_id, bom_type=bom_type)
+
+        # Performance optimization, allow usage of limit and avoid the for loop `bom.product_tmpl_id.product_variant_ids`
+        if len(products) == 1:
+            bom = self.search(domain, order='sequence, product_id, id', limit=1)
+            if bom:
+                bom_by_product[products] = bom
+            return bom_by_product
+
+        boms = self.search(domain, order='sequence, product_id, id')
+
+        products_ids = set(products.ids)
+        for bom in boms:
+            products_implies = bom.product_id or bom.product_tmpl_id.product_variant_ids
+            for product in products_implies:
+                if product.id in products_ids and product not in bom_by_product:
+                    bom_by_product[product] = bom
+
+        return bom_by_product
 
     def explode(self, product, quantity, picking_type=False):
         """
@@ -236,7 +250,7 @@ class MrpBom(models.Model):
                 continue
 
             line_quantity = current_qty * current_line.product_qty
-            bom = self._bom_find(product=current_line.product_id, picking_type=picking_type or self.picking_type_id, company_id=self.company_id.id, bom_type='phantom')
+            bom = self._bom_find(current_line.product_id, picking_type=picking_type or self.picking_type_id, company_id=self.company_id.id, bom_type='phantom')[current_line.product_id]
             if bom:
                 converted_line_quantity = current_line.product_uom_id._compute_quantity(line_quantity / bom.product_qty, bom.product_uom_id)
                 bom_lines = [(line, current_line.product_id, converted_line_quantity, current_line) for line in bom.bom_line_ids] + bom_lines
@@ -331,9 +345,7 @@ class MrpBomLine(models.Model):
             if not line.product_id:
                 line.child_bom_id = False
             else:
-                line.child_bom_id = self.env['mrp.bom']._bom_find(
-                    product_tmpl=line.product_id.product_tmpl_id,
-                    product=line.product_id)
+                line.child_bom_id = self.env['mrp.bom']._bom_find(line.product_id)[line.product_id]
 
     @api.depends('product_id')
     def _compute_attachments_count(self):

addons/mrp/models/mrp_production.py

@@ -540,7 +540,7 @@ class MrpProduction(models.Model):
         if not self.product_id:
             self.bom_id = False
         elif not self.bom_id or self.bom_id.product_tmpl_id != self.product_tmpl_id or (self.bom_id.product_id and self.bom_id.product_id != self.product_id):
-            bom = self.env['mrp.bom']._bom_find(product=self.product_id, picking_type=self.picking_type_id, company_id=self.company_id.id, bom_type='normal')
+            bom = self.env['mrp.bom']._bom_find(self.product_id, picking_type=self.picking_type_id, company_id=self.company_id.id, bom_type='normal')[self.product_id]
             if bom:
                 self.bom_id = bom.id
                 self.product_qty = self.bom_id.product_qty

addons/mrp/models/mrp_unbuild.py

@@ -119,7 +119,7 @@ class MrpUnbuild(models.Model):
     @api.onchange('product_id')
     def _onchange_product_id(self):
         if self.product_id:
-            self.bom_id = self.env['mrp.bom']._bom_find(product=self.product_id, company_id=self.company_id.id)
+            self.bom_id = self.env['mrp.bom']._bom_find(self.product_id, company_id=self.company_id.id)[self.product_id]
             self.product_uom_id = self.product_id.uom_id.id
 
     @api.constrains('product_qty')

addons/mrp/models/product.py

@@ -88,7 +88,7 @@ class ProductProduct(models.Model):
         """ Return the components list ids in case of kit product.
         Return the product itself otherwise"""
         self.ensure_one()
-        bom_kit = self.env['mrp.bom']._bom_find(product=self, bom_type='phantom')
+        bom_kit = self.env['mrp.bom']._bom_find(self, bom_type='phantom')[self]
         if bom_kit:
             boms, bom_sub_lines = bom_kit.explode(self, 1)
             return [bom_line.product_id.id for bom_line, data in bom_sub_lines if bom_line.product_id.type == 'product']
@@ -124,12 +124,7 @@ class ProductProduct(models.Model):
         This override is used to get the correct quantities of products
         with 'phantom' as BoM type.
         """
-        bom_kits = {
-            product: bom
-            for product in self
-            for bom in (self.env['mrp.bom']._bom_find(product=product, bom_type='phantom'),)
-            if bom
-        }
+        bom_kits = self.env['mrp.bom']._bom_find(self, bom_type='phantom')
         kits = self.filtered(lambda p: bom_kits.get(p))
         res = super(ProductProduct, self - kits)._compute_quantities_dict(lot_id, owner_id, package_id, from_date=from_date, to_date=to_date)
         for product in bom_kits:
@@ -198,11 +193,7 @@ class ProductProduct(models.Model):
         return action
 
     def action_open_quants(self):
-        bom_kits = {}
-        for product in self:
-            bom = self.env['mrp.bom']._bom_find(product=product, bom_type='phantom')
-            if bom:
-                bom_kits[product] = bom
+        bom_kits = self.env['mrp.bom']._bom_find(self, bom_type='phantom')
         components = self - self.env['product.product'].concat(*list(bom_kits.keys()))
         for product in bom_kits:
             boms, bom_sub_lines = bom_kits[product].explode(product, 1)

addons/mrp/models/stock_move.py

@@ -219,7 +219,7 @@ class StockMove(models.Model):
             if not move.picking_type_id or (move.production_id and move.production_id.product_id == move.product_id):
                 moves_ids_to_return.add(move.id)
                 continue
-            bom = self.env['mrp.bom'].sudo()._bom_find(product=move.product_id, company_id=move.company_id.id, bom_type='phantom')
+            bom = self.env['mrp.bom'].sudo()._bom_find(move.product_id, company_id=move.company_id.id, bom_type='phantom')[move.product_id]
             if not bom:
                 moves_ids_to_return.add(move.id)
                 continue

addons/mrp/models/stock_orderpoint.py

@@ -44,11 +44,11 @@ class StockWarehouseOrderpoint(models.Model):
             orderpoint.show_bom = orderpoint.route_id.id in manufacture_route
 
     def _quantity_in_progress(self):
+        bom_kits = self.env['mrp.bom']._bom_find(self.product_id, bom_type='phantom')
         bom_kit_orderpoints = {
-            orderpoint: bom
+            orderpoint: bom_kits[orderpoint.product_id]
             for orderpoint in self
-            for bom in self.env['mrp.bom']._bom_find(product=orderpoint.product_id, bom_type='phantom')
-            if bom
+            if orderpoint.product_id in bom_kits
         }
         res = super(StockWarehouseOrderpoint, self.filtered(lambda p: p not in bom_kit_orderpoints))._quantity_in_progress()
         for orderpoint in bom_kit_orderpoints:
@@ -81,7 +81,7 @@ class StockWarehouseOrderpoint(models.Model):
         self.ensure_one()
         qty_multiple_to_order = super()._get_qty_multiple_to_order()
         if 'manufacture' in self.rule_ids.mapped('action'):
-            bom = self.env['mrp.bom']._bom_find(product=self.product_id, bom_type='normal')
+            bom = self.env['mrp.bom']._bom_find(self.product_id, bom_type='normal')[self.product_id]
             return bom.product_uom_id._compute_quantity(bom.product_qty, self.product_uom)
         return qty_multiple_to_order
 

addons/mrp/models/stock_rule.py

@@ -7,6 +7,7 @@ from dateutil.relativedelta import relativedelta
 from odoo import api, fields, models, SUPERUSER_ID, _
 from odoo.osv import expression
 from odoo.addons.stock.models.stock_rule import ProcurementException
+from odoo.tools import OrderedSet
 
 
 class StockRule(models.Model):
@@ -80,8 +81,7 @@ class StockRule(models.Model):
     def _get_matching_bom(self, product_id, company_id, values):
         if values.get('bom_id', False):
             return values['bom_id']
-        return self.env['mrp.bom']._bom_find(
-            product=product_id, picking_type=self.picking_type_id, bom_type='normal', company_id=company_id.id)
+        return self.env['mrp.bom']._bom_find(product_id, picking_type=self.picking_type_id, bom_type='normal', company_id=company_id.id)[product_id]
 
     def _prepare_mo_vals(self, product_id, product_qty, product_uom, location_id, name, origin, company_id, values, bom):
         date_planned = self._get_date_planned(product_id, company_id, values)
@@ -148,12 +148,15 @@ class ProcurementGroup(models.Model):
         the original 'run' method with the values of the components of that kit.
         """
         procurements_without_kit = []
+        product_by_company = defaultdict(OrderedSet)
         for procurement in procurements:
-            bom_kit = self.env['mrp.bom']._bom_find(
-                product=procurement.product_id,
-                company_id=procurement.company_id.id,
-                bom_type='phantom',
-            )
+            product_by_company[procurement.company_id].add(procurement.product_id.id)
+        kits_by_company = {
+            company: self.env['mrp.bom']._bom_find(self.env['product.product'].browse(product_ids), company_id=company.id, bom_type='phantom')
+            for company, product_ids in product_by_company.items()
+        }
+        for procurement in procurements:
+            bom_kit = kits_by_company[procurement.company_id].get(procurement.product_id)
             if bom_kit:
                 order_qty = procurement.product_uom._compute_quantity(procurement.product_qty, bom_kit.product_uom_id, round=False)
                 qty_to_produce = (order_qty / bom_kit.product_qty)

addons/mrp/populate/mrp.py

@@ -85,7 +85,7 @@ class MrpBomLine(models.Model):
 
     def _populate_factories(self):
         # TODO: tree of product by company to be more closer to the reality
-        boms = self.env['mrp.bom'].search([('id', 'in', self.env.registry.populated_models['mrp.bom'])], order='sequence, product_id')
+        boms = self.env['mrp.bom'].search([('id', 'in', self.env.registry.populated_models['mrp.bom'])], order='sequence, product_id, id')
 
         product_manu_ids = OrderedSet()
         for bom in boms:
@@ -236,7 +236,7 @@ class MrpBomByproduct(models.Model):
         boms_ids = self.env.registry.populated_models['mrp.bom']
         boms_ids = random.sample(boms_ids, int(len(boms_ids) * 0.5))
 
-        boms = self.env['mrp.bom'].search([('id', 'in', self.env.registry.populated_models['mrp.bom'])], order='sequence, product_id')
+        boms = self.env['mrp.bom'].search([('id', 'in', self.env.registry.populated_models['mrp.bom'])], order='sequence, product_id, id')
 
         product_manu_ids = OrderedSet()
         for bom in boms:

addons/mrp_account/models/product.py

@@ -36,7 +36,7 @@ class ProductProduct(models.Model):
 
     def _set_price_from_bom(self, boms_to_recompute=False):
         self.ensure_one()
-        bom = self.env['mrp.bom']._bom_find(product=self)
+        bom = self.env['mrp.bom']._bom_find(self)[self]
         if bom:
             self.standard_price = self._compute_bom_price(bom, boms_to_recompute=boms_to_recompute)
 
@@ -44,7 +44,7 @@ class ProductProduct(models.Model):
         self.ensure_one()
         if stock_moves.product_id == self:
             return super()._compute_average_price(qty_invoiced, qty_to_invoice, stock_moves)
-        bom = self.env['mrp.bom']._bom_find(product=self, company_id=stock_moves.company_id.id, bom_type='phantom')
+        bom = self.env['mrp.bom']._bom_find(self, company_id=stock_moves.company_id.id, bom_type='phantom')[self]
         if not bom:
             return super()._compute_average_price(qty_invoiced, qty_to_invoice, stock_moves)
         dummy, bom_lines = bom.explode(self, 1)

addons/mrp_subcontracting/models/mrp_bom.py

@@ -12,10 +12,10 @@ class MrpBom(models.Model):
     ], ondelete={'subcontract': lambda recs: recs.write({'type': 'normal', 'active': False})})
     subcontractor_ids = fields.Many2many('res.partner', 'mrp_bom_subcontractor', string='Subcontractors', check_company=True)
 
-    def _bom_subcontract_find(self, product_tmpl=None, product=None, picking_type=None, company_id=False, bom_type='subcontract', subcontractor=False):
-        domain = self._bom_find_domain(product_tmpl=product_tmpl, product=product, picking_type=picking_type, company_id=company_id, bom_type=bom_type)
+    def _bom_subcontract_find(self, product, picking_type=None, company_id=False, bom_type='subcontract', subcontractor=False):
+        domain = self._bom_find_domain(product, picking_type=picking_type, company_id=company_id, bom_type=bom_type)
         if subcontractor:
             domain = AND([domain, [('subcontractor_ids', 'parent_of', subcontractor.ids)]])
-            return self.search(domain, order='sequence, product_id', limit=1)
+            return self.search(domain, order='sequence, product_id, id', limit=1)
         else:
             return self.env['mrp.bom']

addons/mrp_subcontracting/models/stock_move.py

@@ -155,7 +155,7 @@ class StockMove(models.Model):
     def _get_subcontract_bom(self):
         self.ensure_one()
         bom = self.env['mrp.bom'].sudo()._bom_subcontract_find(
-            product=self.product_id,
+            self.product_id,
             picking_type=self.picking_type_id,
             company_id=self.company_id.id,
             bom_type='subcontract',

addons/purchase_mrp/models/purchase.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+from collections import defaultdict
+
 from odoo import api, fields, models, _
+from odoo.tools import OrderedSet
 
 
 class PurchaseOrder(models.Model):
@@ -44,18 +47,25 @@ class PurchaseOrderLine(models.Model):
 
     def _compute_qty_received(self):
         kit_lines = self.env['purchase.order.line']
-        for line in self:
-            if line.qty_received_method == 'stock_moves' and line.move_ids:
-                kit_bom = self.env['mrp.bom']._bom_find(product=line.product_id, company_id=line.company_id.id, bom_type='phantom')
-                if kit_bom:
-                    moves = line.move_ids.filtered(lambda m: m.state == 'done' and not m.scrapped)
-                    order_qty = line.product_uom._compute_quantity(line.product_uom_qty, kit_bom.product_uom_id)
-                    filters = {
-                        'incoming_moves': lambda m: m.location_id.usage == 'supplier' and (not m.origin_returned_move_id or (m.origin_returned_move_id and m.to_refund)),
-                        'outgoing_moves': lambda m: m.location_id.usage != 'supplier' and m.to_refund
-                    }
-                    line.qty_received = moves._compute_kit_quantities(line.product_id, order_qty, kit_bom, filters)
-                    kit_lines += line
+        lines_stock = self.filtered(lambda l: l.qty_received_method == 'stock_moves' and l.move_ids)
+        product_by_company = defaultdict(OrderedSet)
+        for line in lines_stock:
+            product_by_company[line.company_id].add(line.product_id.id)
+        kits_by_company = {
+            company: self.env['mrp.bom']._bom_find(self.env['product.product'].browse(product_ids), company_id=company.id, bom_type='phantom')
+            for company, product_ids in product_by_company.items()
+        }
+        for line in lines_stock:
+            kit_bom = kits_by_company[line.company_id].get(line.product_id)
+            if kit_bom:
+                moves = line.move_ids.filtered(lambda m: m.state == 'done' and not m.scrapped)
+                order_qty = line.product_uom._compute_quantity(line.product_uom_qty, kit_bom.product_uom_id)
+                filters = {
+                    'incoming_moves': lambda m: m.location_id.usage == 'supplier' and (not m.origin_returned_move_id or (m.origin_returned_move_id and m.to_refund)),
+                    'outgoing_moves': lambda m: m.location_id.usage != 'supplier' and m.to_refund
+                }
+                line.qty_received = moves._compute_kit_quantities(line.product_id, order_qty, kit_bom, filters)
+                kit_lines += line
         super(PurchaseOrderLine, self - kit_lines)._compute_qty_received()
 
     def _get_upstream_documents_and_responsibles(self, visited):

addons/sale_mrp/models/sale.py

@@ -51,7 +51,7 @@ class SaleOrderLine(models.Model):
             if line.state == 'sale':
                 boms = line.move_ids.mapped('bom_line_id.bom_id')
             elif line.state in ['draft', 'sent'] and line.product_id:
-                boms = boms._bom_find(product=line.product_id, company_id=line.company_id.id, bom_type='phantom')
+                boms = boms._bom_find(line.product_id, company_id=line.company_id.id, bom_type='phantom')[line.product_id]
             relevant_bom = boms.filtered(lambda b: b.type == 'phantom' and
                     (b.product_id == line.product_id or
                     (b.product_tmpl_id == line.product_id.product_tmpl_id and not b.product_id)))
@@ -70,7 +70,7 @@ class SaleOrderLine(models.Model):
                 boms = order_line.move_ids.mapped('bom_line_id.bom_id')
                 dropship = False
                 if not boms and any(m._is_dropshipped() for m in order_line.move_ids):
-                    boms = boms._bom_find(product=order_line.product_id, company_id=order_line.company_id.id, bom_type='phantom')
+                    boms = boms._bom_find(order_line.product_id, company_id=order_line.company_id.id, bom_type='phantom')[order_line.product_id]
                     dropship = True
                 # We fetch the BoMs of type kits linked to the order_line,
                 # the we keep only the one related to the finished produst.
@@ -137,7 +137,7 @@ class SaleOrderLine(models.Model):
         # We don't try to be too smart and keep a simple approach: we compare the quantity before
         # and after update, and return the difference. We don't take into account what was already
         # sent, or any other exceptional case.
-        bom = self.env['mrp.bom']._bom_find(product=self.product_id, bom_type='phantom')
+        bom = self.env['mrp.bom']._bom_find(self.product_id, bom_type='phantom')[self.product_id]
         if bom and previous_product_uom_qty:
             return previous_product_uom_qty and previous_product_uom_qty.get(self.id, 0.0)
         return super(SaleOrderLine, self)._get_qty_procurement(previous_product_uom_qty=previous_product_uom_qty)

addons/sale_mrp/tests/test_sale_mrp_flow.py

@@ -734,7 +734,7 @@ class TestSaleMrpFlow(ValuationReconciliationTestCommon):
         self.assertEqual(len(move_lines), 3)
 
         # Check if BoM is created and is for a 'Kit'
-        bom_from_k1 = self.env['mrp.bom']._bom_find(product=self.kit_1)
+        bom_from_k1 = self.env['mrp.bom']._bom_find(self.kit_1)[self.kit_1]
         self.assertEqual(self.bom_kit_1.id, bom_from_k1.id)
         self.assertEqual(bom_from_k1.type, 'phantom')