[FIX] mrp: batch _get_component_data in mrp_report_bom_structure

Currently when showing a BoM Overview, the stock_availability is computed for each bom's component. This means that there will be one read_group on report.stock.quantity by component. As this is a postgresql View, querying it too often can degrade the overall performances. This commit tries to improve on that by computing the stock_availability of all components without bom at the same level at the same time. Let's say there is a BoM with 4 components, none of which has a bill of materials of their own. Before this commit there would have been 4 read_groups, one by component. Now there will only be 1 read_group. Example speedup: customer database with 7M stock.moves, 7M stock.move.lines. BoM Overview with 33 nodes in the component Tree: 3min30s -> 1min30s X-original-commit: 8fa7a11bf41152a06537144c20d04921d3643272 Part-of: odoo/odoo#139067
2023-10-18 20:09:01 +00:00
parent e8fcbeab2d
commit b368860f28
3 changed files with 238 additions and 25 deletions
@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-

+from collections import defaultdict, OrderedDict
+from datetime import date, timedelta
 import json

 from odoo import api, fields, models, _
 from odoo.tools import float_compare, float_round, format_date, float_is_zero
-from datetime import timedelta
-from collections import defaultdict

 class ReportBomStructure(models.AbstractModel):
    _name = 'report.mrp.report_bom_structure'
@@ -128,6 +128,66 @@ class ReportBomStructure(models.AbstractModel):
            'precision': self.env['decimal.precision'].precision_get('Product Unit of Measure'),
        }

+    @api.model
+    def _get_components_closest_forecasted(self, lines, line_quantities, parent_bom, product_info, parent_product, ignore_stock=False):
+        """
+            Returns a dict mapping products to a dict of their corresponding BoM lines,
+            which are mapped to their closest date in the forecast report where consumed quantity >= forecasted quantity.
+
+            E.g. {'product_1_id': {'line_1_id': date_1, line_2_id: date_2}, 'product_2': {line_3_id: date_3}, ...}.
+
+            Note that
+                - if a product is unavailable + not forecasted for a specific bom line => its date will be `date.max`
+                - if a product's type is not `product` or is already in stock for a specific bom line => its date will be `date.min`.
+        """
+        if ignore_stock:
+            return {}
+        # Use defaultdict(OrderedDict) in case there are lines with the same component.
+        closest_forecasted = defaultdict(OrderedDict)
+        remaining_products = []
+        product_quantities_info = defaultdict(OrderedDict)
+        for line in lines:
+            product = line.product_id
+            quantities_info = self._get_quantities_info(product, line.product_uom_id, product_info, parent_bom, parent_product)
+            stock_loc = quantities_info['stock_loc']
+            product_info[product.id]['consumptions'][stock_loc] += line_quantities.get(line.id, 0.0)
+            product_quantities_info[product.id][line.id] = product_info[product.id]['consumptions'][stock_loc]
+            if (product.detailed_type != 'product' or
+                    float_compare(product_info[product.id]['consumptions'][stock_loc], quantities_info['free_qty'], precision_rounding=product.uom_id.rounding) <= 0):
+                # Use date.min as a sentinel value for _get_stock_availability
+                closest_forecasted[product.id][line.id] = date.min
+            elif stock_loc != 'in_stock':
+                closest_forecasted[product.id][line.id] = date.max
+            else:
+                remaining_products.append(product.id)
+                closest_forecasted[product.id][line.id] = None
+        date_today = self.env.context.get('from_date', fields.date.today())
+        domain = [('state', '=', 'forecast'), ('date', '>=', date_today), ('product_id', 'in', list(set(remaining_products)))]
+        if self.env.context.get('warehouse'):
+            domain.append(('warehouse_id', '=', self.env.context.get('warehouse')))
+        if remaining_products:
+            res = self.env['report.stock.quantity']._read_group(
+                domain,
+                groupby=['product_id', 'product_qty'],
+                aggregates=['date:min'],
+                order='product_id asc, date:min asc'
+            )
+            available_quantities = defaultdict(list)
+            for group in res:
+                product_id = group[0].id
+                available_quantities[product_id].append([group[2], group[1]])
+            for product_id in remaining_products:
+                # Find the first empty line_id for the given product_id.
+                line_id = next(filter(lambda k: not closest_forecasted[product_id][k], closest_forecasted[product_id].keys()), None)
+                # Find the first available quantity for the given product and update closest_forecasted
+                for min_date, product_qty in available_quantities[product_id]:
+                    if product_qty >= product_quantities_info[product_id][line_id]:
+                        closest_forecasted[product_id][line_id] = min_date
+                        break
+                if not closest_forecasted[product_id][line_id]:
+                    closest_forecasted[product_id][line_id] = date.max
+        return closest_forecasted
+
    @api.model
    def _get_bom_data(self, bom, warehouse, product=False, line_qty=False, bom_line=False, level=0, parent_bom=False, parent_product=False, index=0, product_info=False, ignore_stock=False):
        """ Gets recursively the BoM and all its subassemblies and computes availibility estimations for each component and their disponibility in stock.
@@ -143,9 +203,6 @@ class ReportBomStructure(models.AbstractModel):

        if not product_info:
            product_info = {}
-        key = product.id
-        if key not in product_info:
-            product_info[key] = {'consumptions': {'in_stock': 0}}

        company = bom.company_id or self.env.company
        current_quantity = line_qty
@@ -165,9 +222,9 @@ class ReportBomStructure(models.AbstractModel):
                prod_cost = bom.product_tmpl_id.uom_id._compute_price(bom.product_tmpl_id.with_company(company).standard_price, bom.product_uom_id) * current_quantity
                attachment_ids = self.env['mrp.document'].search([('res_model', '=', 'product.template'), ('res_id', '=', bom.product_tmpl_id.id)]).ids

+        key = product.id
        bom_key = bom.id
-        if not product_info[key].get(bom_key):
-            product_info[key][bom_key] = self._get_resupply_route_info(warehouse, product, current_quantity, product_info, bom, parent_bom, parent_product)
+        self._update_product_info(product, bom_key, product_info, warehouse, current_quantity, bom=bom, parent_bom=parent_bom, parent_product=parent_product)
        route_info = product_info[key].get(bom_key, {})
        quantities_info = {}
        if not ignore_stock:
@@ -214,17 +271,30 @@ class ReportBomStructure(models.AbstractModel):
            bom_report_line['bom_cost'] += bom_report_line['operations_cost']

        components = []
+        no_bom_lines = self.env['mrp.bom.line']
+        line_quantities = {}
+        for line in bom.bom_line_ids:
+            if product and line._skip_bom_line(product):
+                continue
+            line_quantity = (current_quantity / (bom.product_qty or 1.0)) * line.product_qty
+            line_quantities[line.id] = line_quantity
+            if not line.child_bom_id:
+                no_bom_lines |= line
+                # Update product_info for all the components before computing closest forecasted.
+                self._update_product_info(line.product_id, bom.id, product_info, warehouse, line_quantity, bom=False, parent_bom=bom, parent_product=parent_product)
+        components_closest_forecasted = self._get_components_closest_forecasted(no_bom_lines, line_quantities, bom, product_info, parent_product, ignore_stock)
        for component_index, line in enumerate(bom.bom_line_ids):
            new_index = f"{index}{component_index}"
            if product and line._skip_bom_line(product):
                continue
-            line_quantity = (current_quantity / (bom.product_qty or 1.0)) * line.product_qty
+            line_quantity = line_quantities.get(line.id, 0.0)
            if line.child_bom_id:
                component = self._get_bom_data(line.child_bom_id, warehouse, line.product_id, line_quantity, bom_line=line, level=level + 1, parent_bom=bom,
                                               parent_product=product, index=new_index, product_info=product_info, ignore_stock=ignore_stock)
            else:
-                component = self._get_component_data(bom, product, warehouse, line, line_quantity, level + 1, new_index, product_info, ignore_stock)
-
+                component = self.with_context(
+                    components_closest_forecasted=components_closest_forecasted,
+                )._get_component_data(bom, product, warehouse, line, line_quantity, level + 1, new_index, product_info, ignore_stock)
            for component_bom in components:
                if component['product_id'] == component_bom['product_id'] and component['uom'].id == component_bom['uom'].id:
                    self._merge_components(component_bom, component)
@@ -243,7 +313,7 @@ class ReportBomStructure(models.AbstractModel):
            bom_report_line['byproducts_total'] = sum(byproduct['quantity'] for byproduct in byproducts)
            bom_report_line['bom_cost'] *= bom_report_line['cost_share']

-        availabilities = self._get_availabilities(product, current_quantity, product_info, bom_key, quantities_info, level, ignore_stock, components, bom_report_line)
+        availabilities = self._get_availabilities(product, current_quantity, product_info, bom_key, quantities_info, level, ignore_stock, components, report_line=bom_report_line)
        bom_report_line.update(availabilities)

        if level == 0:
@@ -254,23 +324,19 @@ class ReportBomStructure(models.AbstractModel):
    @api.model
    def _get_component_data(self, parent_bom, parent_product, warehouse, bom_line, line_quantity, level, index, product_info, ignore_stock=False):
        company = parent_bom.company_id or self.env.company
-        key = bom_line.product_id.id
-        if key not in product_info:
-            product_info[key] = {'consumptions': {'in_stock': 0}}
-
        price = bom_line.product_id.uom_id._compute_price(bom_line.product_id.with_company(company).standard_price, bom_line.product_uom_id) * line_quantity
        rounded_price = company.currency_id.round(price)

+        key = bom_line.product_id.id
        bom_key = parent_bom.id
-        if not product_info[key].get(bom_key):
-            product_info[key][bom_key] = self._get_resupply_route_info(warehouse, bom_line.product_id, line_quantity, product_info, parent_bom=parent_bom, parent_product=parent_product)
+        self._update_product_info(bom_line.product_id, bom_key, product_info, warehouse, line_quantity, bom=False, parent_bom=parent_bom, parent_product=parent_product)
        route_info = product_info[key].get(bom_key, {})

        quantities_info = {}
        if not ignore_stock:
            # Useless to compute quantities_info if it's not going to be used later on
            quantities_info = self._get_quantities_info(bom_line.product_id, bom_line.product_uom_id, product_info, parent_bom, parent_product)
-        availabilities = self._get_availabilities(bom_line.product_id, line_quantity, product_info, bom_key, quantities_info, level, ignore_stock)
+        availabilities = self._get_availabilities(bom_line.product_id, line_quantity, product_info, bom_key, quantities_info, level, ignore_stock, bom_line=bom_line)

        attachment_ids = []
        if not self.env.context.get('minimized', False):
@@ -319,6 +385,14 @@ class ReportBomStructure(models.AbstractModel):
            'stock_loc': 'in_stock',
        }

+    @api.model
+    def _update_product_info(self, product, bom_key, product_info, warehouse, quantity, bom, parent_bom, parent_product):
+        key = product.id
+        if key not in product_info:
+            product_info[key] = {'consumptions': {'in_stock': 0}}
+        if not product_info[key].get(bom_key):
+            product_info[key][bom_key] = self._get_resupply_route_info(warehouse, product, quantity, product_info, bom, parent_bom, parent_product)
+
    @api.model
    def _get_byproducts_lines(self, product, bom, bom_quantity, level, total, index):
        byproducts = []
@@ -520,11 +594,11 @@ class ReportBomStructure(models.AbstractModel):
        return {}

    @api.model
-    def _get_availabilities(self, product, quantity, product_info, bom_key, quantities_info, level, ignore_stock=False, components=False, report_line=False):
+    def _get_availabilities(self, product, quantity, product_info, bom_key, quantities_info, level, ignore_stock=False, components=False, bom_line=None, report_line=False):
        # Get availabilities according to stock (today & forecasted).
        stock_state, stock_delay = ('unavailable', False)
        if not ignore_stock:
-            stock_state, stock_delay = self._get_stock_availability(product, quantity, product_info, quantities_info)
+            stock_state, stock_delay = self._get_stock_availability(product, quantity, product_info, quantities_info, bom_line=bom_line)

        # Get availabilities from applied resupply rules
        components = components or []
@@ -556,7 +630,14 @@ class ReportBomStructure(models.AbstractModel):
        }}

    @api.model
-    def _get_stock_availability(self, product, quantity, product_info, quantities_info):
+    def _get_stock_availability(self, product, quantity, product_info, quantities_info, bom_line=None):
+        closest_forecasted = None
+        if bom_line:
+            closest_forecasted = self.env.context.get('components_closest_forecasted', {}).get(product.id, {}).get(bom_line.id)
+        if closest_forecasted == date.min:
+            return ('available', 0)
+        if closest_forecasted == date.max:
+            return ('unavailable', False)
        date_today = self.env.context.get('from_date', fields.date.today())
        if product.detailed_type != 'product':
            return ('available', 0)
@@ -574,9 +655,10 @@ class ReportBomStructure(models.AbstractModel):
                domain.append(('warehouse_id', '=', self.env.context.get('warehouse')))

            # Seek the closest date in the forecast report where consummed quantity >= forecasted quantity
-            [min_date] = self.env['report.stock.quantity']._read_group(domain, aggregates=['date:min'])[0]
-            if min_date:
-                days_to_forecast = (min_date - date_today).days
+            if not closest_forecasted:
+                [closest_forecasted] = self.env['report.stock.quantity']._read_group(domain, aggregates=['date:min'])[0]
+            if closest_forecasted:
+                days_to_forecast = (closest_forecasted - date_today).days
                return ('expected', days_to_forecast)
        return ('unavailable', False)

@@ -1040,6 +1040,39 @@ class TestBoM(TestMrpCommon):
        # Total quantity of components is 4, so shouldn't be able to produce a single one.
        self.assertEqual(report_values['lines']['producible_qty'], 0)

+    def test_bom_report_same_component(self):
+        """ Test report bom structure with duplicated components.
+        """
+        location = self.env.ref('stock.stock_location_stock')
+        uom_unit = self.env.ref('uom.product_uom_unit')
+        final_product_tmpl = self.env['product.template'].create({'name': 'Final Product', 'type': 'product'})
+        component_product = self.env['product.product'].create({'name': 'Compo 1', 'type': 'product'})
+
+        self.env['stock.quant']._update_available_quantity(component_product, location, 3.0)
+
+        bom = self.env['mrp.bom'].create({
+            'product_tmpl_id': final_product_tmpl.id,
+            'product_uom_id': self.uom_unit.id,
+            'product_qty': 1.0,
+            'type': 'normal',
+            'bom_line_ids': [
+                Command.create({
+                    'product_id': component_product.id,
+                    'product_qty': 3,
+                    'product_uom_id': uom_unit.id,
+                }),
+                Command.create({
+                    'product_id': component_product.id,
+                    'product_qty': 3,
+                    'product_uom_id': uom_unit.id,
+                })
+            ]
+        })
+
+        report_values = self.env['report.mrp.report_bom_structure']._get_report_data(bom_id=bom.id)
+        line_values = report_values['lines']['components'][0]
+        self.assertEqual(line_values['availability_state'], 'unavailable', 'The merged components should be unavailable')
+
    def test_validate_no_bom_line_with_same_product(self):
        """
        Cannot set a BOM line on a BOM with the same product as the BOM itself
@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.

+from datetime import timedelta
+
 from odoo.tests.common import Form, TransactionCase
 from odoo.tests import tagged
 from odoo import fields
-from datetime import timedelta
+from odoo.fields import Command


@tagged('post_install', '-at_install')
@@ -835,3 +837,99 @@ class TestPurchaseMrpFlow(TransactionCase):
        report_values = self.env['report.mrp.report_mo_overview']._get_report_data(mo_2.id)['components'][0]['summary']
        self.assertEqual(report_values['quantity'], 4)
        self.assertEqual(report_values['mo_cost'], 40)
+
+    def test_bom_report_incoming_po(self):
+        """ Test report bom structure with duplicated components
+            With enough stock for the first line and two incoming
+            POs for the second line and third line.
+        """
+        location = self.env.ref('stock.stock_location_stock')
+        uom_unit = self.env.ref('uom.product_uom_unit')
+        final_product_tmpl = self.env['product.template'].create({'name': 'Final Product', 'type': 'product'})
+        component_product = self.env['product.product'].create({'name': 'Compo 1', 'type': 'product'})
+
+        self.env['stock.quant']._update_available_quantity(component_product, location, 3.0)
+
+        bom = self.env['mrp.bom'].create({
+            'product_tmpl_id': final_product_tmpl.id,
+            'product_uom_id': self.uom_unit.id,
+            'product_qty': 1.0,
+            'type': 'normal',
+            'bom_line_ids': [
+                Command.create({
+                    'product_id': component_product.id,
+                    'product_qty': 3,
+                    'product_uom_id': uom_unit.id,
+                }),
+                Command.create({
+                    'product_id': component_product.id,
+                    'product_qty': 3,
+                    'product_uom_id': uom_unit.id,
+                }),
+                Command.create({
+                    'product_id': component_product.id,
+                    'product_qty': 4,
+                    'product_uom_id': uom_unit.id,
+                })
+            ]
+        })
+        def create_order(product_id, partner_id, date_order):
+            f = Form(self.env['purchase.order'])
+            f.partner_id = partner_id
+            f.date_order = date_order
+            with f.order_line.new() as line:
+                line.product_id = product_id
+                line.product_qty = 3.0
+                line.price_unit = 10
+            return f.save()
+        partner = self.env['res.partner'].create({'name': 'My Test Partner'})
+        # Create and confirm two POs with 3 component_product at different date
+        po_today = create_order(component_product, partner, fields.Datetime.now())
+        po_5days = create_order(component_product, partner, fields.Datetime.now() + timedelta(days=5))
+
+        po_today.button_confirm()
+        po_5days.button_confirm()
+        report_values = self.env['report.mrp.report_bom_structure']._get_report_data(bom_id=bom.id)
+        line_values = report_values['lines']['components'][0]
+        self.assertEqual(line_values['availability_state'], 'estimated', 'The merged components should be estimated.')
+
+    def test_bom_report_incoming_po2(self):
+        """ Test report bom structure with duplicated components
+            With an incoming PO for the first and second line.
+        """
+        uom_unit = self.env.ref('uom.product_uom_unit')
+        final_product_tmpl = self.env['product.template'].create({'name': 'Final Product', 'type': 'product'})
+        component_product = self.env['product.product'].create({'name': 'Compo 1', 'type': 'product'})
+
+        bom = self.env['mrp.bom'].create({
+            'product_tmpl_id': final_product_tmpl.id,
+            'product_uom_id': self.uom_unit.id,
+            'product_qty': 1.0,
+            'type': 'normal',
+            'bom_line_ids': [
+                Command.create({
+                    'product_id': component_product.id,
+                    'product_qty': 3,
+                    'product_uom_id': uom_unit.id,
+                }),
+                Command.create({
+                    'product_id': component_product.id,
+                    'product_qty': 3,
+                    'product_uom_id': uom_unit.id,
+                }),
+            ]
+        })
+        partner = self.env['res.partner'].create({'name': 'My Test Partner'})
+        # Create and confirm one PO with 6 component_products.
+        f = Form(self.env['purchase.order'])
+        f.partner_id = partner
+        f.date_order = fields.Datetime.now()
+        with f.order_line.new() as line:
+            line.product_id = component_product
+            line.product_qty = 6.0
+            line.price_unit = 10
+        po_today = f.save()
+        po_today.button_confirm()
+        report_values = self.env['report.mrp.report_bom_structure']._get_report_data(bom_id=bom.id)
+        line_values = report_values['lines']['components'][0]
+        self.assertEqual(line_values['availability_state'], 'expected', 'The first component should be expected as there is an incoming PO.')