From 8fd1b20d20eab3a8276ce119bf003638f92fb401 Mon Sep 17 00:00:00 2001 From: "Victor Piryns (pivi)" Date: Mon, 18 Sep 2023 14:37:38 +0200 Subject: [PATCH] [PERF] stock: reduce memory usage of `_get_orderpoint_action` Issue: On databases with lots of products (400k+ `product.product`), the action Inventory > Operations > Replenishment takes a lots of memory to complete, leading to "Out of Memory" errors in some use cases. Analysis: When triggering the action, we need to recompute the `virtual_available` of a lot of products (regular occurrence when a DB for example has a lot of variants). The compute `_compute_quantities` uses the ORM `filtered` method to filter out `services` out of the `product.product` we have. This method is memory hungry, as for each `_prefetch_ids` (capped at 1000, cf. `PREFETCH_MAX`) it will store in cache the `product_tmpl_id.type`, which stores all fields of the product_template + the type, bloating the cache with *useless* data, as it's not used beyond this filtering process. Solution: Disable the prefetcher, the bulk of the work is done in `_compute_quantities_dict`, which makes a lot of `read_group` and doesn't benefit much from fields already cached (this function itself also disables the prefetcher internally at some point). Also correcting the looping order in `_get_orderpoint_action` to avoid multiple cache invalidation when those can be batched in 1 call. The `PREFETCH_MAX` makes sure the compute is only executed with 1k records at a time, so explicitly dividing `all_products_ids` into batches of 5k is unnecessary. Results: - Memory usage of `_get_orderpoint_action`: | Memory usage | Before | After | |--------------|---------|---------| | Total Memory | 1.8 GiB | 363 MiB | | Allocations | 10170 | 2804 | (profiler data available on referenced ticket) - Execution time of `_get_orderpoint_action`: | `product.product` count | Before | After | |-------------------------|----------|----------| | 1k | 1.3s | 1.1s | | 10k | 4.1s | 3.8s | | 100k | 30s | 25.9s | | 400k+ | 2min 25s | 1min 39s | As we can observe, we've reduced drastically the memory usage of the function and also the number of allocations quite drastically. As a side effect there is also a slight speed improvement at small scale, and at large scale the gains are up to ~40% faster. Reference: opw-3415087 closes odoo/odoo#138423 X-original-commit: 41442574159429ce29aa5c5adf0e1a0d8a7046dd Signed-off-by: William Henrotin (whe) Signed-off-by: Piryns Victor (pivi) --- addons/stock/models/product.py | 2 +- addons/stock/models/stock_orderpoint.py | 23 ++++++++++------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/addons/stock/models/product.py b/addons/stock/models/product.py index a2ee31828d4..4d82f507594 100644 --- a/addons/stock/models/product.py +++ b/addons/stock/models/product.py @@ -123,7 +123,7 @@ class Product(models.Model): 'location', 'warehouse', ) def _compute_quantities(self): - products = self.filtered(lambda p: p.type != 'service') + products = self.with_context(prefetch_fields=False).filtered(lambda p: p.type != 'service').with_context(prefetch_fields=True) res = products._compute_quantities_dict(self._context.get('lot_id'), self._context.get('owner_id'), self._context.get('package_id'), self._context.get('from_date'), self._context.get('to_date')) for product in products: product.update(res[product.id]) diff --git a/addons/stock/models/stock_orderpoint.py b/addons/stock/models/stock_orderpoint.py index a3c713787f2..a3757648ed4 100644 --- a/addons/stock/models/stock_orderpoint.py +++ b/addons/stock/models/stock_orderpoint.py @@ -341,22 +341,19 @@ class StockWarehouseOrderpoint(models.Model): orderpoints_removed = orderpoints._unlink_processed_orderpoints() orderpoints = orderpoints - orderpoints_removed to_refill = defaultdict(float) - all_product_ids = self._get_orderpoint_products().ids + all_product_ids = self._get_orderpoint_products() all_replenish_location_ids = self.env['stock.location'].search([('replenish_location', '=', True)]) ploc_per_day = defaultdict(set) # For each replenish location get products with negative virtual_available aka forecast - for products in map(self.env['product.product'].browse, split_every(5000, all_product_ids)): - for loc in all_replenish_location_ids: - quantities = products.with_context(location=loc.id).mapped('virtual_available') - for product, quantity in zip(products, quantities): - if float_compare(quantity, 0, precision_rounding=product.uom_id.rounding) >= 0: - continue - # group product by lead_days and location in order to read virtual_available - # in batch - rules = product._get_rules_from_location(loc) - lead_days = rules.with_context(bypass_delay_description=True)._get_lead_days(product)[0] - ploc_per_day[(lead_days, loc)].add(product.id) - products.invalidate_recordset() + for loc in all_replenish_location_ids: + for product in all_product_ids.with_context(location=loc.id): + if float_compare(product.virtual_available, 0, precision_rounding=product.uom_id.rounding) >= 0: + continue + # group product by lead_days and location in order to read virtual_available + # in batch + rules = product._get_rules_from_location(loc) + lead_days = rules.with_context(bypass_delay_description=True)._get_lead_days(product)[0] + ploc_per_day[(lead_days, loc)].add(product.id) # recompute virtual_available with lead days today = fields.datetime.now().replace(hour=23, minute=59, second=59)