From f0d5c73c57b8be1d59d36d0ff00dc5b1f7f19d91 Mon Sep 17 00:00:00 2001 From: Yannick Tivisse Date: Tue, 14 Sep 2021 10:54:17 +0000 Subject: [PATCH] [IMP] hr_work_entry: Improve work entries generation perf Purpose ======= The work entries generation is not scaling properly for large employee datasets. A lot of improvements have been made to reduce the number of query to the database. However, one of the remaining bottleneck was the new records insertion, made 1 by 1 on the database. Due to the fact that the model doens't have a lot of columns, and doens't have columns containing too much data (HTML fields for example), we could imagine inserting the records by batch, for example 1000 by 1000, as it is done with SELECT. The results weren't satisfying enough, so we prefered using the new cron triggers mechanism, to re-trigger the same cron at the end of its execution in another transaction, to avoid exceeding the non return point from which posgreSQL is pedaling in the semoule. With this commit, it takes less than 30 seconds to generate the work entries for 1000 employees, instead of 3 min, and allows to generate the work entries in a linear execution time instead of a exponential one. Create method analysis: ======================= Note, that only the call to the create method is tracked, not the preprocessing time to retrieve the work entries values. When inserting the records 1 by 1: ---------------------------------- records - Elapsed Time (s) - AVG Time per record (s) 10 - 0.0045862197875976 - 0.00045862197875976 432 - 0.1101946830749511 - 0.00025508028489572 192 - 0.0525383949279785 - 0.00027363747358322 522 - 0.1418645381927490 - 0.00027177114596312 892 - 0.2803149223327636 - 0.00031425439723404 8800 - 3.4928441047668457 - 0.00039691410281441 88000 - 188.82338452339172 - 0.00214572027867490 176000 - 1172.4313135147095 - 0.00666154155406084 We observe that from 10.000 new record, the create method is not scaling anymore before this commit. For a company with 1000 employees, the mean work entries by month is 1000*2*21=42.000 work entries, and the time to create the records is not acceptable. When inserting the records 1000 by 1000 --------------------------------------- records - Elapsed Time (s) - AVG Time per record (s) 10 - 0.003406763076782 - 0.0003406763076782 432 - 0.075028181076049 - 0.0001736763450834 192 - 0.030718326568603 - 0.0001599912842114 522 - 0.084228038787841 - 0.0001613563961452 892 - 0.145947217941284 - 0.0001636179573332 8800 - 2.204506397247314 - 0.0002505120905962 88000 - 181.9736533164978 - 0.0020678824240511 176000 - 1146.928646564483 - 0.0065166400372981 We observe a sligh improvement, but cleary not enough and not worth the complexity of inserting the records in batch on the database. Note: The following explanation is speculative and could be validated with a real analysis, but according to the results we obtained with the cron, this is most likely to be true. In fact, this is a limitation of PosgreSQL. In the implementation that manages the transactions, the number of inserts becomes greater than certain memory limits, suddenly it falls on a slower alternative storage (this is the problem with SELECT and tuples of ids ). When we inject 100,000 ids into a query, PosgreSQL parses the query (already there it could have some trouble), then it stores the ids in a data structure: hash if not too large, other (on the file system) otherwise. Then it executes the query and uses the data structure to check the validity (membership). Specification ============= Instead of inserting the record in batch in the same transaction, it would be more interesting to split the transaction into several transactions: Each transaction process N employees who have not yet been processed, until there are no more. With the new cron trigger mechanism, it is possible to: - When the cron runs, it processes N employees (N to be determined) - If there is still some, he retriggers itself at the end of his transaction Like that, the cron is scheduled 1x / day, but it is retriggered as many times as necessary each month. Regarding the number of employees to process, with 100 employees, we can expect: 100 * 2 (morning / evening) * 21 (working days) = 4200 work entries to generate, which is manageable given the above measures. closes odoo/odoo#76488 Taskid: 2646056 Signed-off-by: Yannick Tivisse (yti) --- addons/hr_work_entry_contract/__manifest__.py | 1 + .../data/ir_cron_data.xml | 17 +++++++++++ .../models/hr_contract.py | 29 ++++++++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 addons/hr_work_entry_contract/data/ir_cron_data.xml diff --git a/addons/hr_work_entry_contract/__manifest__.py b/addons/hr_work_entry_contract/__manifest__.py index 8528cbbe1f7..b57ff5db7c2 100644 --- a/addons/hr_work_entry_contract/__manifest__.py +++ b/addons/hr_work_entry_contract/__manifest__.py @@ -16,6 +16,7 @@ 'security/hr_work_entry_security.xml', 'security/ir.model.access.csv', 'data/hr_work_entry_data.xml', + 'data/ir_cron_data.xml', 'views/hr_work_entry_views.xml', 'wizard/hr_work_entry_regeneration_wizard_views.xml', ], diff --git a/addons/hr_work_entry_contract/data/ir_cron_data.xml b/addons/hr_work_entry_contract/data/ir_cron_data.xml new file mode 100644 index 00000000000..df0921257ed --- /dev/null +++ b/addons/hr_work_entry_contract/data/ir_cron_data.xml @@ -0,0 +1,17 @@ + + + + + Generate Missing Work Entries + + code + model._cron_generate_missing_work_entries() + + + 1 + days + -1 + + + + diff --git a/addons/hr_work_entry_contract/models/hr_contract.py b/addons/hr_work_entry_contract/models/hr_contract.py index 2e0a1e53106..ecfc49af3d2 100644 --- a/addons/hr_work_entry_contract/models/hr_contract.py +++ b/addons/hr_work_entry_contract/models/hr_contract.py @@ -3,7 +3,9 @@ from collections import defaultdict from datetime import datetime, date -from odoo import fields, models +from dateutil.relativedelta import relativedelta + +from odoo import api, fields, models from odoo.addons.resource.models.resource import datetime_to_string, string_to_datetime, Intervals from odoo.osv import expression @@ -277,3 +279,28 @@ class HrContract(models.Model): def _get_fields_that_recompute_we(self): # Returns the fields that should recompute the work entries return ['resource_calendar_id'] + + @api.model + def _cron_generate_missing_work_entries(self): + # retrieve contracts for the current month + today = fields.Date.today() + start = today + relativedelta(day=1) + stop = today + relativedelta(day=31) + contracts = self.env['hr.employee']._get_all_contracts( + start, stop, states=['open', 'close']) + # determine contracts to do (the ones without work entries this month) + work_entry_groups = self.env['hr.work.entry'].read_group([ + ('date_start', '<=', stop), + ('date_stop', '>=', start), + ('contract_id', 'in', contracts.ids) + ], ['contract_id'], ['contract_id']) + contracts_done = self.browse(group['contract_id'][0] for group in work_entry_groups) + contracts_todo = contracts - contracts_done + if not contracts_todo: + return + # generate a batch of work entries + BATCH_SIZE = 100 + contracts_todo[:BATCH_SIZE]._generate_work_entries(start, stop, False) + # if necessary, retrigger the cron to generate more work entries + if len(contracts_todo) > BATCH_SIZE: + self.env.ref('hr_work_entry_contract.ir_cron_generate_missing_work_entries')._trigger()