From faefaf7c39d722020494b5b16c9b6d8a1be3b956 Mon Sep 17 00:00:00 2001 From: William Braeckman Date: Thu, 21 Oct 2021 09:10:59 +0000 Subject: [PATCH] [IMP] resource: improve _attendance_intervals_batch perf Improves on _attendance_intervals_batch to profit more from batching. Resources are grouped by timezone since we know they will all have the same result except for when there are resource assigned attendances. Those have to be handled separately but are incoroporated in the same routine. TaskId-2674527 closes odoo/odoo#78741 Signed-off-by: Yannick Tivisse (yti) --- .../hr_holidays/tests/test_company_leave.py | 2 +- addons/resource/models/resource.py | 68 +++++++++++-------- addons/resource/tests/__init__.py | 2 +- addons/resource/tests/test_performance.py | 44 ++++++++++++ 4 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 addons/resource/tests/test_performance.py diff --git a/addons/hr_holidays/tests/test_company_leave.py b/addons/hr_holidays/tests/test_company_leave.py index 6543ef13c62..ef1ae0448e3 100644 --- a/addons/hr_holidays/tests/test_company_leave.py +++ b/addons/hr_holidays/tests/test_company_leave.py @@ -315,7 +315,7 @@ class TestCompanyLeave(TransactionCase): }) company_leave._compute_date_from_to() - with self.assertQueryCount(__system__=772, admin=865): + with self.assertQueryCount(__system__=773, admin=865): # Original query count: 1987 # Without tracking/activity context keys: 5154 company_leave.action_validate() diff --git a/addons/resource/models/resource.py b/addons/resource/models/resource.py index f2afa6b949e..d7bd6cf3170 100644 --- a/addons/resource/models/resource.py +++ b/addons/resource/models/resource.py @@ -397,24 +397,33 @@ class ResourceCalendar(models.Model): ('display_type', '=', False), ]]) - # for each attendance spec, generate the intervals in the date range + # Since we only have one calendar to take in account + # Group resources per tz they will all have the same result + resources_per_tz = defaultdict(list) + for resource in resources_list: + resources_per_tz[tz or timezone((resource or self).tz)].append(resource) + attendances = self.env['resource.calendar.attendance'].search(domain) cache_dates = defaultdict(dict) cache_deltas = defaultdict(dict) result = defaultdict(list) - for attendance in self.env['resource.calendar.attendance'].search(domain): - for resource in resources_list: + + def date_from_cache(tz, dt): + _key = (tz, dt) + return cache_dates[_key] if _key in cache_dates\ + else cache_dates.setdefault(_key, dt.astimezone(tz)) + + def dt_from_cache(tz, day, hour_from): + _key = (tz, day, hour_from) + return cache_deltas[_key] if _key in cache_deltas\ + else cache_deltas.setdefault(_key, tz.localize(combine(day, float_to_time(hour_from)))) + + for attendance in attendances: + attendance_timezones = [tz or timezone(attendance.resource_id.tz)]\ + if attendance.resource_id else resources_per_tz.keys() + for resource_tz in attendance_timezones: # express all dates and times in specified tz or in the resource's timezone - resource_tz = tz if tz else timezone((resource or self).tz) - if (resource_tz, start_dt) in cache_dates: - start = cache_dates[(resource_tz, start_dt)] - else: - start = start_dt.astimezone(resource_tz) - cache_dates[(resource_tz, start_dt)] = start - if (resource_tz, end_dt) in cache_dates: - end = cache_dates[(resource_tz, end_dt)] - else: - end = end_dt.astimezone(resource_tz) - cache_dates[(resource_tz, end_dt)] = end + start = date_from_cache(resource_tz, start_dt) + end = date_from_cache(resource_tz, end_dt) start = start.date() if attendance.date_from: @@ -435,22 +444,21 @@ class ResourceCalendar(models.Model): else: days = rrule(DAILY, start, until=until, byweekday=weekday) - for day in days: - # attendance hours are interpreted in the resource's timezone - hour_from = attendance.hour_from - if (resource_tz, day, hour_from) in cache_deltas: - dt0 = cache_deltas[(resource_tz, day, hour_from)] - else: - dt0 = resource_tz.localize(combine(day, float_to_time(hour_from))) - cache_deltas[(resource_tz, day, hour_from)] = dt0 - - hour_to = attendance.hour_to - if (resource_tz, day, hour_to) in cache_deltas: - dt1 = cache_deltas[(resource_tz, day, hour_to)] - else: - dt1 = resource_tz.localize(combine(day, float_to_time(hour_to))) - cache_deltas[(resource_tz, day, hour_to)] = dt1 - result[resource.id].append((max(cache_dates[(resource_tz, start_dt)], dt0), min(cache_dates[(resource_tz, end_dt)], dt1), attendance)) + hour_from = attendance.hour_from + hour_to = attendance.hour_to + local_result = [ + ( + max(cache_dates[(resource_tz, start_dt)], dt_from_cache(resource_tz, day, hour_from)), + min(cache_dates[(resource_tz, end_dt)], dt_from_cache(resource_tz, day, hour_to)), + attendance + ) + for day in days + ] + if attendance.resource_id: + result[attendance.resource_id.id].extend(local_result) + else: + for resource in resources_per_tz[resource_tz]: + result[resource.id].extend(local_result) return {r.id: Intervals(result[r.id]) for r in resources_list} def _leave_intervals(self, start_dt, end_dt, resource=None, domain=None, tz=None): diff --git a/addons/resource/tests/__init__.py b/addons/resource/tests/__init__.py index 24b9b87ae93..0a816e9286e 100644 --- a/addons/resource/tests/__init__.py +++ b/addons/resource/tests/__init__.py @@ -2,4 +2,4 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. from . import test_resource - +from . import test_performance diff --git a/addons/resource/tests/test_performance.py b/addons/resource/tests/test_performance.py new file mode 100644 index 00000000000..bb0aedcd00c --- /dev/null +++ b/addons/resource/tests/test_performance.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import logging +import time +import pytz + +from datetime import datetime +from dateutil.relativedelta import relativedelta + +from odoo.tests import TransactionCase, warmup + +_logger = logging.getLogger(__name__) + +class TestResourcePerformance(TransactionCase): + + @warmup + def test_performance_attendance_intervals_batch(self): + # Tests the performance of _attendance_intervals_batch with a batch of 100 resources + calendar = self.env['resource.calendar'].create({ + 'name': 'Calendar', + }) + resources = self.env['resource.test'].create([ + { + 'name': 'Resource ' + str(i), + 'resource_calendar_id': calendar.id, + } + for i in range(100) + ]) + with self.assertQueryCount(__system__=1): + # Generate attendances for a whole year + start = pytz.utc.localize(datetime.now() + relativedelta(month=1, day=1)) + stop = pytz.utc.localize(datetime.now() + relativedelta(month=12, day=31)) + start_time = time.time() + calendar._attendance_intervals_batch(start, stop, resources=resources) + _logger.info('Attendance Intervals Batch (100): --- %s seconds ---', time.time() - start_time) + # Before + #INFO master test_performance: Attendance Intervals Batch (100): --- 2.0667169094085693 seconds --- + #INFO master test_performance: Attendance Intervals Batch (100): --- 2.0868310928344727 seconds --- + #INFO master test_performance: Attendance Intervals Batch (100): --- 1.9209258556365967 seconds --- + #INFO master test_performance: Attendance Intervals Batch (100): --- 1.9474620819091797 seconds --- + # After + #INFO master test_performance: Attendance Intervals Batch (100): --- 0.4092371463775635 seconds --- + #INFO master test_performance: Attendance Intervals Batch (100): --- 0.3598649501800537 seconds ---