From 4a40db209706a5adae879a2b4349f873b336ccb9 Mon Sep 17 00:00:00 2001 From: Francois Volral Date: Mon, 9 Jul 2018 14:46:45 +0200 Subject: [PATCH] [IMP] orm: add support to fill temporal holes (read_group) This commit add an extra parameter 'fill_temporal' to read_group which allows the orm to add missing groups for date intervals. This is useful for charts. Suppose that we are in a use case where data are grouped by a date fields (typically months but it could be another interval) and displayed in a Bar Chart or a Line Char. Let's says a request has to group records by month for August, September and ...December. If we don't changed anything, we would get a Bar Chart looking like this : ___ ___ | | | | | | | | ___ | | | || || | |___||___||___| Aug Sep D December follows directly after September, it can be unintuitive for the user, so we change that. We add some fake records for each missing months between the earliest and the lastest date of the result ___ ___ | | | | | | | | ___ | | | || | | | |___||___| ___ ___ |___| Aug Sep Oct Nov Dec This commit is part of task #1835644 --- odoo/addons/base/tests/test_misc.py | 133 ++++- .../test_read_group/ir.model.access.csv | 1 + odoo/addons/test_read_group/models.py | 8 + odoo/addons/test_read_group/tests/__init__.py | 1 + .../tests/test_fill_temporal.py | 561 ++++++++++++++++++ odoo/models.py | 98 ++- odoo/tools/date_utils.py | 43 +- 7 files changed, 838 insertions(+), 7 deletions(-) create mode 100644 odoo/addons/test_read_group/tests/test_fill_temporal.py diff --git a/odoo/addons/base/tests/test_misc.py b/odoo/addons/base/tests/test_misc.py index 499053a2499..881474f7d08 100644 --- a/odoo/addons/base/tests/test_misc.py +++ b/odoo/addons/base/tests/test_misc.py @@ -2,9 +2,11 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import datetime +from dateutil.relativedelta import relativedelta +import pytz import unittest -from odoo.tools import misc +from odoo.tools import misc, date_utils from odoo.tests.common import TransactionCase, tagged @@ -40,6 +42,135 @@ class TestCountingStream(unittest.TestCase): self.assertEqual(s.index, 0) +@tagged('standard', 'at_install') +class TestDateRangeFunction(unittest.TestCase): + """ Test on date_range generator. """ + + def test_date_range_with_naive_datetimes(self): + """ Check date_range with naive datetimes. """ + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + + expected = [ + datetime.datetime(1985, 1, 1, 0, 0), + datetime.datetime(1985, 2, 1, 0, 0), + datetime.datetime(1985, 3, 1, 0, 0), + datetime.datetime(1985, 4, 1, 0, 0), + datetime.datetime(1985, 5, 1, 0, 0), + datetime.datetime(1985, 6, 1, 0, 0), + datetime.datetime(1985, 7, 1, 0, 0), + datetime.datetime(1985, 8, 1, 0, 0), + datetime.datetime(1985, 9, 1, 0, 0), + datetime.datetime(1985, 10, 1, 0, 0), + datetime.datetime(1985, 11, 1, 0, 0), + datetime.datetime(1985, 12, 1, 0, 0), + datetime.datetime(1986, 1, 1, 0, 0) + ] + + dates = [date for date in date_utils.date_range(start, end)] + + self.assertEqual(dates, expected) + + def test_date_range_with_timezone_aware_datetimes_other_than_utc(self): + """ Check date_range with timezone-aware datetimes other than UTC.""" + timezone = pytz.timezone('Europe/Brussels') + + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + start = timezone.localize(start) + end = timezone.localize(end) + + expected = [datetime.datetime(1985, 1, 1, 0, 0), + datetime.datetime(1985, 2, 1, 0, 0), + datetime.datetime(1985, 3, 1, 0, 0), + datetime.datetime(1985, 4, 1, 0, 0), + datetime.datetime(1985, 5, 1, 0, 0), + datetime.datetime(1985, 6, 1, 0, 0), + datetime.datetime(1985, 7, 1, 0, 0), + datetime.datetime(1985, 8, 1, 0, 0), + datetime.datetime(1985, 9, 1, 0, 0), + datetime.datetime(1985, 10, 1, 0, 0), + datetime.datetime(1985, 11, 1, 0, 0), + datetime.datetime(1985, 12, 1, 0, 0), + datetime.datetime(1986, 1, 1, 0, 0)] + + expected = [timezone.localize(e) for e in expected] + + dates = [date for date in date_utils.date_range(start, end)] + + self.assertEqual(expected, dates) + + def test_date_range_with_mismatching_zones(self): + """ Check date_range with mismatching zone should raise an exception.""" + start_timezone = pytz.timezone('Europe/Brussels') + end_timezone = pytz.timezone('America/Recife') + + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + start = start_timezone.localize(start) + end = end_timezone.localize(end) + + with self.assertRaises(ValueError): + dates = [date for date in date_utils.date_range(start, end)] + + def test_date_range_with_inconsistent_datetimes(self): + """ Check date_range with a timezone-aware datetime and a naive one.""" + context_timezone = pytz.timezone('Europe/Brussels') + + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + end = context_timezone.localize(end) + + with self.assertRaises(ValueError): + dates = [date for date in date_utils.date_range(start, end)] + + def test_date_range_with_hour(self): + """ Test date range with hour and naive datetime.""" + start = datetime.datetime(2018, 3, 25) + end = datetime.datetime(2018, 3, 26) + step = relativedelta(hours=1) + + expected = [ + datetime.datetime(2018, 3, 25, 0, 0), + datetime.datetime(2018, 3, 25, 1, 0), + datetime.datetime(2018, 3, 25, 2, 0), + datetime.datetime(2018, 3, 25, 3, 0), + datetime.datetime(2018, 3, 25, 4, 0), + datetime.datetime(2018, 3, 25, 5, 0), + datetime.datetime(2018, 3, 25, 6, 0), + datetime.datetime(2018, 3, 25, 7, 0), + datetime.datetime(2018, 3, 25, 8, 0), + datetime.datetime(2018, 3, 25, 9, 0), + datetime.datetime(2018, 3, 25, 10, 0), + datetime.datetime(2018, 3, 25, 11, 0), + datetime.datetime(2018, 3, 25, 12, 0), + datetime.datetime(2018, 3, 25, 13, 0), + datetime.datetime(2018, 3, 25, 14, 0), + datetime.datetime(2018, 3, 25, 15, 0), + datetime.datetime(2018, 3, 25, 16, 0), + datetime.datetime(2018, 3, 25, 17, 0), + datetime.datetime(2018, 3, 25, 18, 0), + datetime.datetime(2018, 3, 25, 19, 0), + datetime.datetime(2018, 3, 25, 20, 0), + datetime.datetime(2018, 3, 25, 21, 0), + datetime.datetime(2018, 3, 25, 22, 0), + datetime.datetime(2018, 3, 25, 23, 0), + datetime.datetime(2018, 3, 26, 0, 0) + ] + + dates = [date for date in date_utils.date_range(start, end, step)] + + self.assertEqual(dates, expected) + + def test_date_range_with_identical_boundaries(self): + """ date_range with indentical boundaries. """ + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1985, 1, 1) + + with self.assertRaises(ValueError): + dates = [date for date in date_utils.date_range(start, end)] + + class TestFormatLangDate(TransactionCase): def test_00_accepted_types(self): date_datetime = datetime.datetime.strptime('2017-01-31 12:00:00', "%Y-%m-%d %H:%M:%S") diff --git a/odoo/addons/test_read_group/ir.model.access.csv b/odoo/addons/test_read_group/ir.model.access.csv index 9dc2f6abec2..eee7386d494 100644 --- a/odoo/addons/test_read_group/ir.model.access.csv +++ b/odoo/addons/test_read_group/ir.model.access.csv @@ -3,3 +3,4 @@ access_test_read_group_on_date,access_test_read_group_on_date,model_test_read_gr access_test_read_group_aggregate_boolean,access_test_read_group_aggregate_boolean,model_test_read_group_aggregate_boolean,,1,0,0,0 access_test_read_group_aggregate,access_test_read_group_aggregate,model_test_read_group_aggregate,,1,0,0,0 access_test_read_group_on_selection,access_test_read_group_on_selection,model_test_read_group_on_selection,,1,0,0,0 +access_test_read_group_fill_temporal,access_test_read_group_fill_temporal,model_test_read_group_fill_temporal,,1,0,0,0 diff --git a/odoo/addons/test_read_group/models.py b/odoo/addons/test_read_group/models.py index 97909a83495..38820119f39 100644 --- a/odoo/addons/test_read_group/models.py +++ b/odoo/addons/test_read_group/models.py @@ -37,3 +37,11 @@ class GroupOnSelection(models.Model): def _expand_states(self, states, domain, order): # return all possible states, in order return [key for key, val in type(self).state.selection] + + +class FillTemporal(models.Model): + _name = 'test_read_group.fill_temporal' + + date = fields.Date() + datetime = fields.Datetime() + value = fields.Integer() diff --git a/odoo/addons/test_read_group/tests/__init__.py b/odoo/addons/test_read_group/tests/__init__.py index aef16fccd7a..4821faa7571 100644 --- a/odoo/addons/test_read_group/tests/__init__.py +++ b/odoo/addons/test_read_group/tests/__init__.py @@ -2,3 +2,4 @@ from . import test_empty from . import test_group_expand from . import test_group_operator +from . import test_fill_temporal diff --git a/odoo/addons/test_read_group/tests/test_fill_temporal.py b/odoo/addons/test_read_group/tests/test_fill_temporal.py new file mode 100644 index 00000000000..d7f6fb2856e --- /dev/null +++ b/odoo/addons/test_read_group/tests/test_fill_temporal.py @@ -0,0 +1,561 @@ +# -*- coding: utf-8 -*- +"""Test for fill temporal.""" + +from odoo.tests import common + +class TestFillTemporal(common.TransactionCase): + """Test for fill temporal. + + This feature is mainly used in graph view. For more informations, read the + documentation of models's '_read_group_fill_temporal' method. + """ + + def setUp(self): + super(TestFillTemporal, self).setUp() + self.Model = self.env['test_read_group.fill_temporal'] + + def test_date_range_and_flag(self): + """Simple date range test, the flag is also tested. + + One of the most simple test. It must verify that dates 'holes' are filled + only when the fill_temporal flag is set. + """ + self.Model.create({'date': '1916-08-18', 'value': 2}) + self.Model.create({'date': '1916-10-19', 'value': 3}) + self.Model.create({'date': '1916-12-19', 'value': 5}) + + expected = [{ + '__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')], + 'date': 'August 1916', + 'date_count': 1, + 'value': 2 + }, { + '__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')], + 'date': 'September 1916', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')], + 'date': 'October 1916', + 'date_count': 1, + 'value': 3 + }, { + '__domain': ['&', ('date', '>=', '1916-11-01'), ('date', '<', '1916-12-01')], + 'date': 'November 1916', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-12-01'), ('date', '<', '1917-01-01')], + 'date': 'December 1916', + 'date_count': 1, + 'value': 5 + }] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, [group for group in expected if group['date_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, expected) + + def test_date_range_with_context_timezone(self): + """Test if date are date_trunced correctly by pgres. + + This test was added in attempt to fix a bug appearing with babel that + we use to translate the dates. Typically after a daylight saving, A + whole year was displayed in a graph like this (APR missing and OCT + appearing twice) : + + JAN FEB MAR MAY JUN JUL AUG SEP OCT OCT NOV + ^^^ ^^^ + """ + self.Model.create({'date': '1915-01-01', 'value': 3}) + self.Model.create({'date': '1916-01-01', 'value': 5}) + + expected = [{ + '__domain': ['&', ('date', '>=', '1915-01-01'), ('date', '<', '1915-02-01')], + 'date': 'January 1915', + 'date_count': 1, + 'value': 3 + }, { + '__domain': ['&', ('date', '>=', '1915-02-01'), ('date', '<', '1915-03-01')], + 'date': 'February 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-03-01'), ('date', '<', '1915-04-01')], + 'date': 'March 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-04-01'), ('date', '<', '1915-05-01')], + 'date': 'April 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-05-01'), ('date', '<', '1915-06-01')], + 'date': 'May 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-06-01'), ('date', '<', '1915-07-01')], + 'date': 'June 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-07-01'), ('date', '<', '1915-08-01')], + 'date': 'July 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-08-01'), ('date', '<', '1915-09-01')], + 'date': 'August 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-09-01'), ('date', '<', '1915-10-01')], + 'date': 'September 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-10-01'), ('date', '<', '1915-11-01')], + 'date': 'October 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-11-01'), ('date', '<', '1915-12-01')], + 'date': 'November 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-12-01'), ('date', '<', '1916-01-01')], + 'date': 'December 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-01-01'), ('date', '<', '1916-02-01')], + 'date': 'January 1916', + 'date_count': 1, + 'value': 5 + }] + + # Time Zone UTC UTC DST + tzs = ["America/Anchorage", # −09:00 −08:00 + "Europe/Brussels", # +01:00 +02:00 + "Pacific/Kwajalein"] # +12:00 +12:00 + + for tz in tzs: + model_fill = self.Model.with_context(tz=tz, fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + def test_only_with_only_null_date(self): + """We should have the same result when fill_temporal is set or not.""" + self.Model.create({'date': False, 'value': 13}) + self.Model.create({'date': False, 'value': 11}) + self.Model.create({'date': False, 'value': 17}) + + expected = [{'__domain': [('date', '=', False)], + 'date_count': 3, + 'value': 41, + 'date': False}] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + def test_date_range_and_null_date(self): + """Test data with null and non-null dates.""" + self.Model.create({'date': '1916-08-19', 'value': 4}) + self.Model.create({'date': False, 'value': 13}) + self.Model.create({'date': '1916-10-18', 'value': 5}) + self.Model.create({'date': '1916-08-18', 'value': 3}) + self.Model.create({'date': '1916-10-19', 'value': 4}) + self.Model.create({'date': False, 'value': 11}) + + expected = [{ + '__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')], + 'date': 'August 1916', + 'date_count': 2, + 'value': 7 + }, { + '__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')], + 'date': 'September 1916', + 'date_count': 0, + 'value': 0 + }, { + '__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')], + 'date': 'October 1916', + 'date_count': 2, + 'value': 9 + }, { + '__domain': [('date', '=', False)], + 'date': False, + 'date_count': 2, + 'value': 24 + }] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, [group for group in expected if group['date_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, expected) + + def test_order_date_desc(self): + """Test if changing Model._order has influence on the result.""" + self.Model.create({'date': '1916-08-18', 'value': 3}) + self.Model.create({'date': '1916-08-19', 'value': 4}) + self.Model.create({'date': '1916-10-18', 'value': 5}) + self.Model.create({'date': '1916-10-19', 'value': 4}) + self.patch(type(self.Model), '_order', 'date desc') + + expected = [{ + '__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')], + 'date': 'August 1916', + 'date_count': 2, + 'value': 7 + }, { + '__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')], + 'date': 'September 1916', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')], + 'date': 'October 1916', + 'date_count': 2, + 'value': 9 + }] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, [group for group in expected if group['date_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + def test_timestamp_without_timezone(self): + """Test datetimes. + + Date stored with an hour inside the Odoo model are processed as timestamp + without timezone by postgres. + """ + self.Model.create({'datetime': '1916-08-19 01:30:00', 'value': 7}) + self.Model.create({'datetime': False, 'value': 13}) + self.Model.create({'datetime': '1916-10-18 02:30:00', 'value': 5}) + self.Model.create({'datetime': '1916-08-18 01:50:00', 'value': 3}) + self.Model.create({'datetime': False, 'value': 11}) + self.Model.create({'datetime': '1916-10-19 23:59:59', 'value': 2}) + self.Model.create({'datetime': '1916-10-19', 'value': 19}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '1916-08-01 00:00:00'), + ('datetime', '<', '1916-09-01 00:00:00')], + 'datetime': 'August 1916', + 'datetime_count': 2, + 'value': 10 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-09-01 00:00:00'), + ('datetime', '<', '1916-10-01 00:00:00')], + 'datetime': 'September 1916', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-10-01 00:00:00'), + ('datetime', '<', '1916-11-01 00:00:00')], + 'datetime': 'October 1916', + 'datetime_count': 3, + 'value': 26 + }, { + '__domain': [('datetime', '=', False)], + 'datetime': False, + 'datetime_count': 2, + 'value': 24 + }] + + groups = self.Model.read_group([], fields=['datetime', 'value'], groupby=['datetime']) + + self.assertEqual(groups, [group for group in expected if group['datetime_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime']) + + self.assertEqual(groups, expected) + + def test_with_datetimes_and_groupby_per_hour(self): + """Test with datetimes and groupby per hour. + + Test if datetimes are filled correctly when grouping by hours instead of + months. + """ + self.Model.create({'datetime': '1916-01-01 01:30:00', 'value': 2}) + self.Model.create({'datetime': '1916-01-01 01:50:00', 'value': 8}) + self.Model.create({'datetime': '1916-01-01 02:30:00', 'value': 3}) + self.Model.create({'datetime': '1916-01-01 13:50:00', 'value': 5}) + self.Model.create({'datetime': '1916-01-01 23:50:00', 'value': 7}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '1916-01-01 01:00:00'), + ('datetime', '<', '1916-01-01 02:00:00')], + 'datetime:hour': '01:00 01 Jan', + 'datetime_count': 2, + 'value': 10 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 02:00:00'), + ('datetime', '<', '1916-01-01 03:00:00')], + 'datetime:hour': '02:00 01 Jan', + 'datetime_count': 1, + 'value': 3 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 03:00:00'), + ('datetime', '<', '1916-01-01 04:00:00')], + 'datetime:hour': '03:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 04:00:00'), + ('datetime', '<', '1916-01-01 05:00:00')], + 'datetime:hour': '04:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 05:00:00'), + ('datetime', '<', '1916-01-01 06:00:00')], + 'datetime:hour': '05:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 06:00:00'), + ('datetime', '<', '1916-01-01 07:00:00')], + 'datetime:hour': '06:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 07:00:00'), + ('datetime', '<', '1916-01-01 08:00:00')], + 'datetime:hour': '07:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 08:00:00'), + ('datetime', '<', '1916-01-01 09:00:00')], + 'datetime:hour': '08:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 09:00:00'), + ('datetime', '<', '1916-01-01 10:00:00')], + 'datetime:hour': '09:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 10:00:00'), + ('datetime', '<', '1916-01-01 11:00:00')], + 'datetime:hour': '10:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 11:00:00'), + ('datetime', '<', '1916-01-01 12:00:00')], + 'datetime:hour': '11:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 12:00:00'), + ('datetime', '<', '1916-01-01 13:00:00')], + 'datetime:hour': '12:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 13:00:00'), + ('datetime', '<', '1916-01-01 14:00:00')], + 'datetime:hour': '01:00 01 Jan', + 'datetime_count': 1, + 'value': 5 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 14:00:00'), + ('datetime', '<', '1916-01-01 15:00:00')], + 'datetime:hour': '02:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 15:00:00'), + ('datetime', '<', '1916-01-01 16:00:00')], + 'datetime:hour': '03:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 16:00:00'), + ('datetime', '<', '1916-01-01 17:00:00')], + 'datetime:hour': '04:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 17:00:00'), + ('datetime', '<', '1916-01-01 18:00:00')], + 'datetime:hour': '05:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 18:00:00'), + ('datetime', '<', '1916-01-01 19:00:00')], + 'datetime:hour': '06:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 19:00:00'), + ('datetime', '<', '1916-01-01 20:00:00')], + 'datetime:hour': '07:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 20:00:00'), + ('datetime', '<', '1916-01-01 21:00:00')], + 'datetime:hour': '08:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 21:00:00'), + ('datetime', '<', '1916-01-01 22:00:00')], + 'datetime:hour': '09:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 22:00:00'), + ('datetime', '<', '1916-01-01 23:00:00')], + 'datetime:hour': '10:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 23:00:00'), + ('datetime', '<', '1916-01-02 00:00:00')], + 'datetime:hour': '11:00 01 Jan', + 'datetime_count': 1, + 'value': 7 + }] + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime:hour']) + + self.assertEqual(groups, expected) + + def test_hour_with_timezones(self): + """Test hour with timezones. + + What we do here is similar to test_with_datetimes_and_groupby_per_hour + but with a timezone in the user context. + """ + self.Model.create({'datetime': '1915-12-31 22:30:00', 'value': 2}) + self.Model.create({'datetime': '1916-01-01 03:30:00', 'value': 3}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '1915-12-31 22:00:00'), + ('datetime', '<', '1915-12-31 23:00:00')], + 'datetime:hour': '04:00 01 Jan', + 'datetime_count': 1, + 'value': 2 + }, { + '__domain': ['&', + ('datetime', '>=', '1915-12-31 23:00:00'), + ('datetime', '<', '1916-01-01 00:00:00')], + 'datetime:hour': '05:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 00:00:00'), + ('datetime', '<', '1916-01-01 01:00:00')], + 'datetime:hour': '06:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 01:00:00'), + ('datetime', '<', '1916-01-01 02:00:00')], + 'datetime:hour': '07:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 02:00:00'), + ('datetime', '<', '1916-01-01 03:00:00')], + 'datetime:hour': '08:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 03:00:00'), + ('datetime', '<', '1916-01-01 04:00:00')], + 'datetime:hour': '09:00 01 Jan', + 'datetime_count': 1, + 'value': 3 + }] + + model_fill = self.Model.with_context(tz='Asia/Hovd', fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], + groupby=['datetime:hour']) + + self.assertEqual(groups, expected) + + def test_egde_fx_tz(self): + """We test if different edge effect by using a different timezone from the user context + + Suppose a user resident near Hovd, a city in Mongolia. he sells a product + at exacltly 4:00 AM on 1st January 2018. Using its context, that datetime + is previously converted to UTC time by the ORM so as being stored properly + inside the datebase. We are in winter time so 'Asia/Hovd' is UTC+7 : + + '2018-01-01 04:00:00' --> '2017-12-31 21:00:00' + + If that same user groups by datetime, we must ensure that the last + displayed date is in January and not in December. + """ + self.Model.create({'datetime': '2017-12-31 21:00:00', 'value': 42}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '2017-12-31 17:00:00'), + ('datetime', '<', '2018-01-31 17:00:00')], + 'datetime': 'January 2018', + 'datetime_count': 1, + 'value': 42 + }] + + model_fill = self.Model.with_context(tz='Asia/Hovd', fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime']) + + self.assertEqual(groups, expected) diff --git a/odoo/models.py b/odoo/models.py index 4a610a25176..9c12cc0f5b1 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -58,6 +58,7 @@ from .tools.func import frame_codeinfo from .tools.misc import CountingStream, DEFAULT_SERVER_DATETIME_FORMAT, DEFAULT_SERVER_DATE_FORMAT from .tools.safe_eval import safe_eval from .tools.translate import _ +from .tools import date_utils _logger = logging.getLogger(__name__) _schema = logging.getLogger(__name__ + '.schema') @@ -1755,6 +1756,89 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): return list(result.values()) + @api.model + def _read_group_fill_temporal(self, data, groupby, aggregated_fields, annotated_groupbys, + interval=dateutil.relativedelta.relativedelta(months=1)): + """Helper method for filling date/datetime 'holes' in a result set. + + We are in a use case where data are grouped by a date field (typically + months but it could be any other interval) and displayed in a chart. + + Assume we group records by month, and we only have data for August, + September and December. By default, plotting the result gives something + like: + ___ + ___ | | + | | | | + | | ___ | | + | || || | + |___||___||___| + Aug Sep Dec + + The problem is that December data follows immediately September data, + which is misleading for the user. Adding explicit zeroes for missing data + gives something like: + ___ + ___ | | + | | | | + | | ___ | | + | || | | | + |___||___| ___ ___ |___| + Aug Sep Oct Nov Dec + + :param list data: the data containing groups + :param list groupby: name of the first group by + :param aggregated_fields list: list of aggregated fields in the query + :param relativedelta interval: interval between to temporal groups + expressed as a relativedelta month by default + :rtype: list + :return: list + """ + first_a_gby = annotated_groupbys[0] + if not data: + return + if first_a_gby['type'] not in ('date', 'datetime'): + return data + interval = first_a_gby['interval'] + groupby_name = groupby[0] + + # existing non null datetimes + existing = [d[groupby_name] for d in data if d[groupby_name]] + + if len(existing) < 2: + return data + + # assumption: existing data is sorted by field 'groupby_name' + first, last = existing[0], existing[-1] + + expected = list(date_utils.date_range(first, last, interval)) + + if len(existing) < len(expected): + empty_data = dict.fromkeys(aggregated_fields, False) + empty_data['id'] = False + empty_data[groupby_name.split(':')[0] + '_count'] = 0 + + data = collections.deque(data) + + new_data = [] + + # Note: the list 'expected' contains the the dates that should be + # represented inside data when it is returned. Notice that 'expected' + # is sorted like data, so what we do is empty data progressivly by + # popping its elements from first to last. To do this properly, we + # need to compare dates together. + for dt in expected: + if not data[0][groupby_name] or data[0][groupby_name] == dt: + new_data.append(data.popleft()) + else: + new_data.append(dict(empty_data, **{groupby_name: dt})) + + assert not data + return new_data + + return data + + @api.model def _read_group_prepare(self, orderby, aggregated_fields, annotated_groupbys, query): """ @@ -1845,7 +1929,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): } if tz_convert: qualified_field = "timezone('%s', timezone('UTC',%s))" % (self._context.get('tz', 'UTC'), qualified_field) - qualified_field = "date_trunc('%s', %s)" % (gb_function or 'month', qualified_field) + qualified_field = "date_trunc('%s', %s::timestamp)" % (gb_function or 'month', qualified_field) if field_type == 'boolean': qualified_field = "coalesce(%s,false)" % qualified_field return { @@ -1855,7 +1939,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): 'display_format': display_formats[gb_function or 'month'] if temporal else None, 'interval': time_intervals[gb_function or 'month'] if temporal else None, 'tz_convert': tz_convert, - 'qualified_field': qualified_field + 'qualified_field': qualified_field, } @api.model @@ -1984,7 +2068,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): groupby = [groupby] if isinstance(groupby, pycompat.string_types) else list(OrderedSet(groupby)) dt = [ f for f in groupby - if self._fields[f.split(':')[0]].type in ('date', 'datetime') + if self._fields[f.split(':')[0]].type in ('date', 'datetime') # e.g. 'date:month' ] # iterate on all results and replace the "full" date/datetime value @@ -2100,8 +2184,14 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): self._read_group_resolve_many2one_fields(fetched_data, annotated_groupbys) - data = ({k: self._read_group_prepare_data(k,v, groupby_dict) for k,v in r.items()} for r in fetched_data) + data = [{k: self._read_group_prepare_data(k, v, groupby_dict) for k, v in r.items()} for r in fetched_data] + + if self.env.context.get('fill_temporal'): + data = self._read_group_fill_temporal(data, groupby, aggregated_fields, + annotated_groupbys) + result = [self._read_group_format_result(d, annotated_groupbys, groupby, domain) for d in data] + if lazy: # Right now, read_group only fill results in lazy mode (by default). # If you need to have the empty groups in 'eager' mode, then the diff --git a/odoo/tools/date_utils.py b/odoo/tools/date_utils.py index e7d1aa24ffd..55220ced033 100644 --- a/odoo/tools/date_utils.py +++ b/odoo/tools/date_utils.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- - import math import calendar from datetime import date, datetime, time - +import pytz from dateutil.relativedelta import relativedelta from . import ustr @@ -178,3 +177,43 @@ def json_default(obj): return fields.Datetime.to_string(obj) return fields.Date.to_string(obj) return ustr(obj) + +def date_range(start, end, step=relativedelta(months=1)): + """Date range generator with a step interval. + + :param start datetime: begining date of the range. + :param end datetime: ending date of the range. + :param step relativedelta: interval of the range. + :return: a range of datetime from start to end. + :rtype: Iterator[datetime] + """ + + are_naive = start.tzinfo is None and end.tzinfo is None + are_utc = start.tzinfo == pytz.utc and end.tzinfo == pytz.utc + + # Cases with miscellenous timezone are more complexe because of DST. + are_others = start.tzinfo and end.tzinfo and not are_utc + + if are_others: + if start.tzinfo.zone != end.tzinfo.zone: + raise ValueError("Timezones of start argument and end argument seem inconsistent") + + if not are_naive and not are_utc and not are_others: + raise ValueError("Timezones of start argument and end argument mismatch") + + if start > end: + raise ValueError("start > end, start date must be before end") + + if start == start + step: + raise ValueError("Looks like step is null") + + if start.tzinfo: + localize = start.tzinfo.localize + else: + localize = lambda dt: dt + + dt = start.replace(tzinfo=None) + end = end.replace(tzinfo=None) + while dt <= end: + yield localize(dt) + dt = dt + step