diff --git a/odoo/addons/base/tests/test_misc.py b/odoo/addons/base/tests/test_misc.py index 499053a2499..881474f7d08 100644 --- a/odoo/addons/base/tests/test_misc.py +++ b/odoo/addons/base/tests/test_misc.py @@ -2,9 +2,11 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import datetime +from dateutil.relativedelta import relativedelta +import pytz import unittest -from odoo.tools import misc +from odoo.tools import misc, date_utils from odoo.tests.common import TransactionCase, tagged @@ -40,6 +42,135 @@ class TestCountingStream(unittest.TestCase): self.assertEqual(s.index, 0) +@tagged('standard', 'at_install') +class TestDateRangeFunction(unittest.TestCase): + """ Test on date_range generator. """ + + def test_date_range_with_naive_datetimes(self): + """ Check date_range with naive datetimes. """ + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + + expected = [ + datetime.datetime(1985, 1, 1, 0, 0), + datetime.datetime(1985, 2, 1, 0, 0), + datetime.datetime(1985, 3, 1, 0, 0), + datetime.datetime(1985, 4, 1, 0, 0), + datetime.datetime(1985, 5, 1, 0, 0), + datetime.datetime(1985, 6, 1, 0, 0), + datetime.datetime(1985, 7, 1, 0, 0), + datetime.datetime(1985, 8, 1, 0, 0), + datetime.datetime(1985, 9, 1, 0, 0), + datetime.datetime(1985, 10, 1, 0, 0), + datetime.datetime(1985, 11, 1, 0, 0), + datetime.datetime(1985, 12, 1, 0, 0), + datetime.datetime(1986, 1, 1, 0, 0) + ] + + dates = [date for date in date_utils.date_range(start, end)] + + self.assertEqual(dates, expected) + + def test_date_range_with_timezone_aware_datetimes_other_than_utc(self): + """ Check date_range with timezone-aware datetimes other than UTC.""" + timezone = pytz.timezone('Europe/Brussels') + + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + start = timezone.localize(start) + end = timezone.localize(end) + + expected = [datetime.datetime(1985, 1, 1, 0, 0), + datetime.datetime(1985, 2, 1, 0, 0), + datetime.datetime(1985, 3, 1, 0, 0), + datetime.datetime(1985, 4, 1, 0, 0), + datetime.datetime(1985, 5, 1, 0, 0), + datetime.datetime(1985, 6, 1, 0, 0), + datetime.datetime(1985, 7, 1, 0, 0), + datetime.datetime(1985, 8, 1, 0, 0), + datetime.datetime(1985, 9, 1, 0, 0), + datetime.datetime(1985, 10, 1, 0, 0), + datetime.datetime(1985, 11, 1, 0, 0), + datetime.datetime(1985, 12, 1, 0, 0), + datetime.datetime(1986, 1, 1, 0, 0)] + + expected = [timezone.localize(e) for e in expected] + + dates = [date for date in date_utils.date_range(start, end)] + + self.assertEqual(expected, dates) + + def test_date_range_with_mismatching_zones(self): + """ Check date_range with mismatching zone should raise an exception.""" + start_timezone = pytz.timezone('Europe/Brussels') + end_timezone = pytz.timezone('America/Recife') + + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + start = start_timezone.localize(start) + end = end_timezone.localize(end) + + with self.assertRaises(ValueError): + dates = [date for date in date_utils.date_range(start, end)] + + def test_date_range_with_inconsistent_datetimes(self): + """ Check date_range with a timezone-aware datetime and a naive one.""" + context_timezone = pytz.timezone('Europe/Brussels') + + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1986, 1, 1) + end = context_timezone.localize(end) + + with self.assertRaises(ValueError): + dates = [date for date in date_utils.date_range(start, end)] + + def test_date_range_with_hour(self): + """ Test date range with hour and naive datetime.""" + start = datetime.datetime(2018, 3, 25) + end = datetime.datetime(2018, 3, 26) + step = relativedelta(hours=1) + + expected = [ + datetime.datetime(2018, 3, 25, 0, 0), + datetime.datetime(2018, 3, 25, 1, 0), + datetime.datetime(2018, 3, 25, 2, 0), + datetime.datetime(2018, 3, 25, 3, 0), + datetime.datetime(2018, 3, 25, 4, 0), + datetime.datetime(2018, 3, 25, 5, 0), + datetime.datetime(2018, 3, 25, 6, 0), + datetime.datetime(2018, 3, 25, 7, 0), + datetime.datetime(2018, 3, 25, 8, 0), + datetime.datetime(2018, 3, 25, 9, 0), + datetime.datetime(2018, 3, 25, 10, 0), + datetime.datetime(2018, 3, 25, 11, 0), + datetime.datetime(2018, 3, 25, 12, 0), + datetime.datetime(2018, 3, 25, 13, 0), + datetime.datetime(2018, 3, 25, 14, 0), + datetime.datetime(2018, 3, 25, 15, 0), + datetime.datetime(2018, 3, 25, 16, 0), + datetime.datetime(2018, 3, 25, 17, 0), + datetime.datetime(2018, 3, 25, 18, 0), + datetime.datetime(2018, 3, 25, 19, 0), + datetime.datetime(2018, 3, 25, 20, 0), + datetime.datetime(2018, 3, 25, 21, 0), + datetime.datetime(2018, 3, 25, 22, 0), + datetime.datetime(2018, 3, 25, 23, 0), + datetime.datetime(2018, 3, 26, 0, 0) + ] + + dates = [date for date in date_utils.date_range(start, end, step)] + + self.assertEqual(dates, expected) + + def test_date_range_with_identical_boundaries(self): + """ date_range with indentical boundaries. """ + start = datetime.datetime(1985, 1, 1) + end = datetime.datetime(1985, 1, 1) + + with self.assertRaises(ValueError): + dates = [date for date in date_utils.date_range(start, end)] + + class TestFormatLangDate(TransactionCase): def test_00_accepted_types(self): date_datetime = datetime.datetime.strptime('2017-01-31 12:00:00', "%Y-%m-%d %H:%M:%S") diff --git a/odoo/addons/test_read_group/ir.model.access.csv b/odoo/addons/test_read_group/ir.model.access.csv index 9dc2f6abec2..eee7386d494 100644 --- a/odoo/addons/test_read_group/ir.model.access.csv +++ b/odoo/addons/test_read_group/ir.model.access.csv @@ -3,3 +3,4 @@ access_test_read_group_on_date,access_test_read_group_on_date,model_test_read_gr access_test_read_group_aggregate_boolean,access_test_read_group_aggregate_boolean,model_test_read_group_aggregate_boolean,,1,0,0,0 access_test_read_group_aggregate,access_test_read_group_aggregate,model_test_read_group_aggregate,,1,0,0,0 access_test_read_group_on_selection,access_test_read_group_on_selection,model_test_read_group_on_selection,,1,0,0,0 +access_test_read_group_fill_temporal,access_test_read_group_fill_temporal,model_test_read_group_fill_temporal,,1,0,0,0 diff --git a/odoo/addons/test_read_group/models.py b/odoo/addons/test_read_group/models.py index 97909a83495..38820119f39 100644 --- a/odoo/addons/test_read_group/models.py +++ b/odoo/addons/test_read_group/models.py @@ -37,3 +37,11 @@ class GroupOnSelection(models.Model): def _expand_states(self, states, domain, order): # return all possible states, in order return [key for key, val in type(self).state.selection] + + +class FillTemporal(models.Model): + _name = 'test_read_group.fill_temporal' + + date = fields.Date() + datetime = fields.Datetime() + value = fields.Integer() diff --git a/odoo/addons/test_read_group/tests/__init__.py b/odoo/addons/test_read_group/tests/__init__.py index aef16fccd7a..4821faa7571 100644 --- a/odoo/addons/test_read_group/tests/__init__.py +++ b/odoo/addons/test_read_group/tests/__init__.py @@ -2,3 +2,4 @@ from . import test_empty from . import test_group_expand from . import test_group_operator +from . import test_fill_temporal diff --git a/odoo/addons/test_read_group/tests/test_fill_temporal.py b/odoo/addons/test_read_group/tests/test_fill_temporal.py new file mode 100644 index 00000000000..d7f6fb2856e --- /dev/null +++ b/odoo/addons/test_read_group/tests/test_fill_temporal.py @@ -0,0 +1,561 @@ +# -*- coding: utf-8 -*- +"""Test for fill temporal.""" + +from odoo.tests import common + +class TestFillTemporal(common.TransactionCase): + """Test for fill temporal. + + This feature is mainly used in graph view. For more informations, read the + documentation of models's '_read_group_fill_temporal' method. + """ + + def setUp(self): + super(TestFillTemporal, self).setUp() + self.Model = self.env['test_read_group.fill_temporal'] + + def test_date_range_and_flag(self): + """Simple date range test, the flag is also tested. + + One of the most simple test. It must verify that dates 'holes' are filled + only when the fill_temporal flag is set. + """ + self.Model.create({'date': '1916-08-18', 'value': 2}) + self.Model.create({'date': '1916-10-19', 'value': 3}) + self.Model.create({'date': '1916-12-19', 'value': 5}) + + expected = [{ + '__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')], + 'date': 'August 1916', + 'date_count': 1, + 'value': 2 + }, { + '__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')], + 'date': 'September 1916', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')], + 'date': 'October 1916', + 'date_count': 1, + 'value': 3 + }, { + '__domain': ['&', ('date', '>=', '1916-11-01'), ('date', '<', '1916-12-01')], + 'date': 'November 1916', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-12-01'), ('date', '<', '1917-01-01')], + 'date': 'December 1916', + 'date_count': 1, + 'value': 5 + }] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, [group for group in expected if group['date_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, expected) + + def test_date_range_with_context_timezone(self): + """Test if date are date_trunced correctly by pgres. + + This test was added in attempt to fix a bug appearing with babel that + we use to translate the dates. Typically after a daylight saving, A + whole year was displayed in a graph like this (APR missing and OCT + appearing twice) : + + JAN FEB MAR MAY JUN JUL AUG SEP OCT OCT NOV + ^^^ ^^^ + """ + self.Model.create({'date': '1915-01-01', 'value': 3}) + self.Model.create({'date': '1916-01-01', 'value': 5}) + + expected = [{ + '__domain': ['&', ('date', '>=', '1915-01-01'), ('date', '<', '1915-02-01')], + 'date': 'January 1915', + 'date_count': 1, + 'value': 3 + }, { + '__domain': ['&', ('date', '>=', '1915-02-01'), ('date', '<', '1915-03-01')], + 'date': 'February 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-03-01'), ('date', '<', '1915-04-01')], + 'date': 'March 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-04-01'), ('date', '<', '1915-05-01')], + 'date': 'April 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-05-01'), ('date', '<', '1915-06-01')], + 'date': 'May 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-06-01'), ('date', '<', '1915-07-01')], + 'date': 'June 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-07-01'), ('date', '<', '1915-08-01')], + 'date': 'July 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-08-01'), ('date', '<', '1915-09-01')], + 'date': 'August 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-09-01'), ('date', '<', '1915-10-01')], + 'date': 'September 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-10-01'), ('date', '<', '1915-11-01')], + 'date': 'October 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-11-01'), ('date', '<', '1915-12-01')], + 'date': 'November 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1915-12-01'), ('date', '<', '1916-01-01')], + 'date': 'December 1915', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-01-01'), ('date', '<', '1916-02-01')], + 'date': 'January 1916', + 'date_count': 1, + 'value': 5 + }] + + # Time Zone UTC UTC DST + tzs = ["America/Anchorage", # −09:00 −08:00 + "Europe/Brussels", # +01:00 +02:00 + "Pacific/Kwajalein"] # +12:00 +12:00 + + for tz in tzs: + model_fill = self.Model.with_context(tz=tz, fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + def test_only_with_only_null_date(self): + """We should have the same result when fill_temporal is set or not.""" + self.Model.create({'date': False, 'value': 13}) + self.Model.create({'date': False, 'value': 11}) + self.Model.create({'date': False, 'value': 17}) + + expected = [{'__domain': [('date', '=', False)], + 'date_count': 3, + 'value': 41, + 'date': False}] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + def test_date_range_and_null_date(self): + """Test data with null and non-null dates.""" + self.Model.create({'date': '1916-08-19', 'value': 4}) + self.Model.create({'date': False, 'value': 13}) + self.Model.create({'date': '1916-10-18', 'value': 5}) + self.Model.create({'date': '1916-08-18', 'value': 3}) + self.Model.create({'date': '1916-10-19', 'value': 4}) + self.Model.create({'date': False, 'value': 11}) + + expected = [{ + '__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')], + 'date': 'August 1916', + 'date_count': 2, + 'value': 7 + }, { + '__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')], + 'date': 'September 1916', + 'date_count': 0, + 'value': 0 + }, { + '__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')], + 'date': 'October 1916', + 'date_count': 2, + 'value': 9 + }, { + '__domain': [('date', '=', False)], + 'date': False, + 'date_count': 2, + 'value': 24 + }] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, [group for group in expected if group['date_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + + self.assertEqual(groups, expected) + + def test_order_date_desc(self): + """Test if changing Model._order has influence on the result.""" + self.Model.create({'date': '1916-08-18', 'value': 3}) + self.Model.create({'date': '1916-08-19', 'value': 4}) + self.Model.create({'date': '1916-10-18', 'value': 5}) + self.Model.create({'date': '1916-10-19', 'value': 4}) + self.patch(type(self.Model), '_order', 'date desc') + + expected = [{ + '__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')], + 'date': 'August 1916', + 'date_count': 2, + 'value': 7 + }, { + '__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')], + 'date': 'September 1916', + 'date_count': 0, + 'value': False + }, { + '__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')], + 'date': 'October 1916', + 'date_count': 2, + 'value': 9 + }] + + groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, [group for group in expected if group['date_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date']) + self.assertEqual(groups, expected) + + def test_timestamp_without_timezone(self): + """Test datetimes. + + Date stored with an hour inside the Odoo model are processed as timestamp + without timezone by postgres. + """ + self.Model.create({'datetime': '1916-08-19 01:30:00', 'value': 7}) + self.Model.create({'datetime': False, 'value': 13}) + self.Model.create({'datetime': '1916-10-18 02:30:00', 'value': 5}) + self.Model.create({'datetime': '1916-08-18 01:50:00', 'value': 3}) + self.Model.create({'datetime': False, 'value': 11}) + self.Model.create({'datetime': '1916-10-19 23:59:59', 'value': 2}) + self.Model.create({'datetime': '1916-10-19', 'value': 19}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '1916-08-01 00:00:00'), + ('datetime', '<', '1916-09-01 00:00:00')], + 'datetime': 'August 1916', + 'datetime_count': 2, + 'value': 10 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-09-01 00:00:00'), + ('datetime', '<', '1916-10-01 00:00:00')], + 'datetime': 'September 1916', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-10-01 00:00:00'), + ('datetime', '<', '1916-11-01 00:00:00')], + 'datetime': 'October 1916', + 'datetime_count': 3, + 'value': 26 + }, { + '__domain': [('datetime', '=', False)], + 'datetime': False, + 'datetime_count': 2, + 'value': 24 + }] + + groups = self.Model.read_group([], fields=['datetime', 'value'], groupby=['datetime']) + + self.assertEqual(groups, [group for group in expected if group['datetime_count']]) + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime']) + + self.assertEqual(groups, expected) + + def test_with_datetimes_and_groupby_per_hour(self): + """Test with datetimes and groupby per hour. + + Test if datetimes are filled correctly when grouping by hours instead of + months. + """ + self.Model.create({'datetime': '1916-01-01 01:30:00', 'value': 2}) + self.Model.create({'datetime': '1916-01-01 01:50:00', 'value': 8}) + self.Model.create({'datetime': '1916-01-01 02:30:00', 'value': 3}) + self.Model.create({'datetime': '1916-01-01 13:50:00', 'value': 5}) + self.Model.create({'datetime': '1916-01-01 23:50:00', 'value': 7}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '1916-01-01 01:00:00'), + ('datetime', '<', '1916-01-01 02:00:00')], + 'datetime:hour': '01:00 01 Jan', + 'datetime_count': 2, + 'value': 10 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 02:00:00'), + ('datetime', '<', '1916-01-01 03:00:00')], + 'datetime:hour': '02:00 01 Jan', + 'datetime_count': 1, + 'value': 3 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 03:00:00'), + ('datetime', '<', '1916-01-01 04:00:00')], + 'datetime:hour': '03:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 04:00:00'), + ('datetime', '<', '1916-01-01 05:00:00')], + 'datetime:hour': '04:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 05:00:00'), + ('datetime', '<', '1916-01-01 06:00:00')], + 'datetime:hour': '05:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 06:00:00'), + ('datetime', '<', '1916-01-01 07:00:00')], + 'datetime:hour': '06:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 07:00:00'), + ('datetime', '<', '1916-01-01 08:00:00')], + 'datetime:hour': '07:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 08:00:00'), + ('datetime', '<', '1916-01-01 09:00:00')], + 'datetime:hour': '08:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 09:00:00'), + ('datetime', '<', '1916-01-01 10:00:00')], + 'datetime:hour': '09:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 10:00:00'), + ('datetime', '<', '1916-01-01 11:00:00')], + 'datetime:hour': '10:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 11:00:00'), + ('datetime', '<', '1916-01-01 12:00:00')], + 'datetime:hour': '11:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 12:00:00'), + ('datetime', '<', '1916-01-01 13:00:00')], + 'datetime:hour': '12:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 13:00:00'), + ('datetime', '<', '1916-01-01 14:00:00')], + 'datetime:hour': '01:00 01 Jan', + 'datetime_count': 1, + 'value': 5 + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 14:00:00'), + ('datetime', '<', '1916-01-01 15:00:00')], + 'datetime:hour': '02:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 15:00:00'), + ('datetime', '<', '1916-01-01 16:00:00')], + 'datetime:hour': '03:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 16:00:00'), + ('datetime', '<', '1916-01-01 17:00:00')], + 'datetime:hour': '04:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 17:00:00'), + ('datetime', '<', '1916-01-01 18:00:00')], + 'datetime:hour': '05:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 18:00:00'), + ('datetime', '<', '1916-01-01 19:00:00')], + 'datetime:hour': '06:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 19:00:00'), + ('datetime', '<', '1916-01-01 20:00:00')], + 'datetime:hour': '07:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 20:00:00'), + ('datetime', '<', '1916-01-01 21:00:00')], + 'datetime:hour': '08:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 21:00:00'), + ('datetime', '<', '1916-01-01 22:00:00')], + 'datetime:hour': '09:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 22:00:00'), + ('datetime', '<', '1916-01-01 23:00:00')], + 'datetime:hour': '10:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 23:00:00'), + ('datetime', '<', '1916-01-02 00:00:00')], + 'datetime:hour': '11:00 01 Jan', + 'datetime_count': 1, + 'value': 7 + }] + + model_fill = self.Model.with_context(fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime:hour']) + + self.assertEqual(groups, expected) + + def test_hour_with_timezones(self): + """Test hour with timezones. + + What we do here is similar to test_with_datetimes_and_groupby_per_hour + but with a timezone in the user context. + """ + self.Model.create({'datetime': '1915-12-31 22:30:00', 'value': 2}) + self.Model.create({'datetime': '1916-01-01 03:30:00', 'value': 3}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '1915-12-31 22:00:00'), + ('datetime', '<', '1915-12-31 23:00:00')], + 'datetime:hour': '04:00 01 Jan', + 'datetime_count': 1, + 'value': 2 + }, { + '__domain': ['&', + ('datetime', '>=', '1915-12-31 23:00:00'), + ('datetime', '<', '1916-01-01 00:00:00')], + 'datetime:hour': '05:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 00:00:00'), + ('datetime', '<', '1916-01-01 01:00:00')], + 'datetime:hour': '06:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 01:00:00'), + ('datetime', '<', '1916-01-01 02:00:00')], + 'datetime:hour': '07:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 02:00:00'), + ('datetime', '<', '1916-01-01 03:00:00')], + 'datetime:hour': '08:00 01 Jan', + 'datetime_count': 0, + 'value': False + }, { + '__domain': ['&', + ('datetime', '>=', '1916-01-01 03:00:00'), + ('datetime', '<', '1916-01-01 04:00:00')], + 'datetime:hour': '09:00 01 Jan', + 'datetime_count': 1, + 'value': 3 + }] + + model_fill = self.Model.with_context(tz='Asia/Hovd', fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], + groupby=['datetime:hour']) + + self.assertEqual(groups, expected) + + def test_egde_fx_tz(self): + """We test if different edge effect by using a different timezone from the user context + + Suppose a user resident near Hovd, a city in Mongolia. he sells a product + at exacltly 4:00 AM on 1st January 2018. Using its context, that datetime + is previously converted to UTC time by the ORM so as being stored properly + inside the datebase. We are in winter time so 'Asia/Hovd' is UTC+7 : + + '2018-01-01 04:00:00' --> '2017-12-31 21:00:00' + + If that same user groups by datetime, we must ensure that the last + displayed date is in January and not in December. + """ + self.Model.create({'datetime': '2017-12-31 21:00:00', 'value': 42}) + + expected = [{ + '__domain': ['&', + ('datetime', '>=', '2017-12-31 17:00:00'), + ('datetime', '<', '2018-01-31 17:00:00')], + 'datetime': 'January 2018', + 'datetime_count': 1, + 'value': 42 + }] + + model_fill = self.Model.with_context(tz='Asia/Hovd', fill_temporal=True) + groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime']) + + self.assertEqual(groups, expected) diff --git a/odoo/models.py b/odoo/models.py index 4a610a25176..9c12cc0f5b1 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -58,6 +58,7 @@ from .tools.func import frame_codeinfo from .tools.misc import CountingStream, DEFAULT_SERVER_DATETIME_FORMAT, DEFAULT_SERVER_DATE_FORMAT from .tools.safe_eval import safe_eval from .tools.translate import _ +from .tools import date_utils _logger = logging.getLogger(__name__) _schema = logging.getLogger(__name__ + '.schema') @@ -1755,6 +1756,89 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): return list(result.values()) + @api.model + def _read_group_fill_temporal(self, data, groupby, aggregated_fields, annotated_groupbys, + interval=dateutil.relativedelta.relativedelta(months=1)): + """Helper method for filling date/datetime 'holes' in a result set. + + We are in a use case where data are grouped by a date field (typically + months but it could be any other interval) and displayed in a chart. + + Assume we group records by month, and we only have data for August, + September and December. By default, plotting the result gives something + like: + ___ + ___ | | + | | | | + | | ___ | | + | || || | + |___||___||___| + Aug Sep Dec + + The problem is that December data follows immediately September data, + which is misleading for the user. Adding explicit zeroes for missing data + gives something like: + ___ + ___ | | + | | | | + | | ___ | | + | || | | | + |___||___| ___ ___ |___| + Aug Sep Oct Nov Dec + + :param list data: the data containing groups + :param list groupby: name of the first group by + :param aggregated_fields list: list of aggregated fields in the query + :param relativedelta interval: interval between to temporal groups + expressed as a relativedelta month by default + :rtype: list + :return: list + """ + first_a_gby = annotated_groupbys[0] + if not data: + return + if first_a_gby['type'] not in ('date', 'datetime'): + return data + interval = first_a_gby['interval'] + groupby_name = groupby[0] + + # existing non null datetimes + existing = [d[groupby_name] for d in data if d[groupby_name]] + + if len(existing) < 2: + return data + + # assumption: existing data is sorted by field 'groupby_name' + first, last = existing[0], existing[-1] + + expected = list(date_utils.date_range(first, last, interval)) + + if len(existing) < len(expected): + empty_data = dict.fromkeys(aggregated_fields, False) + empty_data['id'] = False + empty_data[groupby_name.split(':')[0] + '_count'] = 0 + + data = collections.deque(data) + + new_data = [] + + # Note: the list 'expected' contains the the dates that should be + # represented inside data when it is returned. Notice that 'expected' + # is sorted like data, so what we do is empty data progressivly by + # popping its elements from first to last. To do this properly, we + # need to compare dates together. + for dt in expected: + if not data[0][groupby_name] or data[0][groupby_name] == dt: + new_data.append(data.popleft()) + else: + new_data.append(dict(empty_data, **{groupby_name: dt})) + + assert not data + return new_data + + return data + + @api.model def _read_group_prepare(self, orderby, aggregated_fields, annotated_groupbys, query): """ @@ -1845,7 +1929,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): } if tz_convert: qualified_field = "timezone('%s', timezone('UTC',%s))" % (self._context.get('tz', 'UTC'), qualified_field) - qualified_field = "date_trunc('%s', %s)" % (gb_function or 'month', qualified_field) + qualified_field = "date_trunc('%s', %s::timestamp)" % (gb_function or 'month', qualified_field) if field_type == 'boolean': qualified_field = "coalesce(%s,false)" % qualified_field return { @@ -1855,7 +1939,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): 'display_format': display_formats[gb_function or 'month'] if temporal else None, 'interval': time_intervals[gb_function or 'month'] if temporal else None, 'tz_convert': tz_convert, - 'qualified_field': qualified_field + 'qualified_field': qualified_field, } @api.model @@ -1984,7 +2068,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): groupby = [groupby] if isinstance(groupby, pycompat.string_types) else list(OrderedSet(groupby)) dt = [ f for f in groupby - if self._fields[f.split(':')[0]].type in ('date', 'datetime') + if self._fields[f.split(':')[0]].type in ('date', 'datetime') # e.g. 'date:month' ] # iterate on all results and replace the "full" date/datetime value @@ -2100,8 +2184,14 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})): self._read_group_resolve_many2one_fields(fetched_data, annotated_groupbys) - data = ({k: self._read_group_prepare_data(k,v, groupby_dict) for k,v in r.items()} for r in fetched_data) + data = [{k: self._read_group_prepare_data(k, v, groupby_dict) for k, v in r.items()} for r in fetched_data] + + if self.env.context.get('fill_temporal'): + data = self._read_group_fill_temporal(data, groupby, aggregated_fields, + annotated_groupbys) + result = [self._read_group_format_result(d, annotated_groupbys, groupby, domain) for d in data] + if lazy: # Right now, read_group only fill results in lazy mode (by default). # If you need to have the empty groups in 'eager' mode, then the diff --git a/odoo/tools/date_utils.py b/odoo/tools/date_utils.py index e7d1aa24ffd..55220ced033 100644 --- a/odoo/tools/date_utils.py +++ b/odoo/tools/date_utils.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- - import math import calendar from datetime import date, datetime, time - +import pytz from dateutil.relativedelta import relativedelta from . import ustr @@ -178,3 +177,43 @@ def json_default(obj): return fields.Datetime.to_string(obj) return fields.Date.to_string(obj) return ustr(obj) + +def date_range(start, end, step=relativedelta(months=1)): + """Date range generator with a step interval. + + :param start datetime: begining date of the range. + :param end datetime: ending date of the range. + :param step relativedelta: interval of the range. + :return: a range of datetime from start to end. + :rtype: Iterator[datetime] + """ + + are_naive = start.tzinfo is None and end.tzinfo is None + are_utc = start.tzinfo == pytz.utc and end.tzinfo == pytz.utc + + # Cases with miscellenous timezone are more complexe because of DST. + are_others = start.tzinfo and end.tzinfo and not are_utc + + if are_others: + if start.tzinfo.zone != end.tzinfo.zone: + raise ValueError("Timezones of start argument and end argument seem inconsistent") + + if not are_naive and not are_utc and not are_others: + raise ValueError("Timezones of start argument and end argument mismatch") + + if start > end: + raise ValueError("start > end, start date must be before end") + + if start == start + step: + raise ValueError("Looks like step is null") + + if start.tzinfo: + localize = start.tzinfo.localize + else: + localize = lambda dt: dt + + dt = start.replace(tzinfo=None) + end = end.replace(tzinfo=None) + while dt <= end: + yield localize(dt) + dt = dt + step