[IMP] orm: add support to fill temporal holes (read_group)
This commit add an extra parameter 'fill_temporal' to read_group which allows
the orm to add missing groups for date intervals. This is useful for charts.
Suppose that we are in a use case where data are grouped by a date fields
(typically months but it could be another interval) and displayed in a Bar Chart
or a Line Char.
Let's says a request has to group records by month for August, September
and ...December. If we don't changed anything, we would get a Bar Chart
looking like this :
___
___ | |
| | | |
| | ___ | |
| || || |
|___||___||___|
Aug Sep D
December follows directly after September, it can be unintuitive for the
user, so we change that. We add some fake records for each missing months
between the earliest and the lastest date of the result
___
___ | |
| | | |
| | ___ | |
| || | | |
|___||___| ___ ___ |___|
Aug Sep Oct Nov Dec
This commit is part of task #1835644
This commit is contained in:
Francois Volral
@@ -2,9 +2,11 @@
|
||||
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
||||
|
||||
import datetime
|
||||
from dateutil.relativedelta import relativedelta
|
||||
import pytz
|
||||
import unittest
|
||||
|
||||
from odoo.tools import misc
|
||||
from odoo.tools import misc, date_utils
|
||||
from odoo.tests.common import TransactionCase, tagged
|
||||
|
||||
|
||||
@@ -40,6 +42,135 @@ class TestCountingStream(unittest.TestCase):
|
||||
self.assertEqual(s.index, 0)
|
||||
|
||||
|
||||
@tagged('standard', 'at_install')
|
||||
class TestDateRangeFunction(unittest.TestCase):
|
||||
""" Test on date_range generator. """
|
||||
|
||||
def test_date_range_with_naive_datetimes(self):
|
||||
""" Check date_range with naive datetimes. """
|
||||
start = datetime.datetime(1985, 1, 1)
|
||||
end = datetime.datetime(1986, 1, 1)
|
||||
|
||||
expected = [
|
||||
datetime.datetime(1985, 1, 1, 0, 0),
|
||||
datetime.datetime(1985, 2, 1, 0, 0),
|
||||
datetime.datetime(1985, 3, 1, 0, 0),
|
||||
datetime.datetime(1985, 4, 1, 0, 0),
|
||||
datetime.datetime(1985, 5, 1, 0, 0),
|
||||
datetime.datetime(1985, 6, 1, 0, 0),
|
||||
datetime.datetime(1985, 7, 1, 0, 0),
|
||||
datetime.datetime(1985, 8, 1, 0, 0),
|
||||
datetime.datetime(1985, 9, 1, 0, 0),
|
||||
datetime.datetime(1985, 10, 1, 0, 0),
|
||||
datetime.datetime(1985, 11, 1, 0, 0),
|
||||
datetime.datetime(1985, 12, 1, 0, 0),
|
||||
datetime.datetime(1986, 1, 1, 0, 0)
|
||||
]
|
||||
|
||||
dates = [date for date in date_utils.date_range(start, end)]
|
||||
|
||||
self.assertEqual(dates, expected)
|
||||
|
||||
def test_date_range_with_timezone_aware_datetimes_other_than_utc(self):
|
||||
""" Check date_range with timezone-aware datetimes other than UTC."""
|
||||
timezone = pytz.timezone('Europe/Brussels')
|
||||
|
||||
start = datetime.datetime(1985, 1, 1)
|
||||
end = datetime.datetime(1986, 1, 1)
|
||||
start = timezone.localize(start)
|
||||
end = timezone.localize(end)
|
||||
|
||||
expected = [datetime.datetime(1985, 1, 1, 0, 0),
|
||||
datetime.datetime(1985, 2, 1, 0, 0),
|
||||
datetime.datetime(1985, 3, 1, 0, 0),
|
||||
datetime.datetime(1985, 4, 1, 0, 0),
|
||||
datetime.datetime(1985, 5, 1, 0, 0),
|
||||
datetime.datetime(1985, 6, 1, 0, 0),
|
||||
datetime.datetime(1985, 7, 1, 0, 0),
|
||||
datetime.datetime(1985, 8, 1, 0, 0),
|
||||
datetime.datetime(1985, 9, 1, 0, 0),
|
||||
datetime.datetime(1985, 10, 1, 0, 0),
|
||||
datetime.datetime(1985, 11, 1, 0, 0),
|
||||
datetime.datetime(1985, 12, 1, 0, 0),
|
||||
datetime.datetime(1986, 1, 1, 0, 0)]
|
||||
|
||||
expected = [timezone.localize(e) for e in expected]
|
||||
|
||||
dates = [date for date in date_utils.date_range(start, end)]
|
||||
|
||||
self.assertEqual(expected, dates)
|
||||
|
||||
def test_date_range_with_mismatching_zones(self):
|
||||
""" Check date_range with mismatching zone should raise an exception."""
|
||||
start_timezone = pytz.timezone('Europe/Brussels')
|
||||
end_timezone = pytz.timezone('America/Recife')
|
||||
|
||||
start = datetime.datetime(1985, 1, 1)
|
||||
end = datetime.datetime(1986, 1, 1)
|
||||
start = start_timezone.localize(start)
|
||||
end = end_timezone.localize(end)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
dates = [date for date in date_utils.date_range(start, end)]
|
||||
|
||||
def test_date_range_with_inconsistent_datetimes(self):
|
||||
""" Check date_range with a timezone-aware datetime and a naive one."""
|
||||
context_timezone = pytz.timezone('Europe/Brussels')
|
||||
|
||||
start = datetime.datetime(1985, 1, 1)
|
||||
end = datetime.datetime(1986, 1, 1)
|
||||
end = context_timezone.localize(end)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
dates = [date for date in date_utils.date_range(start, end)]
|
||||
|
||||
def test_date_range_with_hour(self):
|
||||
""" Test date range with hour and naive datetime."""
|
||||
start = datetime.datetime(2018, 3, 25)
|
||||
end = datetime.datetime(2018, 3, 26)
|
||||
step = relativedelta(hours=1)
|
||||
|
||||
expected = [
|
||||
datetime.datetime(2018, 3, 25, 0, 0),
|
||||
datetime.datetime(2018, 3, 25, 1, 0),
|
||||
datetime.datetime(2018, 3, 25, 2, 0),
|
||||
datetime.datetime(2018, 3, 25, 3, 0),
|
||||
datetime.datetime(2018, 3, 25, 4, 0),
|
||||
datetime.datetime(2018, 3, 25, 5, 0),
|
||||
datetime.datetime(2018, 3, 25, 6, 0),
|
||||
datetime.datetime(2018, 3, 25, 7, 0),
|
||||
datetime.datetime(2018, 3, 25, 8, 0),
|
||||
datetime.datetime(2018, 3, 25, 9, 0),
|
||||
datetime.datetime(2018, 3, 25, 10, 0),
|
||||
datetime.datetime(2018, 3, 25, 11, 0),
|
||||
datetime.datetime(2018, 3, 25, 12, 0),
|
||||
datetime.datetime(2018, 3, 25, 13, 0),
|
||||
datetime.datetime(2018, 3, 25, 14, 0),
|
||||
datetime.datetime(2018, 3, 25, 15, 0),
|
||||
datetime.datetime(2018, 3, 25, 16, 0),
|
||||
datetime.datetime(2018, 3, 25, 17, 0),
|
||||
datetime.datetime(2018, 3, 25, 18, 0),
|
||||
datetime.datetime(2018, 3, 25, 19, 0),
|
||||
datetime.datetime(2018, 3, 25, 20, 0),
|
||||
datetime.datetime(2018, 3, 25, 21, 0),
|
||||
datetime.datetime(2018, 3, 25, 22, 0),
|
||||
datetime.datetime(2018, 3, 25, 23, 0),
|
||||
datetime.datetime(2018, 3, 26, 0, 0)
|
||||
]
|
||||
|
||||
dates = [date for date in date_utils.date_range(start, end, step)]
|
||||
|
||||
self.assertEqual(dates, expected)
|
||||
|
||||
def test_date_range_with_identical_boundaries(self):
|
||||
""" date_range with indentical boundaries. """
|
||||
start = datetime.datetime(1985, 1, 1)
|
||||
end = datetime.datetime(1985, 1, 1)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
dates = [date for date in date_utils.date_range(start, end)]
|
||||
|
||||
|
||||
class TestFormatLangDate(TransactionCase):
|
||||
def test_00_accepted_types(self):
|
||||
date_datetime = datetime.datetime.strptime('2017-01-31 12:00:00', "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
@@ -3,3 +3,4 @@ access_test_read_group_on_date,access_test_read_group_on_date,model_test_read_gr
|
||||
access_test_read_group_aggregate_boolean,access_test_read_group_aggregate_boolean,model_test_read_group_aggregate_boolean,,1,0,0,0
|
||||
access_test_read_group_aggregate,access_test_read_group_aggregate,model_test_read_group_aggregate,,1,0,0,0
|
||||
access_test_read_group_on_selection,access_test_read_group_on_selection,model_test_read_group_on_selection,,1,0,0,0
|
||||
access_test_read_group_fill_temporal,access_test_read_group_fill_temporal,model_test_read_group_fill_temporal,,1,0,0,0
|
||||
|
||||
|
@@ -37,3 +37,11 @@ class GroupOnSelection(models.Model):
|
||||
def _expand_states(self, states, domain, order):
|
||||
# return all possible states, in order
|
||||
return [key for key, val in type(self).state.selection]
|
||||
|
||||
|
||||
class FillTemporal(models.Model):
|
||||
_name = 'test_read_group.fill_temporal'
|
||||
|
||||
date = fields.Date()
|
||||
datetime = fields.Datetime()
|
||||
value = fields.Integer()
|
||||
|
||||
@@ -2,3 +2,4 @@
|
||||
from . import test_empty
|
||||
from . import test_group_expand
|
||||
from . import test_group_operator
|
||||
from . import test_fill_temporal
|
||||
|
||||
@@ -0,0 +1,561 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Test for fill temporal."""
|
||||
|
||||
from odoo.tests import common
|
||||
|
||||
class TestFillTemporal(common.TransactionCase):
|
||||
"""Test for fill temporal.
|
||||
|
||||
This feature is mainly used in graph view. For more informations, read the
|
||||
documentation of models's '_read_group_fill_temporal' method.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(TestFillTemporal, self).setUp()
|
||||
self.Model = self.env['test_read_group.fill_temporal']
|
||||
|
||||
def test_date_range_and_flag(self):
|
||||
"""Simple date range test, the flag is also tested.
|
||||
|
||||
One of the most simple test. It must verify that dates 'holes' are filled
|
||||
only when the fill_temporal flag is set.
|
||||
"""
|
||||
self.Model.create({'date': '1916-08-18', 'value': 2})
|
||||
self.Model.create({'date': '1916-10-19', 'value': 3})
|
||||
self.Model.create({'date': '1916-12-19', 'value': 5})
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')],
|
||||
'date': 'August 1916',
|
||||
'date_count': 1,
|
||||
'value': 2
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')],
|
||||
'date': 'September 1916',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')],
|
||||
'date': 'October 1916',
|
||||
'date_count': 1,
|
||||
'value': 3
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-11-01'), ('date', '<', '1916-12-01')],
|
||||
'date': 'November 1916',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-12-01'), ('date', '<', '1917-01-01')],
|
||||
'date': 'December 1916',
|
||||
'date_count': 1,
|
||||
'value': 5
|
||||
}]
|
||||
|
||||
groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
|
||||
self.assertEqual(groups, [group for group in expected if group['date_count']])
|
||||
|
||||
model_fill = self.Model.with_context(fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_date_range_with_context_timezone(self):
|
||||
"""Test if date are date_trunced correctly by pgres.
|
||||
|
||||
This test was added in attempt to fix a bug appearing with babel that
|
||||
we use to translate the dates. Typically after a daylight saving, A
|
||||
whole year was displayed in a graph like this (APR missing and OCT
|
||||
appearing twice) :
|
||||
|
||||
JAN FEB MAR MAY JUN JUL AUG SEP OCT OCT NOV
|
||||
^^^ ^^^
|
||||
"""
|
||||
self.Model.create({'date': '1915-01-01', 'value': 3})
|
||||
self.Model.create({'date': '1916-01-01', 'value': 5})
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&', ('date', '>=', '1915-01-01'), ('date', '<', '1915-02-01')],
|
||||
'date': 'January 1915',
|
||||
'date_count': 1,
|
||||
'value': 3
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-02-01'), ('date', '<', '1915-03-01')],
|
||||
'date': 'February 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-03-01'), ('date', '<', '1915-04-01')],
|
||||
'date': 'March 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-04-01'), ('date', '<', '1915-05-01')],
|
||||
'date': 'April 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-05-01'), ('date', '<', '1915-06-01')],
|
||||
'date': 'May 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-06-01'), ('date', '<', '1915-07-01')],
|
||||
'date': 'June 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-07-01'), ('date', '<', '1915-08-01')],
|
||||
'date': 'July 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-08-01'), ('date', '<', '1915-09-01')],
|
||||
'date': 'August 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-09-01'), ('date', '<', '1915-10-01')],
|
||||
'date': 'September 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-10-01'), ('date', '<', '1915-11-01')],
|
||||
'date': 'October 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-11-01'), ('date', '<', '1915-12-01')],
|
||||
'date': 'November 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1915-12-01'), ('date', '<', '1916-01-01')],
|
||||
'date': 'December 1915',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-01-01'), ('date', '<', '1916-02-01')],
|
||||
'date': 'January 1916',
|
||||
'date_count': 1,
|
||||
'value': 5
|
||||
}]
|
||||
|
||||
# Time Zone UTC UTC DST
|
||||
tzs = ["America/Anchorage", # −09:00 −08:00
|
||||
"Europe/Brussels", # +01:00 +02:00
|
||||
"Pacific/Kwajalein"] # +12:00 +12:00
|
||||
|
||||
for tz in tzs:
|
||||
model_fill = self.Model.with_context(tz=tz, fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_only_with_only_null_date(self):
|
||||
"""We should have the same result when fill_temporal is set or not."""
|
||||
self.Model.create({'date': False, 'value': 13})
|
||||
self.Model.create({'date': False, 'value': 11})
|
||||
self.Model.create({'date': False, 'value': 17})
|
||||
|
||||
expected = [{'__domain': [('date', '=', False)],
|
||||
'date_count': 3,
|
||||
'value': 41,
|
||||
'date': False}]
|
||||
|
||||
groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
model_fill = self.Model.with_context(fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_date_range_and_null_date(self):
|
||||
"""Test data with null and non-null dates."""
|
||||
self.Model.create({'date': '1916-08-19', 'value': 4})
|
||||
self.Model.create({'date': False, 'value': 13})
|
||||
self.Model.create({'date': '1916-10-18', 'value': 5})
|
||||
self.Model.create({'date': '1916-08-18', 'value': 3})
|
||||
self.Model.create({'date': '1916-10-19', 'value': 4})
|
||||
self.Model.create({'date': False, 'value': 11})
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')],
|
||||
'date': 'August 1916',
|
||||
'date_count': 2,
|
||||
'value': 7
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')],
|
||||
'date': 'September 1916',
|
||||
'date_count': 0,
|
||||
'value': 0
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')],
|
||||
'date': 'October 1916',
|
||||
'date_count': 2,
|
||||
'value': 9
|
||||
}, {
|
||||
'__domain': [('date', '=', False)],
|
||||
'date': False,
|
||||
'date_count': 2,
|
||||
'value': 24
|
||||
}]
|
||||
|
||||
groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
|
||||
self.assertEqual(groups, [group for group in expected if group['date_count']])
|
||||
|
||||
model_fill = self.Model.with_context(fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_order_date_desc(self):
|
||||
"""Test if changing Model._order has influence on the result."""
|
||||
self.Model.create({'date': '1916-08-18', 'value': 3})
|
||||
self.Model.create({'date': '1916-08-19', 'value': 4})
|
||||
self.Model.create({'date': '1916-10-18', 'value': 5})
|
||||
self.Model.create({'date': '1916-10-19', 'value': 4})
|
||||
self.patch(type(self.Model), '_order', 'date desc')
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&', ('date', '>=', '1916-08-01'), ('date', '<', '1916-09-01')],
|
||||
'date': 'August 1916',
|
||||
'date_count': 2,
|
||||
'value': 7
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-09-01'), ('date', '<', '1916-10-01')],
|
||||
'date': 'September 1916',
|
||||
'date_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&', ('date', '>=', '1916-10-01'), ('date', '<', '1916-11-01')],
|
||||
'date': 'October 1916',
|
||||
'date_count': 2,
|
||||
'value': 9
|
||||
}]
|
||||
|
||||
groups = self.Model.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
self.assertEqual(groups, [group for group in expected if group['date_count']])
|
||||
|
||||
model_fill = self.Model.with_context(fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['date', 'value'], groupby=['date'])
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_timestamp_without_timezone(self):
|
||||
"""Test datetimes.
|
||||
|
||||
Date stored with an hour inside the Odoo model are processed as timestamp
|
||||
without timezone by postgres.
|
||||
"""
|
||||
self.Model.create({'datetime': '1916-08-19 01:30:00', 'value': 7})
|
||||
self.Model.create({'datetime': False, 'value': 13})
|
||||
self.Model.create({'datetime': '1916-10-18 02:30:00', 'value': 5})
|
||||
self.Model.create({'datetime': '1916-08-18 01:50:00', 'value': 3})
|
||||
self.Model.create({'datetime': False, 'value': 11})
|
||||
self.Model.create({'datetime': '1916-10-19 23:59:59', 'value': 2})
|
||||
self.Model.create({'datetime': '1916-10-19', 'value': 19})
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-08-01 00:00:00'),
|
||||
('datetime', '<', '1916-09-01 00:00:00')],
|
||||
'datetime': 'August 1916',
|
||||
'datetime_count': 2,
|
||||
'value': 10
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-09-01 00:00:00'),
|
||||
('datetime', '<', '1916-10-01 00:00:00')],
|
||||
'datetime': 'September 1916',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-10-01 00:00:00'),
|
||||
('datetime', '<', '1916-11-01 00:00:00')],
|
||||
'datetime': 'October 1916',
|
||||
'datetime_count': 3,
|
||||
'value': 26
|
||||
}, {
|
||||
'__domain': [('datetime', '=', False)],
|
||||
'datetime': False,
|
||||
'datetime_count': 2,
|
||||
'value': 24
|
||||
}]
|
||||
|
||||
groups = self.Model.read_group([], fields=['datetime', 'value'], groupby=['datetime'])
|
||||
|
||||
self.assertEqual(groups, [group for group in expected if group['datetime_count']])
|
||||
|
||||
model_fill = self.Model.with_context(fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime'])
|
||||
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_with_datetimes_and_groupby_per_hour(self):
|
||||
"""Test with datetimes and groupby per hour.
|
||||
|
||||
Test if datetimes are filled correctly when grouping by hours instead of
|
||||
months.
|
||||
"""
|
||||
self.Model.create({'datetime': '1916-01-01 01:30:00', 'value': 2})
|
||||
self.Model.create({'datetime': '1916-01-01 01:50:00', 'value': 8})
|
||||
self.Model.create({'datetime': '1916-01-01 02:30:00', 'value': 3})
|
||||
self.Model.create({'datetime': '1916-01-01 13:50:00', 'value': 5})
|
||||
self.Model.create({'datetime': '1916-01-01 23:50:00', 'value': 7})
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 01:00:00'),
|
||||
('datetime', '<', '1916-01-01 02:00:00')],
|
||||
'datetime:hour': '01:00 01 Jan',
|
||||
'datetime_count': 2,
|
||||
'value': 10
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 02:00:00'),
|
||||
('datetime', '<', '1916-01-01 03:00:00')],
|
||||
'datetime:hour': '02:00 01 Jan',
|
||||
'datetime_count': 1,
|
||||
'value': 3
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 03:00:00'),
|
||||
('datetime', '<', '1916-01-01 04:00:00')],
|
||||
'datetime:hour': '03:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 04:00:00'),
|
||||
('datetime', '<', '1916-01-01 05:00:00')],
|
||||
'datetime:hour': '04:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 05:00:00'),
|
||||
('datetime', '<', '1916-01-01 06:00:00')],
|
||||
'datetime:hour': '05:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 06:00:00'),
|
||||
('datetime', '<', '1916-01-01 07:00:00')],
|
||||
'datetime:hour': '06:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 07:00:00'),
|
||||
('datetime', '<', '1916-01-01 08:00:00')],
|
||||
'datetime:hour': '07:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 08:00:00'),
|
||||
('datetime', '<', '1916-01-01 09:00:00')],
|
||||
'datetime:hour': '08:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 09:00:00'),
|
||||
('datetime', '<', '1916-01-01 10:00:00')],
|
||||
'datetime:hour': '09:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 10:00:00'),
|
||||
('datetime', '<', '1916-01-01 11:00:00')],
|
||||
'datetime:hour': '10:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 11:00:00'),
|
||||
('datetime', '<', '1916-01-01 12:00:00')],
|
||||
'datetime:hour': '11:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 12:00:00'),
|
||||
('datetime', '<', '1916-01-01 13:00:00')],
|
||||
'datetime:hour': '12:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 13:00:00'),
|
||||
('datetime', '<', '1916-01-01 14:00:00')],
|
||||
'datetime:hour': '01:00 01 Jan',
|
||||
'datetime_count': 1,
|
||||
'value': 5
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 14:00:00'),
|
||||
('datetime', '<', '1916-01-01 15:00:00')],
|
||||
'datetime:hour': '02:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 15:00:00'),
|
||||
('datetime', '<', '1916-01-01 16:00:00')],
|
||||
'datetime:hour': '03:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 16:00:00'),
|
||||
('datetime', '<', '1916-01-01 17:00:00')],
|
||||
'datetime:hour': '04:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 17:00:00'),
|
||||
('datetime', '<', '1916-01-01 18:00:00')],
|
||||
'datetime:hour': '05:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 18:00:00'),
|
||||
('datetime', '<', '1916-01-01 19:00:00')],
|
||||
'datetime:hour': '06:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 19:00:00'),
|
||||
('datetime', '<', '1916-01-01 20:00:00')],
|
||||
'datetime:hour': '07:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 20:00:00'),
|
||||
('datetime', '<', '1916-01-01 21:00:00')],
|
||||
'datetime:hour': '08:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 21:00:00'),
|
||||
('datetime', '<', '1916-01-01 22:00:00')],
|
||||
'datetime:hour': '09:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 22:00:00'),
|
||||
('datetime', '<', '1916-01-01 23:00:00')],
|
||||
'datetime:hour': '10:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 23:00:00'),
|
||||
('datetime', '<', '1916-01-02 00:00:00')],
|
||||
'datetime:hour': '11:00 01 Jan',
|
||||
'datetime_count': 1,
|
||||
'value': 7
|
||||
}]
|
||||
|
||||
model_fill = self.Model.with_context(fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime:hour'])
|
||||
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_hour_with_timezones(self):
|
||||
"""Test hour with timezones.
|
||||
|
||||
What we do here is similar to test_with_datetimes_and_groupby_per_hour
|
||||
but with a timezone in the user context.
|
||||
"""
|
||||
self.Model.create({'datetime': '1915-12-31 22:30:00', 'value': 2})
|
||||
self.Model.create({'datetime': '1916-01-01 03:30:00', 'value': 3})
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1915-12-31 22:00:00'),
|
||||
('datetime', '<', '1915-12-31 23:00:00')],
|
||||
'datetime:hour': '04:00 01 Jan',
|
||||
'datetime_count': 1,
|
||||
'value': 2
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1915-12-31 23:00:00'),
|
||||
('datetime', '<', '1916-01-01 00:00:00')],
|
||||
'datetime:hour': '05:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 00:00:00'),
|
||||
('datetime', '<', '1916-01-01 01:00:00')],
|
||||
'datetime:hour': '06:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 01:00:00'),
|
||||
('datetime', '<', '1916-01-01 02:00:00')],
|
||||
'datetime:hour': '07:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 02:00:00'),
|
||||
('datetime', '<', '1916-01-01 03:00:00')],
|
||||
'datetime:hour': '08:00 01 Jan',
|
||||
'datetime_count': 0,
|
||||
'value': False
|
||||
}, {
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '1916-01-01 03:00:00'),
|
||||
('datetime', '<', '1916-01-01 04:00:00')],
|
||||
'datetime:hour': '09:00 01 Jan',
|
||||
'datetime_count': 1,
|
||||
'value': 3
|
||||
}]
|
||||
|
||||
model_fill = self.Model.with_context(tz='Asia/Hovd', fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['datetime', 'value'],
|
||||
groupby=['datetime:hour'])
|
||||
|
||||
self.assertEqual(groups, expected)
|
||||
|
||||
def test_egde_fx_tz(self):
|
||||
"""We test if different edge effect by using a different timezone from the user context
|
||||
|
||||
Suppose a user resident near Hovd, a city in Mongolia. he sells a product
|
||||
at exacltly 4:00 AM on 1st January 2018. Using its context, that datetime
|
||||
is previously converted to UTC time by the ORM so as being stored properly
|
||||
inside the datebase. We are in winter time so 'Asia/Hovd' is UTC+7 :
|
||||
|
||||
'2018-01-01 04:00:00' --> '2017-12-31 21:00:00'
|
||||
|
||||
If that same user groups by datetime, we must ensure that the last
|
||||
displayed date is in January and not in December.
|
||||
"""
|
||||
self.Model.create({'datetime': '2017-12-31 21:00:00', 'value': 42})
|
||||
|
||||
expected = [{
|
||||
'__domain': ['&',
|
||||
('datetime', '>=', '2017-12-31 17:00:00'),
|
||||
('datetime', '<', '2018-01-31 17:00:00')],
|
||||
'datetime': 'January 2018',
|
||||
'datetime_count': 1,
|
||||
'value': 42
|
||||
}]
|
||||
|
||||
model_fill = self.Model.with_context(tz='Asia/Hovd', fill_temporal=True)
|
||||
groups = model_fill.read_group([], fields=['datetime', 'value'], groupby=['datetime'])
|
||||
|
||||
self.assertEqual(groups, expected)
|
||||
+94
-4
@@ -58,6 +58,7 @@ from .tools.func import frame_codeinfo
|
||||
from .tools.misc import CountingStream, DEFAULT_SERVER_DATETIME_FORMAT, DEFAULT_SERVER_DATE_FORMAT
|
||||
from .tools.safe_eval import safe_eval
|
||||
from .tools.translate import _
|
||||
from .tools import date_utils
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
_schema = logging.getLogger(__name__ + '.schema')
|
||||
@@ -1755,6 +1756,89 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
|
||||
|
||||
return list(result.values())
|
||||
|
||||
@api.model
|
||||
def _read_group_fill_temporal(self, data, groupby, aggregated_fields, annotated_groupbys,
|
||||
interval=dateutil.relativedelta.relativedelta(months=1)):
|
||||
"""Helper method for filling date/datetime 'holes' in a result set.
|
||||
|
||||
We are in a use case where data are grouped by a date field (typically
|
||||
months but it could be any other interval) and displayed in a chart.
|
||||
|
||||
Assume we group records by month, and we only have data for August,
|
||||
September and December. By default, plotting the result gives something
|
||||
like:
|
||||
___
|
||||
___ | |
|
||||
| | | |
|
||||
| | ___ | |
|
||||
| || || |
|
||||
|___||___||___|
|
||||
Aug Sep Dec
|
||||
|
||||
The problem is that December data follows immediately September data,
|
||||
which is misleading for the user. Adding explicit zeroes for missing data
|
||||
gives something like:
|
||||
___
|
||||
___ | |
|
||||
| | | |
|
||||
| | ___ | |
|
||||
| || | | |
|
||||
|___||___| ___ ___ |___|
|
||||
Aug Sep Oct Nov Dec
|
||||
|
||||
:param list data: the data containing groups
|
||||
:param list groupby: name of the first group by
|
||||
:param aggregated_fields list: list of aggregated fields in the query
|
||||
:param relativedelta interval: interval between to temporal groups
|
||||
expressed as a relativedelta month by default
|
||||
:rtype: list
|
||||
:return: list
|
||||
"""
|
||||
first_a_gby = annotated_groupbys[0]
|
||||
if not data:
|
||||
return
|
||||
if first_a_gby['type'] not in ('date', 'datetime'):
|
||||
return data
|
||||
interval = first_a_gby['interval']
|
||||
groupby_name = groupby[0]
|
||||
|
||||
# existing non null datetimes
|
||||
existing = [d[groupby_name] for d in data if d[groupby_name]]
|
||||
|
||||
if len(existing) < 2:
|
||||
return data
|
||||
|
||||
# assumption: existing data is sorted by field 'groupby_name'
|
||||
first, last = existing[0], existing[-1]
|
||||
|
||||
expected = list(date_utils.date_range(first, last, interval))
|
||||
|
||||
if len(existing) < len(expected):
|
||||
empty_data = dict.fromkeys(aggregated_fields, False)
|
||||
empty_data['id'] = False
|
||||
empty_data[groupby_name.split(':')[0] + '_count'] = 0
|
||||
|
||||
data = collections.deque(data)
|
||||
|
||||
new_data = []
|
||||
|
||||
# Note: the list 'expected' contains the the dates that should be
|
||||
# represented inside data when it is returned. Notice that 'expected'
|
||||
# is sorted like data, so what we do is empty data progressivly by
|
||||
# popping its elements from first to last. To do this properly, we
|
||||
# need to compare dates together.
|
||||
for dt in expected:
|
||||
if not data[0][groupby_name] or data[0][groupby_name] == dt:
|
||||
new_data.append(data.popleft())
|
||||
else:
|
||||
new_data.append(dict(empty_data, **{groupby_name: dt}))
|
||||
|
||||
assert not data
|
||||
return new_data
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@api.model
|
||||
def _read_group_prepare(self, orderby, aggregated_fields, annotated_groupbys, query):
|
||||
"""
|
||||
@@ -1845,7 +1929,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
|
||||
}
|
||||
if tz_convert:
|
||||
qualified_field = "timezone('%s', timezone('UTC',%s))" % (self._context.get('tz', 'UTC'), qualified_field)
|
||||
qualified_field = "date_trunc('%s', %s)" % (gb_function or 'month', qualified_field)
|
||||
qualified_field = "date_trunc('%s', %s::timestamp)" % (gb_function or 'month', qualified_field)
|
||||
if field_type == 'boolean':
|
||||
qualified_field = "coalesce(%s,false)" % qualified_field
|
||||
return {
|
||||
@@ -1855,7 +1939,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
|
||||
'display_format': display_formats[gb_function or 'month'] if temporal else None,
|
||||
'interval': time_intervals[gb_function or 'month'] if temporal else None,
|
||||
'tz_convert': tz_convert,
|
||||
'qualified_field': qualified_field
|
||||
'qualified_field': qualified_field,
|
||||
}
|
||||
|
||||
@api.model
|
||||
@@ -1984,7 +2068,7 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
|
||||
groupby = [groupby] if isinstance(groupby, pycompat.string_types) else list(OrderedSet(groupby))
|
||||
dt = [
|
||||
f for f in groupby
|
||||
if self._fields[f.split(':')[0]].type in ('date', 'datetime')
|
||||
if self._fields[f.split(':')[0]].type in ('date', 'datetime') # e.g. 'date:month'
|
||||
]
|
||||
|
||||
# iterate on all results and replace the "full" date/datetime value
|
||||
@@ -2100,8 +2184,14 @@ class BaseModel(MetaModel('DummyModel', (object,), {'_register': False})):
|
||||
|
||||
self._read_group_resolve_many2one_fields(fetched_data, annotated_groupbys)
|
||||
|
||||
data = ({k: self._read_group_prepare_data(k,v, groupby_dict) for k,v in r.items()} for r in fetched_data)
|
||||
data = [{k: self._read_group_prepare_data(k, v, groupby_dict) for k, v in r.items()} for r in fetched_data]
|
||||
|
||||
if self.env.context.get('fill_temporal'):
|
||||
data = self._read_group_fill_temporal(data, groupby, aggregated_fields,
|
||||
annotated_groupbys)
|
||||
|
||||
result = [self._read_group_format_result(d, annotated_groupbys, groupby, domain) for d in data]
|
||||
|
||||
if lazy:
|
||||
# Right now, read_group only fill results in lazy mode (by default).
|
||||
# If you need to have the empty groups in 'eager' mode, then the
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import math
|
||||
import calendar
|
||||
from datetime import date, datetime, time
|
||||
|
||||
import pytz
|
||||
from dateutil.relativedelta import relativedelta
|
||||
from . import ustr
|
||||
|
||||
@@ -178,3 +177,43 @@ def json_default(obj):
|
||||
return fields.Datetime.to_string(obj)
|
||||
return fields.Date.to_string(obj)
|
||||
return ustr(obj)
|
||||
|
||||
def date_range(start, end, step=relativedelta(months=1)):
|
||||
"""Date range generator with a step interval.
|
||||
|
||||
:param start datetime: begining date of the range.
|
||||
:param end datetime: ending date of the range.
|
||||
:param step relativedelta: interval of the range.
|
||||
:return: a range of datetime from start to end.
|
||||
:rtype: Iterator[datetime]
|
||||
"""
|
||||
|
||||
are_naive = start.tzinfo is None and end.tzinfo is None
|
||||
are_utc = start.tzinfo == pytz.utc and end.tzinfo == pytz.utc
|
||||
|
||||
# Cases with miscellenous timezone are more complexe because of DST.
|
||||
are_others = start.tzinfo and end.tzinfo and not are_utc
|
||||
|
||||
if are_others:
|
||||
if start.tzinfo.zone != end.tzinfo.zone:
|
||||
raise ValueError("Timezones of start argument and end argument seem inconsistent")
|
||||
|
||||
if not are_naive and not are_utc and not are_others:
|
||||
raise ValueError("Timezones of start argument and end argument mismatch")
|
||||
|
||||
if start > end:
|
||||
raise ValueError("start > end, start date must be before end")
|
||||
|
||||
if start == start + step:
|
||||
raise ValueError("Looks like step is null")
|
||||
|
||||
if start.tzinfo:
|
||||
localize = start.tzinfo.localize
|
||||
else:
|
||||
localize = lambda dt: dt
|
||||
|
||||
dt = start.replace(tzinfo=None)
|
||||
end = end.replace(tzinfo=None)
|
||||
while dt <= end:
|
||||
yield localize(dt)
|
||||
dt = dt + step
|
||||
|
||||
Reference in New Issue
Block a user