pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FIX] project: improve performance of burndown chart report

Browse Source 
Before this commit, for the db with old active tasks and many
projects and tasks. The report takes several hours to execute the sql view.
Because we take the create date of the oldest task as the begin date
of each generate_series to calculate the different group bys (day, week,
month, quarter and year) until CURRENT_DATE.

This commit hugely improves the performance of the sql view execution
for the burndown chart report.

closes odoo/odoo#72664

X-original-commit: 89726a22862cf14f7b88e130fb79ddc74bffff5e
Signed-off-by: LTU-Odoo <IT-Ideas@users.noreply.github.com>
Co-authored-by: Nicolas Seinlet (NSE) <nse@openerp.com>
Co-authored-by: Thibault Libioulle (TLE) <tle@odoo.com>
Co-authored-by: Laurent Stukkens (LTU) <ltu@odoo.com>
This commit is contained in:
Xavier BOL (xbo)
2021-06-24 07:22:52 +00:00
co-authored by Nicolas Seinlet Thibault Libioulle Laurent Stukkens
parent 49d311e725
commit f6ab481ad9
3 changed files with 320 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/project/report/project_task_burndown_chart_report.py
+141 -198
View File
@@ -49,204 +49,147 @@ class ReportProjectTaskBurndownChart(models.Model):
def init(self):
query = """
WITH change_stage_tracking AS (
SELECT mm.id as id,
pt.id as task_id,
pt.project_id,
pt.display_project_id,
pt.create_date as date_begin,
mm.date as date_end,
current_stage.id as stage_id,
next_stage.id as next_stage_id,
pt.user_id,
pt.date_assign,
pt.date_deadline,
pt.partner_id
FROM mail_message mm
INNER JOIN mail_tracking_value mtv
ON mm.id = mtv.mail_message_id
INNER JOIN ir_model_fields imf
ON mtv.field = imf.id
AND imf.model = 'project.task'
AND imf.name = 'stage_id'
INNER JOIN project_task_type current_stage
ON mtv.old_value_integer = current_stage.id
INNER JOIN project_task_type next_stage
ON mtv.new_value_integer = next_stage.id
INNER JOIN project_task pt
ON mm.res_id = pt.id
WHERE mm.model = 'project.task'
AND mm.message_type = 'notification'
AND pt.active
),
all_stage_changes AS (
SELECT current_change.id,
current_change.task_id,
current_change.project_id,
current_change.display_project_id,
CASE WHEN previous_change.id IS NULL THEN current_change.date_begin ELSE previous_change.date_end END as date_begin,
current_change.date_end,
current_change.stage_id,
current_change.next_stage_id,
current_change.user_id,
current_change.date_assign,
current_change.date_deadline,
current_change.partner_id
FROM change_stage_tracking current_change
LEFT JOIN change_stage_tracking previous_change
ON previous_change.next_stage_id = current_change.stage_id
AND previous_change.task_id = current_change.task_id
),
all_moves_stage_task AS (
SELECT pt.id AS task_id,
pt.project_id,
pt.display_project_id,
pt.stage_id,
CASE WHEN last_change.id IS NULL THEN pt.create_date ELSE last_change.date_end END AS date_begin,
(CURRENT_DATE + interval '1 year')::date AS date_end,
pt.user_id,
pt.date_assign,
pt.date_deadline,
pt.partner_id
FROM project_task pt
LEFT JOIN all_stage_changes last_change
ON pt.id = last_change.task_id
AND pt.stage_id = last_change.next_stage_id
UNION
SELECT task_id,
project_id,
display_project_id,
stage_id,
date_begin,
date_end,
user_id,
date_assign,
date_deadline,
partner_id
FROM all_stage_changes
),
all_moves_by_day AS (
SELECT project_id,
display_project_id,
stage_id,
date_begin,
date_end,
d as date,
date_trunc('month', d) AS date_month,
date_trunc('week', d) AS date_week,
date_trunc('year', d) + CAST((extract(quarter from d) - 1) * 3 || ' months' AS interval) AS date_quarter,
date_trunc('year', d) AS date_year,
task_id,
user_id,
date_assign,
date_deadline,
partner_id
FROM all_moves_stage_task
CROSS JOIN generate_series(
(
SELECT date_trunc('day', min(create_date))
FROM project_task
WHERE active
),
(CURRENT_DATE + interval '1 day')::date,
'1 day'
) d
WHERE date_begin <= d
AND date_end > d
),
burndown_chart AS (
SELECT DISTINCT project_id,
display_project_id,
stage_id,
date,
user_id,
date_assign,
date_deadline,
partner_id,
'day' AS group_by,
task_id
FROM all_moves_by_day
WHERE date_begin <= date
AND date_end > date
UNION ALL
SELECT DISTINCT project_id,
display_project_id,
stage_id,
date_week AS date,
user_id,
date_assign,
date_deadline,
partner_id,
'week' AS group_by,
task_id
FROM all_moves_by_day
WHERE date_trunc('week', date_begin) <= date_week
AND date_trunc('week', date_end) > date_week
UNION ALL
SELECT DISTINCT project_id,
display_project_id,
stage_id,
date_month AS date,
user_id,
date_assign,
date_deadline,
partner_id,
'month' AS group_by,
task_id
FROM all_moves_by_day
WHERE date_trunc('month', date_begin) <= date_month
AND date_trunc('month', date_end) > date_month
UNION ALL
SELECT DISTINCT project_id,
display_project_id,
stage_id,
date_quarter AS date,
user_id,
date_assign,
date_deadline,
partner_id,
'quarter' AS group_by,
task_id
FROM all_moves_by_day
WHERE date_trunc('month', date_begin) <= date_quarter
AND date_trunc('month', date_end) > date_quarter
UNION ALL
SELECT DISTINCT project_id,
display_project_id,
stage_id,
date_year AS date,
user_id,
date_assign,
date_deadline,
partner_id,
'year' AS group_by,
task_id
FROM all_moves_by_day
WHERE date_trunc('year', date_begin) <= date_year
AND date_trunc('year', date_end) > date_year
)
SELECT row_number() OVER (
ORDER BY project_id,
display_project_id,
stage_id,
date,
user_id,
date_assign,
date_deadline,
partner_id,
group_by
) AS id,
project_id,
display_project_id,
stage_id,
date,
user_id,
date_assign,
date_deadline,
partner_id,
group_by AS date_group_by,
1 AS nb_tasks
FROM burndown_chart
WITH all_moves_stage_task AS (
-- Here we compute all previous stage in tracking values
-- We're missing the last reached stage
-- And the tasks without any stage change (which, by definition, are at the last stage)
SELECT pt.project_id,
pt.id as task_id,
pt.display_project_id,
COALESCE(LAG(mm.date) OVER (PARTITION BY mm.res_id ORDER BY mm.id), pt.create_date) as date_begin,
mm.date as date_end,
mtv.old_value_integer as stage_id,
pt.user_id,
pt.date_assign,
pt.date_deadline,
pt.partner_id
FROM project_task pt
JOIN mail_message mm ON mm.res_id = pt.id
AND mm.message_type = 'notification'
AND mm.model = 'project.task'
JOIN mail_tracking_value mtv ON mm.id = mtv.mail_message_id
JOIN ir_model_fields imf ON mtv.field = imf.id
AND imf.model = 'project.task'
AND imf.name = 'stage_id'
WHERE pt.active
--We compute the last reached stage
UNION ALL
SELECT pt.project_id,
pt.id as task_id,
pt.display_project_id,
COALESCE(md.date, pt.create_date) as date_begin,
(CURRENT_DATE + interval '1 month')::date as date_end,
pt.stage_id,
pt.user_id,
pt.date_assign,
pt.date_deadline,
pt.partner_id
FROM project_task pt
LEFT JOIN LATERAL (SELECT mm.date
FROM mail_message mm
JOIN mail_tracking_value mtv ON mm.id = mtv.mail_message_id
JOIN ir_model_fields imf ON mtv.field = imf.id
AND imf.model = 'project.task'
AND imf.name = 'stage_id'
WHERE mm.res_id = pt.id
AND mm.message_type = 'notification'
AND mm.model = 'project.task'
ORDER BY mm.id DESC
FETCH FIRST ROW ONLY) md ON TRUE
WHERE pt.active
)
SELECT (task_id*10^7 + 10^6 + to_char(d, 'YYMMDD')::integer)::bigint as id,
project_id,
task_id,
display_project_id,
stage_id,
d as date,
user_id,
date_assign,
date_deadline,
partner_id,
'day' AS date_group_by,
1 AS nb_tasks
FROM all_moves_stage_task t
JOIN LATERAL generate_series(t.date_begin, t.date_end-interval '1 day', '1 day') d ON TRUE
UNION ALL
SELECT (task_id*10^7 + 2*10^6 + to_char(d, 'YYMMDD')::integer)::bigint as id,
project_id,
task_id,
display_project_id,
stage_id,
date_trunc('week', d) as date,
user_id,
date_assign,
date_deadline,
partner_id,
'week' AS date_group_by,
1 AS nb_tasks
FROM all_moves_stage_task t
JOIN LATERAL generate_series(t.date_begin, t.date_end, '1 week') d ON TRUE
WHERE date_trunc('week', t.date_begin) <= date_trunc('week', d)
AND date_trunc('week', t.date_end) > date_trunc('week', d)
UNION ALL
SELECT (task_id*10^7 + 3*10^6 + to_char(d, 'YYMMDD')::integer)::bigint as id,
project_id,
task_id,
display_project_id,
stage_id,
date_trunc('month', d) as date,
user_id,
date_assign,
date_deadline,
partner_id,
'month' AS date_group_by,
1 AS nb_tasks
FROM all_moves_stage_task t
JOIN LATERAL generate_series(t.date_begin, t.date_end, '1 month') d ON TRUE
WHERE date_trunc('month', t.date_begin) <= date_trunc('month', d)
AND date_trunc('month', t.date_end) > date_trunc('month', d)
UNION ALL
SELECT (task_id*10^7 + 4*10^6 + to_char(d, 'YYMMDD')::integer)::bigint as id,
project_id,
task_id,
display_project_id,
stage_id,
date_trunc('quarter', d) as date,
user_id,
date_assign,
date_deadline,
partner_id,
'quarter' AS date_group_by,
1 AS nb_tasks
FROM all_moves_stage_task t
JOIN LATERAL generate_series(t.date_begin, t.date_end, '3 month') d ON TRUE
WHERE date_trunc('quarter', t.date_begin) <= date_trunc('quarter', d)
AND date_trunc('quarter', t.date_end) > date_trunc('quarter', d)
UNION ALL
SELECT (task_id*10^7 + 5*10^6 + to_char(d, 'YYMMDD')::integer)::bigint as id,
project_id,
task_id,
display_project_id,
stage_id,
date_trunc('year', d) as date,
user_id,
date_assign,
date_deadline,
partner_id,
'year' AS date_group_by,
1 AS nb_tasks
FROM all_moves_stage_task t
JOIN LATERAL generate_series(t.date_begin, t.date_end, '1 year') d ON TRUE
WHERE date_trunc('year', t.date_begin) <= date_trunc('year', d)
AND date_trunc('year', t.date_end) > date_trunc('year', d)
"""
tools.drop_view_if_exists(self.env.cr, self._table)
addons/project/tests/__init__.py
+1
View File
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
from . import test_access_rights
from . import test_burndown_chart
from . import test_project_base
from . import test_project_config
from . import test_project_flow
addons/project/tests/test_burndown_chart.py
+178
View File
@@ -0,0 +1,178 @@
# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from freezegun import freeze_time
from datetime import datetime
from odoo import Command
from odoo.tests.common import TransactionCase
class TestBurndownChart(TransactionCase):
def set_create_date(self, table, res_id, create_date):
self.env.cr.execute("UPDATE {} SET create_date=%s WHERE id=%s".format(table), (create_date, res_id))
def test_burndown_chart(self):
current_year = datetime.now().year
create_date = datetime(current_year - 1, 1, 1)
kanban_state_vals = {
"legend_blocked": 'Blocked',
"legend_done": 'Ready',
"legend_normal": 'In Progress'
}
Stage = self.env['project.task.type']
todo_stage = Stage.create({
'sequence': 1,
'name': 'TODO',
**kanban_state_vals,
})
self.set_create_date('project_task_type', todo_stage.id, create_date)
in_progress_stage = Stage.create({
'sequence': 10,
'name': 'In Progress',
**kanban_state_vals,
})
self.set_create_date('project_task_type', in_progress_stage.id, create_date)
testing_stage = Stage.create({
'sequence': 20,
'name': 'Testing',
**kanban_state_vals,
})
self.set_create_date('project_task_type', testing_stage.id, create_date)
done_stage = Stage.create({
'sequence': 30,
'name': 'Done',
**kanban_state_vals,
})
self.set_create_date('project_task_type', done_stage.id, create_date)
stages = todo_stage + in_progress_stage + testing_stage + done_stage
project = self.env['project.project'].create({
'name': 'Burndown Chart Test',
'privacy_visibility': 'employees',
'alias_name': 'project+burndown_chart',
'type_ids': [Command.link(stage_id) for stage_id in stages.ids],
})
self.set_create_date('project_project', project.id, create_date)
project.invalidate_cache()
task_a = self.env['project.task'].create({
'name': 'Task A',
'priority': 0,
'project_id': project.id,
'stage_id': todo_stage.id,
})
self.set_create_date('project_task', task_a.id, create_date)
task_b = task_a.copy({
'name': 'Task B',
})
self.set_create_date('project_task', task_b.id, create_date)
task_c = task_a.copy({
'name': 'Task C',
})
self.set_create_date('project_task', task_c.id, create_date)
task_d = task_a.copy({
'name': 'Task D',
})
self.set_create_date('project_task', task_d.id, create_date)
task_e = task_a.copy({
'name': 'Task E',
})
self.set_create_date('project_task', task_e.id, create_date)
# Create a new task to check if a task without changing its stage is taken into account
task_f = self.env['project.task'].create({
'name': 'Task F',
'priority': 0,
'project_id': project.id,
'stage_id': todo_stage.id,
})
self.set_create_date('project_task', task_f.id, datetime(current_year - 1, 12, 20))
# Precommit to have the records in db and allow to rollback at the end of test
self.env.cr.precommit.run()
with freeze_time('%s-02-10' % (current_year - 1)):
(task_a + task_b).write({'stage_id': in_progress_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-02-20' % (current_year - 1)):
task_c.write({'stage_id': in_progress_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-03-15' % (current_year - 1)):
(task_d + task_e).write({'stage_id': in_progress_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-04-10' % (current_year - 1)):
(task_a + task_b).write({'stage_id': testing_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-05-12' % (current_year - 1)):
task_c.write({'stage_id': testing_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-06-25' % (current_year - 1)):
task_d.write({'stage_id': testing_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-07-25' % (current_year - 1)):
task_e.write({'stage_id': testing_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-08-01' % (current_year - 1)):
task_a.write({'stage_id': done_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-09-10' % (current_year - 1)):
task_b.write({'stage_id': done_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-10-05' % (current_year - 1)):
task_c.write({'stage_id': done_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-11-25' % (current_year - 1)):
task_d.write({'stage_id': done_stage.id})
self.env.cr.precommit.run()
with freeze_time('%s-12-12' % (current_year - 1)):
task_e.write({'stage_id': done_stage.id})
self.env.cr.precommit.run()
read_group_result = self.env['project.task.burndown.chart.report'].with_context(fill_temporal=True).read_group([('project_id', '=', project.id), ('display_project_id', '!=', False)], ['date', 'stage_id', 'nb_tasks'], ['date:month', 'stage_id'], lazy=False)
read_group_result_dict = {(res['date:month'], res['stage_id'][0]): res['nb_tasks'] for res in read_group_result}
stages_dict = {stage.id: stage.name for stage in stages}
expected_dict = {
('January %s' % (current_year - 1), todo_stage.id): 5,
('February %s' % (current_year - 1), todo_stage.id): 2,
('February %s' % (current_year - 1), in_progress_stage.id): 3,
('March %s' % (current_year - 1), in_progress_stage.id): 5,
('April %s' % (current_year - 1), in_progress_stage.id): 3,
('April %s' % (current_year - 1), testing_stage.id): 2,
('May %s' % (current_year - 1), in_progress_stage.id): 2,
('May %s' % (current_year - 1), testing_stage.id): 3,
('June %s' % (current_year - 1), in_progress_stage.id): 1,
('June %s' % (current_year - 1), testing_stage.id): 4,
('July %s' % (current_year - 1), testing_stage.id): 5,
('August %s' % (current_year - 1), testing_stage.id): 4,
('August %s' % (current_year - 1), done_stage.id): 1,
('September %s' % (current_year - 1), testing_stage.id): 3,
('September %s' % (current_year - 1), done_stage.id): 2,
('October %s' % (current_year - 1), testing_stage.id): 2,
('October %s' % (current_year - 1), done_stage.id): 3,
('November %s' % (current_year - 1), testing_stage.id): 1,
('November %s' % (current_year - 1), done_stage.id): 4,
('December %s' % (current_year - 1), done_stage.id): 5,
('December %s' % (current_year - 1), todo_stage.id): 1,
('January %s' % (current_year), done_stage.id): 5,
('January %s' % (current_year), todo_stage.id): 1,
}
for (month, stage_id), nb_tasks in read_group_result_dict.items():
# when we don't found any record in the dict then we are in the current_year
# and the number of tasks should always be 5 in Done stage and 1 in Todo Stage
# since we have created the last task without changing its stage.
expected_nb_tasks = expected_dict.get((month, stage_id), 5 if stage_id != todo_stage.id else 1)
self.assertEqual(
nb_tasks,
expected_nb_tasks,
'In %s, the number of tasks should be equal to %s in %s stage.' % (month, expected_nb_tasks, stages_dict.get(stage_id, 'Unknown'))
)