[IMP] web: project unity_read

Introduce an optimised way of reading a graph of data from the webclient.

Before this commit:
When reading data from the webclient it could at most read multiple ids of the same model in one RPC.
This mean that when reading x2many or specific information on many2one, that could only be done after the initial read (when the client knows the ids of the comodels) and model by model.

After this commit:
Introduce methods web_read and web_search_read_unity. Both method receive a specificiation for the fields instead of a list of fields. The specification can request fields from the model, as well as follow relations and request fields for each relations, recursively.

Example of web_read specification for account_move
```python
{'name': {}},
{'date': {}},
{'journal_id': {'fields': {'display_name':{}}}
},
...
{'invoice_line_ids' :
    {
        'fields': {
            'journal_id' : {'fields': {'display_name:{}}},
            'move_name' : {},
            ...
            'tax_ids' : {
                fields: {
                    'display_name':{},
                    ...
                }
            }
        }
    }
}
```

Result for this example with 2 invoice lines
```python
{
    'id': 1234,
    'name' : 'invoice name ABC',
    'journal_id: {
        'id': 999,
        'display_name': 'Customer Invoices'
    },
    ...
    'invoice_line_ids': [
        {
            'id': 666,
            'journal_id': {
                'id': 999,
                'display_name': 'Customer Invoices'
            },
            'move_name': 'a move name',
            'tax_ids': [
                {
                    'id': 333,
                    'display_name: "15% tax",
                    ...
                }
            ]
        },
        {
            'id': 667,
            'journal_id': {
                'id': 999,
                'display_name': 'Customer Invoices'
            },
            'move_name': 'another move name',
            'tax_ids': [
                {
                    'id': 334,
                    'display_name: "21% customer tax",
                    ...
                }
            ]
        }
    ]

}
```

closes odoo/odoo#119034

Signed-off-by: Vincent Schippefilt (vsc) <vsc@odoo.com>
This commit is contained in:
Vincent Schippefilt
2023-04-21 08:02:17 +02:00
parent d5c93c624d
commit 7d2baaa0c7
7 changed files with 647 additions and 6 deletions
+91 -6
View File
@@ -1,14 +1,15 @@
# -*- coding: utf-8 -*-
from typing import Dict, List
import babel.dates
import pytz
from lxml import etree
import base64
import json
from odoo import _, _lt, api, fields, models
from odoo.osv.expression import AND, TRUE_DOMAIN, normalize_domain
from odoo.tools import date_utils, lazy
from odoo.tools.misc import get_lang
from odoo.tools import date_utils
from odoo.tools.misc import OrderedSet, get_lang
from odoo.exceptions import UserError
from collections import defaultdict
@@ -51,11 +52,20 @@ class Base(models.AbstractModel):
'length': number of records matching the domain (result of a call to 'search_count')
}
"""
records = self.search_read(domain, fields, offset=offset, limit=limit, order=order)
values_records = self.search_read(domain, fields, offset=offset, limit=limit, order=order)
return self._format_web_search_read_results(domain, values_records, offset, limit, count_limit)
@api.model
def unity_web_search_read(self, domain, specification, offset=0, limit=None, order=None, count_limit=None):
records = self.search_fetch(domain, specification.keys(), offset=offset, limit=limit, order=order)
values_records = records.web_read(specification)
return self._format_web_search_read_results(domain, values_records, offset, limit, count_limit)
def _format_web_search_read_results(self, domain, records, offset=0, limit=None, count_limit=None):
if not records:
return {
'length': 0,
'records': []
'records': [],
}
current_length = len(records) + offset
limit_reached = len(records) == limit
@@ -67,9 +77,84 @@ class Base(models.AbstractModel):
length = current_length
return {
'length': length,
'records': records
'records': records,
}
def web_read(self, specification: Dict[str, Dict]) -> List[Dict]:
fields_to_read = list(specification) or ['id']
if fields_to_read == ['id']:
# if we request to read only the ids, we have them already so we can build the return dictionaries immediately
# this also avoid a call to read on the co-model that might have different access rules
values_list = [{'id': id_} for id_ in self._ids]
else:
values_list: List[Dict] = self.read(fields_to_read, load=None)
if not values_list:
return values_list
for field_name, field_spec in specification.items():
if not field_spec:
continue
field = self._fields[field_name]
if field.type == 'many2one' and 'fields' in field_spec:
co_records = self[field_name]
if 'context' in field_spec:
co_records = co_records.with_context(**field_spec['context'])
extra_fields = dict(field_spec['fields'])
extra_fields.pop('display_name', None)
many2one_data = {
vals['id']: vals
for vals in co_records.web_read(extra_fields)
}
if 'display_name' in field_spec['fields']:
for rec in co_records.sudo():
many2one_data[rec.id]['display_name'] = rec.display_name
for values in values_list:
if not values[field_name]:
continue
values[field_name] = many2one_data[values[field_name]]
elif field.type in ('one2many', 'many2many'):
co_records = self[field_name]
if 'order' in field_spec and field_spec['order']:
co_records = co_records.search([('id', 'in', co_records.ids)], order=field_spec['order'])
order_key = {
co_record.id: index
for index, co_record in enumerate(co_records)
}
for values in values_list:
values[field_name] = sorted(values[field_name], key=order_key.__getitem__)
if 'context' in field_spec:
co_records = co_records.with_context(**field_spec['context'])
if 'fields' in field_spec:
if field_spec.get('limit') is not None:
limit = field_spec['limit']
ids_to_read = OrderedSet(
id_
for values in values_list
for id_ in values[field_name][:limit]
)
co_records = co_records.browse(ids_to_read)
x2many_data = {
vals['id']: vals
for vals in co_records.web_read(field_spec['fields'])
}
for values in values_list:
values[field_name] = [x2many_data.get(id_) or {'id': id_} for id_ in values[field_name]]
return values_list
@api.model
def web_read_group(self, domain, fields, groupby, limit=None, offset=0, orderby=False,
lazy=True, expand=False, expand_limit=None, expand_orderby=False):
+24
View File
@@ -34,3 +34,27 @@ class TestWebSearchRead(common.TransactionCase):
self.assert_web_search_read(2, 2, limit=2, count_limit=2, expected_search_count_called=False)
self.assert_web_search_read(20, 2, limit=2, offset=10, count_limit=20)
self.assert_web_search_read(12, 2, limit=2, offset=10, count_limit=12, expected_search_count_called=False)
def assert_unity_web_search_read(self, expected_length, expected_records_length, expected_search_count_called=True,
**kwargs):
original_search_count = self.ResCurrency.search_count
search_count_called = [False]
def search_count(obj, *method_args, **method_kwargs):
search_count_called[0] = True
return original_search_count(*method_args, **method_kwargs)
with patch('odoo.addons.base.models.res_currency.Currency.search_count', new=search_count):
results = self.ResCurrency.unity_web_search_read(domain=[], specification={'id':{}}, **kwargs)
self.assertEqual(results['length'], expected_length)
self.assertEqual(len(results['records']), expected_records_length)
self.assertEqual(search_count_called[0], expected_search_count_called)
def test_unity_web_search_read(self):
self.assert_unity_web_search_read(self.max, self.max, expected_search_count_called=False)
self.assert_unity_web_search_read(self.max, 2, limit=2)
self.assert_unity_web_search_read(self.max, 2, limit=2, offset=10)
self.assert_unity_web_search_read(2, 2, limit=2, count_limit=2, expected_search_count_called=False)
self.assert_unity_web_search_read(20, 2, limit=2, offset=10, count_limit=20)
self.assert_unity_web_search_read(12, 2, limit=2, offset=10, count_limit=12, expected_search_count_called=False)
@@ -2,3 +2,4 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from . import test_new_api
from . import test_unity_read
@@ -0,0 +1,55 @@
from odoo import models, fields
class Course(models.Model):
_name = 'test_new_api.course'
_description = 'a course'
name = fields.Char('Name')
lesson_ids = fields.One2many('test_new_api.lesson', 'course_id')
author_id = fields.Many2one('test_new_api.person')
private_field = fields.Char(groups="base.group_no_one")
class Lesson(models.Model):
_name = 'test_new_api.lesson'
_description = 'a lesson of a course (a day typically)'
name = fields.Char('Name')
course_id = fields.Many2one('test_new_api.course')
attendee_ids = fields.Many2many('test_new_api.person', 'lesson_ids')
teacher_id = fields.Many2one('test_new_api.person')
date = fields.Date()
def _compute_display_name(self):
"""
use to check that a context has can still have an impact when reading the names of a many2one
"""
for record in self:
if 'special' in self.env.context:
record.display_name = 'special ' + record.name
else:
record.display_name = record.name
class Person(models.Model):
_name = 'test_new_api.person'
_description = 'a person, can be an author, teacher or attendee of a lesson'
name = fields.Char('Name')
lesson_ids = fields.Many2many('test_new_api.lesson', 'course_id')
employer_id = fields.Many2one('test_new_api.employer')
def name_get(self):
"""
use to check that a context has can still have an impact when reading the names of a many2one
"""
return [(record.id,
("particular " if 'particular' in self.env.context else "") +
record.name +
(" special" if 'special' in self.env.context else ""))
for record in self]
class Employer(models.Model):
_name = 'test_new_api.employer'
_description = 'the employer of a person'
name = fields.Char('Name')
employee_ids = fields.One2many('test_new_api.person', 'employer_id')
@@ -103,3 +103,8 @@ access_test_new_api_related_translation_2,access_test_new_api_related_translatio
access_test_new_api_related_translation_3,access_test_new_api_related_translation_3,model_test_new_api_related_translation_3,,1,1,1,1
access_test_new_api_indexed_translation,access_test_new_api_indexed_translation,model_test_new_api_indexed_translation,,1,1,1,1
access_test_new_api_empty_char,access_test_new_api_empty_char,model_test_new_api_empty_char,,1,1,1,1
access_test_new_api_course,access_test_new_api_course,model_test_new_api_course,base.group_user,1,1,1,1
access_test_new_api_lesson,access_test_new_api_lesson,model_test_new_api_lesson,base.group_user,1,1,1,1
access_test_new_api_person,access_test_new_api_person,model_test_new_api_person,base.group_user,1,1,1,1
access_test_new_api_employer,access_test_new_api_employer,model_test_new_api_employer,base.group_user,1,1,1,1
public_access_test_new_api_course,access_test_new_api_course,model_test_new_api_course,base.group_public,1,0,0,0
1 id name model_id:id group_id:id perm_read perm_write perm_create perm_unlink
103 access_test_new_api_related_translation_3 access_test_new_api_related_translation_3 model_test_new_api_related_translation_3 1 1 1 1
104 access_test_new_api_indexed_translation access_test_new_api_indexed_translation model_test_new_api_indexed_translation 1 1 1 1
105 access_test_new_api_empty_char access_test_new_api_empty_char model_test_new_api_empty_char 1 1 1 1
106 access_test_new_api_course access_test_new_api_course model_test_new_api_course base.group_user 1 1 1 1
107 access_test_new_api_lesson access_test_new_api_lesson model_test_new_api_lesson base.group_user 1 1 1 1
108 access_test_new_api_person access_test_new_api_person model_test_new_api_person base.group_user 1 1 1 1
109 access_test_new_api_employer access_test_new_api_employer model_test_new_api_employer base.group_user 1 1 1 1
110 public_access_test_new_api_course access_test_new_api_course model_test_new_api_course base.group_public 1 0 0 0
@@ -14,3 +14,4 @@ from . import test_ui
from . import test_domain
from . import test_schema
from . import test_company_checks
from . import test_unity_read
@@ -0,0 +1,470 @@
from dateutil.relativedelta import relativedelta
from odoo import Command, fields
from odoo.exceptions import AccessError
from odoo.tests.common import TransactionCase, new_test_user
class TestUnityRead(TransactionCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.only_course_user = new_test_user(cls.env, 'no acc', 'base.group_public')
cls.author = cls.env['test_new_api.person'].create({'name': 'ged'})
cls.teacher = cls.env['test_new_api.person'].create({'name': 'aab'})
cls.course = cls.env['test_new_api.course'].create({
'name': 'introduction to OWL',
'author_id': cls.author.id
})
cls.lesson_day1 = cls.env['test_new_api.lesson'].create({
'name': 'first day',
'date': fields.Date.today(),
'course_id': cls.course.id,
'teacher_id': cls.teacher.id,
'attendee_ids': [Command.create({'name': '123'}),
Command.create({'name': '456'}),
Command.create({'name': '789'})]
})
cls.lesson_day2 = cls.env['test_new_api.lesson'].create({
'name': 'second day',
'date': fields.Date.today() + relativedelta(days=1),
'course_id': cls.course.id,
'teacher_id': cls.teacher.id
})
cls.course_no_author = cls.env['test_new_api.course'].create({'name': 'some other course without author'})
cls.env.invalidate_all()
def test_read_add_id(self):
read = self.course.web_read({'display_name': {}})
self.assertEqual(read, [{'id': self.course.id, 'display_name': 'introduction to OWL'}])
def test_read_many2one_gives_id(self):
read = self.course.web_read({'display_name': {}, 'author_id': {}})
self.assertEqual(read, [
{'id': self.course.id,
'display_name': 'introduction to OWL',
'author_id': self.author.id}])
def test_read_a_model_does_only_1_query(self):
with self.assertQueryCount(1):
self.course.web_read({'display_name': {}, 'author_id': {'fields': {}}})
def test_read_many2one_gives_id_in_dictionary(self):
read = self.course.web_read({'display_name': {}, 'author_id': {'fields': {}}})
self.assertEqual(read, [
{'id': self.course.id,
'display_name': 'introduction to OWL',
'author_id': {'id': self.author.id}}])
def test_read_many2one_can_read_extra_fields(self):
read = self.course.web_read({'display_name': {}, 'author_id': {'fields': {'write_date': {}}}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'author_id': {'id': self.author.id, 'write_date': self.author.write_date}
}
])
def test_many2one_query_count(self):
with self.assertQueryCount(1 # 1 query for the search of the domain and read course fields
+ 1): # 1 query to read the data of the author
self.course.unity_web_search_read(domain=(),
specification={'display_name': {}, 'author_id': {'fields': {'write_date': {}}}})
def test_read_many2one_throws_if_it_cannot_read_extra_fields(self):
with self.assertRaises(AccessError):
self.course.with_user(self.only_course_user).web_read(
{
'display_name': {},
'author_id':
{
'fields': {'write_date': {}}
}
})
def test_read_many2one_gives_false_if_no_value(self):
read = self.course_no_author.web_read({'display_name': {}, 'author_id': {}})
self.assertEqual(read, [
{'id': self.course_no_author.id,
'display_name': 'some other course without author',
'author_id': False}])
def test_read_many2one_gives_id_name(self):
read = self.course.web_read({'display_name': {}, 'author_id': {'fields': {'display_name': {}}}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'author_id': {
'id': self.author.id,
'display_name': 'ged'
}
}
])
def test_read_many2one_gives_id_name_even_if_you_dont_have_access(self):
read = self.course.with_user(self.only_course_user).web_read(
{'display_name': {}, 'author_id': {'fields': {'display_name': {}}}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'author_id': {
'id': self.author.id,
'display_name': 'ged'
}
}
])
def test_many2one_respects_context(self):
read = self.course.web_read(
{
'display_name': {},
'author_id':
{
'fields': {'display_name': {}},
'context': {'special': 'absolutely'}
}
})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'author_id': {
'id': self.author.id,
'display_name': 'ged special'
}
}])
def test_multilevel_query_count(self):
author = self.env['test_new_api.person'].create({'name': 'AAA'})
teacher1 = self.env['test_new_api.person'].create({'name': 'BBB'})
teacher2 = self.env['test_new_api.person'].create({'name': 'FFF'})
course = self.env['test_new_api.course'].create({
'name': 'CCC',
'author_id': author.id
})
self.env['test_new_api.lesson'].create({
'name': 'DDD',
'course_id': course.id,
'teacher_id': teacher1.id,
})
self.env['test_new_api.lesson'].create({
'name': 'EEE',
'course_id': course.id,
'teacher_id': teacher2.id
})
self.env.invalidate_all()
with self.assertQueryCount(1 # read the course with author id
+ 1 # read the lessons of the course
+ 1 # read the author name of course
+ 1 # ids of the teachers of each lesson
+ 1): # read the teacher name of each lessons in one query
course.web_read(
{
'display_name': {},
'author_id': {'fields': {'display_name': {}}},
'lesson_ids':
{
'fields':
{
'teacher_id': {'fields': {'display_name': {}}}
}
}
})
def test_that_contexts_of_many2one_impacts_each_other(self):
read = self.course.web_read(
{
'display_name': {},
'author_id':
{
'fields': {'display_name': {}},
'context': {'special': 'absolutely'}
},
'lesson_ids':
{
'fields':
{
'teacher_id':
{
'fields': {'display_name': {}},
'context': {'particular': 'definitely'}
}
}
}
})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'author_id': {'id': self.author.id, 'display_name': 'ged special'},
'lesson_ids': [
{
'id': self.lesson_day1.id,
'teacher_id': {'id': self.teacher.id, 'display_name': 'particular aab'}
}, {
'id': self.lesson_day2.id,
'teacher_id': {'id': self.teacher.id, 'display_name': 'particular aab'}
},
],
}])
def test_read_one2many_gives_ids(self):
read = self.course.web_read({'display_name': {}, 'lesson_ids': {}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids': [self.lesson_day1.id, self.lesson_day2.id]
}])
def test_specify_fields_one2many(self):
read = self.course.web_read(
{
'display_name': {},
'lesson_ids':
{
'fields': {'display_name': {}}
}
})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids':
[
{'id': self.lesson_day1.id, 'display_name': 'first day'},
{'id': self.lesson_day2.id, 'display_name': 'second day'}
],
}])
def test_one2many_context_have_no_impact_on_name(self):
read = self.course.web_read(
{
'name': {},
'lesson_ids':
{
'fields': {'name': {}},
'context': {'special': 'absolutely'}
}
})
self.assertEqual(read, [
{
'id': self.course.id,
'name': 'introduction to OWL',
'lesson_ids':
[
{'id': self.lesson_day1.id, 'name': 'first day'},
{'id': self.lesson_day2.id, 'name': 'second day'},
]
}])
def test_one2many_respects_context(self):
read = self.course.web_read(
{
'display_name': {},
'lesson_ids':
{
'fields': {'display_name': {}},
'context': {'special': 'absolutely'}
}
})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids':
[
{'id': self.lesson_day1.id, 'display_name': 'special first day'},
{'id': self.lesson_day2.id, 'display_name': 'special second day'}
],
}])
def test_read_many2many_gives_ids(self):
with self.assertQueryCount(1 # 1 query for course
+ 1 # 1 query for the lessons
+ 1): # 1 query for the attendees ids
read = self.course.web_read({'display_name': {},
'lesson_ids': {
'fields': {
'attendee_ids': {}
}
}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids':
[
{'id': self.lesson_day1.id, 'attendee_ids': [*self.lesson_day1.attendee_ids._ids]},
{'id': self.lesson_day2.id, 'attendee_ids': []}
],
}])
def test_specify_fields_many2many(self):
read = self.course.web_read({'display_name': {},
'lesson_ids': {
'fields': {
'attendee_ids': {
'fields': {
'display_name': {}
}
}
}
}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids':
[
{
'id': self.lesson_day1.id,
'attendee_ids':
[
{'id': self.lesson_day1.attendee_ids._ids[0], 'display_name': '123'},
{'id': self.lesson_day1.attendee_ids._ids[1], 'display_name': '456'},
{'id': self.lesson_day1.attendee_ids._ids[2], 'display_name': '789'}
],
},
{'id': self.lesson_day2.id, 'attendee_ids': []}
]
}])
def test_many2many_respects_limit(self):
read = self.course.web_read({'display_name': {},
'lesson_ids': {
'fields': {
'attendee_ids': {
'limit': 2,
'fields': {
'display_name': {}
}
}
}
}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids':
[
{
'id': self.lesson_day1.id,
'attendee_ids':
[
{'id': self.lesson_day1.attendee_ids._ids[0], 'display_name': '123'},
{'id': self.lesson_day1.attendee_ids._ids[1], 'display_name': '456'},
{'id': self.lesson_day1.attendee_ids._ids[2]}
],
},
{'id': self.lesson_day2.id, 'attendee_ids': []}
]
}])
def test_many2many_limit_has_no_effect_when_no_field_requested(self):
read = self.course.web_read({'display_name': {},
'lesson_ids': {
'fields': {
'attendee_ids': {
'limit': 2,
}
}
}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids':
[
{
'id': self.lesson_day1.id,
'attendee_ids':
[
self.lesson_day1.attendee_ids._ids[0],
self.lesson_day1.attendee_ids._ids[1],
self.lesson_day1.attendee_ids._ids[2],
],
},
{'id': self.lesson_day2.id, 'attendee_ids': []}
]
}])
def test_many2many_order_has_effect_when_no_field_requested(self):
read = self.course.web_read({'display_name': {},
'lesson_ids': {
'order': 'name desc'
}})
self.assertEqual(read, [
{
'id': self.course.id,
'display_name': 'introduction to OWL',
'lesson_ids':
[
self.lesson_day2.id,
self.lesson_day1.id
]
}])
def test_many2many_limits_with_deleted_records(self):
# should we ignore this ? in the python we will not use the limits, and in the RPC we won't delete and read in the same transaction with limits
pass
def test_many2many_respects_order(self):
read = self.course.web_read(
{
'name': {},
'lesson_ids':
{
'fields': {'name': {}},
'order': 'name desc'
}
})
self.assertEqual(read, [
{
'id': self.course.id,
'name': 'introduction to OWL',
'lesson_ids':
[
{'id': self.lesson_day2.id, 'name': 'second day'},
{'id': self.lesson_day1.id, 'name': 'first day'},
]
}])
def test_many2many_order_increases_query_count(self):
with self.assertQueryCount(3):
self.course.web_read(
{
'name': {},
'lesson_ids':
{
'fields': {'name': {}},
}
})
self.env.invalidate_all()
with self.assertQueryCount(4):
self.course.web_read(
{
'name': {},
'lesson_ids':
{
'fields': {'name': {}},
'order': 'name desc'
}
})