From 595aa248433246959a5fa9288e477091701c6a35 Mon Sep 17 00:00:00 2001 From: Xavier-Do Date: Wed, 26 Apr 2023 10:07:52 +0000 Subject: [PATCH] [IMP] registry: multiple ormcache One of the main issue with ormcache is that the invalidation clears everything, meaning that some value, slow to compute but with a long lifetime, can be removed from the cache because an easy to invalidate value is cleared, like after writting or creating a product has an example. Most example in the code will try to invalidate the cache of the models doing something like `env['ir.qweb'].clear_caches()` but it is finally equivalent to `env.registry.clear_cache()`, and cross worker. The idea is to have multiple cache, maybe with specific sizes for a specific purpose. Having one per model is maybe a bad idea because it will be difficult to size the LRU correcly, and it is too dynamic. Checking invalidation may be expensive. The proposed solution is closed allow a limited number of named caches, using onse sequence per cache. This is actually close to the cache_longterm. We want to discourage using a specific cache for one use case in the buisness code. Adding a cache shouldn't be something easy, doable in stable. Note that we could also change the invalisation mecanism using an insert only table. We an check the sequence of this table, but also fetch all invalidation messages. Another possible improvement, especially if we have more than x cache is to have a global sequence, checking signaling would mean to check the main sequence, and only the other ones if the main one changed. Note that this poc is inspired from the long term cache but not all use case where applie yet. Part-of: odoo/odoo#119813 --- addons/auth_totp/tests/test_totp.py | 4 +- addons/auth_totp_portal/tests/test_tour.py | 4 +- addons/base_import/models/base_import.py | 4 +- addons/bus/tests/test_assetsbundle.py | 2 +- addons/google_calendar/models/google_sync.py | 4 +- addons/hr_timesheet/tests/test_performance.py | 3 +- addons/http_routing/models/ir_http.py | 13 +- addons/mail/models/mail_message_subtype.py | 6 +- addons/product/models/product_product.py | 8 +- addons/test_mail/tests/test_message_track.py | 2 +- addons/web/controllers/home.py | 2 +- addons/web/tests/test_assets.py | 2 +- addons/web_editor/controllers/main.py | 2 +- addons/web_editor/models/assets.py | 2 +- addons/website/models/ir_http.py | 41 +++-- addons/website/models/ir_ui_view.py | 8 +- addons/website/models/website.py | 8 +- addons/website/models/website_menu.py | 6 +- addons/website/models/website_page.py | 4 +- addons/website/models/website_rewrite.py | 4 +- addons/website/tests/test_http_endpoint.py | 2 +- addons/website/tests/test_views.py | 2 +- .../models/crm_reveal_rule.py | 6 +- .../website_sale/models/product_pricelist.py | 7 +- odoo/addons/base/models/assetsbundle.py | 2 +- odoo/addons/base/models/decimal_precision.py | 6 +- odoo/addons/base/models/ir_actions.py | 10 +- odoo/addons/base/models/ir_asset.py | 8 +- .../addons/base/models/ir_config_parameter.py | 6 +- odoo/addons/base/models/ir_default.py | 6 +- odoo/addons/base/models/ir_http.py | 62 +++---- odoo/addons/base/models/ir_model.py | 27 +-- odoo/addons/base/models/ir_module.py | 2 +- odoo/addons/base/models/ir_property.py | 8 +- odoo/addons/base/models/ir_qweb.py | 15 +- odoo/addons/base/models/ir_rule.py | 11 +- odoo/addons/base/models/ir_ui_menu.py | 6 +- odoo/addons/base/models/ir_ui_view.py | 12 +- odoo/addons/base/models/res_company.py | 8 +- odoo/addons/base/models/res_country.py | 4 +- odoo/addons/base/models/res_lang.py | 6 +- odoo/addons/base/models/res_users.py | 14 +- odoo/addons/base/tests/test_ir_actions.py | 6 +- odoo/addons/base/tests/test_ir_http.py | 4 +- odoo/addons/base/tests/test_ormcache.py | 64 ++++++- odoo/addons/base/tests/test_qweb.py | 4 +- odoo/addons/base/tests/test_res_users.py | 6 +- odoo/addons/base/tests/test_user_has_group.py | 10 +- odoo/addons/base/tests/test_views.py | 12 +- .../test_access_rights/tests/test_ir_rules.py | 4 +- .../tests/test_assetsbundle.py | 20 +- odoo/addons/test_impex/tests/test_load.py | 6 +- .../test_new_api/tests/test_new_fields.py | 2 +- odoo/http.py | 2 +- odoo/models.py | 5 +- odoo/modules/loading.py | 2 - odoo/modules/registry.py | 172 ++++++++++++------ odoo/tests/common.py | 9 +- odoo/tools/cache.py | 102 ++--------- odoo/tools/misc.py | 5 + 60 files changed, 414 insertions(+), 380 deletions(-) diff --git a/addons/auth_totp/tests/test_totp.py b/addons/auth_totp/tests/test_totp.py index d0b2b1dc34c..d287a5ef4ec 100644 --- a/addons/auth_totp/tests/test_totp.py +++ b/addons/auth_totp/tests/test_totp.py @@ -34,14 +34,14 @@ class TestTOTP(HttpCase): return totp.generate(time.time() + 30).token # because not preprocessed by ControllerType metaclass totp_hook.routing_type = 'json' - self.env['ir.http']._clear_routing_map() + self.env.registry.clear_cache('routing') # patch Home to add test endpoint Home.totp_hook = http.route('/totphook', type='json', auth='none')(totp_hook) # remove endpoint and destroy routing map @self.addCleanup def _cleanup(): del Home.totp_hook - self.env['ir.http']._clear_routing_map() + self.env.registry.clear_cache('routing') def test_totp(self): # 1. Enable 2FA diff --git a/addons/auth_totp_portal/tests/test_tour.py b/addons/auth_totp_portal/tests/test_tour.py index 242c9f0dcea..1edac06344a 100644 --- a/addons/auth_totp_portal/tests/test_tour.py +++ b/addons/auth_totp_portal/tests/test_tour.py @@ -31,12 +31,12 @@ class TestTOTPortal(HttpCase): totp_hook.routing_type = 'json' # patch Home to add test endpoint Home.totp_hook = http.route('/totphook', type='json', auth='none')(totp_hook) - self.env['ir.http']._clear_routing_map() + self.env.registry.clear_cache('routing') # remove endpoint and destroy routing map @self.addCleanup def _cleanup(): del Home.totp_hook - self.env['ir.http']._clear_routing_map() + self.env.registry.clear_cache('routing') self.start_tour('/my/security', 'totportal_tour_setup', login='portal') # also disables totp otherwise we can't re-login diff --git a/addons/base_import/models/base_import.py b/addons/base_import/models/base_import.py index 1dce1e4d36e..cd0d3cf6191 100644 --- a/addons/base_import/models/base_import.py +++ b/addons/base_import/models/base_import.py @@ -1339,7 +1339,9 @@ class Import(models.TransientModel): if dryrun: self._cr.execute('ROLLBACK TO SAVEPOINT import') # cancel all changes done to the registry/ormcache - self.pool.clear_caches() + # we need to clear the cache in case any created id was added to an ormcache and would be missing afterward + self.pool.clear_all_caches() + # don't propagate to other workers since it was rollbacked self.pool.reset_changes() else: self._cr.execute('RELEASE SAVEPOINT import') diff --git a/addons/bus/tests/test_assetsbundle.py b/addons/bus/tests/test_assetsbundle.py index d520c1b4aff..e5a246f0cee 100644 --- a/addons/bus/tests/test_assetsbundle.py +++ b/addons/bus/tests/test_assetsbundle.py @@ -25,7 +25,7 @@ class BusWebTests(odoo.tests.HttpCase): ]) # start from a clean slate self.env['ir.attachment'].search(domain).unlink() - self.env.registry._clear_cache() + self.env.registry.clear_cache() sendones = [] def patched_sendone(self, channel, notificationType, message): diff --git a/addons/google_calendar/models/google_sync.py b/addons/google_calendar/models/google_sync.py index 846dd144069..49f5e0edeb4 100644 --- a/addons/google_calendar/models/google_sync.py +++ b/addons/google_calendar/models/google_sync.py @@ -64,7 +64,7 @@ class GoogleSync(models.AbstractModel): def write(self, vals): google_service = GoogleCalendarService(self.env['google.service']) if 'google_id' in vals: - self._event_ids_from_google_ids.clear_cache(self) + self.env.registry.clear_cache() # _event_ids_from_google_ids synced_fields = self._get_google_synced_fields() if 'need_sync' not in vals and vals.keys() & synced_fields and not self.env.user.google_synchronization_stopped: vals['need_sync'] = True @@ -79,7 +79,7 @@ class GoogleSync(models.AbstractModel): @api.model_create_multi def create(self, vals_list): if any(vals.get('google_id') for vals in vals_list): - self._event_ids_from_google_ids.clear_cache(self) + self.env.registry.clear_cache() # _event_ids_from_google_ids if self.env.user.google_synchronization_stopped: for vals in vals_list: vals.update({'need_sync': False}) diff --git a/addons/hr_timesheet/tests/test_performance.py b/addons/hr_timesheet/tests/test_performance.py index bd128cf6551..99c5572d327 100644 --- a/addons/hr_timesheet/tests/test_performance.py +++ b/addons/hr_timesheet/tests/test_performance.py @@ -12,8 +12,7 @@ class TestPerformanceTimesheet(TestCommonTimesheet): 'project_id': project.id, } for i in range(17) for project in projects]) self.env.invalidate_all() - projects.clear_caches() - tasks.clear_caches() + self.env.registry.clear_cache() with self.assertQueryCount(7): self.env['account.analytic.line']._timesheet_preprocess([ {'task_id': task.id} for task in tasks for _i in range(10) diff --git a/addons/http_routing/models/ir_http.py b/addons/http_routing/models/ir_http.py index 4b8f9a8c62a..63ae8c68fbd 100644 --- a/addons/http_routing/models/ir_http.py +++ b/addons/http_routing/models/ir_http.py @@ -185,15 +185,15 @@ def url_for(url_from, lang_code=None, no_rewrite=False): :param no_rewrite: don't try to match route with website.rewrite. ''' new_url = False - + rewrite = not no_rewrite # don't try to match route if we know that no rewrite has been loaded. routing = getattr(request, 'website_routing', None) # not modular, but not overridable - if not getattr(request.env['ir.http'], '_rewrite_len', {}).get(routing): - no_rewrite = True + if not request.env['ir.http']._rewrite_len(routing): + rewrite = False path, _, qs = (url_from or '').partition('?') - if (not no_rewrite and path and ( + if (rewrite and path and ( len(path) > 1 and path.startswith('/') and '/static/' not in path @@ -660,7 +660,7 @@ class IrHttp(models.AbstractModel): return response @api.model - @tools.ormcache('path', 'query_args') + @tools.ormcache('path', 'query_args', cache='routing.rewrites') def url_rewrite(self, path, query_args=None): new_url = False router = http.root.get_db_router(request.db).bind('') @@ -676,3 +676,6 @@ class IrHttp(models.AbstractModel): except werkzeug.exceptions.NotFound: new_url = path return new_url or path, endpoint and endpoint[0] + + def _rewrite_len(self, website_id, rewrites=None): + return 0 diff --git a/addons/mail/models/mail_message_subtype.py b/addons/mail/models/mail_message_subtype.py index 1715d1e9ec7..f75ab4f94d6 100644 --- a/addons/mail/models/mail_message_subtype.py +++ b/addons/mail/models/mail_message_subtype.py @@ -45,15 +45,15 @@ class MailMessageSubtype(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() + self.env.registry.clear_cache() # _get_auto_subscription_subtypes return super(MailMessageSubtype, self).create(vals_list) def write(self, vals): - self.clear_caches() + self.env.registry.clear_cache() # _get_auto_subscription_subtypes return super(MailMessageSubtype, self).write(vals) def unlink(self): - self.clear_caches() + self.env.registry.clear_cache() # _get_auto_subscription_subtypes return super(MailMessageSubtype, self).unlink() @tools.ormcache('model_name') diff --git a/addons/product/models/product_product.py b/addons/product/models/product_product.py index 9a952b3cce7..a65385b74e5 100644 --- a/addons/product/models/product_product.py +++ b/addons/product/models/product_product.py @@ -314,7 +314,7 @@ class ProductProduct(models.Model): self.product_tmpl_id._sanitize_vals(vals) products = super(ProductProduct, self.with_context(create_product_product=True)).create(vals_list) # `_get_variant_id_for_combination` depends on existing variants - self.clear_caches() + self.env.registry.clear_cache() return products def write(self, values): @@ -322,10 +322,10 @@ class ProductProduct(models.Model): res = super(ProductProduct, self).write(values) if 'product_template_attribute_value_ids' in values: # `_get_variant_id_for_combination` depends on `product_template_attribute_value_ids` - self.clear_caches() + self.env.registry.clear_cache() elif 'active' in values: # `_get_first_possible_variant_id` depends on variants active state - self.clear_caches() + self.env.registry.clear_cache() return res def unlink(self): @@ -350,7 +350,7 @@ class ProductProduct(models.Model): # products due to ondelete='cascade' unlink_templates.unlink() # `_get_variant_id_for_combination` depends on existing variants - self.clear_caches() + self.env.registry.clear_cache() return res def _filter_to_unlink(self, check_access=True): diff --git a/addons/test_mail/tests/test_message_track.py b/addons/test_mail/tests/test_message_track.py index 02716f5a94e..ee72f8b628d 100644 --- a/addons/test_mail/tests/test_message_track.py +++ b/addons/test_mail/tests/test_message_track.py @@ -506,7 +506,7 @@ class TestTrackingInternals(MailCommon): # test editing the record with user not in the group of the field self.env.invalidate_all() - self.record.clear_caches() + self.env.registry.clear_cache() record_form = Form(self.record.with_user(self.user_employee)) record_form.name = 'TestDoNoCrash' # the employee user must be able to save the fields on which they can write diff --git a/addons/web/controllers/home.py b/addons/web/controllers/home.py index c02e1313e4e..f82bc1e0678 100644 --- a/addons/web/controllers/home.py +++ b/addons/web/controllers/home.py @@ -141,7 +141,7 @@ class Home(http.Controller): if request.env.user._is_system(): uid = request.session.uid = odoo.SUPERUSER_ID # invalidate session token cache as we've changed the uid - request.env['res.users'].clear_caches() + request.env.registry.clear_cache() request.session.session_token = security.compute_session_token(request.session, request.env) return request.redirect(self._login_redirect(uid)) diff --git a/addons/web/tests/test_assets.py b/addons/web/tests/test_assets.py index dd2c4349148..a1dd4d37977 100644 --- a/addons/web/tests/test_assets.py +++ b/addons/web/tests/test_assets.py @@ -69,7 +69,7 @@ class TestPregenerateTime(HttpCase): def test_logs_pregenerate_time(self): self.env['ir.qweb']._pregenerate_assets_bundles() start = time.time() - self.env.registry.clear_caches() + self.env.registry.clear_cache() self.env.cache.invalidate() with self.profile(collectors=['sql', odoo.tools.profiler.PeriodicCollector(interval=0.01)], disable_gc=True): self.env['ir.qweb']._pregenerate_assets_bundles() diff --git a/addons/web_editor/controllers/main.py b/addons/web_editor/controllers/main.py index a314bf883cf..1c78c74a982 100644 --- a/addons/web_editor/controllers/main.py +++ b/addons/web_editor/controllers/main.py @@ -629,7 +629,7 @@ class Web_Editor(http.Controller): if re.match('^o-color-([1-5])$', css_color_value): if not bundle_css: bundle = 'web.assets_frontend' - asset = request.env["ir.asset"]._get_asset_bundle(bundle) + asset = request.env["ir.qweb"]._get_asset_bundle(bundle) bundle_css = asset.css().index_content color_search = re.search(r'(?i)--%s:\s+(%s|%s)' % (css_color_value, regex_hex, regex_rgba), bundle_css) if not color_search: diff --git a/addons/web_editor/models/assets.py b/addons/web_editor/models/assets.py index e807fa30130..fd418b94b37 100644 --- a/addons/web_editor/models/assets.py +++ b/addons/web_editor/models/assets.py @@ -62,6 +62,7 @@ class Assets(models.AbstractModel): # If it was already modified, simply override the corresponding # attachment content custom_attachment.write({"datas": datas}) + self.env.registry.clear_cache('assets') else: # If not, create a new attachment to copy the original scss/js file # content, with its modifications @@ -93,7 +94,6 @@ class Assets(models.AbstractModel): new_asset['bundle'] = IrAsset._get_related_bundle(url, bundle) IrAsset.create(new_asset) - self.env["ir.qweb"].clear_caches() @api.model def _get_content_from_url(self, url, url_info=None, custom_attachments=None): diff --git a/addons/website/models/ir_http.py b/addons/website/models/ir_http.py index 45443ce5840..c019b88aba5 100644 --- a/addons/website/models/ir_http.py +++ b/addons/website/models/ir_http.py @@ -13,7 +13,7 @@ import werkzeug.utils from functools import partial import odoo -from odoo import api, models +from odoo import api, models, tools from odoo import SUPERUSER_ID from odoo.exceptions import AccessError from odoo.http import request @@ -62,16 +62,10 @@ def get_request_website(): class Http(models.AbstractModel): _inherit = 'ir.http' - @classmethod - def routing_map(cls, key=None): + def routing_map(self, key=None): if not key and request: key = request.website_routing - return super(Http, cls).routing_map(key=key) - - @classmethod - def clear_caches(cls): - super()._clear_routing_map() - return super().clear_caches() + return super().routing_map(key=key) @classmethod def _slug_matching(cls, adapter, endpoint, **kw): @@ -81,18 +75,31 @@ class Http(models.AbstractModel): qs = request.httprequest.query_string.decode('utf-8') return adapter.build(endpoint, kw) + (qs and '?%s' % qs or '') - @classmethod - def _generate_routing_rules(cls, modules, converters): + @tools.ormcache('website_id', cache='routing') + def _rewrite_len(self, website_id, rewrites=None): + # This little hack allow to store a _rewrite_len in cache if we have the value, + # (Populated during routing map computation) + # but also ensures that we will have the correct value if the cache is not populated. + # The cache key actually does not depends on "rewrites" because this is the information + # we are storing, rewrites actually depends on website_id + if rewrites is None: + rewrites = self._get_rewrites(website_id) + return len(rewrites) + + def _get_rewrites(self, website_id): + domain = [('redirect_type', 'in', ('308', '404')), '|', ('website_id', '=', False), ('website_id', '=', website_id)] + rewrites = {x.url_from: x for x in self.env['website.rewrite'].sudo().search(domain)} + self._rewrite_len(website_id, rewrites) # optionnal, update value in cache + #self._rewrite_len.add_cache_value(website_id, value=rewrites) + return rewrites + + def _generate_routing_rules(self, modules, converters): if not request: yield from super()._generate_routing_rules(modules, converters) return - website_id = request.website_routing logger.debug("_generate_routing_rules for website: %s", website_id) - domain = [('redirect_type', 'in', ('308', '404')), '|', ('website_id', '=', False), ('website_id', '=', website_id)] - - rewrites = dict([(x.url_from, x) for x in request.env['website.rewrite'].sudo().search(domain)]) - cls._rewrite_len[website_id] = len(rewrites) + rewrites = self._get_rewrites(website_id) for url, endpoint in super()._generate_routing_rules(modules, converters): if url in rewrites: @@ -104,7 +111,7 @@ class Http(models.AbstractModel): if url != url_to: logger.debug('Redirect from %s to %s for website %s' % (url, url_to, website_id)) - _slug_matching = partial(cls._slug_matching, endpoint=endpoint) + _slug_matching = partial(self._slug_matching, endpoint=endpoint) endpoint.routing['redirect_to'] = _slug_matching yield url, endpoint # yield original redirected to new url elif rewrite.redirect_type == '404': diff --git a/addons/website/models/ir_ui_view.py b/addons/website/models/ir_ui_view.py index faa383ea9f9..949e6d8b304 100644 --- a/addons/website/models/ir_ui_view.py +++ b/addons/website/models/ir_ui_view.py @@ -224,7 +224,7 @@ class View(models.Model): specific_views += view._get_specific_views() result = super(View, self + specific_views).unlink() - self.clear_caches() + self.env.registry.clear_cache('templates') return result def _create_website_specific_pages_for_view(self, new_view, website): @@ -363,7 +363,7 @@ class View(models.Model): """ @api.model - @tools.ormcache_context('self.env.uid', 'self.env.su', 'xml_id', keys=('website_id',)) + @tools.ormcache('self.env.uid', 'self.env.su', 'xml_id', 'self._context.get("website_id")', cache='templates') def _get_view_id(self, xml_id): """If a website_id is in the context and the given xml_id is not an int then try to get the id of the specific view for that website, but @@ -374,7 +374,7 @@ class View(models.Model): method. `viewref` is probably more suitable. Archived views are ignored (unless the active_test context is set, but - then the ormcache_context will not work as expected). + then the ormcache will not work as expected). """ website_id = self._context.get('website_id') if website_id and not isinstance(xml_id, int): @@ -388,7 +388,7 @@ class View(models.Model): return view.id return super(View, self.sudo())._get_view_id(xml_id) - @tools.ormcache('self.id') + @tools.ormcache('self.id', cache='templates') def _get_cached_visibility(self): return self.visibility diff --git a/addons/website/models/website.py b/addons/website/models/website.py index 7f268f7d004..927160afdcb 100644 --- a/addons/website/models/website.py +++ b/addons/website/models/website.py @@ -186,7 +186,7 @@ class Website(models.Model): website.menu_id = top_menus and top_menus[0].id or False # self.env.uid for ir.rule groups on menu - @tools.ormcache('self.env.uid', 'self.id') + @tools.ormcache('self.env.uid', 'self.id', cache='templates') def _get_menu_ids(self): return self.env['website.menu'].search([('website_id', '=', self.id)]).ids @@ -216,7 +216,7 @@ class Website(models.Model): original_company = self.company_id self._handle_create_write(values) - self.clear_caches() + self.env.registry.clear_cache() if 'company_id' in values and 'user_id' not in values: public_user_to_change_websites = self.filtered(lambda w: w.sudo().user_id.company_id.id != values['company_id']) @@ -228,7 +228,7 @@ class Website(models.Model): if 'cdn_activated' in values or 'cdn_url' in values or 'cdn_filters' in values: # invalidate the caches from static node at compile time - self.env['ir.qweb'].clear_caches() + self.env.registry.clear_cache() # invalidate cache for `company.website_id` to be recomputed if 'sequence' in values or 'company_id' in values: @@ -1088,7 +1088,7 @@ class Website(models.Model): return view @api.model - @tools.ormcache_context('key', keys=('website_id',)) + @tools.ormcache('key', 'self._context.get("website_id")', cache='templates') def is_view_active(self, key): """ Return True if active, False if not active, None if not found diff --git a/addons/website/models/website_menu.py b/addons/website/models/website_menu.py index 66106ae6d38..45aeaa032d0 100644 --- a/addons/website/models/website_menu.py +++ b/addons/website/models/website_menu.py @@ -73,7 +73,7 @@ class Menu(models.Model): Be careful to return correct record for ir.model.data xml_id in case of default main menus creation. ''' - self.clear_caches() + self.env.registry.clear_cache('templates') # Only used when creating website_data.xml default menu menus = self.env['website.menu'] for vals in vals_list: @@ -103,11 +103,11 @@ class Menu(models.Model): return menus def write(self, values): - self.clear_caches() + self.env.registry.clear_cache('templates') return super().write(values) def unlink(self): - self.clear_caches() + self.env.registry.clear_cache('templates') default_menu = self.env.ref('website.main_menu', raise_if_not_found=False) menus_to_remove = self for menu in self.filtered(lambda m: default_menu and m.parent_id.id == default_menu.id): diff --git a/addons/website/models/website_page.py b/addons/website/models/website_page.py index 8dc35b53a36..71c2344a48a 100644 --- a/addons/website/models/website_page.py +++ b/addons/website/models/website_page.py @@ -148,7 +148,7 @@ class Page(models.Model): # If there is no other pages linked to that ir_ui_view, we can delete the ir_ui_view page.view_id.unlink() # Make sure website._get_menu_ids() will be recomputed - self.clear_caches() + self.env.registry.clear_cache() return super(Page, self).unlink() def write(self, vals): @@ -186,7 +186,7 @@ class Page(models.Model): if 'visibility' in vals: if vals['visibility'] != 'restricted_group': vals['groups_id'] = False - self.clear_caches() # write on page == write on view that invalid cache + self.env.registry.clear_cache() # write on page == write on view that invalid cache return super(Page, self).write(vals) def get_website_meta(self): diff --git a/addons/website/models/website_rewrite.py b/addons/website/models/website_rewrite.py index f4be519514a..f21439dc2a1 100644 --- a/addons/website/models/website_rewrite.py +++ b/addons/website/models/website_rewrite.py @@ -131,12 +131,12 @@ class WebsiteRewrite(models.Model): return res def _invalidate_routing(self): - # Call clear_caches on this worker to reload routing table. + # Call clear_cache for routing on all workers to reload routing table. # Note that only 404 and 308 redirection alter the routing map: # - 404: remove entry from routing map # - 301/302: served as fallback later if path not found in routing map # - 308: add "alias" (`redirect_to`) in routing map - self.env['ir.http'].clear_caches() + self.env.registry.clear_cache('routing') def refresh_routes(self): self.env['website.route']._refresh() diff --git a/addons/website/tests/test_http_endpoint.py b/addons/website/tests/test_http_endpoint.py index f3845953cf1..d5082eca8e2 100644 --- a/addons/website/tests/test_http_endpoint.py +++ b/addons/website/tests/test_http_endpoint.py @@ -25,7 +25,7 @@ class TestHttpEndPoint(HttpCase): 'inherit_id': homepage_view.id, 'arch_db': """ - + """, }) diff --git a/addons/website/tests/test_views.py b/addons/website/tests/test_views.py index a09c74a77d7..18768afd9bf 100644 --- a/addons/website/tests/test_views.py +++ b/addons/website/tests/test_views.py @@ -1163,7 +1163,7 @@ class TestCowViewSaving(TestViewSavingCommon): french = self.env['res.lang']._activate_lang('fr_FR') self.env['ir.module.module']._load_module_terms(['website'], [french.code]) # Make sure res.lang.get_installed is recomputed - self.env.registry.clear_caches() + self.env.registry.clear_cache() View = self.env['ir.ui.view'].with_context(lang=french.code, website_id=1) old_specific_views = View.search([('website_id', '!=', None)]) diff --git a/addons/website_crm_iap_reveal/models/crm_reveal_rule.py b/addons/website_crm_iap_reveal/models/crm_reveal_rule.py index 732b56fbbbc..bee08966024 100644 --- a/addons/website_crm_iap_reveal/models/crm_reveal_rule.py +++ b/addons/website_crm_iap_reveal/models/crm_reveal_rule.py @@ -85,7 +85,7 @@ class CRMRevealRule(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() # Clear the cache in order to recompute _get_active_rules + self.env.registry.clear_cache() # Clear the cache in order to recompute _get_active_rules return super().create(vals_list) def write(self, vals): @@ -93,11 +93,11 @@ class CRMRevealRule(models.Model): 'country_ids', 'regex_url', 'active' } if set(vals.keys()) & fields_set: - self.clear_caches() # Clear the cache in order to recompute _get_active_rules + self.env.registry.clear_cache() # Clear the cache in order to recompute _get_active_rules return super(CRMRevealRule, self).write(vals) def unlink(self): - self.clear_caches() # Clear the cache in order to recompute _get_active_rules + self.env.registry.clear_cache() # Clear the cache in order to recompute _get_active_rules return super(CRMRevealRule, self).unlink() def action_get_lead_tree_view(self): diff --git a/addons/website_sale/models/product_pricelist.py b/addons/website_sale/models/product_pricelist.py index 1b3e7483c30..eb088aab077 100644 --- a/addons/website_sale/models/product_pricelist.py +++ b/addons/website_sale/models/product_pricelist.py @@ -41,17 +41,18 @@ class ProductPricelist(models.Model): # It be set when we actually create the pricelist self = self.with_context(default_company_id=vals['company_id']) pricelists = super().create(vals_list) - pricelists and pricelists.clear_caches() + if pricelists: + self.env.registry.clear_cache() return pricelists def write(self, data): res = super(ProductPricelist, self).write(data) - self and self.clear_caches() + self and self.env.registry.clear_cache() return res def unlink(self): res = super(ProductPricelist, self).unlink() - self and self.clear_caches() + self and self.env.registry.clear_cache() return res def _get_partner_pricelist_multi_search_domain_hook(self, company_id): diff --git a/odoo/addons/base/models/assetsbundle.py b/odoo/addons/base/models/assetsbundle.py index bfda143199c..fe631138734 100644 --- a/odoo/addons/base/models/assetsbundle.py +++ b/odoo/addons/base/models/assetsbundle.py @@ -302,7 +302,7 @@ class AssetsBundle(object): but the transaction is rollbacked, leading to 404 when getting the attachments. This postrollback hook will help fix this issue by clearing the cache if it isn't commited. """ - self.env.cr.postrollback.add(self.env.registry._Registry__cache.clear) + self.env.cr.postrollback.add(self.env.registry._Registry__caches['assets'].clear) def save_attachment(self, extension, content): """Record the given bundle in an ir.attachment and delete diff --git a/odoo/addons/base/models/decimal_precision.py b/odoo/addons/base/models/decimal_precision.py index 3d01a676f89..373099ebaf7 100644 --- a/odoo/addons/base/models/decimal_precision.py +++ b/odoo/addons/base/models/decimal_precision.py @@ -36,17 +36,17 @@ class DecimalPrecision(models.Model): @api.model_create_multi def create(self, vals_list): res = super(DecimalPrecision, self).create(vals_list) - self.clear_caches() + self.env.registry.clear_cache() return res def write(self, data): res = super(DecimalPrecision, self).write(data) - self.clear_caches() + self.env.registry.clear_cache() return res def unlink(self): res = super(DecimalPrecision, self).unlink() - self.clear_caches() + self.env.registry.clear_cache() return res @api.onchange('digits') diff --git a/odoo/addons/base/models/ir_actions.py b/odoo/addons/base/models/ir_actions.py index 9e375a5237d..34d2ad39293 100644 --- a/odoo/addons/base/models/ir_actions.py +++ b/odoo/addons/base/models/ir_actions.py @@ -73,13 +73,13 @@ class IrActions(models.Model): def create(self, vals_list): res = super(IrActions, self).create(vals_list) # self.get_bindings() depends on action records - self.clear_caches() + self.env.registry.clear_cache() return res def write(self, vals): res = super(IrActions, self).write(vals) # self.get_bindings() depends on action records - self.clear_caches() + self.env.registry.clear_cache() return res def unlink(self): @@ -89,7 +89,7 @@ class IrActions(models.Model): todos.unlink() res = super(IrActions, self).unlink() # self.get_bindings() depends on action records - self.clear_caches() + self.env.registry.clear_cache() return res @api.ondelete(at_uninstall=True) @@ -303,14 +303,14 @@ class IrActionsActWindow(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() + self.env.registry.clear_cache() for vals in vals_list: if not vals.get('name') and vals.get('res_model'): vals['name'] = self.env[vals['res_model']]._description return super(IrActionsActWindow, self).create(vals_list) def unlink(self): - self.clear_caches() + self.env.registry.clear_cache() return super(IrActionsActWindow, self).unlink() def exists(self): diff --git a/odoo/addons/base/models/ir_asset.py b/odoo/addons/base/models/ir_asset.py index deaa7711c5a..d9b8d9e9a02 100644 --- a/odoo/addons/base/models/ir_asset.py +++ b/odoo/addons/base/models/ir_asset.py @@ -64,15 +64,15 @@ class IrAsset(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() + self.env.registry.clear_cache('assets') return super().create(vals_list) def write(self, values): - self.clear_caches() + self.env.registry.clear_cache('assets') return super().write(values) def unlink(self): - self.clear_caches() + self.env.registry.clear_cache('assets') return super().unlink() name = fields.Char(string='Name', required=True) @@ -106,7 +106,7 @@ class IrAsset(models.Model): @tools.conditional( 'xml' not in tools.config['dev_mode'], - tools.ormcache('bundle', 'tuple(sorted(assets_params.items()))'), + tools.ormcache('bundle', 'tuple(sorted(assets_params.items()))', cache='assets'), ) def _get_asset_paths(self, bundle, assets_params): """ diff --git a/odoo/addons/base/models/ir_config_parameter.py b/odoo/addons/base/models/ir_config_parameter.py index db8f38a8b35..a5654a99296 100644 --- a/odoo/addons/base/models/ir_config_parameter.py +++ b/odoo/addons/base/models/ir_config_parameter.py @@ -102,13 +102,13 @@ class IrConfigParameter(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() + self.env.registry.clear_cache() return super(IrConfigParameter, self).create(vals_list) def write(self, vals): - self.clear_caches() + self.env.registry.clear_cache() return super(IrConfigParameter, self).write(vals) def unlink(self): - self.clear_caches() + self.env.registry.clear_cache() return super(IrConfigParameter, self).unlink() diff --git a/odoo/addons/base/models/ir_default.py b/odoo/addons/base/models/ir_default.py index 3c5e6fe36bd..b28c5b62a60 100644 --- a/odoo/addons/base/models/ir_default.py +++ b/odoo/addons/base/models/ir_default.py @@ -32,17 +32,17 @@ class IrDefault(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() + self.env.registry.clear_cache() return super(IrDefault, self).create(vals_list) def write(self, vals): if self: - self.clear_caches() + self.env.registry.clear_cache() return super(IrDefault, self).write(vals) def unlink(self): if self: - self.clear_caches() + self.env.registry.clear_cache() return super(IrDefault, self).unlink() @api.model diff --git a/odoo/addons/base/models/ir_http.py b/odoo/addons/base/models/ir_http.py index 96706f90ab3..d0a9706721e 100644 --- a/odoo/addons/base/models/ir_http.py +++ b/odoo/addons/base/models/ir_http.py @@ -128,8 +128,8 @@ class IrHttp(models.AbstractModel): return {'model': ModelConverter, 'models': ModelsConverter, 'int': SignedIntConverter} @classmethod - def _match(cls, path_info, key=None): - rule, args = cls.routing_map().bind_to_environ(request.httprequest.environ).match(path_info=path_info, return_rule=True) + def _match(cls, path_info): + rule, args = request.env['ir.http'].routing_map().bind_to_environ(request.httprequest.environ).match(path_info=path_info, return_rule=True) return rule, args @classmethod @@ -216,44 +216,30 @@ class IrHttp(models.AbstractModel): def _redirect(cls, location, code=303): return werkzeug.utils.redirect(location, code=code, Response=Response) - @classmethod - def _generate_routing_rules(cls, modules, converters): + def _generate_routing_rules(self, modules, converters): return http._generate_routing_rules(modules, False, converters) - @classmethod - def routing_map(cls, key=None): - - if not hasattr(cls, '_routing_map'): - cls._routing_map = {} - cls._rewrite_len = {} - - if key not in cls._routing_map: - _logger.info("Generating routing map for key %s" % str(key)) - registry = Registry(threading.current_thread().dbname) - installed = registry._init_modules.union(odoo.conf.server_wide_modules) - if tools.config['test_enable'] and odoo.modules.module.current_test: - installed.add(odoo.modules.module.current_test) - mods = sorted(installed) - # Note : when routing map is generated, we put it on the class `cls` - # to make it available for all instance. Since `env` create an new instance - # of the model, each instance will regenared its own routing map and thus - # regenerate its EndPoint. The routing map should be static. - routing_map = werkzeug.routing.Map(strict_slashes=False, converters=cls._get_converters()) - for url, endpoint in cls._generate_routing_rules(mods, converters=cls._get_converters()): - routing = submap(endpoint.routing, ROUTING_KEYS) - if routing['methods'] is not None and 'OPTIONS' not in routing['methods']: - routing['methods'] = routing['methods'] + ['OPTIONS'] - rule = FasterRule(url, endpoint=endpoint, **routing) - rule.merge_slashes = False - routing_map.add(rule) - cls._routing_map[key] = routing_map - return cls._routing_map[key] - - @classmethod - def _clear_routing_map(cls): - if hasattr(cls, '_routing_map'): - cls._routing_map = {} - _logger.debug("Clear routing map") + @tools.ormcache('key', cache='routing') + def routing_map(self, key=None): + _logger.info("Generating routing map for key %s", str(key)) + registry = Registry(threading.current_thread().dbname) + installed = registry._init_modules.union(odoo.conf.server_wide_modules) + if tools.config['test_enable'] and odoo.modules.module.current_test: + installed.add(odoo.modules.module.current_test) + mods = sorted(installed) + # Note : when routing map is generated, we put it on the class `cls` + # to make it available for all instance. Since `env` create an new instance + # of the model, each instance will regenared its own routing map and thus + # regenerate its EndPoint. The routing map should be static. + routing_map = werkzeug.routing.Map(strict_slashes=False, converters=self._get_converters()) + for url, endpoint in self._generate_routing_rules(mods, converters=self._get_converters()): + routing = submap(endpoint.routing, ROUTING_KEYS) + if routing['methods'] is not None and 'OPTIONS' not in routing['methods']: + routing['methods'] = routing['methods'] + ['OPTIONS'] + rule = FasterRule(url, endpoint=endpoint, **routing) + rule.merge_slashes = False + routing_map.add(rule) + return routing_map @api.autovacuum def _gc_sessions(self): diff --git a/odoo/addons/base/models/ir_model.py b/odoo/addons/base/models/ir_model.py index 6e54dbbc01a..1e6e37c0d7f 100644 --- a/odoo/addons/base/models/ir_model.py +++ b/odoo/addons/base/models/ir_model.py @@ -932,7 +932,7 @@ class IrModelFields(models.Model): models.add(vals['model']) # for self._get_ids() in _update_selection() - self.clear_caches() + self.env.registry.clear_cache() res = super(IrModelFields, self).create(vals_list) @@ -1042,7 +1042,7 @@ class IrModelFields(models.Model): def update_field_translations(self, field_name, translations): res = super().update_field_translations(field_name, translations) if res: - self.clear_caches() + self.env.registry.clear_cache() return res @api.depends('field_description', 'model') @@ -1917,23 +1917,11 @@ class IrModelAccess(models.Model): return has_access - __cache_clearing_methods = set() - - @classmethod - def register_cache_clearing_method(cls, model, method): - cls.__cache_clearing_methods.add((model, method)) - - @classmethod - def unregister_cache_clearing_method(cls, model, method): - cls.__cache_clearing_methods.discard((model, method)) @api.model def call_cache_clearing_methods(self): self.env.invalidate_all() - self._get_allowed_models.clear_cache(self) # clear the cache of check function - for model, method in self.__cache_clearing_methods: - if model in self.env: - getattr(self.env[model], method)() + self.env.registry.clear_cache() # mainly _get_allowed_models # # Check rights on actions @@ -2019,7 +2007,7 @@ class IrModelData(models.Model): # NEW V8 API @api.model - @tools.ormcache('xmlid') + @tools.ormcache('xmlid', cache='xmlid') def _xmlid_lookup(self, xmlid: str) -> tuple: """Low level xmlid lookup Return (id, res_model, res_id) or raise ValueError if not found @@ -2066,9 +2054,13 @@ class IrModelData(models.Model): default = dict(default or {}, name="%s_%s" % (self.name, rand)) return super().copy(default) + def write(self, values): + self.env.registry.clear_cache('xmlid') # _xmlid_lookup + return super().write(values) + def unlink(self): """ Regular unlink method, but make sure to clear the caches. """ - self.clear_caches() + self.env.registry.clear_cache('xmlid') # _xmlid_lookup return super(IrModelData, self).unlink() def _lookup_xmlids(self, xml_ids, model): @@ -2120,6 +2112,7 @@ class IrModelData(models.Model): query = self._build_update_xmlids_query(sub_rows, update) try: self.env.cr.execute(query, [arg for row in sub_rows for arg in row]) + self.env.registry.clear_cache('xmlid') except Exception: _logger.error("Failed to insert ir_model_data\n%s", "\n".join(str(row) for row in sub_rows)) raise diff --git a/odoo/addons/base/models/ir_module.py b/odoo/addons/base/models/ir_module.py index 6f304e70c77..0442554cf10 100644 --- a/odoo/addons/base/models/ir_module.py +++ b/odoo/addons/base/models/ir_module.py @@ -311,7 +311,7 @@ class Module(models.Model): raise UserError(_('You are trying to remove a module that is installed or will be installed.')) def unlink(self): - self.clear_caches() + self.env.registry.clear_cache() return super(Module, self).unlink() def _get_modules_to_load_domain(self): diff --git a/odoo/addons/base/models/ir_property.py b/odoo/addons/base/models/ir_property.py index 8a7d4fd6f83..86c53f47272 100644 --- a/odoo/addons/base/models/ir_property.py +++ b/odoo/addons/base/models/ir_property.py @@ -128,9 +128,9 @@ class Property(models.Model): # DLE P44: test `test_27_company_dependent` # Easy solution, need to flush write when changing a property. # Maybe it would be better to be able to compute all impacted cache value and update those instead - # Then clear_caches must be removed as well. + # Then clear_cache must be removed as well. self.env.flush_all() - self.clear_caches() + self.env.registry.clear_cache() return r @api.model_create_multi @@ -141,7 +141,7 @@ class Property(models.Model): if created_default: # DLE P44: test `test_27_company_dependent` self.env.flush_all() - self.clear_caches() + self.env.registry.clear_cache() return r def unlink(self): @@ -154,7 +154,7 @@ class Property(models.Model): default_deleted = self.env.cr.rowcount == 1 r = super().unlink() if default_deleted: - self.clear_caches() + self.env.registry.clear_cache() return r def get_by_record(self): diff --git a/odoo/addons/base/models/ir_qweb.py b/odoo/addons/base/models/ir_qweb.py index 83bf883eac1..08458497316 100644 --- a/odoo/addons/base/models/ir_qweb.py +++ b/odoo/addons/base/models/ir_qweb.py @@ -380,10 +380,12 @@ from dateutil.relativedelta import relativedelta from psycopg2.extensions import TransactionRollbackError from odoo import api, models, tools +from odoo.modules import registry from odoo.tools import config, safe_eval, pycompat from odoo.tools.constants import SUPPORTED_DEBUGGER, EXTERNAL_ASSET from odoo.tools.safe_eval import assert_valid_codeobj, _BUILTINS, to_opcodes, _EXPR_OPCODES, _BLACKLIST from odoo.tools.json import scriptsafe +from odoo.tools.lru import LRU from odoo.tools.misc import str2bool from odoo.tools.image import image_data_uri, FILETYPE_BASE64_MAGICWORD from odoo.http import request @@ -597,7 +599,7 @@ class IrQWeb(models.AbstractModel): @tools.conditional( 'xml' not in tools.config['dev_mode'], - tools.ormcache('template', 'tuple(self.env.context.get(k) for k in self._get_template_cache_keys())'), + tools.ormcache('template', 'tuple(self.env.context.get(k) for k in self._get_template_cache_keys())', cache='templates'), ) def _get_view_id(self, template): try: @@ -2485,7 +2487,6 @@ class IrQWeb(models.AbstractModel): """ generate value from the function if the result is not cached. """ if not cache_key: return get_value() - value = loaded_values and loaded_values.get(cache_key) if not value: value = self._get_cached_values(cache_key, get_value) @@ -2497,7 +2498,7 @@ class IrQWeb(models.AbstractModel): # in '_compile' method contains the write_date of all inherited views. @tools.conditional( 'xml' not in tools.config['dev_mode'], - tools.ormcache('cache_key'), + tools.ormcache('cache_key', cache='templates.cached_values'), ) def _get_cached_values(self, cache_key, get_value): """ generate value from the function if the result is not cached. """ @@ -2508,7 +2509,7 @@ class IrQWeb(models.AbstractModel): # in non-xml-debug mode we want assets to be cached forever, and the admin can force a cache clear # by restarting the server after updating the source code (or using the "Clear server cache" in debug tools) 'xml' not in tools.config['dev_mode'], - tools.ormcache('bundle', 'css', 'js', 'tuple(sorted(assets_params.items()))', 'rtl'), + tools.ormcache('bundle', 'css', 'js', 'tuple(sorted(assets_params.items()))', 'rtl', cache='assets'), ) def _generate_asset_links_cache(self, bundle, css=True, js=True, assets_params=None, rtl=False): return self._generate_asset_links(bundle, css, js, False, assets_params, rtl) @@ -2649,7 +2650,11 @@ def render(template_name, values, load, **options): """ class MockPool: db_name = None - _Registry__cache = {} + _Registry__caches = {cache_name: LRU(cache_size) for cache_name, cache_size in registry._REGISTRY_CACHES.items()} + _Registry__caches_groups = {} + for cache_name, cache in _Registry__caches.items(): + _Registry__caches_groups.setdefault(cache_name.split('.')[0], []).append(cache) + class MockIrQWeb(IrQWeb): _register = False # not visible in real registry diff --git a/odoo/addons/base/models/ir_rule.py b/odoo/addons/base/models/ir_rule.py index 4d713e35712..47a33e097bc 100644 --- a/odoo/addons/base/models/ir_rule.py +++ b/odoo/addons/base/models/ir_rule.py @@ -159,14 +159,9 @@ class IrRule(models.Model): v = tuple(v) yield v - @api.model - def clear_cache(self): - warnings.warn("Deprecated IrRule.clear_cache(), use IrRule.clear_caches() instead", DeprecationWarning) - self.clear_caches() - def unlink(self): res = super(IrRule, self).unlink() - self.clear_caches() + self.env.registry.clear_cache() return res @api.model_create_multi @@ -174,7 +169,7 @@ class IrRule(models.Model): res = super(IrRule, self).create(vals_list) # DLE P33: tests self.env.flush_all() - self.clear_caches() + self.env.registry.clear_cache() return res def write(self, vals): @@ -184,7 +179,7 @@ class IrRule(models.Model): # - odoo/addons/test_access_rights/tests/test_ir_rules.py # - odoo/addons/base/tests/test_orm.py (/home/dle/src/odoo/master-nochange-fp/odoo/addons/base/tests/test_orm.py) self.env.flush_all() - self.clear_caches() + self.env.registry.clear_cache() return res def _make_access_error(self, operation, records): diff --git a/odoo/addons/base/models/ir_ui_menu.py b/odoo/addons/base/models/ir_ui_menu.py index 0f4ab0e4cba..e35f6ff4d31 100644 --- a/odoo/addons/base/models/ir_ui_menu.py +++ b/odoo/addons/base/models/ir_ui_menu.py @@ -162,14 +162,14 @@ class IrUiMenu(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() + self.env.registry.clear_cache() for values in vals_list: if 'web_icon' in values: values['web_icon_data'] = self._compute_web_icon_data(values.get('web_icon')) return super(IrUiMenu, self).create(vals_list) def write(self, values): - self.clear_caches() + self.env.registry.clear_cache() if 'web_icon' in values: values['web_icon_data'] = self._compute_web_icon_data(values.get('web_icon')) return super(IrUiMenu, self).write(values) @@ -194,7 +194,7 @@ class IrUiMenu(models.Model): direct_children = self.with_context(**extra).search([('parent_id', 'in', self.ids)]) direct_children.write({'parent_id': False}) - self.clear_caches() + self.env.registry.clear_cache() return super(IrUiMenu, self).unlink() def copy(self, default=None): diff --git a/odoo/addons/base/models/ir_ui_view.py b/odoo/addons/base/models/ir_ui_view.py index 3dc8833b932..7a5d46ac792 100644 --- a/odoo/addons/base/models/ir_ui_view.py +++ b/odoo/addons/base/models/ir_ui_view.py @@ -564,7 +564,7 @@ actual arch. values['arch_updated'] = False values.update(self._compute_defaults(values)) - self.clear_caches() + self.env.registry.clear_cache('templates') result = super(View, self.with_context(ir_ui_view_partial_validation=True)).create(vals_list) return result.with_env(self.env) @@ -580,7 +580,7 @@ actual arch. if custom_view: custom_view.unlink() - self.clear_caches() + self.env.registry.clear_cache('templates') if 'arch_db' in vals and not self.env.context.get('no_save_prev'): vals['arch_prev'] = self.arch_db @@ -2027,7 +2027,6 @@ actual arch. """ Return the list of context keys to use for caching ``_read_template``. """ return ['lang', 'inherit_branding', 'edit_translations'] - # apply ormcache_context decorator unless in dev mode... @api.model def _read_template(self, view_id): arch_tree = self.browse(view_id)._get_combined_arch() @@ -2058,11 +2057,6 @@ actual arch. """ return self.browse(self._get_view_id(view_ref)) - def clear_cache(self): - """ Deprecated, use `clear_caches` instead. """ - if 'xml' not in config['dev_mode']: - self.clear_caches() - def _contains_branded(self, node): return node.tag == 't'\ or 't-raw' in node.attrib\ @@ -2647,7 +2641,7 @@ class Model(models.AbstractModel): @api.model @tools.conditional( 'xml' not in config['dev_mode'], - tools.ormcache('self._get_view_cache_key(view_id, view_type, **options)'), + tools.ormcache('self._get_view_cache_key(view_id, view_type, **options)', cache='templates'), ) def _get_view_cache(self, view_id=None, view_type='form', **options): """ Get the view information ready to be cached diff --git a/odoo/addons/base/models/res_company.py b/odoo/addons/base/models/res_company.py index b91e16035f1..5a279f9b5a3 100644 --- a/odoo/addons/base/models/res_company.py +++ b/odoo/addons/base/models/res_company.py @@ -181,10 +181,6 @@ class Company(models.Model): for record in self: record.is_company_details_empty = not html2plaintext(record.company_details or '') - def cache_restart(self): - warnings.warn("Since 17.0, deprecated method, use `clear_caches` instead", DeprecationWarning, 2) - self.clear_caches() - @api.model_create_multi def create(self, vals_list): @@ -213,7 +209,7 @@ class Company(models.Model): for vals, partner in zip(no_partner_vals_list, partners): vals['partner_id'] = partner.id - self.clear_caches() + self.env.registry.clear_cache() companies = super().create(vals_list) # The write is made on the user to set it automatically in the multi company group. @@ -228,7 +224,7 @@ class Company(models.Model): return companies def write(self, values): - self.clear_caches() + self.env.registry.clear_cache() # Make sure that the selected currency is enabled if values.get('currency_id'): currency = self.env['res.currency'].browse(values['currency_id']) diff --git a/odoo/addons/base/models/res_country.py b/odoo/addons/base/models/res_country.py index 9ea59cf27de..9e5251f2654 100644 --- a/odoo/addons/base/models/res_country.py +++ b/odoo/addons/base/models/res_country.py @@ -115,11 +115,11 @@ class Country(models.Model): res = super().write(vals) if ('code' in vals or 'phone_code' in vals): # Intentionally simplified by not clearing the cache in create and unlink. - self.clear_caches() + self.env.registry.clear_cache() if 'address_view_id' in vals: # Changing the address view of the company must invalidate the view cached for res.partner # because of _view_get_address - self.env['res.partner'].clear_caches() + self.env.registry.clear_cache('templates') return res def get_address_fields(self): diff --git a/odoo/addons/base/models/res_lang.py b/odoo/addons/base/models/res_lang.py index 48a974735e2..ff8ae31778b 100644 --- a/odoo/addons/base/models/res_lang.py +++ b/odoo/addons/base/models/res_lang.py @@ -281,7 +281,7 @@ class Lang(models.Model): @api.model_create_multi def create(self, vals_list): - self.clear_caches() + self.env.registry.clear_cache() for vals in vals_list: if not vals.get('url_code'): vals['url_code'] = vals.get('iso_code') or vals['code'] @@ -301,7 +301,7 @@ class Lang(models.Model): res = super(Lang, self).write(vals) self.env.flush_all() - self.clear_caches() + self.env.registry.clear_cache() return res @api.ondelete(at_uninstall=True) @@ -316,7 +316,7 @@ class Lang(models.Model): raise UserError(_("You cannot delete the language which is Active!\nPlease de-activate the language first.")) def unlink(self): - self.clear_caches() + self.env.registry.clear_cache() return super(Lang, self).unlink() def format(self, percent, value, grouping=False, monetary=False): diff --git a/odoo/addons/base/models/res_users.py b/odoo/addons/base/models/res_users.py index 4bd88c1dce8..d929f224247 100644 --- a/odoo/addons/base/models/res_users.py +++ b/odoo/addons/base/models/res_users.py @@ -653,10 +653,10 @@ class Users(models.Model): # per-method / per-model caches have been removed so the various # clear_cache/clear_caches methods pretty much just end up calling - # Registry._clear_cache + # Registry.clear_cache invalidation_fields = self._get_invalidation_fields() if (invalidation_fields & values.keys()) or any(key.startswith('context_') for key in values): - self.clear_caches() + self.env.registry.clear_cache() return res @@ -666,7 +666,7 @@ class Users(models.Model): default_user_template = self.env.ref('base.default_user', False) if SUPERUSER_ID in self.ids: raise UserError(_('You can not remove the admin user as it is used internally for resources created by Odoo (updates, module installation, ...)')) - self.clear_caches() + self.env.registry.clear_cache() if (portal_user_template and portal_user_template in self) or (default_user_template and default_user_template in self): raise UserError(_('Deleting the template users is not allowed. Deleting this profile will compromise critical functionalities.')) @@ -848,7 +848,7 @@ class Users(models.Model): FROM res_users WHERE id=%%s""" % (session_fields), (self.id,)) if self.env.cr.rowcount != 1: - self.clear_caches() + self.env.registry.clear_cache() return False data_fields = self.env.cr.fetchone() # generate hmac key @@ -1357,7 +1357,7 @@ class GroupsView(models.Model): groups = super().create(vals_list) self._update_user_groups_view() # actions.get_bindings() depends on action records - self.env['ir.actions.actions'].clear_caches() + self.env.registry.clear_cache() return groups def write(self, values): @@ -1370,14 +1370,14 @@ class GroupsView(models.Model): if view_values0 != view_values1: self._update_user_groups_view() # actions.get_bindings() depends on action records - self.env['ir.actions.actions'].clear_caches() + self.env.registry.clear_cache() return res def unlink(self): res = super(GroupsView, self).unlink() self._update_user_groups_view() # actions.get_bindings() depends on action records - self.env['ir.actions.actions'].clear_caches() + self.env.registry.clear_cache() return res def _get_hidden_extra_categories(self): diff --git a/odoo/addons/base/tests/test_ir_actions.py b/odoo/addons/base/tests/test_ir_actions.py index dd65e50e2bb..689a864cc97 100644 --- a/odoo/addons/base/tests/test_ir_actions.py +++ b/odoo/addons/base/tests/test_ir_actions.py @@ -353,7 +353,7 @@ class TestCustomFields(common.TransactionCase): assert set(self.registry[self.MODEL]._fields) == fnames self.addCleanup(self.registry.reset_changes) - self.addCleanup(self.registry.clear_caches) + self.addCleanup(self.registry.clear_all_caches) super().setUp() @@ -550,7 +550,7 @@ class TestCustomFields(common.TransactionCase): model_id = self.env['ir.model']._get_id('res.partner') query_count = 40 with self.assertQueryCount(query_count): - self.env.registry.clear_caches() + self.env.registry.clear_cache() self.env['ir.model.fields'].create({ 'model_id': model_id, 'name': 'x_oh_box', @@ -561,7 +561,7 @@ class TestCustomFields(common.TransactionCase): # same with a related field, it only takes 8 extra queries with self.assertQueryCount(query_count + 8): - self.env.registry.clear_caches() + self.env.registry.clear_cache() self.env['ir.model.fields'].create({ 'model_id': model_id, 'name': 'x_oh_boy', diff --git a/odoo/addons/base/tests/test_ir_http.py b/odoo/addons/base/tests/test_ir_http.py index 692302c2632..7f894541c20 100644 --- a/odoo/addons/base/tests/test_ir_http.py +++ b/odoo/addons/base/tests/test_ir_http.py @@ -12,12 +12,12 @@ _logger = logging.getLogger(__name__) class TestIrHttpPerformances(TransactionCase): def test_routing_map_performance(self): - self.env['ir.http']._clear_routing_map() + self.env.registry.clear_cache('routing') # if the routing map was already generated it is possible that some compiled regex are in cache. # we want to mesure the cold state, when the worker just spawned, we need to empty the re cache re._cache.clear() - self.env['ir.http']._clear_routing_map() + self.env.registry.clear_cache('routing') start = time.time() self.env['ir.http'].routing_map() duration = time.time() - start diff --git a/odoo/addons/base/tests/test_ormcache.py b/odoo/addons/base/tests/test_ormcache.py index c2a80c06064..e92281ce62f 100644 --- a/odoo/addons/base/tests/test_ormcache.py +++ b/odoo/addons/base/tests/test_ormcache.py @@ -3,7 +3,7 @@ from odoo.tests.common import TransactionCase from odoo.tools import get_cache_key_counter - +from threading import Thread, Barrier class TestOrmcache(TransactionCase): def test_ormcache(self): @@ -17,7 +17,7 @@ class TestOrmcache(TransactionCase): miss = counter.miss # clear the caches of ir.model.data, retrieve its key and - IMD.clear_caches() + self.env.registry.clear_cache() self.assertNotIn(key, cache) # lookup some reference @@ -37,3 +37,63 @@ class TestOrmcache(TransactionCase): self.assertEqual(counter.hit, hit + 2) self.assertEqual(counter.miss, miss + 1) self.assertIn(key, cache) + + def test_invalidation(self): + self.assertEqual(self.env.registry.cache_invalidated, set()) + self.env.registry.clear_cache() + self.env.registry.clear_cache('templates') + self.assertEqual(self.env.registry.cache_invalidated, {'default', 'templates'}) + self.env.registry.reset_changes() + self.assertEqual(self.env.registry.cache_invalidated, set()) + self.env.registry.clear_cache('assets') + self.assertEqual(self.env.registry.cache_invalidated, {'assets'}) + self.env.registry.reset_changes() + self.assertEqual(self.env.registry.cache_invalidated, set()) + + def test_invalidation_thread_local(self): + # this test ensures that the registry.cache_invalidated set is thread local + + caches = ['default', 'templates', 'assets'] + nb_treads = len(caches) + + # use barriers to ensure threads synchronization + sync_clear_cache = Barrier(nb_treads, timeout=5) + sync_assert_equal = Barrier(nb_treads, timeout=5) + sync_reset = Barrier(nb_treads, timeout=5) + + operations = [] + def run(cache): + self.assertEqual(self.env.registry.cache_invalidated, set()) + + self.env.registry.clear_cache(cache) + operations.append('clear_cache') + sync_clear_cache.wait() + + self.assertEqual(self.env.registry.cache_invalidated, {cache}) + operations.append('assert_contains') + sync_assert_equal.wait() + + self.env.registry.reset_changes() + operations.append('reset_changes') + sync_reset.wait() + + self.assertEqual(self.env.registry.cache_invalidated, set()) + operations.append('assert_empty') + + # run all threads + threads = [] + for cache in caches: + threads.append(Thread(target=run, args=(cache,))) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + # ensure that the threads operations where executed in the expected order + self.assertEqual( + operations, + ['clear_cache'] * nb_treads + + ['assert_contains'] * nb_treads + + ['reset_changes'] * nb_treads + + ['assert_empty'] * nb_treads + ) diff --git a/odoo/addons/base/tests/test_qweb.py b/odoo/addons/base/tests/test_qweb.py index d31582ba0c0..a71795f17b3 100644 --- a/odoo/addons/base/tests/test_qweb.py +++ b/odoo/addons/base/tests/test_qweb.py @@ -1618,7 +1618,7 @@ class TestQWebBasic(TransactionCase): QWeb.with_context(preserve_comments=False)._render(view.id), markupsafe.Markup('

'), "Should not have the comment") - QWeb.clear_caches() + self.env.registry.clear_cache('templates') self.assertEqual( QWeb.with_context(preserve_comments=True)._render(view.id), markupsafe.Markup(f'

{comment}

'), @@ -1639,7 +1639,7 @@ class TestQWebBasic(TransactionCase): QWeb.with_context(preserve_comments=False)._render(view.id), markupsafe.Markup('

'), "Should not have the processing instruction") - QWeb.clear_caches() + self.env.registry.clear_cache('templates') self.assertEqual( QWeb.with_context(preserve_comments=True)._render(view.id), markupsafe.Markup(f'

{p_instruction}

'), diff --git a/odoo/addons/base/tests/test_res_users.py b/odoo/addons/base/tests/test_res_users.py index e7b046db9b1..0ce0dcbac19 100644 --- a/odoo/addons/base/tests/test_res_users.py +++ b/odoo/addons/base/tests/test_res_users.py @@ -209,15 +209,15 @@ class TestUsers(TransactionCase): request_patch.start() self.assertEqual(user.context_get()['lang'], 'fr_FR') - self.env.registry.clear_caches() + self.env.registry.clear_cache() user.lang = False self.assertEqual(user.context_get()['lang'], 'es_ES') - self.env.registry.clear_caches() + self.env.registry.clear_cache() request_patch.stop() self.assertEqual(user.context_get()['lang'], 'de_DE') - self.env.registry.clear_caches() + self.env.registry.clear_cache() company.lang = False self.assertEqual(user.context_get()['lang'], 'en_US') diff --git a/odoo/addons/base/tests/test_user_has_group.py b/odoo/addons/base/tests/test_user_has_group.py index 141ca697fa1..2592f24f538 100644 --- a/odoo/addons/base/tests/test_user_has_group.py +++ b/odoo/addons/base/tests/test_user_has_group.py @@ -252,17 +252,17 @@ class TestHasGroup(TransactionCase): user_b.write({"groups_id": [Command.link(group_C.id)]}) def test_has_group_cleared_cache_on_write(self): - self.registry._clear_cache() - self.assertFalse(self.registry._Registry__cache, "Ensure ormcache is empty") + self.env.registry.clear_cache() + self.assertFalse(self.registry._Registry__caches['default'], "Ensure ormcache is empty") def populate_cache(): self.test_user.has_group('test_user_has_group.group0') - self.assertTrue(self.registry._Registry__cache, "user.has_group cache must be populated") + self.assertTrue(self.registry._Registry__caches['default'], "user.has_group cache must be populated") populate_cache() self.env.ref(self.group0).write({"share": True}) - self.assertFalse(self.registry._Registry__cache, "Writing on group must invalidate user.has_group cache") + self.assertFalse(self.registry._Registry__caches['default'], "Writing on group must invalidate user.has_group cache") populate_cache() # call_cache_clearing_methods is called in res.groups.write to invalidate @@ -272,6 +272,6 @@ class TestHasGroup(TransactionCase): # the ormcache of method `user.has_group()` self.env['ir.model.access'].call_cache_clearing_methods() self.assertFalse( - self.registry._Registry__cache, + self.registry._Registry__caches['default'], "call_cache_clearing_methods() must invalidate user.has_group cache" ) diff --git a/odoo/addons/base/tests/test_views.py b/odoo/addons/base/tests/test_views.py index b46b7a93b5d..dae4cf28f83 100644 --- a/odoo/addons/base/tests/test_views.py +++ b/odoo/addons/base/tests/test_views.py @@ -334,12 +334,12 @@ class TestViewInheritance(ViewCase): self.assertEqual(counter.hit, hit) self.assertEqual(counter.miss, miss + 2) - with self.assertQueryCount(9): + with self.assertQueryCount(7): self.assertValid(""" """, inherit_id=base_view.id) - self.assertEqual(counter.hit, hit) - self.assertEqual(counter.miss, miss + 4) + self.assertEqual(counter.hit, hit + 2) + self.assertEqual(counter.miss, miss + 2) def test_view_validate_attrs_groups_query_count(self): _, _, counter = get_cache_key_counter(self.env['ir.model.data']._xmlid_lookup, 'base.group_system') @@ -356,14 +356,14 @@ class TestViewInheritance(ViewCase): self.assertEqual(counter.hit, hit) self.assertEqual(counter.miss, miss + 1) - with self.assertQueryCount(6): + with self.assertQueryCount(5): self.assertValid(""" """, inherit_id=base_view.id) - self.assertEqual(counter.hit, hit) - self.assertEqual(counter.miss, miss + 2) + self.assertEqual(counter.hit, hit + 1) + self.assertEqual(counter.miss, miss + 1) class TestApplyInheritanceSpecs(ViewCase): diff --git a/odoo/addons/test_access_rights/tests/test_ir_rules.py b/odoo/addons/test_access_rights/tests/test_ir_rules.py index dc2826123be..a10d12e7292 100644 --- a/odoo/addons/test_access_rights/tests/test_ir_rules.py +++ b/odoo/addons/test_access_rights/tests/test_ir_rules.py @@ -115,12 +115,12 @@ class TestRules(TransactionCase): self.assertFalse(ObjCateg.with_context(only_media=True).search([])) # record1 is food and is accessible with an empy context - ObjCateg.clear_caches() + self.env.registry.clear_cache() records = SomeObj.search([('id', '=', self.id1)]) self.assertTrue(records) # it should also be accessible as the context is not used when # searching for SomeObjs - ObjCateg.clear_caches() + self.env.registry.clear_cache() records = SomeObj.with_context(only_media=True).search([('id', '=', self.id1)]) self.assertTrue(records) diff --git a/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py b/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py index d5d7a001a19..96213e3ab13 100644 --- a/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py +++ b/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py @@ -125,7 +125,7 @@ class FileTouchable(AddonManifestPatched): @tagged('-at_install', 'post_install') class Test404(HttpCase): def get_assets_node_cache(self): - return [e for e in self.env.registry._Registry__cache.d if e[0] == 'ir.qweb' and '_generate_' in str(e[1])] #_generate_asset_nodes_cache + return [e for e in self.env.registry._Registry__caches['assets'].d if e[0] == 'ir.qweb' and '_generate_' in str(e[1])] #_generate_asset_nodes_cache def test_rollback(self): """ @@ -133,14 +133,14 @@ class Test404(HttpCase): this actually tests the add_post_rollback behaviour """ self.env['ir.attachment'].search([('url', '=like', '/web/assets/%web.assets_frontend%')]).unlink() - self.env.registry._clear_cache() + self.env.registry.clear_cache() self.url_open('/some_404_routes') self.assertFalse(self.get_assets_node_cache(), 'orm_cache should be emptied by rollback') self.assertFalse(self.env['ir.attachment'].search([('url', '=like', '/web/assets/%web.assets_frontend%%')]), "attachments should have been rollbacked") self.url_open('/') self.assertTrue(self.env['ir.attachment'].search([('url', '=like', '/web/assets/%web.assets_frontend%%')]), "attachments should have been generated as expected") self.assertTrue(self.get_assets_node_cache(), 'orm_cache should remain populated') - self.env.registry._clear_cache() + self.env.registry.clear_cache() class TestJavascriptAssetsBundle(FileTouchable): @classmethod @@ -1895,19 +1895,21 @@ class TestAssetsManifest(AddonManifestPatched): class AssetsNodeOrmCacheUsage(TransactionCase): def cache_keys(self): - keys = self.env.registry._Registry__cache.d + keys = self.env.registry._Registry__caches['assets'].d + asset_keys = [key for key in keys if key[0] == 'ir.asset' and '_get_asset_paths' in str(key[1])] # ignore topological sort entry qweb_keys = [key for key in keys if key[0] == 'ir.qweb'] return asset_keys, qweb_keys def test_assets_node_orm_cache_usage_debug(self): - self.env.registry._clear_cache() + self.env.registry.clear_cache('assets') asset_keys, qweb_keys = self.cache_keys() self.assertEqual(len(asset_keys), 0) self.assertEqual(len(qweb_keys), 0) self.env['ir.qweb']._get_asset_nodes('web.assets_backend') + asset_keys, qweb_keys = self.cache_keys() self.assertEqual(len(asset_keys), 1) self.assertEqual(len(qweb_keys), 1) @@ -1929,7 +1931,7 @@ class AssetsNodeOrmCacheUsage(TransactionCase): self.assertEqual(len(qweb_keys), 1) def test_assets_node_orm_cache_usage_file_type(self): - self.env.registry._clear_cache() + self.env.registry.clear_cache('assets') asset_keys, qweb_keys = self.cache_keys() self.assertEqual(len(asset_keys), 0) @@ -1954,7 +1956,7 @@ class AssetsNodeOrmCacheUsage(TransactionCase): def test_assets_node_orm_cache_usage_lang(self): - self.env.registry._clear_cache() + self.env.registry.clear_cache('assets') self.env['res.lang']._activate_lang('ar_SY') self.env['res.lang']._activate_lang('fr_FR') self.env['res.lang']._activate_lang('en_US') @@ -1981,7 +1983,7 @@ class AssetsNodeOrmCacheUsage(TransactionCase): def test_assets_node_orm_cache_usage_website(self): if self.env['ir.module.module'].search([('name', '=', 'website'), ('state', '=', 'uninstalled')]): return # only makes sence if website is installed - self.env.registry._clear_cache() + self.env.registry.clear_cache('assets') asset_keys, qweb_keys = self.cache_keys() self.assertEqual(len(asset_keys), 0) @@ -1998,7 +2000,7 @@ class AssetsNodeOrmCacheUsage(TransactionCase): self.assertEqual(len(qweb_keys), 2) def test_assets_node_orm_cache_usage_node_flags(self): - self.env.registry._clear_cache() + self.env.registry.clear_cache('assets') asset_keys, qweb_keys = self.cache_keys() self.assertEqual(len(asset_keys), 0) diff --git a/odoo/addons/test_impex/tests/test_load.py b/odoo/addons/test_impex/tests/test_load.py index 69e57929594..64c3b69e6f1 100644 --- a/odoo/addons/test_impex/tests/test_load.py +++ b/odoo/addons/test_impex/tests/test_load.py @@ -40,7 +40,7 @@ class ImporterCase(common.TransactionCase): def setUp(self): super(ImporterCase, self).setUp() self.model = self.env[self.model_name] - self.env['ir.model.data'].clear_caches() + self.env.registry.clear_cache() self.cr.cache.clear() def import_(self, fields, rows, context=None): @@ -1114,7 +1114,7 @@ class test_realworld(SavepointCaseWithUserDemo): """ The content of the o2m field's dict needs to go through conversion as it may be composed of convertables or other relational fields """ - self.env['ir.model.data'].clear_caches() + self.env.registry.clear_cache() Model = self.env['export.one2many.recursive'] result = Model.load( ['value', 'child/const', 'child/child1/str', 'child/child2/value'], @@ -1141,7 +1141,7 @@ class test_realworld(SavepointCaseWithUserDemo): [12]) def test_o2m_subfields_fail_by_implicit_id(self): - self.env['ir.model.data'].clear_caches() + self.env.registry.clear_cache() Model = self.env['export.one2many.recursive'] result = Model.with_context(import_file=True).load( ['child/child1/parent_id'], diff --git a/odoo/addons/test_new_api/tests/test_new_fields.py b/odoo/addons/test_new_api/tests/test_new_fields.py index 4f14974ec44..2e3f3fc227b 100644 --- a/odoo/addons/test_new_api/tests/test_new_fields.py +++ b/odoo/addons/test_new_api/tests/test_new_fields.py @@ -25,7 +25,7 @@ class TestFields(TransactionCaseWithUserDemo): def setUp(self): # for tests methods that create custom models/fields self.addCleanup(self.registry.reset_changes) - self.addCleanup(self.registry.clear_caches) + self.addCleanup(self.registry.clear_all_caches) super(TestFields, self).setUp() self.env.ref('test_new_api.discussion_0').write({'participants': [Command.link(self.user_demo.id)]}) # YTI FIX ME: The cache shouldn't be inconsistent (rco is gonna fix it) diff --git a/odoo/http.py b/odoo/http.py index c1240270621..0bc5c7284d7 100644 --- a/odoo/http.py +++ b/odoo/http.py @@ -2054,7 +2054,7 @@ class Application: def get_db_router(self, db): if not db: return self.nodb_routing_map - return request.registry['ir.http'].routing_map() + return request.env['ir.http'].routing_map() @lazy_property def geoip_city_db(self): diff --git a/odoo/models.py b/odoo/models.py index 9ba686f94d1..49996a1cb83 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -1788,9 +1788,10 @@ class BaseModel(metaclass=MetaModel): """ Clear the caches This clears the caches associated to methods decorated with - ``tools.ormcache`` or ``tools.ormcache_multi``. + ``tools.ormcache``. """ - cls.pool._clear_cache() + warnings.warn("Deprecated model.clear_cache(), use registry.clear_cache() instead", DeprecationWarning) + cls.pool.clear_all_cache() @api.model def _read_group(self, domain, groupby=(), aggregates=(), having=(), offset=0, limit=None, order=None): diff --git a/odoo/modules/loading.py b/odoo/modules/loading.py index b2463e75b5a..53dbc7dd48d 100644 --- a/odoo/modules/loading.py +++ b/odoo/modules/loading.py @@ -279,8 +279,6 @@ def load_module_graph(env, graph, status=None, perform_checks=True, if not needs_update: registry.setup_models(env.cr) # Python tests - env['ir.http']._clear_routing_map() # force routing map to be rebuilt - tests_t0, tests_q0 = time.time(), odoo.sql_db.sql_counter test_results = loader.run_suite(suite, module_name) report.update(test_results) diff --git a/odoo/modules/registry.py b/odoo/modules/registry.py index 77f8b1b23b6..55cf2f7f58a 100644 --- a/odoo/modules/registry.py +++ b/odoo/modules/registry.py @@ -9,6 +9,8 @@ from collections.abc import Mapping from contextlib import closing, contextmanager from functools import partial from operator import attrgetter + +import inspect import logging import os import threading @@ -22,8 +24,11 @@ from odoo.modules.db import FunctionStatus from odoo.osv.expression import get_unaccent_wrapper from .. import SUPERUSER_ID from odoo.sql_db import TestCursor -from odoo.tools import (config, existing_tables, lazy_classproperty, - lazy_property, sql, Collector, OrderedSet) +from odoo.tools import ( + config, existing_tables, lazy_classproperty, + lazy_property, sql, Collector, OrderedSet, + format_frame +) from odoo.tools.func import locked from odoo.tools.lru import LRU @@ -31,6 +36,17 @@ _logger = logging.getLogger(__name__) _schema = logging.getLogger('odoo.schema') +_REGISTRY_CACHES = { + 'default': 8192, + 'assets': 512, # arbitrary + 'templates': 1024, # arbitrary + 'templates.cached_values': 2048, # arbitrary + 'routing': 1024, # 2 entries per website + 'routing.rewrites': 8192, # url_rewrite entries + 'xmlid': 1024, +} + + class Registry(Mapping): """ Model registry for a particular database. @@ -118,7 +134,10 @@ class Registry(Mapping): self._fields_by_model = None self._ordinary_tables = None self._constraint_queue = deque() - self.__cache = LRU(8192) + self.__caches = {cache_name: LRU(cache_size) for cache_name, cache_size in _REGISTRY_CACHES.items()} + self.__caches_by_prefix = {} + for cache_name, cache in self.__caches.items(): + self.__caches_by_prefix.setdefault(cache_name.split('.')[0], []).append(cache) # modules fully loaded (maintained during init phase by `loading` module) self._init_modules = set() @@ -151,7 +170,7 @@ class Registry(Mapping): # The `base_cache_signaling sequence` indicates all caches must be # invalidated (i.e. cleared). self.registry_sequence = None - self.cache_sequence = None + self.cache_sequences = {} # Flags indicating invalidation of the registry or the cache. self._invalidation_flags = threading.local() @@ -233,7 +252,8 @@ class Registry(Mapping): from .. import models # clear cache to ensure consistency, but do not signal it - self.__cache.clear() + for cache in self.__caches.values(): + cache.clear() lazy_property.reset_all(self) self._field_trigger_trees.clear() @@ -263,7 +283,8 @@ class Registry(Mapping): model._unregister_hook() # clear cache to ensure consistency, but do not signal it - self.__cache.clear() + for cache in self.__caches.values(): + cache.clear() lazy_property.reset_all(self) self._field_trigger_trees.clear() @@ -677,17 +698,33 @@ class Registry(Mapping): for table in missing_tables: _logger.error("Model %s has no table.", table2model[table]) - def _clear_cache(self): - """ Clear the cache and mark it as invalidated. """ - self.__cache.clear() - self.cache_invalidated = True - - def clear_caches(self): + def clear_cache(self, *cache_names): """ Clear the caches associated to methods decorated with - ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models. + ``tools.ormcache``if cache is in `cache_name` subset. """ + cache_names = cache_names or ('default',) + assert not any('.' in cache_name for cache_name in cache_names) + for cache_name in cache_names: + for cache in self.__caches_by_prefix[cache_name]: + cache.clear() + self.cache_invalidated.add(cache_name) + + # log information about invalidation_cause + caller_info = format_frame(inspect.currentframe().f_back) + log = _logger.info if self.loaded else _logger.debug + log('Invalidating %s model caches from %s', ','.join(cache_names), caller_info) + + def clear_all_caches(self): + """ Clear the caches associated to methods decorated with + ``tools.ormcache``. """ - for model in self.models.values(): - model.clear_caches() + for cache_name, caches in self.__caches_by_prefix.items(): + for cache in caches: + cache.clear() + self.cache_invalidated.add(cache_name) + + caller_info = format_frame(inspect.currentframe().f_back) + log = _logger.info if self.loaded else _logger.debug + log('Invalidating all model caches from %s', caller_info) def is_an_ordinary_table(self, model): """ Return whether the given model has an ordinary table. """ @@ -719,11 +756,11 @@ class Registry(Mapping): @property def cache_invalidated(self): """ Determine whether the current thread has modified the cache. """ - return getattr(self._invalidation_flags, 'cache', False) - - @cache_invalidated.setter - def cache_invalidated(self, value): - self._invalidation_flags.cache = value + try: + return self._invalidation_flags.cache + except AttributeError: + names = self._invalidation_flags.cache = set() + return names def setup_signaling(self): """ Setup the inter-process signaling on this registry. """ @@ -733,21 +770,35 @@ class Registry(Mapping): with self.cursor() as cr: # The `base_registry_signaling` sequence indicates when the registry # must be reloaded. - # The `base_cache_signaling` sequence indicates when all caches must + # The `base_cache_signaling_...` sequences indicates when caches must # be invalidated (i.e. cleared). - cr.execute("SELECT sequence_name FROM information_schema.sequences WHERE sequence_name='base_registry_signaling'") - if not cr.fetchall(): - cr.execute("CREATE SEQUENCE base_registry_signaling INCREMENT BY 1 START WITH 1") - cr.execute("SELECT nextval('base_registry_signaling')") - cr.execute("CREATE SEQUENCE base_cache_signaling INCREMENT BY 1 START WITH 1") - cr.execute("SELECT nextval('base_cache_signaling')") + sequence_names = ('base_registry_signaling', *(f'base_cache_signaling_{cache_name}' for cache_name in self.__caches_by_prefix)) + cr.execute("SELECT sequence_name FROM information_schema.sequences WHERE sequence_name IN %s", [sequence_names]) + existing_sequences = tuple(s[0] for s in cr.fetchall()) # could be a set but not efficient with such a little list - cr.execute(""" SELECT base_registry_signaling.last_value, - base_cache_signaling.last_value - FROM base_registry_signaling, base_cache_signaling""") - self.registry_sequence, self.cache_sequence = cr.fetchone() - _logger.debug("Multiprocess load registry signaling: [Registry: %s] [Cache: %s]", - self.registry_sequence, self.cache_sequence) + for sequence_name in sequence_names: + if sequence_name not in existing_sequences: + _sequence_name = sql.Identifier(sequence_name) # even if sequence_name should be safe, will avoid flagging the sql_injection linter + cr.execute(sql.SQL("CREATE SEQUENCE {} INCREMENT BY 1 START WITH 1").format(_sequence_name)) + cr.execute("SELECT nextval(%s)", [sequence_name]) + + db_registry_sequence, db_cache_sequences = self.get_sequences(cr) + self.registry_sequence = db_registry_sequence + self.cache_sequences.update(db_cache_sequences) + + _logger.debug("Multiprocess load registry signaling: [Registry: %s] %s", + self.registry_sequence, ' '.join('[Cache %s: %s]' % cs for cs in self.cache_sequences.items())) + + def get_sequences(self, cr): + cache_sequences_query = ', '.join([f'base_cache_signaling_{cache_name}' for cache_name in self.__caches_by_prefix]) + cache_sequences_values_query = ',\n'.join([f'base_cache_signaling_{cache_name}.last_value' for cache_name in self.__caches_by_prefix]) + cr.execute(f""" + SELECT base_registry_signaling.last_value, {cache_sequences_values_query} + FROM base_registry_signaling, {cache_sequences_query} + """) + registry_sequence, *cache_sequences_values = cr.fetchone() + cache_sequences = dict(zip(self.__caches_by_prefix, cache_sequences_values)) + return registry_sequence, cache_sequences def check_signaling(self): """ Check whether the registry has changed, and performs all necessary @@ -757,28 +808,28 @@ class Registry(Mapping): return self with closing(self.cursor()) as cr: - cr.execute(""" SELECT base_registry_signaling.last_value, - base_cache_signaling.last_value - FROM base_registry_signaling, base_cache_signaling""") - r, c = cr.fetchone() - _logger.debug("Multiprocess signaling check: [Registry - %s -> %s] [Cache - %s -> %s]", - self.registry_sequence, r, self.cache_sequence, c) + db_registry_sequence, db_cache_sequences = self.get_sequences(cr) + changes = '' # Check if the model registry must be reloaded - if self.registry_sequence != r: + if self.registry_sequence != db_registry_sequence: _logger.info("Reloading the model registry after database signaling.") self = Registry.new(self.db_name) + self.registry_sequence = db_registry_sequence + if _logger.isEnabledFor(logging.DEBUG): + changes += "[Registry - %s -> %s]" % (self.registry_sequence, db_registry_sequence) # Check if the model caches must be invalidated. - elif self.cache_sequence != c: - _logger.info("Invalidating all model caches after database signaling.") - self.clear_caches() - - # prevent re-signaling the clear_caches() above, or any residual one that - # would be inherited from the master process (first request in pre-fork mode) - self.cache_invalidated = False - - self.registry_sequence = r - self.cache_sequence = c - + else: + for cache_name, cache_sequence in self.cache_sequences.items(): + expected_sequence = db_cache_sequences[cache_name] + if cache_sequence != expected_sequence: + _logger.info("Invalidating %s model caches after database signaling.", cache_name) + for cache in self.__caches_by_prefix[cache_name]: # don't call clear_cache to avoid signal loop + cache.clear() + self.cache_sequences[cache_name] = expected_sequence + if _logger.isEnabledFor(logging.DEBUG): + changes += "[Cache %s - %s -> %s]" % (cache_name, cache_sequence, expected_sequence) + if changes: + _logger.debug("Multiprocess signaling check: %s", changes) return self def signal_changes(self): @@ -786,10 +837,10 @@ class Registry(Mapping): if self.in_test_mode(): if self.registry_invalidated: self.registry_sequence += 1 - if self.cache_invalidated: - self.cache_sequence += 1 + for cache_name in self.cache_invalidated or (): + self.cache_sequences[cache_name] += 1 self.registry_invalidated = False - self.cache_invalidated = False + self.cache_invalidated.clear() return if self.registry_invalidated: @@ -803,11 +854,12 @@ class Registry(Mapping): elif self.cache_invalidated: _logger.info("At least one model cache has been invalidated, signaling through the database.") with closing(self.cursor()) as cr: - cr.execute("select nextval('base_cache_signaling')") - self.cache_sequence = cr.fetchone()[0] + for cache_name in self.cache_invalidated: + cr.execute("select nextval(%s)", [f'base_cache_signaling_{cache_name}']) + self.cache_sequences[cache_name] = cr.fetchone()[0] self.registry_invalidated = False - self.cache_invalidated = False + self.cache_invalidated.clear() def reset_changes(self): """ Reset the registry and cancel all invalidations. """ @@ -816,8 +868,10 @@ class Registry(Mapping): self.setup_models(cr) self.registry_invalidated = False if self.cache_invalidated: - self.__cache.clear() - self.cache_invalidated = False + for cache_name in self.cache_invalidated: + for cache in self.__caches_by_prefix[cache_name]: + cache.clear() + self.cache_invalidated.clear() @contextmanager def manage_changes(self): diff --git a/odoo/tests/common.py b/odoo/tests/common.py index 6c6644a4f70..ab381bad1c3 100644 --- a/odoo/tests/common.py +++ b/odoo/tests/common.py @@ -749,10 +749,11 @@ class TransactionCase(BaseCase): if (cls.registry_start_sequence != cls.registry.registry_sequence) or cls.registry.registry_invalidated: with cls.registry.cursor() as cr: cls.registry.setup_models(cr) - cls.registry.registry_invalidated = False + cls.registry.registry_invalidated = False + cls.registry.clear_all_caches() + cls.registry.cache_invalidated.clear() cls.addClassCleanup(reset_changes) - cls.addClassCleanup(cls.registry.clear_caches) cls.cr = cls.registry.cursor() cls.addClassCleanup(cls.cr.close) @@ -771,7 +772,7 @@ class TransactionCase(BaseCase): self.addCleanup(envs.update, list(envs)) self.addCleanup(envs.clear) - self.addCleanup(self.registry.clear_caches) + self.addCleanup(self.registry.clear_all_caches) # This prevents precommit functions and data from piling up # until cr.flush is called in 'assertRaises' clauses @@ -810,7 +811,7 @@ class SingleTransactionCase(BaseCase): super().setUpClass() cls.registry = odoo.registry(get_db_name()) cls.addClassCleanup(cls.registry.reset_changes) - cls.addClassCleanup(cls.registry.clear_caches) + cls.addClassCleanup(cls.registry.clear_all_caches) cls.cr = cls.registry.cursor() cls.addClassCleanup(cls.cr.close) diff --git a/odoo/tools/cache.py b/odoo/tools/cache.py index 912affbba6d..b1939eff696 100644 --- a/odoo/tools/cache.py +++ b/odoo/tools/cache.py @@ -6,6 +6,7 @@ from collections import Counter, defaultdict from decorator import decorator from inspect import signature, Parameter import logging +import warnings unsafe_eval = eval @@ -52,12 +53,13 @@ class ormcache(object): def __init__(self, *args, **kwargs): self.args = args self.skiparg = kwargs.get('skiparg') + self.cache_name = kwargs.get('cache', 'default') def __call__(self, method): self.method = method self.determine_key() lookup = decorator(self.lookup, method) - lookup.clear_cache = self.clear + lookup.ormcache = self return lookup def determine_key(self): @@ -82,7 +84,7 @@ class ormcache(object): def lru(self, model): counter = STAT[(model.pool.db_name, model._name, self.method)] - return model.pool._Registry__cache, (model._name, self.method), counter + return model.pool._Registry__caches[self.cache_name], (model._name, self.method), counter def lookup(self, method, *args, **kwargs): d, key0, counter = self.lru(args[0]) @@ -102,7 +104,8 @@ class ormcache(object): def clear(self, model, *args): """ Clear the registry cache """ - model.pool._clear_cache() + warnings.warn('Deprecated method ormcache.clear(model, *args), use registry.clear_cache() instead') + model.pool.clear_all_cache() class ormcache_context(ormcache): @@ -133,81 +136,6 @@ class ormcache_context(ormcache): self.key = unsafe_eval(code) -class ormcache_multi(ormcache): - """ This LRU cache decorator is a variant of :class:`ormcache`, with an - extra parameter ``multi`` that gives the name of a parameter. Upon call, the - corresponding argument is iterated on, and every value leads to a cache - entry under its own key. - """ - def __init__(self, *args, **kwargs): - super(ormcache_multi, self).__init__(*args, **kwargs) - self.multi = kwargs['multi'] - - def determine_key(self): - """ Determine the function that computes a cache key from arguments. """ - assert self.skiparg is None, "ormcache_multi() no longer supports skiparg" - assert isinstance(self.multi, str), "ormcache_multi() parameter multi must be an argument name" - - super(ormcache_multi, self).determine_key() - - # key_multi computes the extra element added to the key - sign = signature(self.method) - args = ', '.join( - str(params.replace(annotation=Parameter.empty, default=Parameter.empty)) - for params in sign.parameters.values() - ) - code_multi = "lambda %s: %s" % (args, self.multi) - self.key_multi = unsafe_eval(code_multi) - - # self.multi_pos is the position of self.multi in args - self.multi_pos = list(sign.parameters).index(self.multi) - - def lookup(self, method, *args, **kwargs): - d, key0, counter = self.lru(args[0]) - base_key = key0 + self.key(*args, **kwargs) - ids = self.key_multi(*args, **kwargs) - result = {} - missed = [] - - # first take what is available in the cache - for i in ids: - key = base_key + (i,) - try: - result[i] = d[key] - counter.hit += 1 - except Exception: - counter.miss += 1 - missed.append(i) - - if missed: - # call the method for the ids that were not in the cache; note that - # thanks to decorator(), the multi argument will be bound and passed - # positionally in args. - args = list(args) - args[self.multi_pos] = missed - result.update(method(*args, **kwargs)) - - # store those new results back in the cache - for i in missed: - key = base_key + (i,) - d[key] = result[i] - - return result - - -class dummy_cache(object): - """ Cache decorator replacement to actually do no caching. """ - def __init__(self, *l, **kw): - pass - - def __call__(self, fn): - fn.clear_cache = self.clear - return fn - - def clear(self, *l, **kw): - pass - - def log_ormcache_stats(sig=None, frame=None): """ Log statistics of ormcache usage by database, model, and method. """ from odoo.modules.registry import Registry @@ -216,26 +144,30 @@ def log_ormcache_stats(sig=None, frame=None): me = threading.current_thread() me_dbname = getattr(me, 'dbname', 'n/a') - for dbname, reg in sorted(Registry.registries.d.items()): - # set logger prefix to dbname - me.dbname = dbname - entries = Counter(k[:2] for k in reg._Registry__cache.d) + def _log_ormcache_stats(cache_name, cache): + entries = Counter(k[:2] for k in cache.d) # show entries sorted by model name, method name for key in sorted(entries, key=lambda key: (key[0], key[1].__name__)): model, method = key stat = STAT[(dbname, model, method)] _logger.info( - "%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s", - entries[key], stat.hit, stat.miss, stat.err, stat.ratio, model, method.__name__, + "%s, %6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s", + cache_name.rjust(25), entries[key], stat.hit, stat.miss, stat.err, stat.ratio, model, method.__name__, ) + for dbname, reg in sorted(Registry.registries.d.items()): + # set logger prefix to dbname + me.dbname = dbname + for cache_name, cache in reg._Registry__caches.items(): + _log_ormcache_stats(cache_name, cache) + me.dbname = me_dbname def get_cache_key_counter(bound_method, *args, **kwargs): """ Return the cache, key and stat counter for the given call. """ model = bound_method.__self__ - ormcache = bound_method.clear_cache.__self__ + ormcache = bound_method.ormcache cache, key0, counter = ormcache.lru(model) key = key0 + ormcache.key(model, *args, **kwargs) return cache, key, counter diff --git a/odoo/tools/misc.py b/odoo/tools/misc.py index 5c068c11fb0..ee2dfe3d9bd 100644 --- a/odoo/tools/misc.py +++ b/odoo/tools/misc.py @@ -1786,3 +1786,8 @@ def get_flag(country_code: str) -> str: This emoji is composed of the two regional indicator emoji of the country code. """ return "".join(chr(int(f"1f1{ord(c)+165:02x}", base=16)) for c in country_code) + + +def format_frame(frame): + code = frame.f_code + return f'{code.co_name} {code.co_filename}:{frame.f_lineno}'