From 2e29a9350306fd94fff19bc02821d7e7ab9c6f8b Mon Sep 17 00:00:00 2001 From: Julien Castiaux Date: Mon, 22 Nov 2021 15:22:43 +0000 Subject: [PATCH] [REF] core: remove http.addons_manifest The http.addons_manifest is a map {module: manifest_dict} that is populated upon the first http request. This map is basically a module manifest cache with an extra `addons_path` key, the path of the module on the file-system. This cache is eagerly populated upon the first http request, the map is empty in non-http contextes (e.g. cron) which have been a source of bugs (e.g. 50c8eb1). A manifest cache is necessary because reading and parsing python files from the file-system is not that cheap but there is no reason that cache is located in `odoo.http`. A thin cache layer now wraps `load_information_from_description_file()`/`load_manifest()` and is lazily populated. The `http.addons_manifest` have been removed. The extra `addons_path` key is now present in the "normal" manifest. The `read_manifest()` was hardly used so it has been deprecated. The only way to retrieve a manifest is now `load_information_from_description_file()` which was renamed `load_manifest()` (no cache) and `get_manifest()` (cache). Side note about performances, the cache is necessary. Addons manifest are read-only and reading + parsing python files from the file system is not a cheap operation. Running the e-commerce tour `@website_sale.test_04_admin_website_sale_tour` without cache on `load_manifest()` requires 68,29 secs to complete on my laptop, exceeding the default 1-minute time frame allowed in tests. Using a cache the time is down to 36,53 secs. The performance impact is huge. Part-of: odoo/odoo#79977 --- addons/base_import_module/models/ir_module.py | 4 +- addons/hw_drivers/tools/helpers.py | 1 - addons/web/controllers/main.py | 13 +- addons/web/tests/test_assets.py | 14 +- addons/web_editor/controllers/main.py | 6 +- addons/website/models/website.py | 12 +- odoo/addons/base/models/ir_asset.py | 23 +-- odoo/addons/base/models/ir_cron.py | 2 +- odoo/addons/base/models/ir_module.py | 2 +- odoo/addons/base/tests/__init__.py | 1 + odoo/addons/base/tests/test_module.py | 85 +++++++++ .../tests/test_assetsbundle.py | 59 +++--- odoo/http.py | 26 +-- odoo/modules/__init__.py | 2 +- odoo/modules/db.py | 2 +- odoo/modules/graph.py | 5 +- odoo/modules/module.py | 171 +++++++++++------- odoo/tests/common.py | 13 ++ 18 files changed, 267 insertions(+), 174 deletions(-) create mode 100644 odoo/addons/base/tests/test_module.py diff --git a/addons/base_import_module/models/ir_module.py b/addons/base_import_module/models/ir_module.py index 99e12597b5c..3ab7aeb4d68 100644 --- a/addons/base_import_module/models/ir_module.py +++ b/addons/base_import_module/models/ir_module.py @@ -12,7 +12,7 @@ from os.path import join as opj import odoo from odoo import api, fields, models, _ from odoo.exceptions import UserError -from odoo.modules import load_information_from_description_file +from odoo.modules.module import load_manifest from odoo.tools import convert_file, exception_to_unicode _logger = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class IrModule(models.Model): known_mods_names = {m.name: m for m in known_mods} installed_mods = [m.name for m in known_mods if m.state == 'installed'] - terp = load_information_from_description_file(module, mod_path=path) + terp = load_manifest(module, mod_path=path) if not terp: return False values = self.get_values_from_terp(terp) diff --git a/addons/hw_drivers/tools/helpers.py b/addons/hw_drivers/tools/helpers.py index cf4a67b2770..28901368532 100644 --- a/addons/hw_drivers/tools/helpers.py +++ b/addons/hw_drivers/tools/helpers.py @@ -266,7 +266,6 @@ def load_iot_handlers(): if spec: module = util.module_from_spec(spec) spec.loader.exec_module(module) - http.addons_manifest = {} http.root = http.Root() def odoo_restart(delay): diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 1dea7b4e1ac..8a27b7d73d9 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -32,7 +32,7 @@ import odoo import odoo.modules.registry from odoo.api import call_kw from odoo.addons.base.models.qweb import QWeb -from odoo.modules import get_resource_path, module +from odoo.modules import get_resource_path, module, get_manifest from odoo.tools import html_escape, pycompat, ustr, apply_inheritance_specs, lazy_property, float_repr, osutil from odoo.tools.mimetypes import guess_mimetype from odoo.tools.translate import _ @@ -331,7 +331,7 @@ def fix_view_modes(action): def _local_web_translations(trans_file): messages = [] try: - with open(trans_file) as t_file: + with file_open(trans_file, filter_ext=('.po')) as t_file: po = babel.messages.pofile.read_po(t_file) except Exception: return @@ -962,11 +962,10 @@ class WebClient(http.Controller): translations_per_module = {} for addon_name in mods: - manifest = http.addons_manifest.get(addon_name) - if manifest and manifest.get('bootstrap'): - addons_path = http.addons_manifest[addon_name]['addons_path'] - f_name = os.path.join(addons_path, addon_name, "i18n", lang + ".po") - if not os.path.exists(f_name): + manifest = get_manifest(addon_name) + if manifest and manifest['bootstrap']: + f_name = get_resource_path(addon_name, 'i18n', f'{lang}.po') + if not f_name: continue translations_per_module[addon_name] = {'messages': _local_web_translations(f_name)} diff --git a/addons/web/tests/test_assets.py b/addons/web/tests/test_assets.py index bdfbfa9c41c..9c102339ac0 100644 --- a/addons/web/tests/test_assets.py +++ b/addons/web/tests/test_assets.py @@ -6,7 +6,7 @@ import time import odoo import odoo.tests -from odoo.modules.module import read_manifest +from odoo.modules.module import get_manifest from odoo.tools import mute_logger _logger = logging.getLogger(__name__) @@ -15,14 +15,12 @@ _logger = logging.getLogger(__name__) class TestAssetsGenerateTimeCommon(odoo.tests.TransactionCase): def generate_bundles(self): - bundles = set() installed_module_names = self.env['ir.module.module'].search([('state', '=', 'installed')]).mapped('name') - for addon_path in odoo.addons.__path__: - for addon in installed_module_names: - manifest = read_manifest(addon_path, addon) or {} - assets = manifest.get('assets') - if assets: - bundles |= set(assets.keys()) + bundles = { + key + for module in installed_module_names + for key in get_manifest(module)['assets'] + } for bundle in bundles: with mute_logger('odoo.addons.base.models.assetsbundle'): diff --git a/addons/web_editor/controllers/main.py b/addons/web_editor/controllers/main.py index fce6212b224..066860dcc75 100644 --- a/addons/web_editor/controllers/main.py +++ b/addons/web_editor/controllers/main.py @@ -18,6 +18,7 @@ from odoo.addons.http_routing.models.ir_http import slug, unslug from odoo.addons.web_editor.tools import get_video_url_data from odoo.exceptions import UserError from odoo.modules.module import get_resource_path +from odoo.tools import file_open from odoo.tools.mimetypes import guess_mimetype from odoo.tools.image import image_data_uri, base64_to_image from odoo.addons.base.models.assetsbundle import AssetsBundle @@ -56,8 +57,9 @@ class Web_Editor(http.Controller): # Make sure we have at least size=1 size = max(1, min(size, 512)) # Initialize font - addons_path = http.addons_manifest['web']['addons_path'] - font_obj = ImageFont.truetype(addons_path + font, size) + if font.startswith('/'): + font = font[1:] + font_obj = ImageFont.truetype(file_open(font, 'rb'), size) # if received character is not a number, keep old behaviour (icon is character) icon = chr(int(icon)) if icon.isdigit() else icon diff --git a/addons/website/models/website.py b/addons/website/models/website.py index ba97f267009..7f95d721d3a 100644 --- a/addons/website/models/website.py +++ b/addons/website/models/website.py @@ -23,7 +23,7 @@ from odoo.addons.portal.controllers.portal import pager from odoo.addons.iap.tools import iap_tools from odoo.exceptions import UserError, AccessError from odoo.http import request -from odoo.modules.module import get_resource_path +from odoo.modules.module import get_resource_path, get_manifest from odoo.osv.expression import AND, OR, FALSE_DOMAIN, get_unaccent_wrapper from odoo.tools.translate import _ from odoo.tools import escape_psql, pycompat @@ -296,10 +296,10 @@ class Website(models.Model): @api.model def get_theme_snippet_lists(self, theme_name): - default_snippet_lists = http.addons_manifest['theme_default'].get('snippet_lists', {}) - theme_snippet_lists = http.addons_manifest[theme_name].get('snippet_lists', {}) - snippet_lists = {**default_snippet_lists, **theme_snippet_lists} - return snippet_lists + return { + **get_manifest('theme_default')['snippet_lists'], + **get_manifest(theme_name).get('snippet_lists', {}), + } def configurator_set_menu_links(self, menu_company, module_data): menus = self.env['website.menu'].search([('url', 'in', list(module_data.keys())), ('website_id', '=', self.id)]) @@ -339,7 +339,7 @@ class Website(models.Model): def configurator_recommended_themes(self, industry_id, palette): domain = [('name', '=like', 'theme%'), ('name', 'not in', ['theme_default', 'theme_common'])] client_themes = request.env['ir.module.module'].search(domain).mapped('name') - client_themes_img = dict([(t, http.addons_manifest[t].get('images_preview_theme', {})) for t in client_themes]) + client_themes_img = {t: get_manifest(t).get('images_preview_theme', {}) for t in client_themes} themes_suggested = self._website_api_rpc( '/api/website/2/configurator/recommended_themes/%s' % industry_id, {'client_themes': client_themes_img} diff --git a/odoo/addons/base/models/ir_asset.py b/odoo/addons/base/models/ir_asset.py index ab74e233ab4..2824801c5a2 100644 --- a/odoo/addons/base/models/ir_asset.py +++ b/odoo/addons/base/models/ir_asset.py @@ -1,17 +1,14 @@ -# -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. import os - from glob import glob from logging import getLogger from werkzeug import urls import odoo +import odoo.modules.module # get_manifest, don't from-import it +from odoo import api, fields, models, tools from odoo.tools import misc -from odoo import tools -from odoo import api, fields, http, models -from odoo.http import root _logger = getLogger(__name__) @@ -140,10 +137,6 @@ class IrAsset(models.Model): if bundle in seen: raise Exception("Circular assets bundle declaration: %s" % " > ".join(seen + [bundle])) - if not root._loaded: - root.load_addons() - root._loaded = True - manifest_cache = http.addons_manifest exts = [] if js: exts += SCRIPT_EXTENSIONS @@ -163,7 +156,7 @@ class IrAsset(models.Model): accordingly. It is nested inside `_get_asset_paths` since we need the current - list of addons, extensions, asset_paths and manifest_cache. + list of addons, extensions and asset_paths. :param directive: string :param target: string or None or False @@ -209,11 +202,7 @@ class IrAsset(models.Model): # 2. Process all addons' manifests. for addon in self._topological_sort(tuple(addons)): - manifest = manifest_cache.get(addon) - if not manifest: - continue - manifest_assets = manifest.get('assets', {}) - for command in manifest_assets.get(bundle, []): + for command in odoo.modules.module.get_manifest(addon)['assets'].get(bundle, ()): directive, target, path_def = self._process_command(command) process_path(directive, target, path_def) @@ -269,7 +258,7 @@ class IrAsset(models.Model): IrModule = self.env['ir.module.module'] def mapper(addon): - manif = http.addons_manifest.get(addon, {}) + manif = odoo.modules.module.get_manifest(addon) from_terp = IrModule.get_values_from_terp(manif) from_terp['name'] = addon from_terp['depends'] = manif.get('depends', ['base']) @@ -314,7 +303,7 @@ class IrAsset(models.Model): path_url = fs2web(path_def) path_parts = [part for part in path_url.split('/') if part] addon = path_parts[0] - addon_manifest = http.addons_manifest.get(addon) + addon_manifest = odoo.modules.module.get_manifest(addon) safe_path = True if addon_manifest: diff --git a/odoo/addons/base/models/ir_cron.py b/odoo/addons/base/models/ir_cron.py index 7171ee69746..864e38a0723 100644 --- a/odoo/addons/base/models/ir_cron.py +++ b/odoo/addons/base/models/ir_cron.py @@ -14,7 +14,7 @@ from odoo.exceptions import UserError _logger = logging.getLogger(__name__) -BASE_VERSION = odoo.modules.load_information_from_description_file('base')['version'] +BASE_VERSION = odoo.modules.get_manifest('base')['version'] MAX_FAIL_TIME = timedelta(hours=5) # chosen with a fair roll of the dice diff --git a/odoo/addons/base/models/ir_module.py b/odoo/addons/base/models/ir_module.py index 5f46b0d88d5..50e30fcb8e2 100644 --- a/odoo/addons/base/models/ir_module.py +++ b/odoo/addons/base/models/ir_module.py @@ -168,7 +168,7 @@ class Module(models.Model): @classmethod def get_module_info(cls, name): try: - return modules.load_information_from_description_file(name) + return modules.get_manifest(name) except Exception: _logger.debug('Error when trying to fetch information for module %s', name, exc_info=True) return {} diff --git a/odoo/addons/base/tests/__init__.py b/odoo/addons/base/tests/__init__.py index 0c72603cb85..05ba4548bde 100644 --- a/odoo/addons/base/tests/__init__.py +++ b/odoo/addons/base/tests/__init__.py @@ -27,6 +27,7 @@ from . import test_mail from . import test_menu from . import test_mimetypes from . import test_misc +from . import test_module from . import test_orm from . import test_ormcache from . import test_osv diff --git a/odoo/addons/base/tests/test_module.py b/odoo/addons/base/tests/test_module.py new file mode 100644 index 00000000000..a1f78138fdb --- /dev/null +++ b/odoo/addons/base/tests/test_module.py @@ -0,0 +1,85 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import os.path +import tempfile +from os.path import join as opj +from unittest.mock import patch + +import odoo.addons +from odoo.modules.module import load_manifest +from odoo.release import major_version +from odoo.tests.common import BaseCase + + +class TestModuleManifest(BaseCase): + @classmethod + def setUpClass(cls): + cls._tmp_dir = tempfile.TemporaryDirectory(prefix='odoo-test-addons-') + cls.addClassCleanup(cls._tmp_dir.cleanup) + cls.addons_path = cls._tmp_dir.name + + patcher = patch.object(odoo.addons, '__path__', [cls.addons_path]) + patcher.start() + cls.addClassCleanup(patcher.stop) + + def setUp(self): + self.module_root = tempfile.mkdtemp(prefix='odoo-test-module-', dir=self.addons_path) + self.module_name = os.path.basename(self.module_root) + + def test_default_manifest(self): + with open(opj(self.module_root, '__manifest__.py'), 'w') as file: + file.write(str({'name': f'Temp {self.module_name}', 'license': 'MIT'})) + + with self.assertNoLogs('odoo.modules.module', 'WARNING'): + manifest = load_manifest(self.module_name) + + self.maxDiff = None + self.assertDictEqual(manifest, { + 'addons_path': self.addons_path, + 'application': False, + 'assets': {}, + 'author': 'Odoo S.A.', + 'auto_install': False, + 'bootstrap': False, + 'category': 'Uncategorized', + 'data': [], + 'demo': [], + 'demo_xml': [], + 'depends': [], + 'description': '', + 'external_dependencies': [], + 'icon': '/base/static/description/icon.png', + 'init_xml': [], + 'installable': True, + 'images': [], + 'images_preview_theme': {}, + 'license': 'MIT', + 'live_test_url': '', + 'name': f'Temp {self.module_name}', + 'post_init_hook': '', + 'post_load': None, + 'pre_init_hook': '', + 'sequence': 100, + 'snippet_lists': {}, + 'summary': '', + 'test': [], + 'update_xml': [], + 'uninstall_hook': '', + 'version': f'{major_version}.1.0', + 'web': False, + 'website': '', + }) + + def test_missing_manifest(self): + with self.assertLogs('odoo.modules.module', 'DEBUG') as capture: + manifest = load_manifest(self.module_name) + self.assertEqual(manifest, {}) + self.assertIn("no manifest file found", capture.output[0]) + + def test_missing_license(self): + with open(opj(self.module_root, '__manifest__.py'), 'w') as file: + file.write(str({'name': f'Temp {self.module_name}'})) + with self.assertLogs('odoo.modules.module', 'WARNING') as capture: + manifest = load_manifest(self.module_name) + self.assertEqual(manifest['license'], 'LGPL-3') + self.assertIn("Missing `license` key", capture.output[0]) diff --git a/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py b/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py index 9e9dad8f132..3f669a3135b 100644 --- a/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py +++ b/odoo/addons/test_assetsbundle/tests/test_assetsbundle.py @@ -6,18 +6,19 @@ from lxml import etree import os import time from unittest import skip -from unittest.mock import patch +from unittest.mock import Mock, patch import textwrap import pathlib import lxml import base64 +import odoo from odoo import api, http from odoo.addons import __path__ as ADDONS_PATH from odoo.addons.base.models.assetsbundle import AssetsBundle from odoo.addons.base.models.ir_asset import AssetPaths from odoo.addons.base.models.ir_attachment import IrAttachment -from odoo.modules.module import get_resource_path, read_manifest +from odoo.modules.module import get_resource_path, get_manifest from odoo.tests import HttpCase, tagged from odoo.tests.common import TransactionCase from odoo.addons.base.models.qweb import QWebException @@ -80,33 +81,19 @@ class TestAddonPaths(TransactionCase): class AddonManifestPatched(TransactionCase): - - test_assetsbundle_manifest = None - for path in ADDONS_PATH: - manifest = read_manifest(path, 'test_assetsbundle') - if manifest: - manifest['addons_path'] = path - test_assetsbundle_manifest = manifest - break - - def tearDown(self): - super().tearDown() - self.env.registry._init_modules = self.__genuine_registry_modules - http.addons_manifest = self.__genuine_addons_manifest - def setUp(self): super().setUp() - self.__genuine_registry_modules = self.env.registry._init_modules - self.env.registry._init_modules = func.lazy(lambda: set(self.installed_modules)) - self.__genuine_addons_manifest = http.addons_manifest - http.addons_manifest = func.lazy(lambda: self.manifests) - - self.installed_modules = ['base', 'test_assetsbundle'] + self.installed_modules = {'base', 'test_assetsbundle'} self.manifests = { - 'test_assetsbundle': self.test_assetsbundle_manifest, + 'base': get_manifest('base'), + 'web': get_manifest('web'), + 'test_assetsbundle': get_manifest('test_assetsbundle'), } + self.patch(self.env.registry, '_init_modules', self.installed_modules) + self.patch(odoo.modules.module, 'get_manifest', Mock(side_effect=lambda module: self.manifests.get(module, {}))) + class FileTouchable(AddonManifestPatched): def setUp(self): @@ -1205,7 +1192,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_14_other_module(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1229,7 +1216,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_15_other_module_append(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1256,7 +1243,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_16_other_module_prepend(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1283,7 +1270,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_17_other_module_replace(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1307,7 +1294,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_17_other_module_remove(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1332,7 +1319,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_18_other_module_external(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1431,7 +1418,7 @@ class TestAssetsManifest(AddonManifestPatched): def test_21_js_before_css(self): '''Non existing target node: ignore the manifest line''' - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1461,7 +1448,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_22_js_before_js(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1495,7 +1482,7 @@ class TestAssetsManifest(AddonManifestPatched): def test_23_js_after_css(self): '''Non existing target node: ignore the manifest line''' - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1525,7 +1512,7 @@ class TestAssetsManifest(AddonManifestPatched): ) def test_24_js_after_js(self): - self.installed_modules.append('test_other') + self.installed_modules.add('test_other') self.manifests['test_other'] = { 'name': 'test_other', 'depends': ['test_assetsbundle'], @@ -1785,7 +1772,7 @@ class TestAssetsManifest(AddonManifestPatched): 'path': '/test_assetsbundle/%s' % path_to_dummy, }) - files = self.env['ir.asset']._get_asset_paths('test_assetsbundle.irassetsec', addons=self.installed_modules, xml=False) + files = self.env['ir.asset']._get_asset_paths('test_assetsbundle.irassetsec', addons=list(self.installed_modules), xml=False) self.assertFalse(files) def test_33(self): @@ -1834,7 +1821,7 @@ class TestAssetsManifest(AddonManifestPatched): 'bundle': 'test_assetsbundle.irassetsec', 'path': '/test_assetsbundle/data/ir_asset.xml', }) - files = self.env['ir.asset']._get_asset_paths('test_assetsbundle.irassetsec', addons=self.installed_modules, xml=False) + files = self.env['ir.asset']._get_asset_paths('test_assetsbundle.irassetsec', addons=list(self.installed_modules), xml=False) self.assertFalse(files) def test_36(self): @@ -1843,7 +1830,7 @@ class TestAssetsManifest(AddonManifestPatched): 'bundle': 'test_assetsbundle.irassetsec', 'path': '/test_assetsbundle/static/accessible.xml', }) - files = self.env['ir.asset']._get_asset_paths('test_assetsbundle.irassetsec', addons=self.installed_modules, xml=False) + files = self.env['ir.asset']._get_asset_paths('test_assetsbundle.irassetsec', addons=list(self.installed_modules), xml=False) self.assertEqual(len(files), 1) self.assertTrue('test_assetsbundle/static/accessible.xml' in files[0][0]) diff --git a/odoo/http.py b/odoo/http.py index a6b01e5cb8d..20f59ffc1c7 100644 --- a/odoo/http.py +++ b/odoo/http.py @@ -56,7 +56,7 @@ from .tools import ustr, consteq, frozendict, pycompat, unique, date_utils from .tools.mimetypes import guess_mimetype from .tools.misc import str2bool from .tools._vendor import sessions -from .modules.module import read_manifest +from .modules.module import get_manifest _logger = logging.getLogger(__name__) rpc_request = logging.getLogger(__name__ + '.rpc.request') @@ -860,7 +860,6 @@ more details. #---------------------------------------------------------- # Controller and route registration #---------------------------------------------------------- -addons_manifest = {} controllers_per_module = collections.defaultdict(list) class ControllerType(type): @@ -1316,22 +1315,15 @@ class Root(object): controllers and configure them. """ # TODO should we move this to ir.http so that only configured modules are served ? statics = {} - manifests = addons_manifest for addons_path in odoo.addons.__path__: - for module in sorted(os.listdir(str(addons_path))): - if module not in manifests: - # Deal with the manifest first - mod_path = opj(addons_path, module) - manifest = read_manifest(addons_path, module) - if not manifest or (not manifest.get('installable', True) and 'assets' not in manifest): - continue - manifest['addons_path'] = addons_path - manifests[module] = manifest - # Then deal with the statics - path_static = opj(addons_path, module, 'static') - if os.path.isdir(path_static): - _logger.debug("Loading %s", module) - statics['/%s/static' % module] = path_static + for module in os.listdir(addons_path): + manifest = get_manifest(module) + static_path = opj(addons_path, module, 'static') + if (manifest + and manifest['installable'] + and manifest['assets'] + and os.path.isdir(static_path)): + statics[f'/{module}/static'] = static_path if statics: _logger.info("HTTP Configuring static files") diff --git a/odoo/modules/__init__.py b/odoo/modules/__init__.py index 3a754a4e171..4203a92a102 100644 --- a/odoo/modules/__init__.py +++ b/odoo/modules/__init__.py @@ -19,6 +19,6 @@ from odoo.modules.module import ( get_resource_path, check_resource_path, initialize_sys_path, - load_information_from_description_file, + get_manifest, load_openerp_module, ) diff --git a/odoo/modules/db.py b/odoo/modules/db.py index fda05783e56..475205cdfc0 100644 --- a/odoo/modules/db.py +++ b/odoo/modules/db.py @@ -37,7 +37,7 @@ def initialize(cr): continue # This will raise an exception if no/unreadable descriptor file. - info = odoo.modules.load_information_from_description_file(i) + info = odoo.modules.get_manifest(i) if not info: continue diff --git a/odoo/modules/graph.py b/odoo/modules/graph.py index 664212ec84a..efe8629b466 100644 --- a/odoo/modules/graph.py +++ b/odoo/modules/graph.py @@ -58,10 +58,7 @@ class Graph(dict): packages = [] len_graph = len(self) for module in module_list: - # This will raise an exception if no/unreadable descriptor file. - # NOTE The call to load_information_from_description_file is already - # done by db.initialize, so it is possible to not do it again here. - info = odoo.modules.module.load_information_from_description_file(module) + info = odoo.modules.module.get_manifest(module) if info and info['installable']: packages.append((module, info)) # TODO directly a dict, like in get_modules_with_version elif module != 'studio_customization': diff --git a/odoo/modules/module.py b/odoo/modules/module.py index 32af0b256ab..1b60622653b 100644 --- a/odoo/modules/module.py +++ b/odoo/modules/module.py @@ -3,6 +3,8 @@ import ast import collections.abc +import copy +import functools import importlib import logging import os @@ -10,7 +12,7 @@ import pkg_resources import re import sys import warnings -from os.path import join as opj +from os.path import join as opj, realpath import odoo import odoo.tools as tools @@ -20,6 +22,42 @@ from odoo.tools import pycompat MANIFEST_NAMES = ('__manifest__.py', '__openerp__.py') README = ['README.rst', 'README.md', 'README.txt'] +_DEFAULT_MANIFEST = { + #addons_path: f'/path/to/the/addons/path/of/{module}', # automatic + 'application': False, + 'bootstrap': False, # web + 'assets': {}, + 'author': 'Odoo S.A.', + 'auto_install': False, + 'category': 'Uncategorized', + 'data': [], + 'demo': [], + 'demo_xml': [], + 'depends': [], + 'description': '', + 'external_dependencies': [], + #icon: f'/{module}/static/description/icon.png', # automatic + 'init_xml': [], + 'installable': True, + 'images': [], # website + 'images_preview_theme': {}, # website themes + #license, mandatory + 'live_test_url': '', # website themes + #name, mandatory + 'post_init_hook': '', + 'post_load': None, + 'pre_init_hook': '', + 'sequence': 100, + 'snippet_lists': {}, # website themes + 'summary': '', + 'test': [], + 'update_xml': [], + 'uninstall_hook': '', + 'version': '1.0', + 'web': False, + 'website': '', +} + _logger = logging.getLogger(__name__) # addons path as a list @@ -276,14 +314,6 @@ def module_manifest(path): if os.path.isfile(opj(path, manifest_name)): return opj(path, manifest_name) -def read_manifest(addons_path, module): - mod_path = opj(addons_path, module) - manifest_path = module_manifest(mod_path) - if manifest_path: - with tools.file_open(manifest_path, 'r') as fd: - manifest_data = fd.read() - return ast.literal_eval(manifest_data) - def get_module_root(path): """ Get closest module's root beginning from path @@ -311,74 +341,75 @@ def get_module_root(path): path = new_path return path -def load_information_from_description_file(module, mod_path=None): - """ - :param module: The name of the module (sale, purchase, ...) - :param mod_path: Physical path of module, if not providedThe name of the module (sale, purchase, ...) - """ +def load_manifest(module, mod_path=None): + """ Load the module manifest from the file system. """ + if not mod_path: mod_path = get_module_path(module, downloaded=True) manifest_file = module_manifest(mod_path) - if manifest_file: - # default values for descriptor - info = { - 'application': False, - 'author': 'Odoo S.A.', - 'auto_install': False, - 'category': 'Uncategorized', - 'depends': [], - 'description': '', - 'icon': get_module_icon(module), - 'installable': True, - 'post_load': None, - 'version': '1.0', - 'web': False, - 'sequence': 100, - 'summary': '', - 'website': '', - } - info.update(zip( - 'depends data demo test init_xml update_xml demo_xml'.split(), - iter(list, None))) - f = tools.file_open(manifest_file, mode='rb') - try: - info.update(ast.literal_eval(pycompat.to_text(f.read()))) - finally: - f.close() + if not manifest_file: + _logger.debug('module %s: no manifest file found %s', module, MANIFEST_NAMES) + return {} - if not info.get('description'): - readme_path = [opj(mod_path, x) for x in README - if os.path.isfile(opj(mod_path, x))] - if readme_path: - with tools.file_open(readme_path[0]) as fd: - info['description'] = fd.read() + manifest = copy.deepcopy(_DEFAULT_MANIFEST) + manifest['icon'] = get_module_icon(module) - if not info.get('license'): - info['license'] = 'LGPL-3' - _logger.warning("Missing `license` key in manifest for '%s', defaulting to LGPL-3", module) + with tools.file_open(manifest_file, mode='r') as f: + manifest.update(ast.literal_eval(f.read())) - # auto_install is either `False` (by default) in which case the module - # is opt-in, either a list of dependencies in which case the module is - # automatically installed if all dependencies are (special case: [] to - # always install the module), either `True` to auto-install the module - # in case all dependencies declared in `depends` are installed. - if isinstance(info['auto_install'], collections.abc.Iterable): - info['auto_install'] = set(info['auto_install']) - non_dependencies = info['auto_install'].difference(info['depends']) - assert not non_dependencies,\ - "auto_install triggers must be dependencies, found " \ - "non-dependencies [%s] for module %s" % ( - ', '.join(non_dependencies), module - ) - elif info['auto_install']: - info['auto_install'] = set(info['depends']) + if not manifest['description']: + readme_path = [opj(mod_path, x) for x in README + if os.path.isfile(opj(mod_path, x))] + if readme_path: + with tools.file_open(readme_path[0]) as fd: + manifest['description'] = fd.read() - info['version'] = adapt_version(info['version']) - return info + if not manifest.get('license'): + manifest['license'] = 'LGPL-3' + _logger.warning("Missing `license` key in manifest for %r, defaulting to LGPL-3", module) - _logger.debug('module %s: no manifest file found %s', module, MANIFEST_NAMES) - return {} + # auto_install is either `False` (by default) in which case the module + # is opt-in, either a list of dependencies in which case the module is + # automatically installed if all dependencies are (special case: [] to + # always install the module), either `True` to auto-install the module + # in case all dependencies declared in `depends` are installed. + if isinstance(manifest['auto_install'], collections.abc.Iterable): + manifest['auto_install'] = set(manifest['auto_install']) + non_dependencies = manifest['auto_install'].difference(manifest['depends']) + assert not non_dependencies,\ + "auto_install triggers must be dependencies, found " \ + "non-dependencies [%s] for module %s" % ( + ', '.join(non_dependencies), module + ) + elif manifest['auto_install']: + manifest['auto_install'] = set(manifest['depends']) + + manifest['version'] = adapt_version(manifest['version']) + manifest['addons_path'] = realpath(opj(mod_path, os.pardir)) + + return manifest + +@functools.lru_cache(maxsize=None) +def get_manifest(module, mod_path=None): + """ + Get the module manifest. + + :param str module: The name of the module (sale, purchase, ...). + :param Optional[str] mod_path: The optional path to the module on + the file-system. If not set, it is determined by scanning the + addons-paths. + :returns: The module manifest as a dict or an empty dict + when the manifest was not found. + :rtype: dict + """ + return load_manifest(module, mod_path) + +def load_information_from_description_file(module, mod_path=None): + warnings.warn( + 'load_information_from_description_file() is a deprecated ' + 'alias to get_manifest()', DeprecationWarning, stacklevel=2) + return get_manifest(module, mod_path) def load_openerp_module(module_name): """ Load an OpenERP module, if not already loaded. @@ -398,7 +429,7 @@ def load_openerp_module(module_name): # Call the module's post-load hook. This can done before any model or # data has been initialized. This is ok as the post-load hook is for # server-wide (instead of registry-specific) functionalities. - info = load_information_from_description_file(module_name) + info = get_manifest(module_name) if info['post_load']: getattr(sys.modules['odoo.addons.' + module_name], info['post_load'])() @@ -440,7 +471,7 @@ def get_modules_with_version(): res = dict.fromkeys(modules, adapt_version('1.0')) for module in modules: try: - info = load_information_from_description_file(module) + info = get_manifest(module) res[module] = info['version'] except Exception: continue diff --git a/odoo/tests/common.py b/odoo/tests/common.py index 96a021ea92a..e918223de26 100644 --- a/odoo/tests/common.py +++ b/odoo/tests/common.py @@ -455,6 +455,19 @@ class BaseCase(unittest.TestCase, metaclass=MetaCase): else: return self._assertRaises(exception, **kwargs) + if sys.version_info < (3, 10): + # simplified backport of assertNoLogs() + @contextmanager + def assertNoLogs(self, logger: str, level: str): + # assertLogs ensures there is at least one log record when + # exiting the context manager. We insert one dummy record just + # so we pass that silly test while still capturing the logs. + with self.assertLogs(logger, level) as capture: + logging.getLogger(logger).log(getattr(logging, level), "Dummy log record") + yield + if len(capture.output) > 1: + raise self.failureException(f"Unexpected logs found: {capture.output[1:]}") + @contextmanager def assertQueries(self, expected, flush=True): """ Check the queries made by the current cursor. ``expected`` is a list