From d1834758ea4a42fa9d817d30735a47c4bbc7a4c9 Mon Sep 17 00:00:00 2001 From: "Antoine Dupuis (andu)" Date: Thu, 6 Apr 2023 13:01:01 +0000 Subject: [PATCH] [IMP] tools, account: remove XSD crons; download XSD button At the moment, XSD files are automatically downloaded at database initialization, which is quite unnecessary. The idea is to change Odoo's use of XSDs from being systematically downloaded and used for validation to simply being available if desired (e.g. for development or for customers who want them). To achieve this, this commit does the following: - remove the XSD download crons; - provide a 'download XSDs' button in the Settings (next to the debug mode button) which is available in debug mode; - skip XSD validation if any required XSD file is not present; and - deprecate the 'force_reload' option. Entreprise PR: odoo/enterprise#38350 Task id: 3010716 closes odoo/odoo#118790 X-original-commit: c8174c7914e567489de74b574b076467440bdb2d Related: odoo/enterprise#39879 Signed-off-by: Josse Colpaert Signed-off-by: Antoine Dupuis (andu) --- addons/account/models/ir_attachment.py | 10 ++ .../settings_form_view/res_config_dev_tool.js | 38 ++++++ .../res_config_dev_tool.xml | 8 ++ addons/account/tests/__init__.py | 1 + addons/account/tests/test_download_xsds.py | 7 + .../widgets/res_config_dev_tool.js | 2 +- odoo/tools/xml_utils.py | 122 ++++++++---------- 7 files changed, 122 insertions(+), 66 deletions(-) create mode 100644 addons/account/static/src/components/settings_form_view/res_config_dev_tool.js create mode 100644 addons/account/static/src/components/settings_form_view/res_config_dev_tool.xml create mode 100644 addons/account/tests/test_download_xsds.py diff --git a/addons/account/models/ir_attachment.py b/addons/account/models/ir_attachment.py index 9e75a0ca00b..ddaf4b40ac7 100644 --- a/addons/account/models/ir_attachment.py +++ b/addons/account/models/ir_attachment.py @@ -141,3 +141,13 @@ class IrAttachment(models.Model): to_process.sort(key=lambda x: x['sort_weight']) return to_process + + # ------------------------------------------------------------------------- + # XSD validation + # ------------------------------------------------------------------------- + + @api.model + def action_download_xsd_files(self): + # To be extended by localisations, where they can download their necessary XSD files + # Note: they should always return super().action_download_xsd_files() + return diff --git a/addons/account/static/src/components/settings_form_view/res_config_dev_tool.js b/addons/account/static/src/components/settings_form_view/res_config_dev_tool.js new file mode 100644 index 00000000000..aeb71b95278 --- /dev/null +++ b/addons/account/static/src/components/settings_form_view/res_config_dev_tool.js @@ -0,0 +1,38 @@ +/** @odoo-module */ + +import { registry } from "@web/core/registry"; +import { useService } from "@web/core/utils/hooks"; + +import { ResConfigDevTool } from "@web/webclient/settings_form_view/widgets/res_config_dev_tool"; + +/** + * Override of the widget in the settings that handles the "Developer Tools" section. + * Provides a button to download XSD files for XML validation. + */ +class ResConfigDevToolDownloadXsd extends ResConfigDevTool { + /** + * Downloads every XSD file, based on installed localisations. + */ + + static template = "res_config_dev_tool"; + + setup() { + super.setup(); + this.rpc = useService("rpc"); + } + + async onClickDownloadXSD() { + await this.rpc("/web/dataset/call_kw/ir.attachment/action_download_xsd_files", { + model: 'ir.attachment', + method: 'action_download_xsd_files', + args: [], + kwargs: {} + }) + } +} + +const resConfigDevToolDownloadXsd = { + component: ResConfigDevToolDownloadXsd, +}; + +registry.category("view_widgets").add("res_config_dev_tool", resConfigDevToolDownloadXsd, {force: true}); diff --git a/addons/account/static/src/components/settings_form_view/res_config_dev_tool.xml b/addons/account/static/src/components/settings_form_view/res_config_dev_tool.xml new file mode 100644 index 00000000000..2429848c949 --- /dev/null +++ b/addons/account/static/src/components/settings_form_view/res_config_dev_tool.xml @@ -0,0 +1,8 @@ + + diff --git a/addons/account/tests/__init__.py b/addons/account/tests/__init__.py index b81052eff79..982d2354d8e 100644 --- a/addons/account/tests/__init__.py +++ b/addons/account/tests/__init__.py @@ -39,3 +39,4 @@ from . import test_account_payment_register from . import test_tour from . import test_early_payment_discount from . import test_ir_actions_report +from . import test_download_xsds diff --git a/addons/account/tests/test_download_xsds.py b/addons/account/tests/test_download_xsds.py new file mode 100644 index 00000000000..46615ad8869 --- /dev/null +++ b/addons/account/tests/test_download_xsds.py @@ -0,0 +1,7 @@ +from odoo.tests import tagged +from odoo.tests.common import TransactionCase + +@tagged('external_l10n', '-at_install', 'post_install', '-standard', 'external') +class TestDownloadXsds(TransactionCase): + def test_download_xsds(self): + self.env['ir.attachment'].action_download_xsd_files() diff --git a/addons/web/static/src/webclient/settings_form_view/widgets/res_config_dev_tool.js b/addons/web/static/src/webclient/settings_form_view/widgets/res_config_dev_tool.js index a2d04b31bb9..ee5f5769f98 100644 --- a/addons/web/static/src/webclient/settings_form_view/widgets/res_config_dev_tool.js +++ b/addons/web/static/src/webclient/settings_form_view/widgets/res_config_dev_tool.js @@ -13,7 +13,7 @@ import { standardWidgetProps } from "@web/views/widgets/standard_widget_props"; * Can be used to enable/disable the debug modes. * Can be used to load the demo data. */ -class ResConfigDevTool extends Component { +export class ResConfigDevTool extends Component { static template = "res_config_dev_tool"; static components = { SettingsBlock, diff --git a/odoo/tools/xml_utils.py b/odoo/tools/xml_utils.py index 245d2c80daf..957bcadde2d 100644 --- a/odoo/tools/xml_utils.py +++ b/odoo/tools/xml_utils.py @@ -151,16 +151,15 @@ def cleanup_xml_node(xml_node_or_string, remove_blank_text=True, remove_blank_no return xml_node -def load_xsd_files_from_url(env, url, file_name, force_reload=False, +def load_xsd_files_from_url(env, url, file_name=None, force_reload=False, request_max_timeout=10, xsd_name_prefix='', xsd_names_filter=None, modify_xsd_content=None): - """Load XSD file or ZIP archive and save it as ir.attachment. + """Load XSD file or ZIP archive. Save XSD files as ir.attachment. - If the XSD file/archive has already been saved in database, then just return the attachment. - In such a case, the attachment content can also be updated by force if desired. - If the attachment is a ZIP archive, then a force reload will also update all attachments from the archive. + An XSD attachment from the database is identified by its prefixed `file_name`. A typical prefix is the + calling module name. - When the attachment is a ZIP archive, every file inside will also be saved as attachments. - Filtering which file will be saved can be done by providing a list of `xsd_names` + For ZIP archives, XSD files inside it will be saved as attachments, depending on the provided list of XSD names. + ZIP archive themselves are not saved. The XSD files content can be modified by providing the `modify_xsd_content` function as argument. Typically, this is used when XSD files depend on each other (with the schemaLocation attribute), @@ -169,27 +168,19 @@ def load_xsd_files_from_url(env, url, file_name, force_reload=False, :param odoo.api.Environment env: environment of calling module :param str url: URL of XSD file/ZIP archive :param str file_name: the name given to the XSD attachment - :param bool force_reload: if True, reload the attachment from URL, even if it is already cached + :param bool force_reload: Deprecated. :param int request_max_timeout: maximum time (in seconds) before the request times out :param str xsd_name_prefix: if provided, will be added as a prefix to every XSD file name :param list | str xsd_names_filter: if provided, will only save the XSD files with these names :param func modify_xsd_content: function that takes the xsd content as argument and returns a modified version of it :rtype: odoo.api.ir.attachment | bool - :return: the main attachment or False if an error occurred (see warning logs) + :return: every XSD attachment created/fetched or False if an error occurred (see warning logs) """ if not url.endswith(('.xsd', '.zip')): _logger.warning("The given URL (%s) needs to lead to an XSD file or a ZIP archive", url) return False - is_zip = url.endswith('.zip') - fetched_attachment = env['ir.attachment'].search([('name', '=', file_name)]) - if fetched_attachment: - if not force_reload: - _logger.info("Retrieved attachment from database, with name: %s", fetched_attachment.name) - return fetched_attachment - _logger.info("Found the attachment with name %s in database, but forcing the reloading.", fetched_attachment.name) - try: _logger.info("Fetching file/archive from given URL: %s", url) response = requests.get(url, timeout=request_max_timeout) @@ -205,29 +196,34 @@ def load_xsd_files_from_url(env, url, file_name, force_reload=False, return False content = response.content + if not content: + _logger.warning("The HTTP response from %s is empty (no content)", url) + return False if modify_xsd_content and not is_zip: content = modify_xsd_content(content) - if fetched_attachment: - _logger.info("Updating the content of ir.attachment with name: %s", file_name) - fetched_attachment.raw = content - return fetched_attachment - - _logger.info("Saving XSD file as ir.attachment, with name: %s", file_name) - main_attachment = env['ir.attachment'].create({ - 'name': file_name, - 'raw': content, - 'public': True, - }) - if not is_zip: - return main_attachment + if not file_name: + file_name = f"{url.split('/')[-1]}" + _logger.info("XSD name not provided, defaulting to %s", file_name) - _logger.info("Unzipping loaded archive, with name %s", file_name) - if xsd_names_filter and not isinstance(xsd_names_filter, list): - xsd_names_filter = [xsd_names_filter] + prefixed_xsd_name = f"{xsd_name_prefix}.{file_name}" if xsd_name_prefix else file_name + fetched_attachment = env['ir.attachment'].search([('name', '=', prefixed_xsd_name)], limit=1) + if fetched_attachment: + _logger.info("Updating the content of ir.attachment with name: %s", prefixed_xsd_name) + fetched_attachment.raw = content + return fetched_attachment + else: + _logger.info("Saving XSD file as ir.attachment, with name: %s", prefixed_xsd_name) + return env['ir.attachment'].create({ + 'name': prefixed_xsd_name, + 'raw': content, + 'public': True, + }) + _logger.info("Unzipping loaded archive") archive = zipfile.ZipFile(BytesIO(content)) + saved_attachments = env['ir.attachment'] for file_path in archive.namelist(): if not file_path.endswith('.xsd'): continue @@ -235,60 +231,56 @@ def load_xsd_files_from_url(env, url, file_name, force_reload=False, file_name = file_path.rsplit('/', 1)[-1] if xsd_names_filter and file_name not in xsd_names_filter: + _logger.info("Skipping file with name %s in ZIP archive", file_name) continue - if xsd_name_prefix: - file_name = f'{xsd_name_prefix}.{file_name}' - - attachment = env['ir.attachment'].search([('name', '=', file_name)]) - if attachment and not force_reload: - continue - - if force_reload: - _logger.info("Updating the content of ir.attachment with name: %s", file_name) - else: - _logger.info("Saving XSD file as ir.attachment, with name: %s", file_name) try: content = archive.read(file_path) - if modify_xsd_content: - content = modify_xsd_content(content) - env['ir.attachment'].create({ - 'name': file_name, + except KeyError: + _logger.warning("Failed to retrieve XSD file with name %s from ZIP archive", file_name) + continue + if modify_xsd_content: + content = modify_xsd_content(content) + + prefixed_xsd_name = f"{xsd_name_prefix}.{file_name}" if xsd_name_prefix else file_name + fetched_attachment = env['ir.attachment'].search([('name', '=', prefixed_xsd_name)], limit=1) + if fetched_attachment: + _logger.info("Updating the content of ir.attachment with name: %s", prefixed_xsd_name) + fetched_attachment.raw = content + saved_attachments |= fetched_attachment + + else: + _logger.info("Saving XSD file as ir.attachment, with name: %s", prefixed_xsd_name) + saved_attachments |= env['ir.attachment'].create({ + 'name': prefixed_xsd_name, 'raw': content, 'public': True, }) - except KeyError: - _logger.warning("Failed to retrieve XSD file with name %s from ZIP archive", file_name) - return fetched_attachment + return saved_attachments def validate_xml_from_attachment(env, xml_content, xsd_name, reload_files_function=None, prefix=None): """Try and validate the XML content with an XSD attachment. - If the XSD attachment cannot be found in database, (re)load it. - - A skip_xsd key can be provided in the context in order to skip the XSD validation. - This should be used during tests to avoid loading XSD files (and making http requests every time). + If the XSD attachment cannot be found in database, skip validation without raising. + If the skip_xsd context key is truthy, skip validation. :param odoo.api.Environment env: environment of calling module :param xml_content: the XML content to validate :param xsd_name: the XSD file name in database - :param reload_files_function: function that will be called to try and (re)load XSD files + :param reload_files_function: Deprecated. :return: the result of the function :func:`odoo.tools.xml_utils._check_with_xsd` """ if env.context.get('skip_xsd', False): return + + prefixed_xsd_name = f"{prefix}.{xsd_name}" if prefix else xsd_name try: - _check_with_xsd(xml_content, xsd_name, env, prefix) + _logger.info("Validating with XSD...") + _check_with_xsd(xml_content, prefixed_xsd_name, env, prefix) + _logger.info("XSD validation successful!") except FileNotFoundError: - if not reload_files_function: - _logger.warning("You need to provide a function used to (re)load XSD files") - return - reload_files_function() - try: - _check_with_xsd(xml_content, xsd_name, env) - except FileNotFoundError: - _logger.warning("The XSD file(s) could not be found, even after a reload") + _logger.info("XSD file not found, skipping validation") def find_xml_value(xpath, xml_element, namespaces=None):