From 234590f3db5b9e8518bf476e34591f5ab504328b Mon Sep 17 00:00:00 2001 From: Pierre Masereel Date: Thu, 9 Mar 2023 17:11:39 +0100 Subject: [PATCH] [IMP] base_import_module: add industry modules availability We now have the possbility to publish modules that only contains XML on apps.odoo.com and we want to display in the apps of the database to ease the installation. TASK-3186716 closes odoo/odoo#140245 Signed-off-by: Pierre Masereel (pim) --- addons/base_import_module/__manifest__.py | 4 +- addons/base_import_module/controllers/main.py | 2 +- .../i18n/base_import_module.pot | 55 ++++++- .../models/base_import_module.py | 14 +- addons/base_import_module/models/ir_module.py | 154 ++++++++++++++++-- addons/base_import_module/tests/test_cloc.py | 4 +- .../tests/test_import_module.py | 8 +- .../views/base_import_module_view.xml | 10 +- .../views/ir_module_views.xml | 65 ++++++++ 9 files changed, 282 insertions(+), 34 deletions(-) create mode 100644 addons/base_import_module/views/ir_module_views.xml diff --git a/addons/base_import_module/__manifest__.py b/addons/base_import_module/__manifest__.py index 477f1c1dead..deac67c7389 100644 --- a/addons/base_import_module/__manifest__.py +++ b/addons/base_import_module/__manifest__.py @@ -12,9 +12,11 @@ for customization purpose. 'category': 'Hidden/Tools', 'depends': ['web'], 'installable': True, + 'auto_install': True, 'data': [ 'security/ir.model.access.csv', - 'views/base_import_module_view.xml' + 'views/base_import_module_view.xml', + 'views/ir_module_views.xml', ], 'license': 'LGPL-3', } diff --git a/addons/base_import_module/controllers/main.py b/addons/base_import_module/controllers/main.py index c0e6b64da2b..61ddfa5ee42 100644 --- a/addons/base_import_module/controllers/main.py +++ b/addons/base_import_module/controllers/main.py @@ -17,7 +17,7 @@ class ImportModule(Controller): request.session.authenticate(request.db, login, password) # request.uid is None in case of MFA if request.uid and request.env.user._is_admin(): - return request.env['ir.module.module'].import_zipfile(mod_file, force=force == '1')[0] + return request.env['ir.module.module']._import_zipfile(mod_file, force=force == '1')[0] raise AccessError(_("Only administrators can upload a module")) except Exception as e: return Response(response=str(e), status=500) diff --git a/addons/base_import_module/i18n/base_import_module.pot b/addons/base_import_module/i18n/base_import_module.pot index bb5ccdd8a56..793a1fe5f7f 100644 --- a/addons/base_import_module/i18n/base_import_module.pot +++ b/addons/base_import_module/i18n/base_import_module.pot @@ -15,6 +15,19 @@ msgstr "" "Content-Transfer-Encoding: \n" "Plural-Forms: \n" +#. module: base_import_module +#: model_terms:ir.ui.view,arch_db:base_import_module.module_view_kanban_apps_inherit +msgid "Activate" +msgstr "" + +#. module: base_import_module +#. odoo-python +#: code:addons/base_import_module/models/ir_module.py:0 +#: model_terms:ir.ui.view,arch_db:base_import_module.view_module_filter_apps_inherit +#, python-format +msgid "Apps" +msgstr "" + #. module: base_import_module #: model_terms:ir.ui.view,arch_db:base_import_module.view_base_module_import msgid "Cancel" @@ -25,6 +38,20 @@ msgstr "" msgid "Close" msgstr "" +#. module: base_import_module +#. odoo-python +#: code:addons/base_import_module/models/ir_module.py:0 +#, python-format +msgid "Connection to %s failed The list of industry modules cannot be fetched" +msgstr "" + +#. module: base_import_module +#. odoo-python +#: code:addons/base_import_module/models/ir_module.py:0 +#, python-format +msgid "Connection to %s failed, the module %s cannot be downloaded." +msgstr "" + #. module: base_import_module #. odoo-python #: code:addons/base_import_module/controllers/main.py:0 @@ -82,6 +109,11 @@ msgstr "" msgid "Import Module" msgstr "" +#. module: base_import_module +#: model_terms:ir.ui.view,arch_db:base_import_module.view_base_module_import +msgid "Import demo data" +msgstr "" + #. module: base_import_module #: model:ir.model.fields,field_description:base_import_module.field_base_import_module__with_demo msgid "Import demo data of module" @@ -192,14 +224,31 @@ msgstr "" msgid "Studio customizations require the Odoo Studio app." msgstr "" +#. module: base_import_module +#. odoo-python +#: code:addons/base_import_module/models/ir_module.py:0 +#, python-format +msgid "The following modules will be also installed:\n" +msgstr "" + #. module: base_import_module #. odoo-python #: code:addons/base_import_module/models/ir_module.py:0 #, python-format msgid "" -"Unmet module dependencies: \n" -"\n" -" - %s" +"The list of industry applications cannot be fetched. Please try again later" +msgstr "" + +#. module: base_import_module +#. odoo-python +#: code:addons/base_import_module/models/ir_module.py:0 +#, python-format +msgid "The module %s cannot be downloaded" +msgstr "" + +#. module: base_import_module +#: model_terms:ir.ui.view,arch_db:base_import_module.module_view_kanban_apps_inherit +msgid "Upgrade" msgstr "" #. module: base_import_module diff --git a/addons/base_import_module/models/base_import_module.py b/addons/base_import_module/models/base_import_module.py index 64b6a597636..e01b06612f4 100644 --- a/addons/base_import_module/models/base_import_module.py +++ b/addons/base_import_module/models/base_import_module.py @@ -22,18 +22,12 @@ class BaseImportModule(models.TransientModel): zip_data = base64.decodebytes(self.module_file) fp = BytesIO() fp.write(zip_data) - res = IrModule.import_zipfile(fp, force=self.force) + res = IrModule._import_zipfile(fp, force=self.force, with_demo=self.with_demo) self.write({'state': 'done', 'import_message': res[0]}) - context = dict(self.env.context, module_name=res[1]) - # Return wizard otherwise it will close wizard and will not show result message to user. return { - 'name': 'Import Module', - 'view_mode': 'form', - 'target': 'new', - 'res_id': self.id, - 'res_model': 'base.import.module', - 'type': 'ir.actions.act_window', - 'context': context, + 'type': 'ir.actions.act_url', + 'target': 'self', + 'url': '/web', } def action_module_open(self): diff --git a/addons/base_import_module/models/ir_module.py b/addons/base_import_module/models/ir_module.py index dea5ef9c0e5..b31ed8ca00e 100644 --- a/addons/base_import_module/models/ir_module.py +++ b/addons/base_import_module/models/ir_module.py @@ -4,20 +4,24 @@ import base64 import logging import lxml import os +import requests import sys import tempfile import zipfile from collections import defaultdict +from io import BytesIO from os.path import join as opj from odoo import api, fields, models, _ -from odoo.exceptions import UserError +from odoo.exceptions import AccessDenied, UserError from odoo.modules.module import MANIFEST_NAMES +from odoo.release import major_version from odoo.tools import convert_csv_import, convert_sql_import, convert_xml_import, exception_to_unicode from odoo.tools import file_open, file_open_temporary_directory _logger = logging.getLogger(__name__) +APPS_URL = "https://apps.odoo.com" MAX_FILE_SIZE = 100 * 1024 * 1024 # in megabytes @@ -41,7 +45,7 @@ class IrModule(models.Model): module.installed_version = module.latest_version super(IrModule, self - imported_modules)._get_latest_version() - def _import_module(self, module, path, force=False): + def _import_module(self, module, path, force=False, with_demo=False): known_mods = self.search([]) known_mods_names = {m.name: m for m in known_mods} installed_mods = [m.name for m in known_mods if m.state == 'installed'] @@ -68,10 +72,8 @@ class IrModule(models.Model): _is_studio_custom(path)): err = _("Studio customizations require Studio") else: - err = _("Unmet module dependencies: \n\n - %s", '\n - '.join( - known_mods.filtered(lambda mod: mod.name in unmet_dependencies).mapped('shortdesc') - )) - raise UserError(err) + to_install = known_mods.filtered(lambda mod: mod.name in unmet_dependencies) + to_install.button_immediate_install() elif 'web_studio' not in installed_mods and _is_studio_custom(path): raise UserError(_("Studio customizations require the Odoo Studio app.")) @@ -84,7 +86,10 @@ class IrModule(models.Model): self.create(dict(name=module, state='installed', imported=True, **values)) mode = 'init' - for kind in ['data', 'init_xml', 'update_xml']: + kind_of_files = ['data', 'init_xml', 'update_xml'] + if with_demo: + kind_of_files.append('demo') + for kind in kind_of_files: for filename in terp.get(kind, []): ext = os.path.splitext(filename)[1].lower() if ext not in ('.xml', '.csv', '.sql'): @@ -177,7 +182,7 @@ class IrModule(models.Model): return True @api.model - def import_zipfile(self, module_file, force=False): + def _import_zipfile(self, module_file, force=False, with_demo=False): if not module_file: raise Exception(_("No file sent.")) if not zipfile.is_zipfile(module_file): @@ -207,7 +212,10 @@ class IrModule(models.Model): terp = ast.literal_eval(f.read().decode()) except Exception: continue - for filename in terp.get('data', []) + terp.get('init_xml', []) + terp.get('update_xml', []): + files_to_import = terp.get('data', []) + terp.get('init_xml', []) + terp.get('update_xml', []) + if with_demo: + files_to_import += terp.get('demo', []) + for filename in files_to_import: if os.path.splitext(filename)[1].lower() not in ('.xml', '.csv', '.sql'): continue module_data_files[mod_name].append('%s/%s' % (mod_name, filename)) @@ -225,7 +233,7 @@ class IrModule(models.Model): try: # assert mod_name.startswith('theme_') path = opj(module_dir, mod_name) - if self._import_module(mod_name, path, force=force): + if self._import_module(mod_name, path, force=force, with_demo=with_demo): success.append(mod_name) except Exception as e: _logger.exception('Error while importing module') @@ -258,6 +266,132 @@ class IrModule(models.Model): modules_to_delete.unlink() return res + @api.model + def web_search_read(self, domain, specification, offset=0, limit=None, order=None, count_limit=None): + res = super().web_search_read(domain, specification, offset=offset, limit=limit, order=order, count_limit=count_limit) + if any(dom[0] == 'module_type' and dom[-1] == 'industries' for dom in domain): + fields_name = list(specification.keys()) + modules_list = self._get_modules_from_apps(fields_name, 'industries', False) + res['length'] += len(modules_list) + res['records'].extend(modules_list) + return res + + def more_info(self): + return { + 'name': _('Apps'), + 'type': 'ir.actions.act_window', + 'res_model': 'ir.module.module', + 'view_mode': 'form', + 'res_id': self.id, + 'context': self.env.context, + } + + def web_read(self, specification): + fields = list(specification.keys()) + module_type = self.env.context.get('module_type', 'official') + if module_type != 'official': + modules_list = self._get_modules_from_apps(fields, module_type, self.env.context.get('module_name')) + return modules_list + else: + return super().web_read(specification) + + @api.model + def _get_modules_from_apps(self, fields, module_type, module_name): + payload = { + 'series': major_version, + 'module_fields': fields, + 'module_type':module_type, + 'module_name': module_name + } + try: + resp = requests.post( + f"{APPS_URL}/loempia/listdatamodules", + json={'params': payload}, + timeout=5.0, + ) + if resp.status_code == 200: + modules_list = resp.json().get('result', []) + for mod in modules_list: + mod['id'] = -1 + module_name = mod.get('name', module_name) + if 'icon' in fields: + mod['icon'] = f"{APPS_URL}{mod['icon']}" + if 'state' in fields: + existing_mod = self.search([('name', '=', module_name), ('state', '=', 'installed')]) + if existing_mod: + mod['state'] = 'installed' + else: + mod['state'] = 'uninstalled' + if 'module_type' in fields: + mod['module_type'] = module_type + if 'website' in fields: + mod['website'] = f"{APPS_URL}/apps/modules/{major_version}/{module_name}/" + return modules_list + resp.raise_for_status() + except requests.exceptions.HTTPError: + raise UserError(_('The list of industry applications cannot be fetched. Please try again later')) + except requests.exceptions.ConnectionError: + raise UserError(_('Connection to %s failed The list of industry modules cannot be fetched') % APPS_URL) + + def button_immediate_install_app(self): + if not self.env.is_admin(): + raise AccessDenied() + module_name = self.env.context.get('module_name') + try: + resp = requests.get( + f"{APPS_URL}/loempia/download/data_app/{module_name}/{major_version}", + timeout=5.0, + ) + if resp.status_code == 200: + import_module = self.env['base.import.module'].create({ + 'module_file': base64.b64encode(resp.content), + 'state': 'init', + 'modules_dependencies': self._get_missing_dependencies(resp.content) + }) + return { + 'name': 'Install an App', + 'view_mode': 'form', + 'target': 'new', + 'res_id': import_module.id, + 'res_model': 'base.import.module', + 'type': 'ir.actions.act_window', + 'context': {'data_module': True} + } + resp.raise_for_status() + except requests.exceptions.HTTPError: + raise UserError(_('The module %s cannot be downloaded') % module_name) + except requests.exceptions.ConnectionError: + raise UserError(_('Connection to %s failed, the module %s cannot be downloaded.', APPS_URL, module_name)) + + @api.model + def _get_missing_dependencies(self, zip_data): + dependencies_to_install = self.env['ir.module.module'] + known_mods = self.search([]) + installed_mods = [m.name for m in known_mods if m.state == 'installed'] + with zipfile.ZipFile(BytesIO(zip_data), "r") as z: + manifest_files = [ + file + for file in z.filelist + if file.filename.count('/') == 1 + and file.filename.split('/')[1] in MANIFEST_NAMES + ] + for manifest_file in manifest_files: + if manifest_file.file_size > MAX_FILE_SIZE: + raise UserError(_("File '%s' exceed maximum allowed file size", manifest_file.filename)) + try: + with z.open(manifest_file) as manifest: + terp = ast.literal_eval(manifest.read().decode()) + except Exception: + continue + unmet_dependencies = set(terp.get('depends', [])).difference(installed_mods) + dependencies_to_install |= known_mods.filtered(lambda m: m.name in unmet_dependencies) + description = '' + if dependencies_to_install: + description = _('The following modules will be also installed:\n') + for mod in dependencies_to_install: + description += "- " + mod.shortdesc + "\n" + return description + def _is_studio_custom(path): """ diff --git a/addons/base_import_module/tests/test_cloc.py b/addons/base_import_module/tests/test_cloc.py index 978b4a38084..bdc736513b0 100644 --- a/addons/base_import_module/tests/test_cloc.py +++ b/addons/base_import_module/tests/test_cloc.py @@ -146,7 +146,7 @@ class TestClocFields(test_cloc.TestClocCustomization): archive.writestr('test_imported_module/static/src/js/test.xml', VALID_XML) # Import test module - self.env['ir.module.module'].import_zipfile(stream) + self.env['ir.module.module']._import_zipfile(stream) cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('test_imported_module', 0), 35) @@ -201,7 +201,7 @@ class TestClocFields(test_cloc.TestClocCustomization): ] # Import test module - self.env['ir.module.module'].import_zipfile(stream) + self.env['ir.module.module']._import_zipfile(stream) # Create exclude xml_id for name in id_names: rec = self.env.ref(f'test_imported_module.{name}') diff --git a/addons/base_import_module/tests/test_import_module.py b/addons/base_import_module/tests/test_import_module.py index 488267d8100..65c10ba0327 100644 --- a/addons/base_import_module/tests/test_import_module.py +++ b/addons/base_import_module/tests/test_import_module.py @@ -25,7 +25,7 @@ class TestImportModule(odoo.tests.TransactionCase): with ZipFile(archive, 'w') as zipf: for path, data in files: zipf.writestr(path, data) - return self.env['ir.module.module'].import_zipfile(archive) + return self.env['ir.module.module']._import_zipfile(archive) def test_import_zip(self): """Assert the behaviors expected by the module import feature using a ZIP archive""" @@ -187,7 +187,7 @@ class TestImportModule(odoo.tests.TransactionCase): archive.writestr('test_module/static/src/js/test.js', "console.log('AAA');") # Import test module - self.env['ir.module.module'].import_zipfile(stream) + self.env['ir.module.module']._import_zipfile(stream) attachment = self.env['ir.attachment'].search([('url', '=', path)]) self.assertEqual(attachment.name, 'test.js') @@ -240,7 +240,7 @@ class TestImportModule(odoo.tests.TransactionCase): archive.writestr(path, "console.log('AAA');") # Import test module - self.env['ir.module.module'].with_user(self.test_user).import_zipfile(stream) + self.env['ir.module.module'].with_user(self.test_user)._import_zipfile(stream) attachment = self.env['ir.attachment'].search([('url', '=', f'/{path}')]) self.assertEqual(attachment.name, 'test.js') @@ -264,7 +264,7 @@ class TestImportModule(odoo.tests.TransactionCase): archive.writestr(path, "console.log('BBB');") # Import test module - self.env['ir.module.module'].with_user(self.test_user).import_zipfile(stream) + self.env['ir.module.module'].with_user(self.test_user)._import_zipfile(stream) attachment = self.env['ir.attachment'].search([('url', '=', f'/{path}')]) self.assertEqual(attachment.name, 'test.js') diff --git a/addons/base_import_module/views/base_import_module_view.xml b/addons/base_import_module/views/base_import_module_view.xml index 851d796bc0f..ac3f6635127 100644 --- a/addons/base_import_module/views/base_import_module_view.xml +++ b/addons/base_import_module/views/base_import_module_view.xml @@ -7,10 +7,14 @@
- + + + + - - + + + diff --git a/addons/base_import_module/views/ir_module_views.xml b/addons/base_import_module/views/ir_module_views.xml new file mode 100644 index 00000000000..298cebf4eb6 --- /dev/null +++ b/addons/base_import_module/views/ir_module_views.xml @@ -0,0 +1,65 @@ + + + + + Apps Kanban Data Modules + ir.module.module + + + + + + + + + + + state != 'uninstalled' or module_type != 'official' + + + {'module_type': module_type, 'module_name':name} + object + more_info + + + {'module_type': module_type, 'module_name':name} + object + more_info + + + + + Apps List Data Modules + ir.module.module + + + + + + + + + Apps + ir.module.module + + + + + + + module_type != 'official' + + + + + Search Data Modules + ir.module.module + + + + + + + + +