[IMP] base_import_module: add industry modules availability

We now have the possbility to publish modules that only contains XML on
apps.odoo.com and we want to display in the apps of the database to ease
the installation.

TASK-3186716

closes odoo/odoo#140245

Signed-off-by: Pierre Masereel (pim) <pim@odoo.com>

addons/base_import_module/__manifest__.py

@@ -12,9 +12,11 @@ for customization purpose.
     'category': 'Hidden/Tools',
     'depends': ['web'],
     'installable': True,
+    'auto_install': True,
     'data': [
         'security/ir.model.access.csv',
-        'views/base_import_module_view.xml'
+        'views/base_import_module_view.xml',
+        'views/ir_module_views.xml',
     ],
     'license': 'LGPL-3',
 }

addons/base_import_module/controllers/main.py

@@ -17,7 +17,7 @@ class ImportModule(Controller):
             request.session.authenticate(request.db, login, password)
             # request.uid is None in case of MFA
             if request.uid and request.env.user._is_admin():
-                return request.env['ir.module.module'].import_zipfile(mod_file, force=force == '1')[0]
+                return request.env['ir.module.module']._import_zipfile(mod_file, force=force == '1')[0]
             raise AccessError(_("Only administrators can upload a module"))
         except Exception as e:
             return Response(response=str(e), status=500)

addons/base_import_module/i18n/base_import_module.pot

@@ -15,6 +15,19 @@ msgstr ""
 "Content-Transfer-Encoding: \n"
 "Plural-Forms: \n"
 
+#. module: base_import_module
+#: model_terms:ir.ui.view,arch_db:base_import_module.module_view_kanban_apps_inherit
+msgid "Activate"
+msgstr ""
+
+#. module: base_import_module
+#. odoo-python
+#: code:addons/base_import_module/models/ir_module.py:0
+#: model_terms:ir.ui.view,arch_db:base_import_module.view_module_filter_apps_inherit
+#, python-format
+msgid "Apps"
+msgstr ""
+
 #. module: base_import_module
 #: model_terms:ir.ui.view,arch_db:base_import_module.view_base_module_import
 msgid "Cancel"
@@ -25,6 +38,20 @@ msgstr ""
 msgid "Close"
 msgstr ""
 
+#. module: base_import_module
+#. odoo-python
+#: code:addons/base_import_module/models/ir_module.py:0
+#, python-format
+msgid "Connection to %s failed The list of industry modules cannot be fetched"
+msgstr ""
+
+#. module: base_import_module
+#. odoo-python
+#: code:addons/base_import_module/models/ir_module.py:0
+#, python-format
+msgid "Connection to %s failed, the module %s cannot be downloaded."
+msgstr ""
+
 #. module: base_import_module
 #. odoo-python
 #: code:addons/base_import_module/controllers/main.py:0
@@ -82,6 +109,11 @@ msgstr ""
 msgid "Import Module"
 msgstr ""
 
+#. module: base_import_module
+#: model_terms:ir.ui.view,arch_db:base_import_module.view_base_module_import
+msgid "Import demo data"
+msgstr ""
+
 #. module: base_import_module
 #: model:ir.model.fields,field_description:base_import_module.field_base_import_module__with_demo
 msgid "Import demo data of module"
@@ -192,14 +224,31 @@ msgstr ""
 msgid "Studio customizations require the Odoo Studio app."
 msgstr ""
 
+#. module: base_import_module
+#. odoo-python
+#: code:addons/base_import_module/models/ir_module.py:0
+#, python-format
+msgid "The following modules will be also installed:\n"
+msgstr ""
+
 #. module: base_import_module
 #. odoo-python
 #: code:addons/base_import_module/models/ir_module.py:0
 #, python-format
 msgid ""
-"Unmet module dependencies: \n"
-"\n"
-" - %s"
+"The list of industry applications cannot be fetched. Please try again later"
+msgstr ""
+
+#. module: base_import_module
+#. odoo-python
+#: code:addons/base_import_module/models/ir_module.py:0
+#, python-format
+msgid "The module %s cannot be downloaded"
+msgstr ""
+
+#. module: base_import_module
+#: model_terms:ir.ui.view,arch_db:base_import_module.module_view_kanban_apps_inherit
+msgid "Upgrade"
 msgstr ""
 
 #. module: base_import_module

addons/base_import_module/models/base_import_module.py

@@ -22,18 +22,12 @@ class BaseImportModule(models.TransientModel):
         zip_data = base64.decodebytes(self.module_file)
         fp = BytesIO()
         fp.write(zip_data)
-        res = IrModule.import_zipfile(fp, force=self.force)
+        res = IrModule._import_zipfile(fp, force=self.force, with_demo=self.with_demo)
         self.write({'state': 'done', 'import_message': res[0]})
-        context = dict(self.env.context, module_name=res[1])
-        # Return wizard otherwise it will close wizard and will not show result message to user.
         return {
-            'name': 'Import Module',
-            'view_mode': 'form',
-            'target': 'new',
-            'res_id': self.id,
-            'res_model': 'base.import.module',
-            'type': 'ir.actions.act_window',
-            'context': context,
+            'type': 'ir.actions.act_url',
+            'target': 'self',
+            'url': '/web',
         }
 
     def action_module_open(self):

addons/base_import_module/models/ir_module.py

@@ -4,20 +4,24 @@ import base64
 import logging
 import lxml
 import os
+import requests
 import sys
 import tempfile
 import zipfile
 from collections import defaultdict
+from io import BytesIO
 from os.path import join as opj
 
 from odoo import api, fields, models, _
-from odoo.exceptions import UserError
+from odoo.exceptions import AccessDenied, UserError
 from odoo.modules.module import MANIFEST_NAMES
+from odoo.release import major_version
 from odoo.tools import convert_csv_import, convert_sql_import, convert_xml_import, exception_to_unicode
 from odoo.tools import file_open, file_open_temporary_directory
 
 _logger = logging.getLogger(__name__)
 
+APPS_URL = "https://apps.odoo.com"
 MAX_FILE_SIZE = 100 * 1024 * 1024  # in megabytes
 
 
@@ -41,7 +45,7 @@ class IrModule(models.Model):
             module.installed_version = module.latest_version
         super(IrModule, self - imported_modules)._get_latest_version()
 
-    def _import_module(self, module, path, force=False):
+    def _import_module(self, module, path, force=False, with_demo=False):
         known_mods = self.search([])
         known_mods_names = {m.name: m for m in known_mods}
         installed_mods = [m.name for m in known_mods if m.state == 'installed']
@@ -68,10 +72,8 @@ class IrModule(models.Model):
                     _is_studio_custom(path)):
                 err = _("Studio customizations require Studio")
             else:
-                err = _("Unmet module dependencies: \n\n - %s", '\n - '.join(
-                    known_mods.filtered(lambda mod: mod.name in unmet_dependencies).mapped('shortdesc')
-                ))
-            raise UserError(err)
+                to_install = known_mods.filtered(lambda mod: mod.name in unmet_dependencies)
+                to_install.button_immediate_install()
         elif 'web_studio' not in installed_mods and _is_studio_custom(path):
             raise UserError(_("Studio customizations require the Odoo Studio app."))
 
@@ -84,7 +86,10 @@ class IrModule(models.Model):
             self.create(dict(name=module, state='installed', imported=True, **values))
             mode = 'init'
 
-        for kind in ['data', 'init_xml', 'update_xml']:
+        kind_of_files = ['data', 'init_xml', 'update_xml']
+        if with_demo:
+            kind_of_files.append('demo')
+        for kind in kind_of_files:
             for filename in terp.get(kind, []):
                 ext = os.path.splitext(filename)[1].lower()
                 if ext not in ('.xml', '.csv', '.sql'):
@@ -177,7 +182,7 @@ class IrModule(models.Model):
         return True
 
     @api.model
-    def import_zipfile(self, module_file, force=False):
+    def _import_zipfile(self, module_file, force=False, with_demo=False):
         if not module_file:
             raise Exception(_("No file sent."))
         if not zipfile.is_zipfile(module_file):
@@ -207,7 +212,10 @@ class IrModule(models.Model):
                             terp = ast.literal_eval(f.read().decode())
                     except Exception:
                         continue
-                    for filename in terp.get('data', []) + terp.get('init_xml', []) + terp.get('update_xml', []):
+                    files_to_import = terp.get('data', []) + terp.get('init_xml', []) + terp.get('update_xml', [])
+                    if with_demo:
+                        files_to_import += terp.get('demo', [])
+                    for filename in files_to_import:
                         if os.path.splitext(filename)[1].lower() not in ('.xml', '.csv', '.sql'):
                             continue
                         module_data_files[mod_name].append('%s/%s' % (mod_name, filename))
@@ -225,7 +233,7 @@ class IrModule(models.Model):
                     try:
                         # assert mod_name.startswith('theme_')
                         path = opj(module_dir, mod_name)
-                        if self._import_module(mod_name, path, force=force):
+                        if self._import_module(mod_name, path, force=force, with_demo=with_demo):
                             success.append(mod_name)
                     except Exception as e:
                         _logger.exception('Error while importing module')
@@ -258,6 +266,132 @@ class IrModule(models.Model):
             modules_to_delete.unlink()
         return res
 
+    @api.model
+    def web_search_read(self, domain, specification, offset=0, limit=None, order=None, count_limit=None):
+        res = super().web_search_read(domain, specification, offset=offset, limit=limit, order=order, count_limit=count_limit)
+        if any(dom[0] == 'module_type' and dom[-1] == 'industries' for dom in domain):
+            fields_name = list(specification.keys())
+            modules_list = self._get_modules_from_apps(fields_name, 'industries', False)
+            res['length'] += len(modules_list)
+            res['records'].extend(modules_list)
+        return res
+
+    def more_info(self):
+        return {
+            'name': _('Apps'),
+            'type': 'ir.actions.act_window',
+            'res_model': 'ir.module.module',
+            'view_mode': 'form',
+            'res_id': self.id,
+            'context': self.env.context,
+        }
+
+    def web_read(self, specification):
+        fields = list(specification.keys())
+        module_type = self.env.context.get('module_type', 'official')
+        if module_type != 'official':
+            modules_list = self._get_modules_from_apps(fields, module_type, self.env.context.get('module_name'))
+            return modules_list
+        else:
+            return super().web_read(specification)
+
+    @api.model
+    def _get_modules_from_apps(self, fields, module_type, module_name):
+        payload = {
+            'series': major_version,
+            'module_fields': fields,
+            'module_type':module_type,
+            'module_name': module_name
+        }
+        try:
+            resp = requests.post(
+                f"{APPS_URL}/loempia/listdatamodules",
+                json={'params': payload},
+                timeout=5.0,
+            )
+            if resp.status_code == 200:
+                modules_list = resp.json().get('result', [])
+                for mod in modules_list:
+                    mod['id'] = -1
+                    module_name = mod.get('name', module_name)
+                    if 'icon' in fields:
+                        mod['icon'] = f"{APPS_URL}{mod['icon']}"
+                    if 'state' in fields:
+                        existing_mod = self.search([('name', '=', module_name), ('state', '=', 'installed')])
+                        if existing_mod:
+                            mod['state'] = 'installed'
+                        else:
+                            mod['state'] = 'uninstalled'
+                    if 'module_type' in fields:
+                        mod['module_type'] = module_type
+                    if 'website' in fields:
+                        mod['website'] = f"{APPS_URL}/apps/modules/{major_version}/{module_name}/"
+                return modules_list
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError:
+            raise UserError(_('The list of industry applications cannot be fetched. Please try again later'))
+        except requests.exceptions.ConnectionError:
+            raise UserError(_('Connection to %s failed The list of industry modules cannot be fetched') % APPS_URL)
+
+    def button_immediate_install_app(self):
+        if not self.env.is_admin():
+            raise AccessDenied()
+        module_name = self.env.context.get('module_name')
+        try:
+            resp = requests.get(
+                f"{APPS_URL}/loempia/download/data_app/{module_name}/{major_version}",
+                timeout=5.0,
+            )
+            if resp.status_code == 200:
+                import_module = self.env['base.import.module'].create({
+                    'module_file': base64.b64encode(resp.content),
+                    'state': 'init',
+                    'modules_dependencies': self._get_missing_dependencies(resp.content)
+                })
+                return {
+                    'name': 'Install an App',
+                    'view_mode': 'form',
+                    'target': 'new',
+                    'res_id': import_module.id,
+                    'res_model': 'base.import.module',
+                    'type': 'ir.actions.act_window',
+                    'context': {'data_module': True}
+                }
+            resp.raise_for_status()
+        except requests.exceptions.HTTPError:
+            raise UserError(_('The module %s cannot be downloaded') % module_name)
+        except requests.exceptions.ConnectionError:
+            raise UserError(_('Connection to %s failed, the module %s cannot be downloaded.', APPS_URL, module_name))
+
+    @api.model
+    def _get_missing_dependencies(self, zip_data):
+        dependencies_to_install = self.env['ir.module.module']
+        known_mods = self.search([])
+        installed_mods = [m.name for m in known_mods if m.state == 'installed']
+        with zipfile.ZipFile(BytesIO(zip_data), "r") as z:
+            manifest_files = [
+                file
+                for file in z.filelist
+                if file.filename.count('/') == 1
+                and file.filename.split('/')[1] in MANIFEST_NAMES
+            ]
+            for manifest_file in manifest_files:
+                if manifest_file.file_size > MAX_FILE_SIZE:
+                    raise UserError(_("File '%s' exceed maximum allowed file size", manifest_file.filename))
+                try:
+                    with z.open(manifest_file) as manifest:
+                        terp = ast.literal_eval(manifest.read().decode())
+                except Exception:
+                    continue
+                unmet_dependencies = set(terp.get('depends', [])).difference(installed_mods)
+                dependencies_to_install |= known_mods.filtered(lambda m: m.name in unmet_dependencies)
+        description = ''
+        if dependencies_to_install:
+            description = _('The following modules will be also installed:\n')
+            for mod in dependencies_to_install:
+                description += "- " + mod.shortdesc + "\n"
+        return description
+
 
 def _is_studio_custom(path):
     """

addons/base_import_module/tests/test_cloc.py

@@ -146,7 +146,7 @@ class TestClocFields(test_cloc.TestClocCustomization):
             archive.writestr('test_imported_module/static/src/js/test.xml', VALID_XML)
 
         # Import test module
-        self.env['ir.module.module'].import_zipfile(stream)
+        self.env['ir.module.module']._import_zipfile(stream)
         cl = cloc.Cloc()
         cl.count_customization(self.env)
         self.assertEqual(cl.code.get('test_imported_module', 0), 35)
@@ -201,7 +201,7 @@ class TestClocFields(test_cloc.TestClocCustomization):
         ]
 
         # Import test module
-        self.env['ir.module.module'].import_zipfile(stream)
+        self.env['ir.module.module']._import_zipfile(stream)
         # Create exclude xml_id
         for name in id_names:
             rec = self.env.ref(f'test_imported_module.{name}')

addons/base_import_module/tests/test_import_module.py

@@ -25,7 +25,7 @@ class TestImportModule(odoo.tests.TransactionCase):
         with ZipFile(archive, 'w') as zipf:
             for path, data in files:
                 zipf.writestr(path, data)
-        return self.env['ir.module.module'].import_zipfile(archive)
+        return self.env['ir.module.module']._import_zipfile(archive)
 
     def test_import_zip(self):
         """Assert the behaviors expected by the module import feature using a ZIP archive"""
@@ -187,7 +187,7 @@ class TestImportModule(odoo.tests.TransactionCase):
             archive.writestr('test_module/static/src/js/test.js', "console.log('AAA');")
 
         # Import test module
-        self.env['ir.module.module'].import_zipfile(stream)
+        self.env['ir.module.module']._import_zipfile(stream)
 
         attachment = self.env['ir.attachment'].search([('url', '=', path)])
         self.assertEqual(attachment.name, 'test.js')
@@ -240,7 +240,7 @@ class TestImportModule(odoo.tests.TransactionCase):
             archive.writestr(path, "console.log('AAA');")
 
         # Import test module
-        self.env['ir.module.module'].with_user(self.test_user).import_zipfile(stream)
+        self.env['ir.module.module'].with_user(self.test_user)._import_zipfile(stream)
 
         attachment = self.env['ir.attachment'].search([('url', '=', f'/{path}')])
         self.assertEqual(attachment.name, 'test.js')
@@ -264,7 +264,7 @@ class TestImportModule(odoo.tests.TransactionCase):
             archive.writestr(path, "console.log('BBB');")
 
         # Import test module
-        self.env['ir.module.module'].with_user(self.test_user).import_zipfile(stream)
+        self.env['ir.module.module'].with_user(self.test_user)._import_zipfile(stream)
 
         attachment = self.env['ir.attachment'].search([('url', '=', f'/{path}')])
         self.assertEqual(attachment.name, 'test.js')

addons/base_import_module/views/base_import_module_view.xml

@@ -7,10 +7,14 @@
             <field name="arch" type="xml">
                 <form string="Install the application">
                     <field name="state" invisible="1"/>
-                    <p class="alert alert-warning" role="alert" invisible="state == 'done'">Note: you can only import data modules (.xml files and static assets)</p>
+                    <p class="alert alert-warning" role="alert" invisible="state == 'done' or context.get('data_module')">Note: you can only import data modules (.xml files and static assets)</p>
+                    <group col="4">
+                    <field name="modules_dependencies" readonly="1" colspan="4" nolabel="1" invisible="state == 'done'"/>
+                    </group>
                     <group invisible="state != 'init'" col="4">
-                        <field name="module_file" string="Module file (.zip)" colspan="4" options="{'accepted_file_extensions': '.zip'}"/>
-                        <field name="force" colspan="4" groups="base.group_no_one"/>
+                        <field name="module_file" string="Module file (.zip)" colspan="4" options="{'accepted_file_extensions': '.zip'}" invisible="context.get('data_module')"/>
+                        <field name="force" groups="base.group_no_one"/>
+                        <field name="with_demo" string="Import demo data"/>
                     </group>
                     <group invisible="state != 'done'" col="4">
                         <field name="import_message" colspan="4" nolabel="1" readonly="1"/>

addons/base_import_module/views/ir_module_views.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <data>
+        <record model="ir.ui.view" id="module_view_kanban_apps_inherit">
+            <field name="name">Apps Kanban Data Modules</field>
+            <field name="model">ir.module.module</field>
+            <field name="inherit_id" ref="base.module_view_kanban"/>
+            <field name="arch" type="xml">
+                <xpath expr="//field[@name='application']" position="after">
+                    <field name="module_type"/>
+                </xpath>
+                <xpath expr="//button[@name='button_immediate_install']" position="after">
+                    <button type="object" class="btn btn-primary btn-sm" name="button_immediate_install_app" invisible="state != 'uninstalled' or module_type == 'official'" groups="base.group_system" context="{'module_name':name}">Activate</button>
+                    <button type="object" class="btn btn-primary btn-sm" name="button_immediate_install_app" invisible="state == 'uninstalled' or module_type == 'official'" groups="base.group_system" context="{'module_name':name}">Upgrade</button>
+                </xpath>
+                <xpath expr="//button[@name='button_immediate_install']" position="attributes">
+                    <attribute name="invisible">state != 'uninstalled' or module_type != 'official'</attribute>
+                </xpath>
+                <xpath expr="//t[@t-name='kanban-menu']/a[@type='edit']" position="attributes">
+                    <attribute name="context">{'module_type': module_type, 'module_name':name}</attribute>
+                    <attribute name="type">object</attribute>
+                    <attribute name="name">more_info</attribute>
+                </xpath>
+                <xpath expr="//div[hasclass('oe_module_action')]/a[@type='edit']" position="attributes">
+                    <attribute name="context">{'module_type': module_type, 'module_name':name}</attribute>
+                    <attribute name="type">object</attribute>
+                    <attribute name="name">more_info</attribute>
+                </xpath>
+            </field>
+        </record>
+        <record model="ir.ui.view" id="module_tree_apps_inherit">
+            <field name="name">Apps List Data Modules</field>
+            <field name="model">ir.module.module</field>
+            <field name="inherit_id" ref="base.module_tree"/>
+            <field name="arch" type="xml">
+                <xpath expr="//field[@name='category_id']" position="after">
+                    <field name="module_type" invisible="1"/>
+                </xpath>
+            </field>
+        </record>
+        <record model="ir.ui.view" id="module_form_apps_inherit">
+            <field name="name">Apps</field>
+            <field name="model">ir.module.module</field>
+            <field name="inherit_id" ref="base.module_form"/>
+            <field name="arch" type="xml">
+                <xpath expr="//field[@name='has_iap']" position="after">
+                    <field name="module_type" invisible="1"/>
+                </xpath>
+                <xpath expr="//page[@name='technical_data']" position="attributes">
+                    <attribute name="invisible">module_type != 'official'</attribute>
+                </xpath>
+            </field>
+        </record>
+        <record model="ir.ui.view" id="view_module_filter_apps_inherit">
+            <field name="name">Search Data Modules</field>
+            <field name="model">ir.module.module</field>
+            <field name="inherit_id" ref="base.view_module_filter"/>
+            <field name="arch" type="xml">
+                <xpath expr="//searchpanel/field[@name='category_id']" position="before">
+                    <field name="module_type" string="Apps" expand="1"/>
+                </xpath>
+            </field>
+        </record>
+    </data>
+</odoo>