[FIX] cloc: fix count of studio field + exclude sa

This commit fix 3 problem Don't count studio field ------------------------ the commit https://github.com/odoo/odoo/commit/75b8c4eb1ed6e44f427886eea06ad216995e16c8 was supposed to stop counting count field generate by addins a smart button with studio. It works fine when base import module is not installed which is never the case. When base_import_module the field is counted as a field that belong to a imported module installed : studio_customization studio_customization should not be counted as an imported module installed Count Field with standard xml_id -------------------------------- With https://github.com/odoo/odoo/commit/9afce4805fc8bac45fdba817488aa867fddff69b Each manual can end up with an xml_id from a standard module: the original module of the model A standard module should never create a manual field so we can consider they should be counted unless they match the criteria of the first problem If a field has a standard module xml_id and no other the module should be odoo/studio and not the name of the standard module Make possible to exclude some db record from cloc ------------------------------------------------- It's possible to exclude some file in python module but it's not possible to exclude some field or SA in the database from the count Make it possible if they are link to an xml_id from the module __cloc_exclude__ The exclude record will be shown in cloc report with the verbose mode closes odoo/odoo#95028 X-original-commit: db52fb174461107e26b066147801f54a89d0dbf9 Related: odoo/enterprise#29007 Signed-off-by: Christophe Simonis <chs@odoo.com>
2022-07-01 16:36:18 +02:00
parent 5c8441debb
commit 226df213f2
3 changed files with 259 additions and 59 deletions
@@ -20,6 +20,20 @@ VALID_XML = """

 class TestClocFields(test_cloc.TestClocCustomization):

+    def create_studio_module(self):
+        # Studio module does not exist at this stage, so we simulate it
+        # Check for existing module in case the test run on an existing database
+        if not self.env['ir.module.module'].search([('name', '=', 'studio_customization')]):
+            self.env['ir.module.module'].create({
+                'author': 'Odoo',
+                'imported': True,
+                'latest_version': '13.0.1.0.0',
+                'name': 'studio_customization',
+                'state': 'installed',
+                'summary': 'Studio Customization',
+            })
+
+
    def test_fields_from_import_module(self):
        """
            Check that custom computed fields installed with an imported module
@@ -31,15 +45,44 @@ class TestClocFields(test_cloc.TestClocCustomization):
            'imported': True,
        })
        f1 = self.create_field('x_imported_field')
-        self.create_xml_id('import_field', 'ir.model.fields', f1.id, 'imported_module')
+        self.create_xml_id('imported_module', 'import_field', f1)
        cl = cloc.Cloc()
        cl.count_customization(self.env)
        self.assertEqual(cl.code.get('imported_module', 0), 1, 'Count fields with xml_id of imported module')
-        f2 = self.create_field('x_base_field')
-        self.create_xml_id('base_field', 'ir.model.fields', f2.id, 'base')
+
+    def test_fields_from_studio(self):
+        self.create_studio_module()
+        f1 = self.create_field('x_field_count')
+        self.create_xml_id('studio_customization', 'field_count', f1)
        cl = cloc.Cloc()
        cl.count_customization(self.env)
-        self.assertEqual(cl.code.get('base', 0), 0, "Don't count fields from standard module")
+        self.assertEqual(cl.code.get('studio_customization', 0), 0, "Don't count field generated by studio")
+        f2 = self.create_field('x_studio_manual_field')
+        self.create_xml_id('studio_customization', 'manual_field', f2)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('studio_customization', 0), 1, "Count manual field created via studio")
+
+    def test_fields_module_name(self):
+        """
+            Check that custom computed fields installed with an imported module
+            is counted as customization
+        """
+        self.env['ir.module.module'].create({
+            'name': 'imported_module',
+            'state': 'installed',
+            'imported': True,
+        })
+        f1 = self.create_field('x_imported_field')
+        self.create_xml_id('imported_module', 'import_field', f1)
+        self.create_xml_id('__export__', 'import_field', f1)
+
+        sa = self.create_server_action("Test imported double xml_id")
+        self.create_xml_id("imported_module", "first", sa)
+        self.create_xml_id("__export__", "second", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('imported_module', 0), 3)

    def test_count_qweb_imported_module(self):
        self.env['ir.module.module'].create({
@@ -50,24 +93,15 @@ class TestClocFields(test_cloc.TestClocCustomization):
            'state': 'installed',
            'summary': 'Test imported module for cloc',
        })
-        # Studio module does not exist at this stage, so we simulate it
-        # Check for existing module in case the test run on an existing database
-        if not self.env['ir.module.module'].search([('name', '=', 'studio_customization')]):
-            self.env['ir.module.module'].create({
-                'author': 'Odoo',
-                'imported': True,
-                'latest_version': '15.0.1.0.0',
-                'name': 'studio_customization',
-                'state': 'installed',
-                'summary': 'Studio Customization',
-            })
+        self.create_studio_module()
+
        qweb_view = self.env['ir.ui.view'].create({
            "name": "Qweb Test",
            "type": "qweb",
            "mode": "primary",
            "arch_base": "<html>\n  <body>\n    <t t-out=\"Hello World\"/>\n  </body>\n</html>",
        })
-        self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id, 'test_imported_module')
+        self.create_xml_id('test_imported_module', "qweb_view_test", qweb_view)

        # Add qweb view from non imported module
        qweb_view = self.env['ir.ui.view'].create({
@@ -75,7 +109,7 @@ class TestClocFields(test_cloc.TestClocCustomization):
            "type": "qweb",
            "arch_base": "<html>\n  <body>\n    <t t-out=\"Hello World\"/>\n  </body>\n</html>",
        })
-        self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id)
+        self.create_xml_id("studio_customization", "qweb_view_test", qweb_view)

        # Add form view from module
        form_view = self.env['ir.ui.view'].create({
@@ -84,7 +118,7 @@ class TestClocFields(test_cloc.TestClocCustomization):
            "model": "res.partner",
            "arch_base": "<form>\n  <field name=\"name\" \n         invisible=\"1\" />\n</form>",
        })
-        self.create_xml_id("form_view_test", 'ir.ui.view', form_view.id, 'test_imported_module')
+        self.create_xml_id("test_imported_module", "form_view_test", form_view)

        cl = cloc.Cloc()
        cl.count_customization(self.env)
@@ -116,3 +150,63 @@ class TestClocFields(test_cloc.TestClocCustomization):
        cl = cloc.Cloc()
        cl.count_customization(self.env)
        self.assertEqual(cl.code.get('test_imported_module', 0), 35)
+
+    def test_exclude_qweb(self):
+        self.env['ir.module.module'].create({
+            'author': 'Odoo',
+            'imported': True,
+            'latest_version': '15.0.1.0.0',
+            'name': 'test_imported_module',
+            'state': 'installed',
+            'summary': 'Test imported module for cloc',
+        })
+
+        qweb_view = self.env['ir.ui.view'].create({
+            "name": "Qweb Test",
+            "type": "qweb",
+            "mode": "primary",
+            "arch_base": "<html>\n  <body>\n    <t t-out=\"Hello World\"/>\n  </body>\n</html>",
+        })
+        self.create_xml_id('test_imported_module', "qweb_view_test", qweb_view)
+        self.create_xml_id('__cloc_exclude__', "qweb_view_test", qweb_view)
+
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('test_imported_module', 0), 0, "Do not count view with cloc_exclude")
+
+    def test_exclude_attachment_imported_module(self):
+        manifest_content = json.dumps({
+            'name': 'test_imported_module',
+            'description': 'Test',
+            'assets': {
+                'web.assets_backend': [
+                    'test_imported_module/static/src/js/test.js',
+                    'test_imported_module/static/src/css/test.scss',
+                ]
+            },
+            'license': 'LGPL-3',
+        })
+
+        stream = BytesIO()
+        with ZipFile(stream, 'w') as archive:
+            archive.writestr('test_imported_module/__manifest__.py', manifest_content)
+            archive.writestr('test_imported_module/static/src/js/test.js', test_cloc.JS_TEST)
+            archive.writestr('test_imported_module/static/src/js/test.scss', test_cloc.SCSS_TEST)
+            archive.writestr('test_imported_module/static/src/js/test.xml', VALID_XML)
+
+        id_names = [
+            'attachment_/test_imported_module/static/src/js/test_js',
+            'attachment_/test_imported_module/static/src/js/test_scss',
+            'attachment_/test_imported_module/static/src/js/test_xml',
+        ]
+
+        # Import test module
+        self.env['ir.module.module'].import_zipfile(stream)
+        # Create exclude xml_id
+        for name in id_names:
+            rec = self.env.ref(f'test_imported_module.{name}')
+            self.create_xml_id('__cloc_exclude__', name, rec)
+
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('test_imported_module', 0), 0)
@@ -129,16 +129,16 @@ SCSS_TEST = '''
 '''

 class TestClocCustomization(TransactionCase):
-    def create_xml_id(self, name, model, res_id, module='studio_customization'):
+    def create_xml_id(self, module, name, rec):
        self.env['ir.model.data'].create({
            'name': name,
-            'model': model,
-            'res_id': res_id,
+            'model': rec._name,
+            'res_id': rec.id,
            'module': module,
        })

    def create_field(self, name):
-        return self.env['ir.model.fields'].with_context(studio=True).create({
+        field = self.env['ir.model.fields'].with_context(studio=True).create({
            'name': name,
            'field_description': name,
            'model': 'res.partner',
@@ -147,6 +147,23 @@ class TestClocCustomization(TransactionCase):
            'store': False,
            'compute': "for rec in self: rec['x_invoice_count'] = 10",
        })
+        # Simulate the effect of https://github.com/odoo/odoo/commit/9afce4805fc8bac45fdba817488aa867fddff69b
+        # Updating a module create xml_id of the module even for manual field if it's the original module
+        # of the model
+        self.create_xml_id('base', name, field)
+        return field
+
+    def create_server_action(self, name):
+        return self.env['ir.actions.server'].create({
+            'name': name,
+            'code': """
+for rec in records:
+    rec['name'] = test
+            """,
+            'state': 'code',
+            'type': 'ir.actions.server',
+            'model_id': self.env.ref('base.model_res_partner').id,
+        })

    def test_ignore_auto_generated_computed_field(self):
        """
@@ -154,12 +171,12 @@ class TestClocCustomization(TransactionCase):
            Having an xml_id but no existing module is consider as not belonging to a module
        """
        f1 = self.create_field('x_invoice_count')
-        self.create_xml_id('invoice_count', 'ir.model.fields', f1.id)
+        self.create_xml_id('studio_customization', 'invoice_count', f1)
        cl = cloc.Cloc()
        cl.count_customization(self.env)
        self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Studio auto generated count field should not be counted in cloc')
        f2 = self.create_field('x_studio_custom_field')
-        self.create_xml_id('studio_custom', 'ir.model.fields', f2.id)
+        self.create_xml_id('studio_customization', 'studio_custom', f2)
        cl = cloc.Cloc()
        cl.count_customization(self.env)
        self.assertEqual(cl.code.get('odoo/studio', 0), 1, 'Count other studio computed field')
@@ -168,11 +185,52 @@ class TestClocCustomization(TransactionCase):
        cl.count_customization(self.env)
        self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'Count fields without xml_id')
        f4 = self.create_field('x_custom_field_export')
-        self.create_xml_id('studio_custom', 'ir.model.fields', f4.id, '__export__')
+        self.create_xml_id('__export__', 'studio_custom', f4)
        cl = cloc.Cloc()
        cl.count_customization(self.env)
        self.assertEqual(cl.code.get('odoo/studio', 0), 3, 'Count fields with xml_id but without module')

+    def test_several_xml_id(self):
+        sa = self.create_server_action("Test double xml_id")
+        self.create_xml_id("__export__", "first", sa)
+        self.create_xml_id("base", "second", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'Count Should count SA with a non standard xml_id')
+        self.create_xml_id("__import__", "third", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'SA with several xml_id should be counted only once')
+
+    def test_cloc_exclude_xml_id(self):
+        sa = self.create_server_action("Test double xml_id")
+        self.create_xml_id("__cloc_exclude__", "sa_first", sa)
+        self.create_xml_id("__upgrade__", "sa_second", sa)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Should not count SA with cloc_exclude xml_id')
+
+        f1 = self.create_field('x_invoice_count')
+        self.create_xml_id("__cloc_exclude__", "field_first", f1)
+        self.create_xml_id("__upgrade__", "field_second", f1)
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Should not count Field with cloc_exclude xml_id')
+
+    def test_field_no_xml_id(self):
+        self.env['ir.model.fields'].create({
+            'name': "x_no_xml_id",
+            'field_description': "no_xml_id",
+            'model': 'res.partner',
+            'model_id': self.env.ref('base.model_res_partner').id,
+            'ttype': 'integer',
+            'store': False,
+            'compute': "for rec in self: rec['x_invoice_count'] = 10",
+        })
+        cl = cloc.Cloc()
+        cl.count_customization(self.env)
+        self.assertEqual(cl.code.get('odoo/studio', 0), 1, 'Should count field with no xml_id at all')
+

 class TestClocParser(TransactionCase):

@@ -30,6 +30,7 @@ class Cloc(object):
        self.code = {}
        self.total = {}
        self.errors = {}
+        self.excluded = {}
        self.max_width = 70

    #------------------------------------------------------
@@ -95,10 +96,13 @@ class Cloc(object):
    #------------------------------------------------------
    # Enumeration
    #------------------------------------------------------
-    def book(self, module, item='', count=(0, 0)):
+    def book(self, module, item='', count=(0, 0), exclude=False):
        if count[0] == -1:
            self.errors.setdefault(module, {})
            self.errors[module][item] = count[1]
+        elif exclude and item:
+            self.excluded.setdefault(module, {})
+            self.excluded[module][item] = count
        else:
            self.modules.setdefault(module, {})
            if item:
@@ -168,65 +172,93 @@ class Cloc(object):
                self.count_path(module_path)

    def count_customization(self, env):
-        imported_module = ""
+        imported_module_sa = ""
        if env['ir.module.module']._fields.get('imported'):
-            imported_module = "OR (m.imported = TRUE AND m.state = 'installed')"
+            imported_module_sa = "OR (m.imported = TRUE AND m.state = 'installed')"
        query = """
-            SELECT s.id, m.name FROM ir_act_server AS s
-                LEFT JOIN ir_model_data AS d ON (d.res_id = s.id AND d.model = 'ir.actions.server')
-                LEFT JOIN ir_module_module AS m ON m.name = d.module
-            WHERE s.state = 'code' AND (m.name IS null {})
-        """.format(imported_module)
+                SELECT s.id, min(m.name), array_agg(d.module)
+                  FROM ir_act_server AS s
+             LEFT JOIN ir_model_data AS d
+                    ON (d.res_id = s.id AND d.model = 'ir.actions.server')
+             LEFT JOIN ir_module_module AS m
+                    ON m.name = d.module
+                 WHERE s.state = 'code' AND (m.name IS null {})
+              GROUP BY s.id
+        """.format(imported_module_sa)
        env.cr.execute(query)
-        data = {r[0]: r[1] for r in env.cr.fetchall()}
+        data = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
        for a in env['ir.actions.server'].browse(data.keys()):
-            self.book(data[a.id] or "odoo/studio", "ir.actions.server/%s: %s" % (a.id, a.name), self.parse_py(a.code))
+            self.book(
+                data[a.id][0] or "odoo/studio",
+                "ir.actions.server/%s: %s" % (a.id, a.name),
+                self.parse_py(a.code),
+                '__cloc_exclude__' in data[a.id][1]
+            )

+        imported_module_field = ("'odoo/studio'", "")
+        if env['ir.module.module']._fields.get('imported'):
+            imported_module_field = ("min(m.name)", "AND m.imported = TRUE AND m.state = 'installed'")
+        # We always want to count manual compute field unless they are generated by studio
+        # the module should be odoo/studio unless it comes from an imported module install
+        # because manual field get an external id from the original module of the model
        query = r"""
-            SELECT f.id, m.name FROM ir_model_fields AS f
-                LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields')
-                LEFT JOIN ir_module_module AS m ON m.name = d.module
-            WHERE f.compute IS NOT null AND (
-                -- Do not count studio field
-                (m.name IS null AND (d.module != 'studio_customization' or d.module is NULL))
-                -- Unless they start with x_studio
-                OR (d.module = 'studio_customization' AND f.name ilike 'x\_studio%')
-                {})
-        """.format(imported_module)
+                SELECT f.id, f.name, {}, array_agg(d.module)
+                  FROM ir_model_fields AS f
+             LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields')
+             LEFT JOIN ir_module_module AS m ON m.name = d.module {}
+                 WHERE f.compute IS NOT null AND f.state = 'manual'
+              GROUP BY f.id, f.name
+        """.format(*imported_module_field)
        env.cr.execute(query)
-        data = {r[0]: r[1] for r in env.cr.fetchall()}
+        # Do not count field generated by studio
+        all_data = env.cr.fetchall()
+        data = {r[0]: (r[2], r[3]) for r in all_data if not ("studio_customization" in r[3] and not r[1].startswith('x_studio'))}
        for f in env['ir.model.fields'].browse(data.keys()):
-            self.book(data[f.id] or "odoo/studio", "ir.model.fields/%s: %s" % (f.id, f.name), self.parse_py(f.compute))
+            self.book(
+                data[f.id][0] or "odoo/studio",
+                "ir.model.fields/%s: %s" % (f.id, f.name),
+                self.parse_py(f.compute),
+                '__cloc_exclude__' in data[f.id][1]
+            )

-        if not imported_module:
+        if not env['ir.module.module']._fields.get('imported'):
            return

        # Count qweb view only from imported module and not studio
        query = """
-            SELECT view.id, data.module
+            SELECT view.id, min(mod.name), array_agg(data.module)
              FROM ir_ui_view view
        INNER JOIN ir_model_data data ON view.id = data.res_id AND data.model = 'ir.ui.view'
-        INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
-             WHERE view.type = 'qweb' AND mod.name != 'studio_customization'
+         LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
+             WHERE view.type = 'qweb' AND data.module != 'studio_customization'
+          GROUP BY view.id
+            HAVING count(mod.name) > 0
        """
        env.cr.execute(query)
-        custom_views = dict(env.cr.fetchall())
+        custom_views = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
        for view in env['ir.ui.view'].browse(custom_views.keys()):
-            modue_name = custom_views[view.id]
-            self.book(modue_name, "/%s/views/%s.xml" % (modue_name, view.name), self.parse_xml(view.arch_base))
+            module_name = custom_views[view.id][0]
+            self.book(
+                module_name,
+                "/%s/views/%s.xml" % (module_name, view.name),
+                self.parse_xml(view.arch_base),
+                '__cloc_exclude__' in custom_views[view.id][1]
+            )

        # Count js, xml, css/scss file from imported module
        query = r"""
-            SELECT attach.id, data.module
+            SELECT attach.id, min(mod.name), array_agg(data.module)
              FROM ir_attachment attach
        INNER JOIN ir_model_data data ON attach.id = data.res_id AND data.model = 'ir.attachment'
-        INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
+         LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
             WHERE attach.name ~ '.*\.(js|xml|css|scss)$'
+          GROUP BY attach.id
+            HAVING count(mod.name) > 0
        """
        env.cr.execute(query)
-        uploaded_file = dict(env.cr.fetchall())
+        uploaded_file = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
        for attach in env['ir.attachment'].browse(uploaded_file.keys()):
-            module_name = uploaded_file[attach.id]
+            module_name = uploaded_file[attach.id][0]
            ext = os.path.splitext(attach.url)[1].lower()
            if ext not in VALID_EXTENSION:
                continue
@@ -238,7 +270,12 @@ class Cloc(object):
            # Decode using latin1 to avoid error that may raise by decoding with utf8
            # The chars not correctly decoded in latin1 have no impact on how many lines will be counted
            content = attach.raw.decode('latin1')
-            self.book(module_name, attach.url, self.parse(content, ext))
+            self.book(
+                module_name,
+                attach.url,
+                self.parse(content, ext),
+                '__cloc_exclude__' in uploaded_file[attach.id][1],
+            )

    def count_env(self, env):
        self.count_modules(env)
@@ -254,6 +291,7 @@ class Cloc(object):
    #------------------------------------------------------
    # Report
    #------------------------------------------------------
+    # pylint: disable=W0141
    def report(self, verbose=False, width=None):
        # Prepare format
        if not width:
@@ -276,6 +314,16 @@ class Cloc(object):
        s += fmt.format(k='', lines=total, other=total - code, code=code)
        print(s)

+        if self.excluded and verbose:
+            ex = fmt.format(k="Excluded", lines="Line", other="Other", code="Code")
+            ex += hr
+            for m in sorted(self.excluded):
+                for i in sorted(self.excluded[m], key=lambda i: self.excluded[m][i][0], reverse=True):
+                    code, total = self.excluded[m][i]
+                    ex += fmt.format(k='    ' + i, lines=total, other=total - code, code=code)
+            ex += hr
+            print(ex)
+
        if self.errors:
            e = "\nErrors\n\n"
            for m in sorted(self.errors):