diff --git a/addons/base_import_module/tests/test_cloc.py b/addons/base_import_module/tests/test_cloc.py index 352d42dde38..978b4a38084 100644 --- a/addons/base_import_module/tests/test_cloc.py +++ b/addons/base_import_module/tests/test_cloc.py @@ -20,6 +20,20 @@ VALID_XML = """ class TestClocFields(test_cloc.TestClocCustomization): + def create_studio_module(self): + # Studio module does not exist at this stage, so we simulate it + # Check for existing module in case the test run on an existing database + if not self.env['ir.module.module'].search([('name', '=', 'studio_customization')]): + self.env['ir.module.module'].create({ + 'author': 'Odoo', + 'imported': True, + 'latest_version': '13.0.1.0.0', + 'name': 'studio_customization', + 'state': 'installed', + 'summary': 'Studio Customization', + }) + + def test_fields_from_import_module(self): """ Check that custom computed fields installed with an imported module @@ -31,15 +45,44 @@ class TestClocFields(test_cloc.TestClocCustomization): 'imported': True, }) f1 = self.create_field('x_imported_field') - self.create_xml_id('import_field', 'ir.model.fields', f1.id, 'imported_module') + self.create_xml_id('imported_module', 'import_field', f1) cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('imported_module', 0), 1, 'Count fields with xml_id of imported module') - f2 = self.create_field('x_base_field') - self.create_xml_id('base_field', 'ir.model.fields', f2.id, 'base') + + def test_fields_from_studio(self): + self.create_studio_module() + f1 = self.create_field('x_field_count') + self.create_xml_id('studio_customization', 'field_count', f1) cl = cloc.Cloc() cl.count_customization(self.env) - self.assertEqual(cl.code.get('base', 0), 0, "Don't count fields from standard module") + self.assertEqual(cl.code.get('studio_customization', 0), 0, "Don't count field generated by studio") + f2 = self.create_field('x_studio_manual_field') + self.create_xml_id('studio_customization', 'manual_field', f2) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('studio_customization', 0), 1, "Count manual field created via studio") + + def test_fields_module_name(self): + """ + Check that custom computed fields installed with an imported module + is counted as customization + """ + self.env['ir.module.module'].create({ + 'name': 'imported_module', + 'state': 'installed', + 'imported': True, + }) + f1 = self.create_field('x_imported_field') + self.create_xml_id('imported_module', 'import_field', f1) + self.create_xml_id('__export__', 'import_field', f1) + + sa = self.create_server_action("Test imported double xml_id") + self.create_xml_id("imported_module", "first", sa) + self.create_xml_id("__export__", "second", sa) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('imported_module', 0), 3) def test_count_qweb_imported_module(self): self.env['ir.module.module'].create({ @@ -50,24 +93,15 @@ class TestClocFields(test_cloc.TestClocCustomization): 'state': 'installed', 'summary': 'Test imported module for cloc', }) - # Studio module does not exist at this stage, so we simulate it - # Check for existing module in case the test run on an existing database - if not self.env['ir.module.module'].search([('name', '=', 'studio_customization')]): - self.env['ir.module.module'].create({ - 'author': 'Odoo', - 'imported': True, - 'latest_version': '15.0.1.0.0', - 'name': 'studio_customization', - 'state': 'installed', - 'summary': 'Studio Customization', - }) + self.create_studio_module() + qweb_view = self.env['ir.ui.view'].create({ "name": "Qweb Test", "type": "qweb", "mode": "primary", "arch_base": "\n \n \n \n", }) - self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id, 'test_imported_module') + self.create_xml_id('test_imported_module', "qweb_view_test", qweb_view) # Add qweb view from non imported module qweb_view = self.env['ir.ui.view'].create({ @@ -75,7 +109,7 @@ class TestClocFields(test_cloc.TestClocCustomization): "type": "qweb", "arch_base": "\n \n \n \n", }) - self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id) + self.create_xml_id("studio_customization", "qweb_view_test", qweb_view) # Add form view from module form_view = self.env['ir.ui.view'].create({ @@ -84,7 +118,7 @@ class TestClocFields(test_cloc.TestClocCustomization): "model": "res.partner", "arch_base": "
\n \n", }) - self.create_xml_id("form_view_test", 'ir.ui.view', form_view.id, 'test_imported_module') + self.create_xml_id("test_imported_module", "form_view_test", form_view) cl = cloc.Cloc() cl.count_customization(self.env) @@ -116,3 +150,63 @@ class TestClocFields(test_cloc.TestClocCustomization): cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('test_imported_module', 0), 35) + + def test_exclude_qweb(self): + self.env['ir.module.module'].create({ + 'author': 'Odoo', + 'imported': True, + 'latest_version': '15.0.1.0.0', + 'name': 'test_imported_module', + 'state': 'installed', + 'summary': 'Test imported module for cloc', + }) + + qweb_view = self.env['ir.ui.view'].create({ + "name": "Qweb Test", + "type": "qweb", + "mode": "primary", + "arch_base": "\n \n \n \n", + }) + self.create_xml_id('test_imported_module', "qweb_view_test", qweb_view) + self.create_xml_id('__cloc_exclude__', "qweb_view_test", qweb_view) + + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('test_imported_module', 0), 0, "Do not count view with cloc_exclude") + + def test_exclude_attachment_imported_module(self): + manifest_content = json.dumps({ + 'name': 'test_imported_module', + 'description': 'Test', + 'assets': { + 'web.assets_backend': [ + 'test_imported_module/static/src/js/test.js', + 'test_imported_module/static/src/css/test.scss', + ] + }, + 'license': 'LGPL-3', + }) + + stream = BytesIO() + with ZipFile(stream, 'w') as archive: + archive.writestr('test_imported_module/__manifest__.py', manifest_content) + archive.writestr('test_imported_module/static/src/js/test.js', test_cloc.JS_TEST) + archive.writestr('test_imported_module/static/src/js/test.scss', test_cloc.SCSS_TEST) + archive.writestr('test_imported_module/static/src/js/test.xml', VALID_XML) + + id_names = [ + 'attachment_/test_imported_module/static/src/js/test_js', + 'attachment_/test_imported_module/static/src/js/test_scss', + 'attachment_/test_imported_module/static/src/js/test_xml', + ] + + # Import test module + self.env['ir.module.module'].import_zipfile(stream) + # Create exclude xml_id + for name in id_names: + rec = self.env.ref(f'test_imported_module.{name}') + self.create_xml_id('__cloc_exclude__', name, rec) + + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('test_imported_module', 0), 0) diff --git a/odoo/addons/base/tests/test_cloc.py b/odoo/addons/base/tests/test_cloc.py index 6d48eed8835..29c4c0ec74b 100644 --- a/odoo/addons/base/tests/test_cloc.py +++ b/odoo/addons/base/tests/test_cloc.py @@ -129,16 +129,16 @@ SCSS_TEST = ''' ''' class TestClocCustomization(TransactionCase): - def create_xml_id(self, name, model, res_id, module='studio_customization'): + def create_xml_id(self, module, name, rec): self.env['ir.model.data'].create({ 'name': name, - 'model': model, - 'res_id': res_id, + 'model': rec._name, + 'res_id': rec.id, 'module': module, }) def create_field(self, name): - return self.env['ir.model.fields'].with_context(studio=True).create({ + field = self.env['ir.model.fields'].with_context(studio=True).create({ 'name': name, 'field_description': name, 'model': 'res.partner', @@ -147,6 +147,23 @@ class TestClocCustomization(TransactionCase): 'store': False, 'compute': "for rec in self: rec['x_invoice_count'] = 10", }) + # Simulate the effect of https://github.com/odoo/odoo/commit/9afce4805fc8bac45fdba817488aa867fddff69b + # Updating a module create xml_id of the module even for manual field if it's the original module + # of the model + self.create_xml_id('base', name, field) + return field + + def create_server_action(self, name): + return self.env['ir.actions.server'].create({ + 'name': name, + 'code': """ +for rec in records: + rec['name'] = test + """, + 'state': 'code', + 'type': 'ir.actions.server', + 'model_id': self.env.ref('base.model_res_partner').id, + }) def test_ignore_auto_generated_computed_field(self): """ @@ -154,12 +171,12 @@ class TestClocCustomization(TransactionCase): Having an xml_id but no existing module is consider as not belonging to a module """ f1 = self.create_field('x_invoice_count') - self.create_xml_id('invoice_count', 'ir.model.fields', f1.id) + self.create_xml_id('studio_customization', 'invoice_count', f1) cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Studio auto generated count field should not be counted in cloc') f2 = self.create_field('x_studio_custom_field') - self.create_xml_id('studio_custom', 'ir.model.fields', f2.id) + self.create_xml_id('studio_customization', 'studio_custom', f2) cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 1, 'Count other studio computed field') @@ -168,11 +185,52 @@ class TestClocCustomization(TransactionCase): cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'Count fields without xml_id') f4 = self.create_field('x_custom_field_export') - self.create_xml_id('studio_custom', 'ir.model.fields', f4.id, '__export__') + self.create_xml_id('__export__', 'studio_custom', f4) cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 3, 'Count fields with xml_id but without module') + def test_several_xml_id(self): + sa = self.create_server_action("Test double xml_id") + self.create_xml_id("__export__", "first", sa) + self.create_xml_id("base", "second", sa) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'Count Should count SA with a non standard xml_id') + self.create_xml_id("__import__", "third", sa) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'SA with several xml_id should be counted only once') + + def test_cloc_exclude_xml_id(self): + sa = self.create_server_action("Test double xml_id") + self.create_xml_id("__cloc_exclude__", "sa_first", sa) + self.create_xml_id("__upgrade__", "sa_second", sa) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Should not count SA with cloc_exclude xml_id') + + f1 = self.create_field('x_invoice_count') + self.create_xml_id("__cloc_exclude__", "field_first", f1) + self.create_xml_id("__upgrade__", "field_second", f1) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Should not count Field with cloc_exclude xml_id') + + def test_field_no_xml_id(self): + self.env['ir.model.fields'].create({ + 'name': "x_no_xml_id", + 'field_description': "no_xml_id", + 'model': 'res.partner', + 'model_id': self.env.ref('base.model_res_partner').id, + 'ttype': 'integer', + 'store': False, + 'compute': "for rec in self: rec['x_invoice_count'] = 10", + }) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('odoo/studio', 0), 1, 'Should count field with no xml_id at all') + class TestClocParser(TransactionCase): diff --git a/odoo/tools/cloc.py b/odoo/tools/cloc.py index b93fb08bcd0..83f6cf73927 100644 --- a/odoo/tools/cloc.py +++ b/odoo/tools/cloc.py @@ -30,6 +30,7 @@ class Cloc(object): self.code = {} self.total = {} self.errors = {} + self.excluded = {} self.max_width = 70 #------------------------------------------------------ @@ -95,10 +96,13 @@ class Cloc(object): #------------------------------------------------------ # Enumeration #------------------------------------------------------ - def book(self, module, item='', count=(0, 0)): + def book(self, module, item='', count=(0, 0), exclude=False): if count[0] == -1: self.errors.setdefault(module, {}) self.errors[module][item] = count[1] + elif exclude and item: + self.excluded.setdefault(module, {}) + self.excluded[module][item] = count else: self.modules.setdefault(module, {}) if item: @@ -168,65 +172,93 @@ class Cloc(object): self.count_path(module_path) def count_customization(self, env): - imported_module = "" + imported_module_sa = "" if env['ir.module.module']._fields.get('imported'): - imported_module = "OR (m.imported = TRUE AND m.state = 'installed')" + imported_module_sa = "OR (m.imported = TRUE AND m.state = 'installed')" query = """ - SELECT s.id, m.name FROM ir_act_server AS s - LEFT JOIN ir_model_data AS d ON (d.res_id = s.id AND d.model = 'ir.actions.server') - LEFT JOIN ir_module_module AS m ON m.name = d.module - WHERE s.state = 'code' AND (m.name IS null {}) - """.format(imported_module) + SELECT s.id, min(m.name), array_agg(d.module) + FROM ir_act_server AS s + LEFT JOIN ir_model_data AS d + ON (d.res_id = s.id AND d.model = 'ir.actions.server') + LEFT JOIN ir_module_module AS m + ON m.name = d.module + WHERE s.state = 'code' AND (m.name IS null {}) + GROUP BY s.id + """.format(imported_module_sa) env.cr.execute(query) - data = {r[0]: r[1] for r in env.cr.fetchall()} + data = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()} for a in env['ir.actions.server'].browse(data.keys()): - self.book(data[a.id] or "odoo/studio", "ir.actions.server/%s: %s" % (a.id, a.name), self.parse_py(a.code)) + self.book( + data[a.id][0] or "odoo/studio", + "ir.actions.server/%s: %s" % (a.id, a.name), + self.parse_py(a.code), + '__cloc_exclude__' in data[a.id][1] + ) + imported_module_field = ("'odoo/studio'", "") + if env['ir.module.module']._fields.get('imported'): + imported_module_field = ("min(m.name)", "AND m.imported = TRUE AND m.state = 'installed'") + # We always want to count manual compute field unless they are generated by studio + # the module should be odoo/studio unless it comes from an imported module install + # because manual field get an external id from the original module of the model query = r""" - SELECT f.id, m.name FROM ir_model_fields AS f - LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields') - LEFT JOIN ir_module_module AS m ON m.name = d.module - WHERE f.compute IS NOT null AND ( - -- Do not count studio field - (m.name IS null AND (d.module != 'studio_customization' or d.module is NULL)) - -- Unless they start with x_studio - OR (d.module = 'studio_customization' AND f.name ilike 'x\_studio%') - {}) - """.format(imported_module) + SELECT f.id, f.name, {}, array_agg(d.module) + FROM ir_model_fields AS f + LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields') + LEFT JOIN ir_module_module AS m ON m.name = d.module {} + WHERE f.compute IS NOT null AND f.state = 'manual' + GROUP BY f.id, f.name + """.format(*imported_module_field) env.cr.execute(query) - data = {r[0]: r[1] for r in env.cr.fetchall()} + # Do not count field generated by studio + all_data = env.cr.fetchall() + data = {r[0]: (r[2], r[3]) for r in all_data if not ("studio_customization" in r[3] and not r[1].startswith('x_studio'))} for f in env['ir.model.fields'].browse(data.keys()): - self.book(data[f.id] or "odoo/studio", "ir.model.fields/%s: %s" % (f.id, f.name), self.parse_py(f.compute)) + self.book( + data[f.id][0] or "odoo/studio", + "ir.model.fields/%s: %s" % (f.id, f.name), + self.parse_py(f.compute), + '__cloc_exclude__' in data[f.id][1] + ) - if not imported_module: + if not env['ir.module.module']._fields.get('imported'): return # Count qweb view only from imported module and not studio query = """ - SELECT view.id, data.module + SELECT view.id, min(mod.name), array_agg(data.module) FROM ir_ui_view view INNER JOIN ir_model_data data ON view.id = data.res_id AND data.model = 'ir.ui.view' - INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True - WHERE view.type = 'qweb' AND mod.name != 'studio_customization' + LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True + WHERE view.type = 'qweb' AND data.module != 'studio_customization' + GROUP BY view.id + HAVING count(mod.name) > 0 """ env.cr.execute(query) - custom_views = dict(env.cr.fetchall()) + custom_views = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()} for view in env['ir.ui.view'].browse(custom_views.keys()): - modue_name = custom_views[view.id] - self.book(modue_name, "/%s/views/%s.xml" % (modue_name, view.name), self.parse_xml(view.arch_base)) + module_name = custom_views[view.id][0] + self.book( + module_name, + "/%s/views/%s.xml" % (module_name, view.name), + self.parse_xml(view.arch_base), + '__cloc_exclude__' in custom_views[view.id][1] + ) # Count js, xml, css/scss file from imported module query = r""" - SELECT attach.id, data.module + SELECT attach.id, min(mod.name), array_agg(data.module) FROM ir_attachment attach INNER JOIN ir_model_data data ON attach.id = data.res_id AND data.model = 'ir.attachment' - INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True + LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True WHERE attach.name ~ '.*\.(js|xml|css|scss)$' + GROUP BY attach.id + HAVING count(mod.name) > 0 """ env.cr.execute(query) - uploaded_file = dict(env.cr.fetchall()) + uploaded_file = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()} for attach in env['ir.attachment'].browse(uploaded_file.keys()): - module_name = uploaded_file[attach.id] + module_name = uploaded_file[attach.id][0] ext = os.path.splitext(attach.url)[1].lower() if ext not in VALID_EXTENSION: continue @@ -238,7 +270,12 @@ class Cloc(object): # Decode using latin1 to avoid error that may raise by decoding with utf8 # The chars not correctly decoded in latin1 have no impact on how many lines will be counted content = attach.raw.decode('latin1') - self.book(module_name, attach.url, self.parse(content, ext)) + self.book( + module_name, + attach.url, + self.parse(content, ext), + '__cloc_exclude__' in uploaded_file[attach.id][1], + ) def count_env(self, env): self.count_modules(env) @@ -254,6 +291,7 @@ class Cloc(object): #------------------------------------------------------ # Report #------------------------------------------------------ + # pylint: disable=W0141 def report(self, verbose=False, width=None): # Prepare format if not width: @@ -276,6 +314,16 @@ class Cloc(object): s += fmt.format(k='', lines=total, other=total - code, code=code) print(s) + if self.excluded and verbose: + ex = fmt.format(k="Excluded", lines="Line", other="Other", code="Code") + ex += hr + for m in sorted(self.excluded): + for i in sorted(self.excluded[m], key=lambda i: self.excluded[m][i][0], reverse=True): + code, total = self.excluded[m][i] + ex += fmt.format(k=' ' + i, lines=total, other=total - code, code=code) + ex += hr + print(ex) + if self.errors: e = "\nErrors\n\n" for m in sorted(self.errors):