From dc8f06624f684ebc78ef7e1d77e7583fdc4d83ec Mon Sep 17 00:00:00 2001 From: Thibault Francois Date: Fri, 4 Mar 2022 11:01:30 +0000 Subject: [PATCH] [ADD] cloc: count loc in style file and from imported module Context ------- Web design specific require a lot of work and need maintenance. So far the style files were not taken into account despite the amount of maintenance they require. With this commit they are going to be counted, knowing they still can be excluded in the manifest. Imported module allow to deploy frontend assets: stylesheet, javascript, xml template and Qweb view that require as well some maintenance. They are going to be counted Implementation -------------- - Add method to parse css and scss file - Include .scss and .css file in the count - Add external_id to attachment that store frontend asset of imported module - Find all attachment with .js, .css, .scss, .xml from imported module - Find qweb view from imported module - cound the content of the attachment and the qweb view closes odoo/odoo#86816 X-original-commit: d2e0b48271dbd8cd7bbdbc5c2c1649c55c51417b Signed-off-by: Christophe Simonis Signed-off-by: Thibault Francois --- addons/base_import_module/models/ir_module.py | 10 +- addons/base_import_module/tests/test_cloc.py | 82 +++++++++++++++- .../tests/test_import_module.py | 4 +- odoo/addons/base/tests/test_cloc.py | 63 ++++++++++++- odoo/tools/cloc.py | 93 ++++++++++++++++--- 5 files changed, 227 insertions(+), 25 deletions(-) diff --git a/addons/base_import_module/models/ir_module.py b/addons/base_import_module/models/ir_module.py index 3ab7aeb4d68..7bca006c1f9 100644 --- a/addons/base_import_module/models/ir_module.py +++ b/addons/base_import_module/models/ir_module.py @@ -104,7 +104,13 @@ class IrModule(models.Model): if attachment: attachment.write(values) else: - IrAttachment.create(values) + attachment = IrAttachment.create(values) + self.env['ir.model.data'].create({ + 'name': f"attachment_{url_path}".replace('.', '_'), + 'model': 'ir.attachment', + 'module': module, + 'res_id': attachment.id, + }) IrAsset = self.env['ir.asset'] assets_vals = [] @@ -139,7 +145,7 @@ class IrModule(models.Model): # Create new assets and attach 'ir.model.data' records to them created_assets = IrAsset.create(assets_to_create) self.env['ir.model.data'].create([{ - 'name': f"{asset['bundle']}.{asset['path']}", + 'name': f"{asset['bundle']}_{asset['path']}".replace(".", "_"), 'model': 'ir.asset', 'module': module, 'res_id': asset.id, diff --git a/addons/base_import_module/tests/test_cloc.py b/addons/base_import_module/tests/test_cloc.py index 05611a0cc6f..7b393e71573 100644 --- a/addons/base_import_module/tests/test_cloc.py +++ b/addons/base_import_module/tests/test_cloc.py @@ -1,9 +1,23 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +import json +from io import BytesIO +from zipfile import ZipFile from odoo.tools import cloc from odoo.addons.base.tests import test_cloc +VALID_XML = """ + + +
+ + +
+ + +""" + class TestClocFields(test_cloc.TestClocCustomization): def test_fields_from_import_module(self): @@ -17,12 +31,76 @@ class TestClocFields(test_cloc.TestClocCustomization): 'imported': True, }) f1 = self.create_field('x_imported_field') - self.create_xml_id('import_field', f1.id, 'imported_module') + self.create_xml_id('import_field', 'ir.model.fields', f1.id, 'imported_module') cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('imported_module', 0), 1, 'Count fields with xml_id of imported module') f2 = self.create_field('x_base_field') - self.create_xml_id('base_field', f2.id, 'base') + self.create_xml_id('base_field', 'ir.model.fields', f2.id, 'base') cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('base', 0), 0, "Don't count fields from standard module") + + def test_count_qweb_imported_module(self): + self.env['ir.module.module'].create({ + 'author': 'Odoo', + 'imported': True, + 'latest_version': '15.0.1.0.0', + 'name': 'test_imported_module', + 'state': 'installed', + 'summary': 'Test imported module for cloc', + }) + qweb_view = self.env['ir.ui.view'].create({ + "name": "Qweb Test", + "type": "qweb", + "mode": "primary", + "arch_base": "\n \n \n \n", + }) + self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id, 'test_imported_module') + + # Add qweb view from non imported module + qweb_view = self.env['ir.ui.view'].create({ + "name": "Qweb Test", + "type": "qweb", + "arch_base": "\n \n \n \n", + }) + self.create_xml_id("qweb_view_test", 'ir.ui.view', qweb_view.id) + + # Add form view from module + form_view = self.env['ir.ui.view'].create({ + "name": "Test partner", + "type": "form", + "model": "res.partner", + "arch_base": "
\n \n", + }) + self.create_xml_id("form_view_test", 'ir.ui.view', form_view.id, 'test_imported_module') + + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('test_imported_module', 0), 5, "Count only qweb view from imported module") + + def test_count_attachment_imported_module(self): + manifest_content = json.dumps({ + 'name': 'test_imported_module', + 'description': 'Test', + 'assets': { + 'web.assets_backend': [ + 'test_imported_module/static/src/js/test.js', + 'test_imported_module/static/src/css/test.scss', + ] + }, + 'license': 'LGPL-3', + }) + + stream = BytesIO() + with ZipFile(stream, 'w') as archive: + archive.writestr('test_imported_module/__manifest__.py', manifest_content) + archive.writestr('test_imported_module/static/src/js/test.js', test_cloc.JS_TEST) + archive.writestr('test_imported_module/static/src/js/test.scss', test_cloc.SCSS_TEST) + archive.writestr('test_imported_module/static/src/js/test.xml', VALID_XML) + + # Import test module + self.env['ir.module.module'].import_zipfile(stream) + cl = cloc.Cloc() + cl.count_customization(self.env) + self.assertEqual(cl.code.get('test_imported_module', 0), 35) diff --git a/addons/base_import_module/tests/test_import_module.py b/addons/base_import_module/tests/test_import_module.py index 2b9398999d5..af0a5271d4d 100644 --- a/addons/base_import_module/tests/test_import_module.py +++ b/addons/base_import_module/tests/test_import_module.py @@ -44,13 +44,13 @@ class TestImportModule(odoo.tests.TransactionCase): asset_data = self.env['ir.model.data'].search([('model', '=', 'ir.asset'), ('res_id', '=', asset.id)]) self.assertEqual(asset_data.module, 'test_module') - self.assertEqual(asset_data.name, f'{bundle}.{path}') + self.assertEqual(asset_data.name, f'{bundle}_{path}'.replace(".", "_")) # Uninstall test module self.env['ir.module.module'].search([('name', '=', 'test_module')]).module_uninstall() attachment = self.env['ir.attachment'].search([('url', '=', path)]) - self.assertEqual(len(attachment), 1) + self.assertEqual(len(attachment), 0) asset = self.env['ir.asset'].search([('name', '=', f'test_module.{bundle}.{path}')]) self.assertEqual(len(asset), 0) diff --git a/odoo/addons/base/tests/test_cloc.py b/odoo/addons/base/tests/test_cloc.py index 94965dc9e05..6d48eed8835 100644 --- a/odoo/addons/base/tests/test_cloc.py +++ b/odoo/addons/base/tests/test_cloc.py @@ -79,11 +79,60 @@ function() { } ''' +CSS_TEST = ''' +/* + Comment +*/ + +p { + text-align: center; + color: red; + text-overflow: ' /* '; +} + + +#content, #footer, #supplement { + position: absolute; + left: 510px; + width: 200px; + text-overflow: ' */ '; +} +''' + +SCSS_TEST = ''' +/* + Comment +*/ + +// Standalone list views +.o_content > .o_list_view > .table-responsive > .table { + // List views always have the table-sm class, maybe we should remove + // it (and consider it does not exist) and change the default table paddings + @include o-list-view-full-width-padding($base: $table-cell-padding-sm, $ratio: 2); + &:not(.o_list_table_grouped) { + @include media-breakpoint-up(xl) { + @include o-list-view-full-width-padding($base: $table-cell-padding-sm, $ratio: 2.5); + } + } + + .o_optional_columns_dropdown_toggle { + padding: 8px 10px; + } +} + +#content, #footer, #supplement { + text-overflow: '/*'; + left: 510px; + width: 200px; + text-overflow: '*/'; +} +''' + class TestClocCustomization(TransactionCase): - def create_xml_id(self, name, res_id, module='studio_customization'): + def create_xml_id(self, name, model, res_id, module='studio_customization'): self.env['ir.model.data'].create({ 'name': name, - 'model': 'ir.model.fields', + 'model': model, 'res_id': res_id, 'module': module, }) @@ -105,12 +154,12 @@ class TestClocCustomization(TransactionCase): Having an xml_id but no existing module is consider as not belonging to a module """ f1 = self.create_field('x_invoice_count') - self.create_xml_id('invoice_count', f1.id) + self.create_xml_id('invoice_count', 'ir.model.fields', f1.id) cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 0, 'Studio auto generated count field should not be counted in cloc') f2 = self.create_field('x_studio_custom_field') - self.create_xml_id('studio_custom', f2.id) + self.create_xml_id('studio_custom', 'ir.model.fields', f2.id) cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 1, 'Count other studio computed field') @@ -119,7 +168,7 @@ class TestClocCustomization(TransactionCase): cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 2, 'Count fields without xml_id') f4 = self.create_field('x_custom_field_export') - self.create_xml_id('studio_custom', f4.id, '__export__') + self.create_xml_id('studio_custom', 'ir.model.fields', f4.id, '__export__') cl = cloc.Cloc() cl.count_customization(self.env) self.assertEqual(cl.code.get('odoo/studio', 0), 3, 'Count fields with xml_id but without module') @@ -143,6 +192,10 @@ class TestClocParser(TransactionCase): self.assertEqual(py_count, (8, 16)) js_count = cl.parse_js(JS_TEST) self.assertEqual(js_count, (10, 17)) + css_count = cl.parse_css(CSS_TEST) + self.assertEqual(css_count, (11, 17)) + scss_count = cl.parse_scss(SCSS_TEST) + self.assertEqual(scss_count, (17, 26)) @tagged('post_install', '-at_install') diff --git a/odoo/tools/cloc.py b/odoo/tools/cloc.py index 0b40a43be8f..34836b66648 100644 --- a/odoo/tools/cloc.py +++ b/odoo/tools/cloc.py @@ -22,6 +22,7 @@ DEFAULT_EXCLUDE = [ STANDARD_MODULES = ['web', 'web_enterprise', 'theme_common', 'base'] MAX_FILE_SIZE = 25 * 2**20 # 25 MB +VALID_EXTENSION = ['.py', '.js', '.xml', '.css', '.scss'] class Cloc(object): def __init__(self): @@ -56,18 +57,41 @@ class Cloc(object): except Exception: return (-1, "Syntax Error") - def parse_js(self, s): + def parse_c_like(self, s, regex): # Based on https://stackoverflow.com/questions/241327 s = s.strip() + "\n" total = s.count("\n") + def replacer(match): s = match.group(0) return " " if s.startswith('/') else s - comments_re = re.compile(r'//.*?$|(? MAX_FILE_SIZE: - self.book(module_name, file_path, (-1, "Max file size exceeded")) - continue + if ext not in VALID_EXTENSION: + continue - with open(file_path, 'rb') as f: - content = f.read().decode('latin1') - if ext == '.py': - self.book(module_name, file_path, self.parse_py(content)) - elif ext == '.js': - self.book(module_name, file_path, self.parse_js(content)) - elif ext == '.xml': - self.book(module_name, file_path, self.parse_xml(content)) + if os.path.getsize(file_path) > MAX_FILE_SIZE: + self.book(module_name, file_path, (-1, "Max file size exceeded")) + continue + + with open(file_path, 'rb') as f: + # Decode using latin1 to avoid error that may raise by decoding with utf8 + # The chars not correctly decoded in latin1 have no impact on how many lines will be counted + content = f.read().decode('latin1') + self.book(module_name, file_path, self.parse(content, ext)) def count_modules(self, env): # Exclude standard addons paths @@ -175,6 +198,48 @@ class Cloc(object): for f in env['ir.model.fields'].browse(data.keys()): self.book(data[f.id] or "odoo/studio", "ir.model.fields/%s: %s" % (f.id, f.name), self.parse_py(f.compute)) + if not imported_module: + return + + # Count qweb view only from imported module + query = """ + SELECT view.id, data.module + FROM ir_ui_view view + INNER JOIN ir_model_data data ON view.id = data.res_id AND data.model = 'ir.ui.view' + INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True + WHERE view.type = 'qweb' + """ + env.cr.execute(query) + custom_views = dict(env.cr.fetchall()) + for view in env['ir.ui.view'].browse(custom_views.keys()): + modue_name = custom_views[view.id] + self.book(modue_name, "/%s/views/%s.xml" % (modue_name, view.name), self.parse_xml(view.arch_base)) + + # Count js, xml, css/scss file from imported module + query = r""" + SELECT attach.id, data.module + FROM ir_attachment attach + INNER JOIN ir_model_data data ON attach.id = data.res_id AND data.model = 'ir.attachment' + INNER JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True + WHERE attach.name ~ '.*\.(js|xml|css|scss)$' + """ + env.cr.execute(query) + uploaded_file = dict(env.cr.fetchall()) + for attach in env['ir.attachment'].browse(uploaded_file.keys()): + module_name = uploaded_file[attach.id] + ext = os.path.splitext(attach.url)[1].lower() + if ext not in VALID_EXTENSION: + continue + + if len(attach.datas) > MAX_FILE_SIZE: + self.book(module_name, attach.url, (-1, "Max file size exceeded")) + continue + + # Decode using latin1 to avoid error that may raise by decoding with utf8 + # The chars not correctly decoded in latin1 have no impact on how many lines will be counted + content = attach.raw.decode('latin1') + self.book(module_name, attach.url, self.parse(content, ext)) + def count_env(self, env): self.count_modules(env) self.count_customization(env)