From 1c522ee778b520a215963922f5b17d2790c31907 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Mon, 25 May 2020 03:23:28 +0000 Subject: [PATCH] [ADD] odoo-bin cloc cli subcommand. Odoo cloc is a tool to count the number of relevant lines written in Python, Javascript or XML. This can be used as rough metric for pricing maintenance of customizations. It has two modes of operation, either by providing a path: odoo-bin cloc -p module_path Or by providing the name of a database: odoo-bin cloc --addons-path=dirs -d database In the latter mode, only the custom code is accounted for. Both modes can be used simultaneously. Files that cannot be parsed are shown at the end of the report. Parsing can fail due to syntax errors or excessive file size. closes odoo/odoo#52635 X-original-commit: ae858c3ac66267b4726db459032b91a6be1cc1d6 Related: odoo/enterprise#11018 Signed-off-by: Olivier Dony (odo) Co-authored-by: Thibault Francois Co-authored-by: Antoine Vandevenne (anv) --- doc/reference/cmdline.rst | 156 ++++++++++++++++++++ odoo/addons/base/tests/__init__.py | 1 + odoo/addons/base/tests/test_cloc.py | 90 ++++++++++++ odoo/cli/__init__.py | 1 + odoo/cli/cloc.py | 48 +++++++ odoo/tools/__init__.py | 1 + odoo/tools/cloc.py | 214 ++++++++++++++++++++++++++++ 7 files changed, 511 insertions(+) create mode 100644 odoo/addons/base/tests/test_cloc.py create mode 100644 odoo/cli/cloc.py create mode 100644 odoo/tools/cloc.py diff --git a/doc/reference/cmdline.rst b/doc/reference/cmdline.rst index 71b52bce8bb..9e90d9616d6 100644 --- a/doc/reference/cmdline.rst +++ b/doc/reference/cmdline.rst @@ -586,3 +586,159 @@ generation of the model's records to test your modules in databases containing n .. seealso:: :ref:`reference/testing/populate` + + +Cloc +==== + +.. program:: odoo-bin cloc + +Odoo Cloc is a tool to count the number of relevant lines written in +Python, Javascript or XML. This can be used as a rough metric for pricing +maintenance of extra modules. + +Command-line options +-------------------- +.. option:: -d , --database + +| Process the code of all extra modules installed on the provided database, + and of all server actions and computed fields manually created in the provided + database. +| The :option:`--addons-path` option is required to specify the path(s) to the + module folder(s). +| If combined with :option:`--path`, the count will be that of the sum of both + options' results (with possible overlaps). At least one of these two options is + required to specify which code to process. + +.. code-block:: console + + $ odoo-bin cloc --addons-path=addons -d my_database + +.. seealso:: + - :ref:`reference/cmdline/cloc/database-option` + + +.. option:: -p , --path + +| Process the files in the provided path. +| If combined with :option:`--database`, the count will be that of the sum of both + options' results (with possible overlaps). At least one of these two options is + required to specify which code to process. + +.. code-block:: console + + $ odoo-bin cloc -p addons/account + + +Multiple paths can be provided by repeating the option. + +.. code-block:: console + + $ odoo-bin cloc -p addons/account -p addons/sale + +.. seealso:: + - :ref:`reference/cmdline/cloc/path-option` + + +.. option:: --addons-path + +| Comma-separated list of directories in which modules are stored. These directories + are scanned for modules. +| Required if the :option:`--database` option is used. + + +.. option:: -c + +Specify a configuration file to use in place of the :option:`--addons-path` option. + +.. code-block:: console + + $ odoo-bin cloc -c config.conf -d my_database + + +.. option:: -v, --verbose + +Show the details of lines counted for each file. + + +Processed files +--------------- + +.. _reference/cmdline/cloc/database-option: + +With the :option:`--database` option +'''''''''''''''''''''''''''''''''''' + +Odoo Cloc counts the lines in each file of extra installed modules in a +given database. In addition, it counts the Python lines of server actions and +custom computed fields that have been directly created in the database or +imported. + +Some files are excluded from the count by default: + +- The manifest (:file:`__manifest__.py` or :file:`__openerp__.py`) +- The contents of the folder :file:`static/lib` +- The tests defined in the folder :file:`tests` and :file:`static/tests` +- The XML files declared in the ``demo`` or ``demo_xml`` sections of the manifest + +For special cases, a list of files that should be ignored by Odoo Cloc can be defined +per module. This is specified by the ``cloc_exclude`` entry of the manifest: + +.. code-block:: python + + "cloc_exclude": [ + "lib/common.py", # exclude a single file + "data/*.xml", # exclude all XML files in a specific folder + "example/**/*", # exclude all files in a folder hierarchy recursively + ] + +| The pattern ``**/*`` can be used to ignore an entire module. This can be useful + to exclude a module from maintenance service costs. +| For more information about the pattern syntax, see `glob + `_. + + + +.. _reference/cmdline/cloc/path-option: + +With the :option:`--path` option +'''''''''''''''''''''''''''''''' + +This method works the same as with the :ref:`--database option +` if a manifest file is present in the given +folder. Otherwise, it counts all files. + + +Identifying Extra Modules +------------------------- + +To distinguish between standard and extra modules, Odoo Cloc uses the following heuristic: +modules that are located (real file system path, after following symbolic links) +in the same parent directory as the ``base``, ``web`` or ``web_enterprise`` +standard modules are considered standard. Other modules are treated as extra modules. + + +Error Handling +-------------- + +Some file cannot be counted by Odoo Cloc. +Those file are reported at the end of the output. + +Max file size exceeded +'''''''''''''''''''''' + +Odoo Cloc rejects any file larger than 25MB. Usually, source files are smaller +than 1 MB. If a file is rejected, it may be: + +- A generated XML file that contains lots of data. It should be excluded in the manifest. +- A JavaScript library that should be placed in the :file:`static/lib` folder. + +Syntax Error +'''''''''''' + +Odoo Cloc cannot count the lines of code of a Python file with a syntax problem. +If an extra module contains such files, they should be fixed to allow the module to +load. If the module works despite the presence of those files, they are probably +not loaded and should therefore be removed from the module, or at least excluded +in the manifest via ``cloc_exclude``. + diff --git a/odoo/addons/base/tests/__init__.py b/odoo/addons/base/tests/__init__.py index e74868e89c7..b08e17ca0ae 100644 --- a/odoo/addons/base/tests/__init__.py +++ b/odoo/addons/base/tests/__init__.py @@ -44,3 +44,4 @@ from . import test_reports from . import test_tests_tags from . import test_base_document_layout from . import test_form_create +from . import test_cloc diff --git a/odoo/addons/base/tests/test_cloc.py b/odoo/addons/base/tests/test_cloc.py new file mode 100644 index 00000000000..4aaeb2be537 --- /dev/null +++ b/odoo/addons/base/tests/test_cloc.py @@ -0,0 +1,90 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. +from odoo.tools import cloc +from odoo.tests.common import TransactionCase + +XML_TEST = """ + + + Line + + Line + + + + cdata Line + yes6]]> + ]]> + ]]> + + + + name + + + not a comment but found as is + + After closed comment back to normal + +""" + +PY_TEST_NO_RETURN = '''line = 1 +line = 2''' + +PY_TEST = ''' +# comment 1 + +def func(): # eol comment 3 + """ docstring + """ + pass + +def query(): + long_query = """ + SELECT * + FROM table + WHERE id = 1; + """ + return query + +print(i.lineno, i, getattr(i,'s',None), getattr(i,'value',None)) +''' + +JS_TEST = ''' +/* +comment +*/ + +function() { + return 1+2; // comment +} + +function() { + hello = 4; /* + comment + */ + console.log(hello); + regex = /\/*h/; + legit_code_counted = 1; + regex2 = /.*/; +} +''' + +class TestCloc(TransactionCase): + def test_parser(self): + cl = cloc.Cloc() + xml_count = cl.parse_xml(XML_TEST) + self.assertEqual(xml_count, (18, 31)) + py_count = cl.parse_py(PY_TEST_NO_RETURN) + self.assertEqual(py_count, (2, 2)) + py_count = cl.parse_py(PY_TEST) + self.assertEqual(py_count, (8, 16)) + js_count = cl.parse_js(JS_TEST) + self.assertEqual(js_count, (10, 17)) diff --git a/odoo/cli/__init__.py b/odoo/cli/__init__.py index 78a080eca1c..3734b533354 100644 --- a/odoo/cli/__init__.py +++ b/odoo/cli/__init__.py @@ -6,6 +6,7 @@ import odoo from .command import Command, main +from . import cloc from . import deploy from . import scaffold from . import server diff --git a/odoo/cli/cloc.py b/odoo/cli/cloc.py new file mode 100644 index 00000000000..30b2dde8305 --- /dev/null +++ b/odoo/cli/cloc.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. +import argparse +import os +import sys +import textwrap + +from odoo.tools import cloc, config +from . import Command + +class Cloc(Command): + """\ + Odoo cloc is a tool to count the number of relevant lines written in + Python, Javascript or XML. This can be used as rough metric for pricing + maintenance of customizations. + + It has two modes of operation, either by providing a path: + + odoo-bin cloc -p module_path + + Or by providing the name of a database: + + odoo-bin cloc --addons-path=dirs -d database + + In the latter mode, only the custom code is accounted for. + """ + def run(self, args): + parser = argparse.ArgumentParser( + prog="%s cloc" % sys.argv[0].split(os.path.sep)[-1], + description=textwrap.dedent(self.__doc__), + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('--database', '-d', dest="database", help="Database name") + parser.add_argument('--path', '-p', action='append', help="File or directory path") + parser.add_argument('--verbose', '-v', action='count', default=0) + opt, unknown = parser.parse_known_args(args) + if not opt.database and not opt.path: + parser.print_help() + sys.exit() + + c = cloc.Cloc() + if opt.database: + config.parse_config(['-d', opt.database] + unknown) + c.count_database(opt.database) + if opt.path: + for i in opt.path: + c.count_path(i) + c.report(opt.verbose) diff --git a/odoo/tools/__init__.py b/odoo/tools/__init__.py index b3619c88213..1e23e499d67 100644 --- a/odoo/tools/__init__.py +++ b/odoo/tools/__init__.py @@ -6,6 +6,7 @@ from . import win32 from . import appdirs from . import osutil from . import pdf +from . import cloc from .config import config from .misc import * from .translate import * diff --git a/odoo/tools/cloc.py b/odoo/tools/cloc.py new file mode 100644 index 00000000000..3f753ea678f --- /dev/null +++ b/odoo/tools/cloc.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. +import ast +import pathlib +import os +import re +import shutil + +import odoo +from odoo.tools.config import config + +VERSION = 1 +DEFAULT_EXCLUDE = [ + "__manifest__.py", + "__openerp__.py", + "tests/**/*", + "static/lib/**/*", + "static/tests/**/*" +] + +STANDARD_MODULES = ['web', 'web_enterprise', 'website_animate', 'base'] +MAX_FILE_SIZE = 25 * 2**20 # 25 MB + +class Cloc(object): + def __init__(self): + self.modules = {} + self.code = {} + self.total = {} + self.errors = {} + self.max_width = 70 + + #------------------------------------------------------ + # Parse + #------------------------------------------------------ + def parse_xml(self, s): + s = s.strip() + "\n" + # Unbalanced xml comments inside a CDATA are not supported, and xml + # comments inside a CDATA will (wrongly) be considered as comment + total = s.count("\n") + s = re.sub("()", "", s, flags=re.DOTALL) + s = re.sub(r"\s*\n\s*", r"\n", s).lstrip() + return s.count("\n"), total + + def parse_py(self, s): + try: + s = s.strip() + "\n" + total = s.count("\n") + lines = set() + for i in ast.walk(ast.parse(s)): + # we only count 1 for a long string or a docstring + if hasattr(i, 'lineno'): + lines.add(i.lineno) + return len(lines), total + except Exception: + return (-1, "Syntax Error") + + def parse_js(self, s): + # Based on https://stackoverflow.com/questions/241327 + s = s.strip() + "\n" + total = s.count("\n") + def replacer(match): + s = match.group(0) + return " " if s.startswith('/') else s + comments_re = re.compile(r'//.*?$|(? MAX_FILE_SIZE: + self.book(module_name, file_path, (-1, "Max file size exceeded")) + continue + + with open(file_path, 'rb') as f: + content = f.read().decode('latin1') + if ext == '.py': + self.book(module_name, file_path, self.parse_py(content)) + elif ext == '.js': + self.book(module_name, file_path, self.parse_js(content)) + elif ext == '.xml': + self.book(module_name, file_path, self.parse_xml(content)) + + def count_modules(self, env): + # Exclude standard addons paths + exclude_heuristic = [odoo.modules.get_module_path(m, display_warning=False) for m in STANDARD_MODULES] + exclude_path = set([os.path.dirname(os.path.realpath(m)) for m in exclude_heuristic if m]) + + domain = [('state', '=', 'installed')] + # if base_import_module is present + if env['ir.module.module']._fields.get('imported'): + domain.append(('imported', '=', False)) + module_list = env['ir.module.module'].search(domain).mapped('name') + + for module_name in module_list: + module_path = os.path.realpath(odoo.modules.get_module_path(module_name)) + if module_path: + if any(module_path.startswith(i) for i in exclude_path): + continue + self.count_path(module_path) + + def count_customization(self, env): + imported_module = "" + if env['ir.module.module']._fields.get('imported'): + imported_module = "OR (m.imported = TRUE AND m.state = 'installed')" + query = """ + SELECT s.id, m.name FROM ir_act_server AS s + LEFT JOIN ir_model_data AS d ON (d.res_id = s.id AND d.model = 'ir.actions.server') + LEFT JOIN ir_module_module AS m ON m.name = d.module + WHERE s.state = 'code' AND (m.name IS null {}) + """.format(imported_module) + env.cr.execute(query) + data = {r[0]: r[1] for r in env.cr.fetchall()} + for a in env['ir.actions.server'].browse(data.keys()): + self.book(data[a.id] or "odoo/studio", "ir.actions.server/%s: %s" % (a.id, a.name), self.parse_py(a.code)) + + query = """ + SELECT f.id, m.name FROM ir_model_fields AS f + LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields') + LEFT JOIN ir_module_module AS m ON m.name = d.module + WHERE f.compute IS NOT null AND (m.name IS null {}) + """.format(imported_module) + env.cr.execute(query) + data = {r[0]: r[1] for r in env.cr.fetchall()} + for f in env['ir.model.fields'].browse(data.keys()): + self.book(data[f.id] or "odoo/studio", "ir.model.fields/%s: %s" % (f.id, f.name), self.parse_py(f.compute)) + + def count_env(self, env): + self.count_modules(env) + self.count_customization(env) + + def count_database(self, database): + with odoo.api.Environment.manage(): + registry = odoo.registry(config['db_name']) + with registry.cursor() as cr: + uid = odoo.SUPERUSER_ID + env = odoo.api.Environment(cr, uid, {}) + self.count_env(env) + + #------------------------------------------------------ + # Report + #------------------------------------------------------ + def report(self, verbose=False, width=None): + # Prepare format + if not width: + width = min(self.max_width, shutil.get_terminal_size()[0] - 24) + hr = "-" * (width + 24) + "\n" + fmt = '{k:%d}{lines:>8}{other:>8}{code:>8}\n' % (width,) + + # Render + s = fmt.format(k="Odoo cloc", lines="Line", other="Other", code="Code") + s += hr + for m in sorted(self.modules): + s += fmt.format(k=m, lines=self.total[m], other=self.total[m]-self.code[m], code=self.code[m]) + if verbose: + for i in sorted(self.modules[m], key=lambda i: self.modules[m][i][0], reverse=True): + code, total = self.modules[m][i] + s += fmt.format(k=' ' + i, lines=total, other=total - code, code=code) + s += hr + total = sum(self.total.values()) + code = sum(self.code.values()) + s += fmt.format(k='', lines=total, other=total - code, code=code) + print(s) + + if self.errors: + e = "\nErrors\n\n" + for m in sorted(self.errors): + e += "{}\n".format(m) + for i in sorted(self.errors[m]): + e += fmt.format(k=' ' + i, lines=self.errors[m][i], other='', code='') + print(e)