diff --git a/doc/reference/cmdline.rst b/doc/reference/cmdline.rst index 71b52bce8bb..9e90d9616d6 100644 --- a/doc/reference/cmdline.rst +++ b/doc/reference/cmdline.rst @@ -586,3 +586,159 @@ generation of the model's records to test your modules in databases containing n .. seealso:: :ref:`reference/testing/populate` + + +Cloc +==== + +.. program:: odoo-bin cloc + +Odoo Cloc is a tool to count the number of relevant lines written in +Python, Javascript or XML. This can be used as a rough metric for pricing +maintenance of extra modules. + +Command-line options +-------------------- +.. option:: -d , --database + +| Process the code of all extra modules installed on the provided database, + and of all server actions and computed fields manually created in the provided + database. +| The :option:`--addons-path` option is required to specify the path(s) to the + module folder(s). +| If combined with :option:`--path`, the count will be that of the sum of both + options' results (with possible overlaps). At least one of these two options is + required to specify which code to process. + +.. code-block:: console + + $ odoo-bin cloc --addons-path=addons -d my_database + +.. seealso:: + - :ref:`reference/cmdline/cloc/database-option` + + +.. option:: -p , --path + +| Process the files in the provided path. +| If combined with :option:`--database`, the count will be that of the sum of both + options' results (with possible overlaps). At least one of these two options is + required to specify which code to process. + +.. code-block:: console + + $ odoo-bin cloc -p addons/account + + +Multiple paths can be provided by repeating the option. + +.. code-block:: console + + $ odoo-bin cloc -p addons/account -p addons/sale + +.. seealso:: + - :ref:`reference/cmdline/cloc/path-option` + + +.. option:: --addons-path + +| Comma-separated list of directories in which modules are stored. These directories + are scanned for modules. +| Required if the :option:`--database` option is used. + + +.. option:: -c + +Specify a configuration file to use in place of the :option:`--addons-path` option. + +.. code-block:: console + + $ odoo-bin cloc -c config.conf -d my_database + + +.. option:: -v, --verbose + +Show the details of lines counted for each file. + + +Processed files +--------------- + +.. _reference/cmdline/cloc/database-option: + +With the :option:`--database` option +'''''''''''''''''''''''''''''''''''' + +Odoo Cloc counts the lines in each file of extra installed modules in a +given database. In addition, it counts the Python lines of server actions and +custom computed fields that have been directly created in the database or +imported. + +Some files are excluded from the count by default: + +- The manifest (:file:`__manifest__.py` or :file:`__openerp__.py`) +- The contents of the folder :file:`static/lib` +- The tests defined in the folder :file:`tests` and :file:`static/tests` +- The XML files declared in the ``demo`` or ``demo_xml`` sections of the manifest + +For special cases, a list of files that should be ignored by Odoo Cloc can be defined +per module. This is specified by the ``cloc_exclude`` entry of the manifest: + +.. code-block:: python + + "cloc_exclude": [ + "lib/common.py", # exclude a single file + "data/*.xml", # exclude all XML files in a specific folder + "example/**/*", # exclude all files in a folder hierarchy recursively + ] + +| The pattern ``**/*`` can be used to ignore an entire module. This can be useful + to exclude a module from maintenance service costs. +| For more information about the pattern syntax, see `glob + `_. + + + +.. _reference/cmdline/cloc/path-option: + +With the :option:`--path` option +'''''''''''''''''''''''''''''''' + +This method works the same as with the :ref:`--database option +` if a manifest file is present in the given +folder. Otherwise, it counts all files. + + +Identifying Extra Modules +------------------------- + +To distinguish between standard and extra modules, Odoo Cloc uses the following heuristic: +modules that are located (real file system path, after following symbolic links) +in the same parent directory as the ``base``, ``web`` or ``web_enterprise`` +standard modules are considered standard. Other modules are treated as extra modules. + + +Error Handling +-------------- + +Some file cannot be counted by Odoo Cloc. +Those file are reported at the end of the output. + +Max file size exceeded +'''''''''''''''''''''' + +Odoo Cloc rejects any file larger than 25MB. Usually, source files are smaller +than 1 MB. If a file is rejected, it may be: + +- A generated XML file that contains lots of data. It should be excluded in the manifest. +- A JavaScript library that should be placed in the :file:`static/lib` folder. + +Syntax Error +'''''''''''' + +Odoo Cloc cannot count the lines of code of a Python file with a syntax problem. +If an extra module contains such files, they should be fixed to allow the module to +load. If the module works despite the presence of those files, they are probably +not loaded and should therefore be removed from the module, or at least excluded +in the manifest via ``cloc_exclude``. + diff --git a/odoo/addons/base/tests/__init__.py b/odoo/addons/base/tests/__init__.py index e74868e89c7..b08e17ca0ae 100644 --- a/odoo/addons/base/tests/__init__.py +++ b/odoo/addons/base/tests/__init__.py @@ -44,3 +44,4 @@ from . import test_reports from . import test_tests_tags from . import test_base_document_layout from . import test_form_create +from . import test_cloc diff --git a/odoo/addons/base/tests/test_cloc.py b/odoo/addons/base/tests/test_cloc.py new file mode 100644 index 00000000000..4aaeb2be537 --- /dev/null +++ b/odoo/addons/base/tests/test_cloc.py @@ -0,0 +1,90 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. +from odoo.tools import cloc +from odoo.tests.common import TransactionCase + +XML_TEST = """ + + + Line + + Line + + + + cdata Line + yes6]]> + ]]> + ]]> + + + + name + + + not a comment but found as is + + After closed comment back to normal + +""" + +PY_TEST_NO_RETURN = '''line = 1 +line = 2''' + +PY_TEST = ''' +# comment 1 + +def func(): # eol comment 3 + """ docstring + """ + pass + +def query(): + long_query = """ + SELECT * + FROM table + WHERE id = 1; + """ + return query + +print(i.lineno, i, getattr(i,'s',None), getattr(i,'value',None)) +''' + +JS_TEST = ''' +/* +comment +*/ + +function() { + return 1+2; // comment +} + +function() { + hello = 4; /* + comment + */ + console.log(hello); + regex = /\/*h/; + legit_code_counted = 1; + regex2 = /.*/; +} +''' + +class TestCloc(TransactionCase): + def test_parser(self): + cl = cloc.Cloc() + xml_count = cl.parse_xml(XML_TEST) + self.assertEqual(xml_count, (18, 31)) + py_count = cl.parse_py(PY_TEST_NO_RETURN) + self.assertEqual(py_count, (2, 2)) + py_count = cl.parse_py(PY_TEST) + self.assertEqual(py_count, (8, 16)) + js_count = cl.parse_js(JS_TEST) + self.assertEqual(js_count, (10, 17)) diff --git a/odoo/cli/__init__.py b/odoo/cli/__init__.py index 78a080eca1c..3734b533354 100644 --- a/odoo/cli/__init__.py +++ b/odoo/cli/__init__.py @@ -6,6 +6,7 @@ import odoo from .command import Command, main +from . import cloc from . import deploy from . import scaffold from . import server diff --git a/odoo/cli/cloc.py b/odoo/cli/cloc.py new file mode 100644 index 00000000000..30b2dde8305 --- /dev/null +++ b/odoo/cli/cloc.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. +import argparse +import os +import sys +import textwrap + +from odoo.tools import cloc, config +from . import Command + +class Cloc(Command): + """\ + Odoo cloc is a tool to count the number of relevant lines written in + Python, Javascript or XML. This can be used as rough metric for pricing + maintenance of customizations. + + It has two modes of operation, either by providing a path: + + odoo-bin cloc -p module_path + + Or by providing the name of a database: + + odoo-bin cloc --addons-path=dirs -d database + + In the latter mode, only the custom code is accounted for. + """ + def run(self, args): + parser = argparse.ArgumentParser( + prog="%s cloc" % sys.argv[0].split(os.path.sep)[-1], + description=textwrap.dedent(self.__doc__), + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument('--database', '-d', dest="database", help="Database name") + parser.add_argument('--path', '-p', action='append', help="File or directory path") + parser.add_argument('--verbose', '-v', action='count', default=0) + opt, unknown = parser.parse_known_args(args) + if not opt.database and not opt.path: + parser.print_help() + sys.exit() + + c = cloc.Cloc() + if opt.database: + config.parse_config(['-d', opt.database] + unknown) + c.count_database(opt.database) + if opt.path: + for i in opt.path: + c.count_path(i) + c.report(opt.verbose) diff --git a/odoo/tools/__init__.py b/odoo/tools/__init__.py index b3619c88213..1e23e499d67 100644 --- a/odoo/tools/__init__.py +++ b/odoo/tools/__init__.py @@ -6,6 +6,7 @@ from . import win32 from . import appdirs from . import osutil from . import pdf +from . import cloc from .config import config from .misc import * from .translate import * diff --git a/odoo/tools/cloc.py b/odoo/tools/cloc.py new file mode 100644 index 00000000000..3f753ea678f --- /dev/null +++ b/odoo/tools/cloc.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. +import ast +import pathlib +import os +import re +import shutil + +import odoo +from odoo.tools.config import config + +VERSION = 1 +DEFAULT_EXCLUDE = [ + "__manifest__.py", + "__openerp__.py", + "tests/**/*", + "static/lib/**/*", + "static/tests/**/*" +] + +STANDARD_MODULES = ['web', 'web_enterprise', 'website_animate', 'base'] +MAX_FILE_SIZE = 25 * 2**20 # 25 MB + +class Cloc(object): + def __init__(self): + self.modules = {} + self.code = {} + self.total = {} + self.errors = {} + self.max_width = 70 + + #------------------------------------------------------ + # Parse + #------------------------------------------------------ + def parse_xml(self, s): + s = s.strip() + "\n" + # Unbalanced xml comments inside a CDATA are not supported, and xml + # comments inside a CDATA will (wrongly) be considered as comment + total = s.count("\n") + s = re.sub("()", "", s, flags=re.DOTALL) + s = re.sub(r"\s*\n\s*", r"\n", s).lstrip() + return s.count("\n"), total + + def parse_py(self, s): + try: + s = s.strip() + "\n" + total = s.count("\n") + lines = set() + for i in ast.walk(ast.parse(s)): + # we only count 1 for a long string or a docstring + if hasattr(i, 'lineno'): + lines.add(i.lineno) + return len(lines), total + except Exception: + return (-1, "Syntax Error") + + def parse_js(self, s): + # Based on https://stackoverflow.com/questions/241327 + s = s.strip() + "\n" + total = s.count("\n") + def replacer(match): + s = match.group(0) + return " " if s.startswith('/') else s + comments_re = re.compile(r'//.*?$|(? MAX_FILE_SIZE: + self.book(module_name, file_path, (-1, "Max file size exceeded")) + continue + + with open(file_path, 'rb') as f: + content = f.read().decode('latin1') + if ext == '.py': + self.book(module_name, file_path, self.parse_py(content)) + elif ext == '.js': + self.book(module_name, file_path, self.parse_js(content)) + elif ext == '.xml': + self.book(module_name, file_path, self.parse_xml(content)) + + def count_modules(self, env): + # Exclude standard addons paths + exclude_heuristic = [odoo.modules.get_module_path(m, display_warning=False) for m in STANDARD_MODULES] + exclude_path = set([os.path.dirname(os.path.realpath(m)) for m in exclude_heuristic if m]) + + domain = [('state', '=', 'installed')] + # if base_import_module is present + if env['ir.module.module']._fields.get('imported'): + domain.append(('imported', '=', False)) + module_list = env['ir.module.module'].search(domain).mapped('name') + + for module_name in module_list: + module_path = os.path.realpath(odoo.modules.get_module_path(module_name)) + if module_path: + if any(module_path.startswith(i) for i in exclude_path): + continue + self.count_path(module_path) + + def count_customization(self, env): + imported_module = "" + if env['ir.module.module']._fields.get('imported'): + imported_module = "OR (m.imported = TRUE AND m.state = 'installed')" + query = """ + SELECT s.id, m.name FROM ir_act_server AS s + LEFT JOIN ir_model_data AS d ON (d.res_id = s.id AND d.model = 'ir.actions.server') + LEFT JOIN ir_module_module AS m ON m.name = d.module + WHERE s.state = 'code' AND (m.name IS null {}) + """.format(imported_module) + env.cr.execute(query) + data = {r[0]: r[1] for r in env.cr.fetchall()} + for a in env['ir.actions.server'].browse(data.keys()): + self.book(data[a.id] or "odoo/studio", "ir.actions.server/%s: %s" % (a.id, a.name), self.parse_py(a.code)) + + query = """ + SELECT f.id, m.name FROM ir_model_fields AS f + LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields') + LEFT JOIN ir_module_module AS m ON m.name = d.module + WHERE f.compute IS NOT null AND (m.name IS null {}) + """.format(imported_module) + env.cr.execute(query) + data = {r[0]: r[1] for r in env.cr.fetchall()} + for f in env['ir.model.fields'].browse(data.keys()): + self.book(data[f.id] or "odoo/studio", "ir.model.fields/%s: %s" % (f.id, f.name), self.parse_py(f.compute)) + + def count_env(self, env): + self.count_modules(env) + self.count_customization(env) + + def count_database(self, database): + with odoo.api.Environment.manage(): + registry = odoo.registry(config['db_name']) + with registry.cursor() as cr: + uid = odoo.SUPERUSER_ID + env = odoo.api.Environment(cr, uid, {}) + self.count_env(env) + + #------------------------------------------------------ + # Report + #------------------------------------------------------ + def report(self, verbose=False, width=None): + # Prepare format + if not width: + width = min(self.max_width, shutil.get_terminal_size()[0] - 24) + hr = "-" * (width + 24) + "\n" + fmt = '{k:%d}{lines:>8}{other:>8}{code:>8}\n' % (width,) + + # Render + s = fmt.format(k="Odoo cloc", lines="Line", other="Other", code="Code") + s += hr + for m in sorted(self.modules): + s += fmt.format(k=m, lines=self.total[m], other=self.total[m]-self.code[m], code=self.code[m]) + if verbose: + for i in sorted(self.modules[m], key=lambda i: self.modules[m][i][0], reverse=True): + code, total = self.modules[m][i] + s += fmt.format(k=' ' + i, lines=total, other=total - code, code=code) + s += hr + total = sum(self.total.values()) + code = sum(self.code.values()) + s += fmt.format(k='', lines=total, other=total - code, code=code) + print(s) + + if self.errors: + e = "\nErrors\n\n" + for m in sorted(self.errors): + e += "{}\n".format(m) + for i in sorted(self.errors[m]): + e += fmt.format(k=' ' + i, lines=self.errors[m][i], other='', code='') + print(e)