odoo_source/odoo/tools/cloc.py

# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import ast
import pathlib
import os
import re
import shutil

import odoo
from odoo.tools.config import config

VERSION = 1
DEFAULT_EXCLUDE = [
    "__manifest__.py",
    "__openerp__.py",
    "tests/**/*",
    "static/lib/**/*",
    "static/tests/**/*",
    "migrations/**/*",
    "upgrades/**/*",
]

STANDARD_MODULES = ['web', 'web_enterprise', 'theme_common', 'base']
MAX_FILE_SIZE = 25 * 2**20 # 25 MB
MAX_LINE_SIZE = 100000
VALID_EXTENSION = ['.py', '.js', '.xml', '.css', '.scss']

class Cloc(object):
    def __init__(self):
        self.modules = {}
        self.code = {}
        self.total = {}
        self.errors = {}
        self.excluded = {}
        self.max_width = 70

    #------------------------------------------------------
    # Parse
    #------------------------------------------------------
    def parse_xml(self, s):
        s = s.strip() + "\n"
        # Unbalanced xml comments inside a CDATA are not supported, and xml
        # comments inside a CDATA will (wrongly) be considered as comment
        total = s.count("\n")
        s = re.sub("(<!--.*?-->)", "", s, flags=re.DOTALL)
        s = re.sub(r"\s*\n\s*", r"\n", s).lstrip()
        return s.count("\n"), total

    def parse_py(self, s):
        try:
            s = s.strip() + "\n"
            total = s.count("\n")
            lines = set()
            for i in ast.walk(ast.parse(s)):
                # we only count 1 for a long string or a docstring
                if hasattr(i, 'lineno'):
                    lines.add(i.lineno)
            return len(lines), total
        except Exception:
            return (-1, "Syntax Error")

    def parse_c_like(self, s, regex):
        # Based on https://stackoverflow.com/questions/241327
        s = s.strip() + "\n"
        total = s.count("\n")
        # To avoid to use too much memory we don't try to count file
        # with very large line, usually minified file
        if max(len(l) for l in s.split('\n')) > MAX_LINE_SIZE:
            return -1, "Max line size exceeded"

        def replacer(match):
            s = match.group(0)
            return " " if s.startswith('/') else s

        comments_re = re.compile(regex, re.DOTALL | re.MULTILINE)
        s = re.sub(comments_re, replacer, s)
        s = re.sub(r"\s*\n\s*", r"\n", s).lstrip()
        return s.count("\n"), total

    def parse_js(self, s):
        return self.parse_c_like(s, r'//.*?$|(?<!\\)/\*.*?\*/|\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"')

    def parse_scss(self, s):
        return self.parse_c_like(s, r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"')

    def parse_css(self, s):
        return self.parse_c_like(s, r'/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"')

    def parse(self, s, ext):
        if ext == '.py':
            return self.parse_py(s)
        elif ext == '.js':
            return self.parse_js(s)
        elif ext == '.xml':
            return self.parse_xml(s)
        elif ext == '.css':
            return self.parse_css(s)
        elif ext == '.scss':
            return self.parse_scss(s)

    #------------------------------------------------------
    # Enumeration
    #------------------------------------------------------
    def book(self, module, item='', count=(0, 0), exclude=False):
        if count[0] == -1:
            self.errors.setdefault(module, {})
            self.errors[module][item] = count[1]
        elif exclude and item:
            self.excluded.setdefault(module, {})
            self.excluded[module][item] = count
        else:
            self.modules.setdefault(module, {})
            if item:
                self.modules[module][item] = count
            self.code[module] = self.code.get(module, 0) + count[0]
            self.total[module] = self.total.get(module, 0) + count[1]
            self.max_width = max(self.max_width, len(module), len(item) + 4)

    def count_path(self, path, exclude=None):
        path = path.rstrip('/')
        exclude_list = []
        for i in odoo.modules.module.MANIFEST_NAMES:
            manifest_path = os.path.join(path, i)
            try:
                with open(manifest_path, 'rb') as manifest:
                    exclude_list.extend(DEFAULT_EXCLUDE)
                    d = ast.literal_eval(manifest.read().decode('latin1'))
                    for j in ['cloc_exclude', 'demo', 'demo_xml']:
                        exclude_list.extend(d.get(j, []))
                    break
            except Exception:
                pass
        if not exclude:
            exclude = set()
        for i in filter(None, exclude_list):
            exclude.update(str(p) for p in pathlib.Path(path).glob(i))

        module_name = os.path.basename(path)
        self.book(module_name)
        for root, dirs, files in os.walk(path):
            for file_name in files:
                file_path = os.path.join(root, file_name)

                if file_path in exclude:
                    continue

                ext = os.path.splitext(file_path)[1].lower()
                if ext not in VALID_EXTENSION:
                    continue

                if os.path.getsize(file_path) > MAX_FILE_SIZE:
                    self.book(module_name, file_path, (-1, "Max file size exceeded"))
                    continue

                with open(file_path, 'rb') as f:
                    # Decode using latin1 to avoid error that may raise by decoding with utf8
                    # The chars not correctly decoded in latin1 have no impact on how many lines will be counted
                    content = f.read().decode('latin1')
                self.book(module_name, file_path, self.parse(content, ext))

    def count_modules(self, env):
        # Exclude standard addons paths
        exclude_heuristic = [odoo.modules.get_module_path(m, display_warning=False) for m in STANDARD_MODULES]
        exclude_path = set([os.path.dirname(os.path.realpath(m)) for m in exclude_heuristic if m])

        domain = [('state', '=', 'installed')]
        # if base_import_module is present
        if env['ir.module.module']._fields.get('imported'):
            domain.append(('imported', '=', False))
        module_list = env['ir.module.module'].search(domain).mapped('name')

        for module_name in module_list:
            module_path = os.path.realpath(odoo.modules.get_module_path(module_name))
            if module_path:
                if any(module_path.startswith(i) for i in exclude_path):
                    continue
                self.count_path(module_path)

    def count_customization(self, env):
        imported_module_sa = ""
        if env['ir.module.module']._fields.get('imported'):
            imported_module_sa = "OR (m.imported = TRUE AND m.state = 'installed')"
        query = """
                SELECT s.id, min(m.name), array_agg(d.module)
                  FROM ir_act_server AS s
             LEFT JOIN ir_model_data AS d
                    ON (d.res_id = s.id AND d.model = 'ir.actions.server')
             LEFT JOIN ir_module_module AS m
                    ON m.name = d.module
                 WHERE s.state = 'code' AND (m.name IS null {})
              GROUP BY s.id
        """.format(imported_module_sa)
        env.cr.execute(query)
        data = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
        for a in env['ir.actions.server'].browse(data.keys()):
            self.book(
                data[a.id][0] or "odoo/studio",
                "ir.actions.server/%s: %s" % (a.id, a.name),
                self.parse_py(a.code),
                '__cloc_exclude__' in data[a.id][1]
            )

        imported_module_field = ("'odoo/studio'", "")
        if env['ir.module.module']._fields.get('imported'):
            imported_module_field = ("min(m.name)", "AND m.imported = TRUE AND m.state = 'installed'")
        # We always want to count manual compute field unless they are generated by studio
        # the module should be odoo/studio unless it comes from an imported module install
        # because manual field get an external id from the original module of the model
        query = r"""
                SELECT f.id, f.name, {}, array_agg(d.module)
                  FROM ir_model_fields AS f
             LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields')
             LEFT JOIN ir_module_module AS m ON m.name = d.module {}
                 WHERE f.compute IS NOT null AND f.state = 'manual'
              GROUP BY f.id, f.name
        """.format(*imported_module_field)
        env.cr.execute(query)
        # Do not count field generated by studio
        all_data = env.cr.fetchall()
        data = {r[0]: (r[2], r[3]) for r in all_data if not ("studio_customization" in r[3] and not r[1].startswith('x_studio'))}
        for f in env['ir.model.fields'].browse(data.keys()):
            self.book(
                data[f.id][0] or "odoo/studio",
                "ir.model.fields/%s: %s" % (f.id, f.name),
                self.parse_py(f.compute),
                '__cloc_exclude__' in data[f.id][1]
            )

        if not env['ir.module.module']._fields.get('imported'):
            return

        # Count qweb view only from imported module and not studio
        query = """
            SELECT view.id, min(mod.name), array_agg(data.module)
              FROM ir_ui_view view
        INNER JOIN ir_model_data data ON view.id = data.res_id AND data.model = 'ir.ui.view'
         LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
             WHERE view.type = 'qweb' AND data.module != 'studio_customization'
          GROUP BY view.id
            HAVING count(mod.name) > 0
        """
        env.cr.execute(query)
        custom_views = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
        for view in env['ir.ui.view'].browse(custom_views.keys()):
            module_name = custom_views[view.id][0]
            self.book(
                module_name,
                "/%s/views/%s.xml" % (module_name, view.name),
                self.parse_xml(view.arch_base),
                '__cloc_exclude__' in custom_views[view.id][1]
            )

        # Count js, xml, css/scss file from imported module
        query = r"""
            SELECT attach.id, min(mod.name), array_agg(data.module)
              FROM ir_attachment attach
        INNER JOIN ir_model_data data ON attach.id = data.res_id AND data.model = 'ir.attachment'
         LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
             WHERE attach.name ~ '.*\.(js|xml|css|scss)$'
          GROUP BY attach.id
            HAVING count(mod.name) > 0
        """
        env.cr.execute(query)
        uploaded_file = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
        for attach in env['ir.attachment'].browse(uploaded_file.keys()):
            module_name = uploaded_file[attach.id][0]
            ext = os.path.splitext(attach.url)[1].lower()
            if ext not in VALID_EXTENSION:
                continue

            if len(attach.datas) > MAX_FILE_SIZE:
                self.book(module_name, attach.url, (-1, "Max file size exceeded"))
                continue

            # Decode using latin1 to avoid error that may raise by decoding with utf8
            # The chars not correctly decoded in latin1 have no impact on how many lines will be counted
            content = attach.raw.decode('latin1')
            self.book(
                module_name,
                attach.url,
                self.parse(content, ext),
                '__cloc_exclude__' in uploaded_file[attach.id][1],
            )

    def count_env(self, env):
        self.count_modules(env)
        self.count_customization(env)

    def count_database(self, database):
        registry = odoo.registry(config['db_name'])
        with registry.cursor() as cr:
            uid = odoo.SUPERUSER_ID
            env = odoo.api.Environment(cr, uid, {})
            self.count_env(env)

    #------------------------------------------------------
    # Report
    #------------------------------------------------------
    # pylint: disable=W0141
    def report(self, verbose=False, width=None):
        # Prepare format
        if not width:
            width = min(self.max_width, shutil.get_terminal_size()[0] - 24)
        hr = "-" * (width + 24) + "\n"
        fmt = '{k:%d}{lines:>8}{other:>8}{code:>8}\n' % (width,)

        # Render
        s = fmt.format(k="Odoo cloc", lines="Line", other="Other", code="Code")
        s += hr
        for m in sorted(self.modules):
            s += fmt.format(k=m, lines=self.total[m], other=self.total[m]-self.code[m], code=self.code[m])
            if verbose:
                for i in sorted(self.modules[m], key=lambda i: self.modules[m][i][0], reverse=True):
                    code, total = self.modules[m][i]
                    s += fmt.format(k='    ' + i, lines=total, other=total - code, code=code)
        s += hr
        total = sum(self.total.values())
        code = sum(self.code.values())
        s += fmt.format(k='', lines=total, other=total - code, code=code)
        print(s)

        if self.excluded and verbose:
            ex = fmt.format(k="Excluded", lines="Line", other="Other", code="Code")
            ex += hr
            for m in sorted(self.excluded):
                for i in sorted(self.excluded[m], key=lambda i: self.excluded[m][i][0], reverse=True):
                    code, total = self.excluded[m][i]
                    ex += fmt.format(k='    ' + i, lines=total, other=total - code, code=code)
            ex += hr
            print(ex)

        if self.errors:
            e = "\nErrors\n\n"
            for m in sorted(self.errors):
                e += "{}\n".format(m)
                for i in sorted(self.errors[m]):
                    e += fmt.format(k='    ' + i, lines=self.errors[m][i], other='', code='')
            print(e)