Thibault Francois
226df213f2
This commit fix 3 problem Don't count studio field ------------------------ the commit https://github.com/odoo/odoo/commit/75b8c4eb1ed6e44f427886eea06ad216995e16c8 was supposed to stop counting count field generate by addins a smart button with studio. It works fine when base import module is not installed which is never the case. When base_import_module the field is counted as a field that belong to a imported module installed : studio_customization studio_customization should not be counted as an imported module installed Count Field with standard xml_id -------------------------------- With https://github.com/odoo/odoo/commit/9afce4805fc8bac45fdba817488aa867fddff69b Each manual can end up with an xml_id from a standard module: the original module of the model A standard module should never create a manual field so we can consider they should be counted unless they match the criteria of the first problem If a field has a standard module xml_id and no other the module should be odoo/studio and not the name of the standard module Make possible to exclude some db record from cloc ------------------------------------------------- It's possible to exclude some file in python module but it's not possible to exclude some field or SA in the database from the count Make it possible if they are link to an xml_id from the module __cloc_exclude__ The exclude record will be shown in cloc report with the verbose mode closes odoo/odoo#95028 X-original-commit: db52fb174461107e26b066147801f54a89d0dbf9 Related: odoo/enterprise#29007 Signed-off-by: Christophe Simonis <chs@odoo.com>
334 lines
13 KiB
Python
334 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
|
import ast
|
|
import pathlib
|
|
import os
|
|
import re
|
|
import shutil
|
|
|
|
import odoo
|
|
from odoo.tools.config import config
|
|
|
|
VERSION = 1
|
|
DEFAULT_EXCLUDE = [
|
|
"__manifest__.py",
|
|
"__openerp__.py",
|
|
"tests/**/*",
|
|
"static/lib/**/*",
|
|
"static/tests/**/*",
|
|
"migrations/**/*",
|
|
"upgrades/**/*",
|
|
]
|
|
|
|
STANDARD_MODULES = ['web', 'web_enterprise', 'theme_common', 'base']
|
|
MAX_FILE_SIZE = 25 * 2**20 # 25 MB
|
|
VALID_EXTENSION = ['.py', '.js', '.xml', '.css', '.scss']
|
|
|
|
class Cloc(object):
|
|
def __init__(self):
|
|
self.modules = {}
|
|
self.code = {}
|
|
self.total = {}
|
|
self.errors = {}
|
|
self.excluded = {}
|
|
self.max_width = 70
|
|
|
|
#------------------------------------------------------
|
|
# Parse
|
|
#------------------------------------------------------
|
|
def parse_xml(self, s):
|
|
s = s.strip() + "\n"
|
|
# Unbalanced xml comments inside a CDATA are not supported, and xml
|
|
# comments inside a CDATA will (wrongly) be considered as comment
|
|
total = s.count("\n")
|
|
s = re.sub("(<!--.*?-->)", "", s, flags=re.DOTALL)
|
|
s = re.sub(r"\s*\n\s*", r"\n", s).lstrip()
|
|
return s.count("\n"), total
|
|
|
|
def parse_py(self, s):
|
|
try:
|
|
s = s.strip() + "\n"
|
|
total = s.count("\n")
|
|
lines = set()
|
|
for i in ast.walk(ast.parse(s)):
|
|
# we only count 1 for a long string or a docstring
|
|
if hasattr(i, 'lineno'):
|
|
lines.add(i.lineno)
|
|
return len(lines), total
|
|
except Exception:
|
|
return (-1, "Syntax Error")
|
|
|
|
def parse_c_like(self, s, regex):
|
|
# Based on https://stackoverflow.com/questions/241327
|
|
s = s.strip() + "\n"
|
|
total = s.count("\n")
|
|
|
|
def replacer(match):
|
|
s = match.group(0)
|
|
return " " if s.startswith('/') else s
|
|
|
|
comments_re = re.compile(regex, re.DOTALL | re.MULTILINE)
|
|
s = re.sub(comments_re, replacer, s)
|
|
s = re.sub(r"\s*\n\s*", r"\n", s).lstrip()
|
|
return s.count("\n"), total
|
|
|
|
def parse_js(self, s):
|
|
return self.parse_c_like(s, r'//.*?$|(?<!\\)/\*.*?\*/|\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"')
|
|
|
|
def parse_scss(self, s):
|
|
return self.parse_c_like(s, r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"')
|
|
|
|
def parse_css(self, s):
|
|
return self.parse_c_like(s, r'/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"')
|
|
|
|
def parse(self, s, ext):
|
|
if ext == '.py':
|
|
return self.parse_py(s)
|
|
elif ext == '.js':
|
|
return self.parse_js(s)
|
|
elif ext == '.xml':
|
|
return self.parse_xml(s)
|
|
elif ext == '.css':
|
|
return self.parse_css(s)
|
|
elif ext == '.scss':
|
|
return self.parse_scss(s)
|
|
|
|
#------------------------------------------------------
|
|
# Enumeration
|
|
#------------------------------------------------------
|
|
def book(self, module, item='', count=(0, 0), exclude=False):
|
|
if count[0] == -1:
|
|
self.errors.setdefault(module, {})
|
|
self.errors[module][item] = count[1]
|
|
elif exclude and item:
|
|
self.excluded.setdefault(module, {})
|
|
self.excluded[module][item] = count
|
|
else:
|
|
self.modules.setdefault(module, {})
|
|
if item:
|
|
self.modules[module][item] = count
|
|
self.code[module] = self.code.get(module, 0) + count[0]
|
|
self.total[module] = self.total.get(module, 0) + count[1]
|
|
self.max_width = max(self.max_width, len(module), len(item) + 4)
|
|
|
|
def count_path(self, path, exclude=None):
|
|
path = path.rstrip('/')
|
|
exclude_list = []
|
|
for i in odoo.modules.module.MANIFEST_NAMES:
|
|
manifest_path = os.path.join(path, i)
|
|
try:
|
|
with open(manifest_path, 'rb') as manifest:
|
|
exclude_list.extend(DEFAULT_EXCLUDE)
|
|
d = ast.literal_eval(manifest.read().decode('latin1'))
|
|
for j in ['cloc_exclude', 'demo', 'demo_xml']:
|
|
exclude_list.extend(d.get(j, []))
|
|
break
|
|
except Exception:
|
|
pass
|
|
if not exclude:
|
|
exclude = set()
|
|
for i in filter(None, exclude_list):
|
|
exclude.update(str(p) for p in pathlib.Path(path).glob(i))
|
|
|
|
module_name = os.path.basename(path)
|
|
self.book(module_name)
|
|
for root, dirs, files in os.walk(path):
|
|
for file_name in files:
|
|
file_path = os.path.join(root, file_name)
|
|
|
|
if file_path in exclude:
|
|
continue
|
|
|
|
ext = os.path.splitext(file_path)[1].lower()
|
|
if ext not in VALID_EXTENSION:
|
|
continue
|
|
|
|
if os.path.getsize(file_path) > MAX_FILE_SIZE:
|
|
self.book(module_name, file_path, (-1, "Max file size exceeded"))
|
|
continue
|
|
|
|
with open(file_path, 'rb') as f:
|
|
# Decode using latin1 to avoid error that may raise by decoding with utf8
|
|
# The chars not correctly decoded in latin1 have no impact on how many lines will be counted
|
|
content = f.read().decode('latin1')
|
|
self.book(module_name, file_path, self.parse(content, ext))
|
|
|
|
def count_modules(self, env):
|
|
# Exclude standard addons paths
|
|
exclude_heuristic = [odoo.modules.get_module_path(m, display_warning=False) for m in STANDARD_MODULES]
|
|
exclude_path = set([os.path.dirname(os.path.realpath(m)) for m in exclude_heuristic if m])
|
|
|
|
domain = [('state', '=', 'installed')]
|
|
# if base_import_module is present
|
|
if env['ir.module.module']._fields.get('imported'):
|
|
domain.append(('imported', '=', False))
|
|
module_list = env['ir.module.module'].search(domain).mapped('name')
|
|
|
|
for module_name in module_list:
|
|
module_path = os.path.realpath(odoo.modules.get_module_path(module_name))
|
|
if module_path:
|
|
if any(module_path.startswith(i) for i in exclude_path):
|
|
continue
|
|
self.count_path(module_path)
|
|
|
|
def count_customization(self, env):
|
|
imported_module_sa = ""
|
|
if env['ir.module.module']._fields.get('imported'):
|
|
imported_module_sa = "OR (m.imported = TRUE AND m.state = 'installed')"
|
|
query = """
|
|
SELECT s.id, min(m.name), array_agg(d.module)
|
|
FROM ir_act_server AS s
|
|
LEFT JOIN ir_model_data AS d
|
|
ON (d.res_id = s.id AND d.model = 'ir.actions.server')
|
|
LEFT JOIN ir_module_module AS m
|
|
ON m.name = d.module
|
|
WHERE s.state = 'code' AND (m.name IS null {})
|
|
GROUP BY s.id
|
|
""".format(imported_module_sa)
|
|
env.cr.execute(query)
|
|
data = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
|
|
for a in env['ir.actions.server'].browse(data.keys()):
|
|
self.book(
|
|
data[a.id][0] or "odoo/studio",
|
|
"ir.actions.server/%s: %s" % (a.id, a.name),
|
|
self.parse_py(a.code),
|
|
'__cloc_exclude__' in data[a.id][1]
|
|
)
|
|
|
|
imported_module_field = ("'odoo/studio'", "")
|
|
if env['ir.module.module']._fields.get('imported'):
|
|
imported_module_field = ("min(m.name)", "AND m.imported = TRUE AND m.state = 'installed'")
|
|
# We always want to count manual compute field unless they are generated by studio
|
|
# the module should be odoo/studio unless it comes from an imported module install
|
|
# because manual field get an external id from the original module of the model
|
|
query = r"""
|
|
SELECT f.id, f.name, {}, array_agg(d.module)
|
|
FROM ir_model_fields AS f
|
|
LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields')
|
|
LEFT JOIN ir_module_module AS m ON m.name = d.module {}
|
|
WHERE f.compute IS NOT null AND f.state = 'manual'
|
|
GROUP BY f.id, f.name
|
|
""".format(*imported_module_field)
|
|
env.cr.execute(query)
|
|
# Do not count field generated by studio
|
|
all_data = env.cr.fetchall()
|
|
data = {r[0]: (r[2], r[3]) for r in all_data if not ("studio_customization" in r[3] and not r[1].startswith('x_studio'))}
|
|
for f in env['ir.model.fields'].browse(data.keys()):
|
|
self.book(
|
|
data[f.id][0] or "odoo/studio",
|
|
"ir.model.fields/%s: %s" % (f.id, f.name),
|
|
self.parse_py(f.compute),
|
|
'__cloc_exclude__' in data[f.id][1]
|
|
)
|
|
|
|
if not env['ir.module.module']._fields.get('imported'):
|
|
return
|
|
|
|
# Count qweb view only from imported module and not studio
|
|
query = """
|
|
SELECT view.id, min(mod.name), array_agg(data.module)
|
|
FROM ir_ui_view view
|
|
INNER JOIN ir_model_data data ON view.id = data.res_id AND data.model = 'ir.ui.view'
|
|
LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
|
|
WHERE view.type = 'qweb' AND data.module != 'studio_customization'
|
|
GROUP BY view.id
|
|
HAVING count(mod.name) > 0
|
|
"""
|
|
env.cr.execute(query)
|
|
custom_views = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
|
|
for view in env['ir.ui.view'].browse(custom_views.keys()):
|
|
module_name = custom_views[view.id][0]
|
|
self.book(
|
|
module_name,
|
|
"/%s/views/%s.xml" % (module_name, view.name),
|
|
self.parse_xml(view.arch_base),
|
|
'__cloc_exclude__' in custom_views[view.id][1]
|
|
)
|
|
|
|
# Count js, xml, css/scss file from imported module
|
|
query = r"""
|
|
SELECT attach.id, min(mod.name), array_agg(data.module)
|
|
FROM ir_attachment attach
|
|
INNER JOIN ir_model_data data ON attach.id = data.res_id AND data.model = 'ir.attachment'
|
|
LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
|
|
WHERE attach.name ~ '.*\.(js|xml|css|scss)$'
|
|
GROUP BY attach.id
|
|
HAVING count(mod.name) > 0
|
|
"""
|
|
env.cr.execute(query)
|
|
uploaded_file = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
|
|
for attach in env['ir.attachment'].browse(uploaded_file.keys()):
|
|
module_name = uploaded_file[attach.id][0]
|
|
ext = os.path.splitext(attach.url)[1].lower()
|
|
if ext not in VALID_EXTENSION:
|
|
continue
|
|
|
|
if len(attach.datas) > MAX_FILE_SIZE:
|
|
self.book(module_name, attach.url, (-1, "Max file size exceeded"))
|
|
continue
|
|
|
|
# Decode using latin1 to avoid error that may raise by decoding with utf8
|
|
# The chars not correctly decoded in latin1 have no impact on how many lines will be counted
|
|
content = attach.raw.decode('latin1')
|
|
self.book(
|
|
module_name,
|
|
attach.url,
|
|
self.parse(content, ext),
|
|
'__cloc_exclude__' in uploaded_file[attach.id][1],
|
|
)
|
|
|
|
def count_env(self, env):
|
|
self.count_modules(env)
|
|
self.count_customization(env)
|
|
|
|
def count_database(self, database):
|
|
registry = odoo.registry(config['db_name'])
|
|
with registry.cursor() as cr:
|
|
uid = odoo.SUPERUSER_ID
|
|
env = odoo.api.Environment(cr, uid, {})
|
|
self.count_env(env)
|
|
|
|
#------------------------------------------------------
|
|
# Report
|
|
#------------------------------------------------------
|
|
# pylint: disable=W0141
|
|
def report(self, verbose=False, width=None):
|
|
# Prepare format
|
|
if not width:
|
|
width = min(self.max_width, shutil.get_terminal_size()[0] - 24)
|
|
hr = "-" * (width + 24) + "\n"
|
|
fmt = '{k:%d}{lines:>8}{other:>8}{code:>8}\n' % (width,)
|
|
|
|
# Render
|
|
s = fmt.format(k="Odoo cloc", lines="Line", other="Other", code="Code")
|
|
s += hr
|
|
for m in sorted(self.modules):
|
|
s += fmt.format(k=m, lines=self.total[m], other=self.total[m]-self.code[m], code=self.code[m])
|
|
if verbose:
|
|
for i in sorted(self.modules[m], key=lambda i: self.modules[m][i][0], reverse=True):
|
|
code, total = self.modules[m][i]
|
|
s += fmt.format(k=' ' + i, lines=total, other=total - code, code=code)
|
|
s += hr
|
|
total = sum(self.total.values())
|
|
code = sum(self.code.values())
|
|
s += fmt.format(k='', lines=total, other=total - code, code=code)
|
|
print(s)
|
|
|
|
if self.excluded and verbose:
|
|
ex = fmt.format(k="Excluded", lines="Line", other="Other", code="Code")
|
|
ex += hr
|
|
for m in sorted(self.excluded):
|
|
for i in sorted(self.excluded[m], key=lambda i: self.excluded[m][i][0], reverse=True):
|
|
code, total = self.excluded[m][i]
|
|
ex += fmt.format(k=' ' + i, lines=total, other=total - code, code=code)
|
|
ex += hr
|
|
print(ex)
|
|
|
|
if self.errors:
|
|
e = "\nErrors\n\n"
|
|
for m in sorted(self.errors):
|
|
e += "{}\n".format(m)
|
|
for i in sorted(self.errors[m]):
|
|
e += fmt.format(k=' ' + i, lines=self.errors[m][i], other='', code='')
|
|
print(e)
|