* remove useless OdooTestRunner * don't log results & time per-file, log a module-level tally instead * add number of tests to post-test results * generate a single test suite per module (see note) * use the previous item to split out the at_install test-running in two steps: generating the suite for the module then running that suite, this way for modules which have no test, or for which all tests have been deselected by test tags, we can avoid some of the setup necessary to prepare for running tests but possibly quite expensive (e.g. `setup_models`) Note: single test suite per module I wanted to stop creating a test result for (essentially) every file in the module, however because of the class-level ``addCleanup``, a TestResult can't be reused by independent suites: In order to run class-level cleanup, the test suite checks between tests if the test it's *preparing* to run is in the same class as the last test it ran, and if not applies the class-level cleanup. The problem is that the "previous test class" is stored on the result object, which is never cleaned up, and the "between tests" check is really performed *before each test*. This means when reusing results across suites it will run the class-level cleanup at the end of one suite and immediately at the start of the next, which will cause issues if class-level cleanups are not idempotent (thankfully ``TestTestCursor`` has a non-idempotent ``tearDownClass` which let me discover the error). Possible fixes are: * don't reuse results * clear the relevant states / attributes between suites * put individual suites in a Big Suite for running The latter seems simpler: just create a single suite for the entire odoo-level module instead of creating one suite per test module. Note to the note: the case of nested suite is taken in account, the "end of suite" cleanup only runs at the end of the top-level suite, so technically we don't have to unwrap suites for *that* purpose, we're doing so in order to filter the test cases inside the suites. But maybe we could integrate this feature to the suites themselves... closes odoo/odoo#55185 Signed-off-by: Xavier Morel (xmo) <xmo@odoo.com>
132 lines
5.5 KiB
Python
132 lines
5.5 KiB
Python
import logging
|
|
import time
|
|
import unittest
|
|
|
|
from .. import sql_db
|
|
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
class OdooTestResult(unittest.result.TestResult):
|
|
"""
|
|
This class in inspired from TextTestResult (https://github.com/python/cpython/blob/master/Lib/unittest/runner.py)
|
|
Instead of using a stream, we are using the logger,
|
|
but replacing the "findCaller" in order to give the information we
|
|
have based on the test object that is running.
|
|
"""
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.time_start = None
|
|
self.queries_start = None
|
|
|
|
def __str__(self):
|
|
return f'{len(self.failures)} failed, {len(self.errors)} error(s) of {self.testsRun} tests'
|
|
|
|
def update(self, other):
|
|
""" Merges an other test result into this one, only updates contents
|
|
|
|
:type other: OdooTestResult
|
|
"""
|
|
self.failures.extend(other.failures)
|
|
self.errors.extend(other.errors)
|
|
self.testsRun += other.testsRun
|
|
self.skipped.extend(other.skipped)
|
|
self.expectedFailures.extend(other.expectedFailures)
|
|
self.unexpectedSuccesses.extend(other.unexpectedSuccesses)
|
|
self.shouldStop = self.shouldStop or other.shouldStop
|
|
|
|
def log(self, level, msg, *args, test=None, exc_info=None, extra=None, stack_info=False, caller_infos=None):
|
|
"""
|
|
``test`` is the running test case, ``caller_infos`` is
|
|
(fn, lno, func, sinfo) (logger.findCaller format), see logger.log for
|
|
the other parameters.
|
|
"""
|
|
test = test or self
|
|
if isinstance(test, unittest.case._SubTest) and test.test_case:
|
|
test = test.test_case
|
|
logger = logging.getLogger(test.__module__)
|
|
try:
|
|
caller_infos = caller_infos or logger.findCaller(stack_info)
|
|
except ValueError:
|
|
caller_infos = "(unknown file)", 0, "(unknown function)", None
|
|
(fn, lno, func, sinfo) = caller_infos
|
|
# using logger.log makes it difficult to spot-replace findCaller in
|
|
# order to provide useful location information (the problematic spot
|
|
# inside the test function), so use lower-level functions instead
|
|
if logger.isEnabledFor(level):
|
|
record = logger.makeRecord(logger.name, level, fn, lno, msg, args, exc_info, func, extra, sinfo)
|
|
logger.handle(record)
|
|
|
|
def getDescription(self, test):
|
|
if isinstance(test, unittest.case._SubTest):
|
|
return 'Subtest %s' % test._subDescription()
|
|
if isinstance(test, unittest.TestCase):
|
|
# since we have the module name in the logger, this will avoid to duplicate module info in log line
|
|
# we only apply this for TestCase since we can receive error handler or other special case
|
|
return "%s.%s" % (test.__class__.__qualname__, test._testMethodName)
|
|
return str(test)
|
|
|
|
def startTest(self, test):
|
|
super().startTest(test)
|
|
self.log(logging.INFO, 'Starting %s ...', self.getDescription(test), test=test)
|
|
self.time_start = time.time()
|
|
self.queries_start = sql_db.sql_counter
|
|
|
|
def addError(self, test, err):
|
|
super().addError(test, err)
|
|
self.logError("ERROR", test, err)
|
|
|
|
def addFailure(self, test, err):
|
|
super().addFailure(test, err)
|
|
self.logError("FAIL", test, err)
|
|
|
|
def addSubTest(self, test, subtest, err):
|
|
# since addSubTest is not making a call to addFailure or addError we need to manage it too
|
|
# https://github.com/python/cpython/blob/3.7/Lib/unittest/result.py#L136
|
|
if err is not None:
|
|
if issubclass(err[0], test.failureException):
|
|
flavour = "FAIL"
|
|
else:
|
|
flavour = "ERROR"
|
|
self.logError(flavour, subtest, err)
|
|
super().addSubTest(test, subtest, err)
|
|
|
|
def addSkip(self, test, reason):
|
|
super().addSkip(test, reason)
|
|
self.log(logging.INFO, 'skipped %s', self.getDescription(test), test=test)
|
|
|
|
def addUnexpectedSuccess(self, test):
|
|
super().addUnexpectedSuccess(test)
|
|
self.log(logging.ERROR, 'unexpected success for %s', self.getDescription(test), test=test)
|
|
|
|
def logError(self, flavour, test, error):
|
|
err = self._exc_info_to_string(error, test)
|
|
caller_infos = self.getErrorCallerInfo(error, test)
|
|
self.log(logging.INFO, '=' * 70, test=test, caller_infos=caller_infos) # keep this as info !!!!!!
|
|
self.log(logging.ERROR, "%s: %s\n%s", flavour, self.getDescription(test), err, test=test, caller_infos=caller_infos)
|
|
|
|
def getErrorCallerInfo(self, error, test):
|
|
"""
|
|
:param error: A tuple (exctype, value, tb) as returned by sys.exc_info().
|
|
:param test: A TestCase that created this error.
|
|
:returns: a tuple (fn, lno, func, sinfo) matching the logger findCaller format or None
|
|
"""
|
|
|
|
# only test case should be executed in odoo, this is only a safe guard
|
|
if isinstance(test, unittest.suite._ErrorHolder):
|
|
return
|
|
if not isinstance(test, unittest.TestCase):
|
|
_logger.warning('%r is not a TestCase' % test)
|
|
return
|
|
_, _, error_traceback = error
|
|
|
|
while error_traceback:
|
|
code = error_traceback.tb_frame.f_code
|
|
if code.co_name == test._testMethodName:
|
|
lineno = error_traceback.tb_lineno
|
|
filename = code.co_filename
|
|
method = test._testMethodName
|
|
infos = (filename, lineno, method, None)
|
|
return infos
|
|
error_traceback = error_traceback.tb_next
|