[FIX] tests: fix test retry mechanism for subtests

When a subtest fails, the failure or error is taken into account and even if
the subtest was silenced the test itself logs an ERROR with the count of
failures.

With this commit, at the first try, the subtest result is not taken into
account.

closes odoo/odoo#91974

X-original-commit: 96ca5bb6d7aeb54dfb8d5453edaf04b2ea903d41
Signed-off-by: Christophe Monniez (moc) <moc@odoo.com>

odoo/addons/base/tests/test_test_retry.py

@@ -7,8 +7,17 @@ import os
 
 _logger = logging.getLogger(__name__)
 
+
+class TestRetryCommon(BaseCase):
+    def get_tests_run_count(self):
+        return int(os.environ.get('ODOO_TEST_FAILURE_RETRIES', 0)) + 1
+
+    def update_count(self):
+        self.count = getattr(self, 'count', 0) + 1
+
+
 @tagged('-standard', 'test_retry', 'test_retry_success')
-class TestRetry(BaseCase):
+class TestRetry(TestRetryCommon):
     """ Check some tests behaviour when ODOO_TEST_FAILURE_RETRIES is set"""
 
     def test_log_levels(self):
@@ -17,14 +26,57 @@ class TestRetry(BaseCase):
         _logger.runbot('test 25')
 
     def test_retry_success(self):
-        tests_run_count = int(os.environ.get('ODOO_TEST_FAILURE_RETRIES', 0)) + 1
-        self.count = getattr(self, 'count', 0) + 1
+        tests_run_count = self.get_tests_run_count()
+        self.update_count()
+        if tests_run_count != self.count:
+            _logger.error('Failure')
         self.assertEqual(tests_run_count, self.count)
 
+
 @tagged('-standard', 'test_retry', 'test_retry_failures')
-class TestRetryFailures(BaseCase):
+class TestRetryFailures(TestRetryCommon):
     def test_retry_failure_assert(self):
         self.assertFalse(1 == 1)
 
     def test_retry_failure_log(self):
         _logger.error('Failure')
+
+
+@tagged('-standard', 'test_retry', 'test_retry_success')
+class TestRetrySubtest(TestRetryCommon):
+
+    def test_retry_subtest_success_one(self):
+        tests_run_count = self.get_tests_run_count()
+        self.update_count()
+        for i in range(3):
+            if i == 1:
+                with self.subTest():
+                    if tests_run_count != self.count:
+                        _logger.error('Failure')
+                    self.assertEqual(tests_run_count, self.count)
+
+    def test_retry_subtest_success_all(self):
+        tests_run_count = self.get_tests_run_count()
+        self.update_count()
+        for _ in range(3):
+            with self.subTest():
+                if tests_run_count != self.count:
+                    _logger.error('Failure')
+                self.assertEqual(tests_run_count, self.count)
+
+
+@tagged('-standard', 'test_retry', 'test_retry_failures')
+class TestRetrySubtestFailures(TestRetryCommon):
+
+    def test_retry_subtest_failure_one(self):
+        for i in range(3):
+            if i == 1:
+                with self.subTest():
+                    _logger.error('Failure')
+                    self.assertFalse(1 == 1)
+
+    def test_retry_subtest_failure_all(self):
+        for _ in range(3):
+            with self.subTest():
+                _logger.error('Failure')
+                self.assertFalse(1 == 1)

odoo/tests/runner.py

@@ -108,6 +108,9 @@ class OdooTestResult(unittest.result.TestResult):
             else:
                 flavour = "ERROR"
             self.logError(flavour, subtest, err)
+            if self._soft_fail:
+                self.had_failure = True
+                err = None
         super().addSubTest(test, subtest, err)
 
     def addSkip(self, test, reason):