From c16f35387f601fdc013e66b67773d8600c52bf7e Mon Sep 17 00:00:00 2001
From: Christophe Monniez <moc@odoo.com>
Date: Tue, 18 Jan 2022 06:28:31 +0000
Subject: [PATCH] [FIX] tests: fix test retry mechanism for subtests

When a subtest fails, the failure or error is taken into account and even if
the subtest was silenced the test itself logs an ERROR with the count of
failures.

With this commit, at the first try, the subtest result is not taken into
account.

closes odoo/odoo#91974

X-original-commit: 96ca5bb6d7aeb54dfb8d5453edaf04b2ea903d41
Signed-off-by: Christophe Monniez (moc) <moc@odoo.com>
---
 odoo/addons/base/tests/test_test_retry.py | 60 +++++++++++++++++++++--
 odoo/tests/runner.py                      |  3 ++
 2 files changed, 59 insertions(+), 4 deletions(-)

diff --git a/odoo/addons/base/tests/test_test_retry.py b/odoo/addons/base/tests/test_test_retry.py
index 317c491ecf7..848082aeb9a 100644
--- a/odoo/addons/base/tests/test_test_retry.py
+++ b/odoo/addons/base/tests/test_test_retry.py
@@ -7,8 +7,17 @@ import os
 
 _logger = logging.getLogger(__name__)
 
+
+class TestRetryCommon(BaseCase):
+    def get_tests_run_count(self):
+        return int(os.environ.get('ODOO_TEST_FAILURE_RETRIES', 0)) + 1
+
+    def update_count(self):
+        self.count = getattr(self, 'count', 0) + 1
+
+
 @tagged('-standard', 'test_retry', 'test_retry_success')
-class TestRetry(BaseCase):
+class TestRetry(TestRetryCommon):
     """ Check some tests behaviour when ODOO_TEST_FAILURE_RETRIES is set"""
 
     def test_log_levels(self):
@@ -17,14 +26,57 @@ class TestRetry(BaseCase):
         _logger.runbot('test 25')
 
     def test_retry_success(self):
-        tests_run_count = int(os.environ.get('ODOO_TEST_FAILURE_RETRIES', 0)) + 1
-        self.count = getattr(self, 'count', 0) + 1
+        tests_run_count = self.get_tests_run_count()
+        self.update_count()
+        if tests_run_count != self.count:
+            _logger.error('Failure')
         self.assertEqual(tests_run_count, self.count)
 
+
 @tagged('-standard', 'test_retry', 'test_retry_failures')
-class TestRetryFailures(BaseCase):
+class TestRetryFailures(TestRetryCommon):
     def test_retry_failure_assert(self):
         self.assertFalse(1 == 1)
 
     def test_retry_failure_log(self):
         _logger.error('Failure')
+
+
+@tagged('-standard', 'test_retry', 'test_retry_success')
+class TestRetrySubtest(TestRetryCommon):
+
+    def test_retry_subtest_success_one(self):
+        tests_run_count = self.get_tests_run_count()
+        self.update_count()
+        for i in range(3):
+            if i == 1:
+                with self.subTest():
+                    if tests_run_count != self.count:
+                        _logger.error('Failure')
+                    self.assertEqual(tests_run_count, self.count)
+
+    def test_retry_subtest_success_all(self):
+        tests_run_count = self.get_tests_run_count()
+        self.update_count()
+        for _ in range(3):
+            with self.subTest():
+                if tests_run_count != self.count:
+                    _logger.error('Failure')
+                self.assertEqual(tests_run_count, self.count)
+
+
+@tagged('-standard', 'test_retry', 'test_retry_failures')
+class TestRetrySubtestFailures(TestRetryCommon):
+
+    def test_retry_subtest_failure_one(self):
+        for i in range(3):
+            if i == 1:
+                with self.subTest():
+                    _logger.error('Failure')
+                    self.assertFalse(1 == 1)
+
+    def test_retry_subtest_failure_all(self):
+        for _ in range(3):
+            with self.subTest():
+                _logger.error('Failure')
+                self.assertFalse(1 == 1)
diff --git a/odoo/tests/runner.py b/odoo/tests/runner.py
index 86483bf9e0b..d7db4327d16 100644
--- a/odoo/tests/runner.py
+++ b/odoo/tests/runner.py
@@ -108,6 +108,9 @@ class OdooTestResult(unittest.result.TestResult):
             else:
                 flavour = "ERROR"
             self.logError(flavour, subtest, err)
+            if self._soft_fail:
+                self.had_failure = True
+                err = None
         super().addSubTest(test, subtest, err)
 
     def addSkip(self, test, reason):