Detect and report "Rotten Green Tests"

.gitignore

@@ -11,6 +11,9 @@ bazel-testlogs
 # python
 *.pyc
 
+# PTR: emacs
+*~
+
 # Visual Studio files
 .vs
 *.sdf

googletest/CMakeLists.txt

@@ -223,6 +223,7 @@ if (gtest_build_tests)
   cxx_test(gtest_prod_test gtest_main
     test/production.cc)
   cxx_test(gtest_repeat_test gtest)
+  cxx_test(gtest_rgt_parser_test gtest_main)
   cxx_test(gtest_sole_header_test gtest_main)
   cxx_test(gtest_stress_test gtest)
   cxx_test(googletest-test-part-test gtest_main)
@@ -327,6 +328,9 @@ if (gtest_build_tests)
   cxx_executable(gtest_list_output_unittest_ test gtest)
   py_test(gtest_list_output_unittest)
 
+ cxx_executable(gtest_rgt_output_test_ test gtest_main)
+ py_test(gtest_rgt_output_test)
+
   cxx_executable(gtest_xml_outfile1_test_ test gtest_main)
   cxx_executable(gtest_xml_outfile2_test_ test gtest_main)
   py_test(gtest_xml_outfiles_test)

googletest/include/gtest/gtest-spi.h

@@ -152,6 +152,7 @@ GTEST_DISABLE_MSC_WARNINGS_POP_()  //  4251
      public:                                                                  \
       static void Execute() { statement; }                                    \
     };                                                                        \
+    GTEST_RGT_EXPECT_FAILURE_;                                                \
     ::testing::TestPartResultArray gtest_failures;                            \
     ::testing::internal::SingleFailureChecker gtest_checker(                  \
         &gtest_failures, ::testing::TestPartResult::kFatalFailure, (substr)); \
@@ -170,6 +171,7 @@ GTEST_DISABLE_MSC_WARNINGS_POP_()  //  4251
      public:                                                                  \
       static void Execute() { statement; }                                    \
     };                                                                        \
+    GTEST_RGT_EXPECT_FAILURE_;                                                \
     ::testing::TestPartResultArray gtest_failures;                            \
     ::testing::internal::SingleFailureChecker gtest_checker(                  \
         &gtest_failures, ::testing::TestPartResult::kFatalFailure, (substr)); \
@@ -216,6 +218,7 @@ GTEST_DISABLE_MSC_WARNINGS_POP_()  //  4251
 // to avoid an MSVC warning on unreachable code.
 #define EXPECT_NONFATAL_FAILURE(statement, substr)                    \
   do {                                                                \
+    GTEST_RGT_EXPECT_FAILURE_;                                        \
     ::testing::TestPartResultArray gtest_failures;                    \
     ::testing::internal::SingleFailureChecker gtest_checker(          \
         &gtest_failures, ::testing::TestPartResult::kNonFatalFailure, \
@@ -233,6 +236,7 @@ GTEST_DISABLE_MSC_WARNINGS_POP_()  //  4251
 
 #define EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(statement, substr)             \
   do {                                                                        \
+    GTEST_RGT_EXPECT_FAILURE_;                                                \
     ::testing::TestPartResultArray gtest_failures;                            \
     ::testing::internal::SingleFailureChecker gtest_checker(                  \
         &gtest_failures, ::testing::TestPartResult::kNonFatalFailure,         \

googletest/include/gtest/gtest-test-part.h

@@ -59,7 +59,8 @@ class GTEST_API_ TestPartResult {
     kSuccess,          // Succeeded.
     kNonFatalFailure,  // Failed but the test can continue.
     kFatalFailure,     // Failed and the test should be terminated.
-    kSkip              // Skipped.
+    kSkip,             // Skipped.
+    kRotten            // Not executed but should have.
   };
 
   // C'tor.  TestPartResult does NOT have a default constructor.
@@ -95,6 +96,9 @@ class GTEST_API_ TestPartResult {
   // Returns true if and only if the test part was skipped.
   bool skipped() const { return type_ == kSkip; }
 
+  // Returns true if and only if the test part didn't execute but should have.
+  bool rotten() const { return type_ == kRotten; }
+
   // Returns true if and only if the test part passed.
   bool passed() const { return type_ == kSuccess; }
 

googletest/include/gtest/gtest.h

@@ -153,6 +153,10 @@ GTEST_DECLARE_int32_(stack_trace_depth);
 // non-zero code otherwise. For use with an external test framework.
 GTEST_DECLARE_bool_(throw_on_failure);
 
+// This flag controls whether an un-executed assertion within an otherwise
+// passing test is treated as pass or fail.
+GTEST_DECLARE_bool_(treat_rotten_as_pass);
+
 // When this flag is set with a "host:port" string, on supported
 // platforms test results are streamed to the specified port on
 // the specified host machine.
@@ -418,6 +422,9 @@ class GTEST_API_ TestResult {
   // Returns true if and only if the test was skipped.
   bool Skipped() const;
 
+  // Returns true if and only if the test passed but had a rotten assertion.
+  bool Rotten() const;
+
   // Returns true if and only if the test failed.
   bool Failed() const;
 
@@ -599,6 +606,9 @@ class GTEST_API_ TestInfo {
   // Returns the result of the test.
   const TestResult* result() const { return &result_; }
 
+  // Report a rotten assertion location, after the test is done.
+  void ReportRotten(const char *file, int line);
+
  private:
 #ifdef GTEST_HAS_DEATH_TEST
   friend class internal::DefaultDeathTestFactory;
@@ -710,6 +720,9 @@ class GTEST_API_ TestSuite {
   // Gets the number of skipped tests in this test suite.
   int skipped_test_count() const;
 
+  // Gets the number of rotten tests in this test suite.
+  int rotten_test_count() const;
+
   // Gets the number of failed tests in this test suite.
   int failed_test_count() const;
 
@@ -731,6 +744,10 @@ class GTEST_API_ TestSuite {
   // Returns true if and only if the test suite passed.
   bool Passed() const { return !Failed(); }
 
+  // Returns true if and only if the test suite passed but there is at least
+  // one rotten assertion.
+  bool Rotten() const { return Passed() && rotten_test_count() > 0; }
+
   // Returns true if and only if the test suite failed.
   bool Failed() const {
     return failed_test_count() > 0 || ad_hoc_test_result().Failed();
@@ -814,6 +831,11 @@ class GTEST_API_ TestSuite {
     return test_info->should_run() && test_info->result()->Skipped();
   }
 
+  // Returns true if and only if test was rotten.
+  static bool TestRotten(const TestInfo* test_info) {
+    return test_info->should_run() && test_info->result()->Rotten();
+  }
+
   // Returns true if and only if test failed.
   static bool TestFailed(const TestInfo* test_info) {
     return test_info->should_run() && test_info->result()->Failed();
@@ -1153,6 +1175,9 @@ class GTEST_API_ UnitTest {
   // Gets the number of successful test suites.
   int successful_test_suite_count() const;
 
+  // Gets the number of test suites with rotten assertions.
+  int rotten_test_suite_count() const;
+
   // Gets the number of failed test suites.
   int failed_test_suite_count() const;
 
@@ -1177,6 +1202,9 @@ class GTEST_API_ UnitTest {
   // Gets the number of skipped tests.
   int skipped_test_count() const;
 
+  // Gets the number of rotten tests.
+  int rotten_test_count() const;
+
   // Gets the number of failed tests.
   int failed_test_count() const;
 
@@ -1206,6 +1234,10 @@ class GTEST_API_ UnitTest {
   // passed).
   bool Passed() const;
 
+  // Returns true if and only if the unit test passed but had at least one
+  // rotten assertion.
+  bool Rotten() const;
+
   // Returns true if and only if the unit test failed (i.e. some test suite
   // failed or something outside of all tests failed).
   bool Failed() const;

googletest/include/gtest/gtest_pred_impl.h

@@ -77,7 +77,7 @@ namespace testing {
 #define GTEST_ASSERT_(expression, on_failure)                   \
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                 \
   if (const ::testing::AssertionResult gtest_ar = (expression)) \
-    ;                                                           \
+    { GTEST_RGT_DECLARE_ }                                      \
   else                                                          \
     on_failure(gtest_ar.failure_message())
 

googletest/include/gtest/internal/custom/gtest-port.h

@@ -34,4 +34,7 @@
 #ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_
 #define GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_
 
+// ** Hack so I can test gcc-built gtest with gcc7.5
+#define GTEST_HAS_RGT 1
+
 #endif  // GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_

googletest/include/gtest/internal/gtest-death-test-internal.h

@@ -222,6 +222,7 @@ inline Matcher<const ::std::string&> MakeDeathTestMatcher(
 #define GTEST_DEATH_TEST_(statement, predicate, regex_or_matcher, fail)        \
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                                \
   if (::testing::internal::AlwaysTrue()) {                                     \
+    { GTEST_RGT_DECLARE_ }                                                     \
     ::testing::internal::DeathTest* gtest_dt;                                  \
     if (!::testing::internal::DeathTest::Create(                               \
             #statement,                                                        \
@@ -260,6 +261,7 @@ inline Matcher<const ::std::string&> MakeDeathTestMatcher(
 #define GTEST_EXECUTE_STATEMENT_(statement, regex_or_matcher)    \
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                  \
   if (::testing::internal::AlwaysTrue()) {                       \
+    { GTEST_RGT_DECLARE_ }                                       \
     GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement);   \
   } else if (!::testing::internal::AlwaysTrue()) {               \
     ::testing::internal::MakeDeathTestMatcher(regex_or_matcher); \

googletest/include/gtest/internal/gtest-internal.h

@@ -69,6 +69,7 @@
 
 #include "gtest/gtest-message.h"
 #include "gtest/internal/gtest-filepath.h"
+#include "gtest/internal/gtest-rgt.h"
 #include "gtest/internal/gtest-string.h"
 #include "gtest/internal/gtest-type-util.h"
 
@@ -1408,6 +1409,7 @@ class NeverThrown {
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                             \
   if (::testing::internal::TrueWithString gtest_msg{}) {                    \
     bool gtest_caught_expected = false;                                     \
+    { GTEST_RGT_DECLARE_ }                                                  \
     try {                                                                   \
       GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement);            \
     } catch (expected_exception const&) {                                   \
@@ -1451,6 +1453,7 @@ class NeverThrown {
 #define GTEST_TEST_NO_THROW_(statement, fail)                            \
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                          \
   if (::testing::internal::TrueWithString gtest_msg{}) {                 \
+    { GTEST_RGT_DECLARE_ }                                               \
     try {                                                                \
       GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement);         \
     }                                                                    \
@@ -1470,6 +1473,7 @@ class NeverThrown {
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                      \
   if (::testing::internal::AlwaysTrue()) {                           \
     bool gtest_caught_any = false;                                   \
+    { GTEST_RGT_DECLARE_ }                                           \
     try {                                                            \
       GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement);     \
     } catch (...) {                                                  \
@@ -1491,7 +1495,7 @@ class NeverThrown {
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                       \
   if (const ::testing::AssertionResult gtest_ar_ =                    \
           ::testing::AssertionResult(expression))                     \
-    ;                                                                 \
+    { GTEST_RGT_DECLARE_ }                                            \
   else                                                                \
     fail(::testing::internal::GetBoolAssertionFailureMessage(         \
              gtest_ar_, text, #actual, #expected)                     \
@@ -1500,6 +1504,7 @@ class NeverThrown {
 #define GTEST_TEST_NO_FATAL_FAILURE_(statement, fail)                          \
   GTEST_AMBIGUOUS_ELSE_BLOCKER_                                                \
   if (::testing::internal::AlwaysTrue()) {                                     \
+    { GTEST_RGT_DECLARE_ }                                                     \
     ::testing::internal::HasNewFatalFailureHelper gtest_fatal_failure_checker; \
     GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement);                 \
     if (gtest_fatal_failure_checker.has_new_fatal_failure()) {                 \

googletest/include/gtest/internal/gtest-port.h

@@ -78,6 +78,12 @@
 //                              expressions are/aren't available.
 //   GTEST_HAS_PTHREAD        - Define it to 1/0 to indicate that <pthread.h>
 //                              is/isn't available.
+//   GTEST_HAS_RGT            - Define it to 1/0 to indicate that Rotten Green
+//                              Test detection is/isn't enabled. On by default.
+//   GTEST_DEFAULT_RGT_PASS   - The default for --gtest_treat_rotten_as_pass.
+//                              Set true here so existing test suites don't fail
+//                              en masse. The recommended value is false and can
+//                              be set in custom/gtest-port.h.
 //   GTEST_HAS_RTTI           - Define it to 1/0 to indicate that RTTI is/isn't
 //                              enabled.
 //   GTEST_HAS_STD_WSTRING    - Define it to 1/0 to indicate that
@@ -1959,6 +1965,18 @@ class GTEST_API_ ThreadLocal {
 // we cannot detect it.
 GTEST_API_ size_t GetThreadCount();
 
+// Determine whether the compiler can support Rotten Green Test detection.
+// The definition below is guarded by #ifndef to give embedders a chance to
+// define GTEST_HAS_RGT in gtest/internal/custom/gtest-port.h
+#ifndef GTEST_HAS_RGT
+#define GTEST_HAS_RGT 1
+#endif // GTEST_HAS_RGT
+
+#ifndef GTEST_DEFAULT_RGT_PASS
+#define GTEST_DEFAULT_RGT_PASS true
+#endif // GTEST_DEFAULT_RGT_PASS
+
+
 #ifdef GTEST_OS_WINDOWS
 #define GTEST_PATH_SEP_ "\\"
 #define GTEST_HAS_ALT_PATH_SEP_ 1