gtest: Output a canned test case for test suite setup / teardown failures in XML/JSON

This surfaces useful information about the environment failure in a structured form.

As we can see from the updated test, previously unsurfaced information is now present.

PiperOrigin-RevId: 709892315
Change-Id: I2656294d50c33f995bef5c96195a66cff3c4b907

googletest/src/gtest.cc

@@ -3989,6 +3989,12 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener {
   static void OutputXmlTestSuiteForTestResult(::std::ostream* stream,
                                               const TestResult& result);
 
+  // Streams a test case XML stanza containing the given test result.
+  //
+  // Requires: result.Failed()
+  static void OutputXmlTestCaseForTestResult(::std::ostream* stream,
+                                             const TestResult& result);
+
   // Streams an XML representation of a TestResult object.
   static void OutputXmlTestResult(::std::ostream* stream,
                                   const TestResult& result);
@@ -4236,6 +4242,15 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
       FormatEpochTimeInMillisAsIso8601(result.start_timestamp()));
   *stream << ">";
 
+  OutputXmlTestCaseForTestResult(stream, result);
+
+  // Complete the test suite.
+  *stream << "  </testsuite>\n";
+}
+
+// Streams a test case XML stanza containing the given test result.
+void XmlUnitTestResultPrinter::OutputXmlTestCaseForTestResult(
+    ::std::ostream* stream, const TestResult& result) {
   // Output the boilerplate for a minimal test case with a single test.
   *stream << "    <testcase";
   OutputXmlAttribute(stream, "testcase", "name", "");
@@ -4250,9 +4265,6 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
 
   // Output the actual test result.
   OutputXmlTestResult(stream, result);
-
-  // Complete the test suite.
-  *stream << "  </testsuite>\n";
 }
 
 // Prints an XML representation of a TestInfo object.
@@ -4379,6 +4391,10 @@ void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream,
     if (test_suite.GetTestInfo(i)->is_reportable())
       OutputXmlTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
   }
+  if (test_suite.ad_hoc_test_result().Failed()) {
+    OutputXmlTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
+  }
+
   *stream << "  </" << kTestsuite << ">\n";
 }
 
@@ -4518,6 +4534,12 @@ class JsonUnitTestResultPrinter : public EmptyTestEventListener {
   static void OutputJsonTestSuiteForTestResult(::std::ostream* stream,
                                                const TestResult& result);
 
+  // Streams a test case JSON stanza containing the given test result.
+  //
+  // Requires: result.Failed()
+  static void OutputJsonTestCaseForTestResult(::std::ostream* stream,
+                                              const TestResult& result);
+
   // Streams a JSON representation of a TestResult object.
   static void OutputJsonTestResult(::std::ostream* stream,
                                    const TestResult& result);
@@ -4688,6 +4710,15 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
   }
   *stream << Indent(6) << "\"testsuite\": [\n";
 
+  OutputJsonTestCaseForTestResult(stream, result);
+
+  // Finish the test suite.
+  *stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
+}
+
+// Streams a test case JSON stanza containing the given test result.
+void JsonUnitTestResultPrinter::OutputJsonTestCaseForTestResult(
+    ::std::ostream* stream, const TestResult& result) {
   // Output the boilerplate for a new test case.
   *stream << Indent(8) << "{\n";
   OutputJsonKey(stream, "testcase", "name", "", Indent(10));
@@ -4704,9 +4735,6 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
 
   // Output the actual test result.
   OutputJsonTestResult(stream, result);
-
-  // Finish the test suite.
-  *stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
 }
 
 // Prints a JSON representation of a TestInfo object.
@@ -4851,6 +4879,16 @@ void JsonUnitTestResultPrinter::PrintJsonTestSuite(
       OutputJsonTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
     }
   }
+
+  // If there was a failure in the test suite setup or teardown include that in
+  // the output.
+  if (test_suite.ad_hoc_test_result().Failed()) {
+    if (comma) {
+      *stream << ",\n";
+    }
+    OutputJsonTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
+  }
+
   *stream << "\n" << kIndent << "]\n" << Indent(4) << "}";
 }
 

googletest/test/googletest-json-output-unittest.py

@@ -57,7 +57,7 @@ else:
   STACK_TRACE_TEMPLATE = '\n'
 
 EXPECTED_NON_EMPTY = {
-    'tests': 26,
+    'tests': 28,
     'failures': 5,
     'disabled': 2,
     'errors': 0,
@@ -419,6 +419,83 @@ EXPECTED_NON_EMPTY = {
                 },
             ],
         },
+        {
+            'name': 'SetupFailTest',
+            'tests': 1,
+            'failures': 0,
+            'disabled': 0,
+            'errors': 0,
+            'time': '*',
+            'timestamp': '*',
+            'testsuite': [
+                {
+                    'name': 'NoopPassingTest',
+                    'file': 'gtest_xml_output_unittest_.cc',
+                    'line': 168,
+                    'status': 'RUN',
+                    'result': 'SKIPPED',
+                    'timestamp': '*',
+                    'time': '*',
+                    'classname': 'SetupFailTest',
+                    'skipped': [
+                        {'message': 'gtest_xml_output_unittest_.cc:*\n'}
+                    ],
+                },
+                {
+                    'name': '',
+                    'status': 'RUN',
+                    'result': 'COMPLETED',
+                    'timestamp': '*',
+                    'time': '*',
+                    'classname': '',
+                    'failures': [{
+                        'failure': (
+                            'gtest_xml_output_unittest_.cc:*\nExpected equality'
+                            ' of these values:\n  1\n  2'
+                            + STACK_TRACE_TEMPLATE
+                        ),
+                        'type': '',
+                    }],
+                },
+            ],
+        },
+        {
+            'name': 'TearDownFailTest',
+            'tests': 1,
+            'failures': 0,
+            'disabled': 0,
+            'errors': 0,
+            'timestamp': '*',
+            'time': '*',
+            'testsuite': [
+                {
+                    'name': 'NoopPassingTest',
+                    'file': 'gtest_xml_output_unittest_.cc',
+                    'line': 175,
+                    'status': 'RUN',
+                    'result': 'COMPLETED',
+                    'timestamp': '*',
+                    'time': '*',
+                    'classname': 'TearDownFailTest',
+                },
+                {
+                    'name': '',
+                    'status': 'RUN',
+                    'result': 'COMPLETED',
+                    'timestamp': '*',
+                    'time': '*',
+                    'classname': '',
+                    'failures': [{
+                        'failure': (
+                            'gtest_xml_output_unittest_.cc:*\nExpected equality'
+                            ' of these values:\n  1\n  2'
+                            + STACK_TRACE_TEMPLATE
+                        ),
+                        'type': '',
+                    }],
+                },
+            ],
+        },
         {
             'name': 'TypedTest/0',
             'tests': 1,
@@ -431,7 +508,7 @@ EXPECTED_NON_EMPTY = {
                 'name': 'HasTypeParamAttribute',
                 'type_param': 'int',
                 'file': 'gtest_xml_output_unittest_.cc',
-                'line': 173,
+                'line': 189,
                 'status': 'RUN',
                 'result': 'COMPLETED',
                 'time': '*',
@@ -451,7 +528,7 @@ EXPECTED_NON_EMPTY = {
                 'name': 'HasTypeParamAttribute',
                 'type_param': 'long',
                 'file': 'gtest_xml_output_unittest_.cc',
-                'line': 173,
+                'line': 189,
                 'status': 'RUN',
                 'result': 'COMPLETED',
                 'time': '*',
@@ -471,7 +548,7 @@ EXPECTED_NON_EMPTY = {
                 'name': 'HasTypeParamAttribute',
                 'type_param': 'int',
                 'file': 'gtest_xml_output_unittest_.cc',
-                'line': 180,
+                'line': 196,
                 'status': 'RUN',
                 'result': 'COMPLETED',
                 'time': '*',
@@ -491,7 +568,7 @@ EXPECTED_NON_EMPTY = {
                 'name': 'HasTypeParamAttribute',
                 'type_param': 'long',
                 'file': 'gtest_xml_output_unittest_.cc',
-                'line': 180,
+                'line': 196,
                 'status': 'RUN',
                 'result': 'COMPLETED',
                 'time': '*',
@@ -512,7 +589,7 @@ EXPECTED_NON_EMPTY = {
                     'name': 'HasValueParamAttribute/0',
                     'value_param': '33',
                     'file': 'gtest_xml_output_unittest_.cc',
-                    'line': 164,
+                    'line': 180,
                     'status': 'RUN',
                     'result': 'COMPLETED',
                     'time': '*',
@@ -523,7 +600,7 @@ EXPECTED_NON_EMPTY = {
                     'name': 'HasValueParamAttribute/1',
                     'value_param': '42',
                     'file': 'gtest_xml_output_unittest_.cc',
-                    'line': 164,
+                    'line': 180,
                     'status': 'RUN',
                     'result': 'COMPLETED',
                     'time': '*',
@@ -534,7 +611,7 @@ EXPECTED_NON_EMPTY = {
                     'name': 'AnotherTestThatHasValueParamAttribute/0',
                     'value_param': '33',
                     'file': 'gtest_xml_output_unittest_.cc',
-                    'line': 165,
+                    'line': 181,
                     'status': 'RUN',
                     'result': 'COMPLETED',
                     'time': '*',
@@ -545,7 +622,7 @@ EXPECTED_NON_EMPTY = {
                     'name': 'AnotherTestThatHasValueParamAttribute/1',
                     'value_param': '42',
                     'file': 'gtest_xml_output_unittest_.cc',
-                    'line': 165,
+                    'line': 181,
                     'status': 'RUN',
                     'result': 'COMPLETED',
                     'time': '*',

googletest/test/gtest_xml_output_unittest.py

@@ -29,14 +29,14 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-"""Unit test for the gtest_xml_output module"""
+"""Unit test for the gtest_xml_output module."""
 
 import datetime
 import errno
 import os
 import re
 import sys
-from xml.dom import minidom, Node
+from xml.dom import minidom
 
 from googletest.test import gtest_test_utils
 from googletest.test import gtest_xml_test_utils
@@ -67,7 +67,7 @@ else:
   sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
 
 EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="26" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
+<testsuites tests="28" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
   <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
     <testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
   </testsuite>
@@ -173,23 +173,44 @@ It is good practice to tell why you skip a test.
        </properties>
      </testcase>
   </testsuite>
+  <testsuite name="SetupFailTest" tests="1" failures="0" disabled="0" skipped="1" errors="0" time="*" timestamp="*">
+    <testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="168" status="run" result="skipped" time="*" timestamp="*" classname="SetupFailTest">
+      <skipped message="gtest_xml_output_unittest_.cc:*&#x0A;"><![CDATA[gtest_xml_output_unittest_.cc:*
+]]></skipped>
+    </testcase>
+    <testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
+      <failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A;  1&#x0A;  2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
+Expected equality of these values:
+  1
+  2%(stack)s]]></failure>
+    </testcase>
+  </testsuite>
+  <testsuite name="TearDownFailTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
+    <testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="175" status="run" result="completed" time="*" timestamp="*" classname="TearDownFailTest"/>
+    <testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
+      <failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A;  1&#x0A;  2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
+Expected equality of these values:
+  1
+  2%(stack)s]]></failure>
+    </testcase>
+  </testsuite>
   <testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
-    <testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
+    <testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="180" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
-    <testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="164" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
+    <testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="180" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
-    <testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="165" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
+    <testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="181" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
-    <testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="165" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
+    <testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="181" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
   </testsuite>
   <testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
-    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
+    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
   </testsuite>
   <testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
-    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
+    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
   </testsuite>
   <testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
-    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
+    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
   </testsuite>
   <testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
-    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
+    <testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
   </testsuite>
 </testsuites>""" % {
     'stack': STACK_TRACE_TEMPLATE,
@@ -205,6 +226,24 @@ EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
   </testsuite>
 </testsuites>"""
 
+ACTUAL_OUTPUT = """<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" ad_hoc_property="42" name="AllTests">
+  <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
+    <testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
+  </testsuite>
+  <testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
+    <testcase name="IntValuedProperty" file="gtest_xml_output_unittest_.cc" line="125" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
+      <properties>
+        <property name="key_int" value="1"/>
+      </properties>
+    </testcase>
+  </testsuite>
+  <testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
+    <testcase name="HasTypeParamAttribute" type_param="int" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0"/>
+  </testsuite>
+</testsuites>
+"""
+
 EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
 <testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
   <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
@@ -217,8 +256,8 @@ EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
       </properties>
     </testcase>
   </testsuite>
-  <testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
+  <testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
-    <testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
+    <testcase name="HasTypeParamAttribute" type_param="*" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
   </testsuite>
 </testsuites>"""
 

googletest/test/gtest_xml_output_unittest_.cc

@@ -158,6 +158,22 @@ TEST(NoFixtureTest, ExternalUtilityThatCallsRecordStringValuedProperty) {
   ExternalUtilityThatCallsRecordProperty("key_for_utility_string", "1");
 }
 
+// Ensures that SetUpTestSuite and TearDownTestSuite failures are reported in
+// the XML output.
+class SetupFailTest : public ::testing::Test {
+ protected:
+  static void SetUpTestSuite() { ASSERT_EQ(1, 2); }
+};
+
+TEST_F(SetupFailTest, NoopPassingTest) {}
+
+class TearDownFailTest : public ::testing::Test {
+ protected:
+  static void TearDownTestSuite() { ASSERT_EQ(1, 2); }
+};
+
+TEST_F(TearDownFailTest, NoopPassingTest) {}
+
 // Verifies that the test parameter value is output in the 'value_param'
 // XML attribute for value-parameterized tests.
 class ValueParamTest : public TestWithParam<int> {};