From 7d76a231b0e29caf86e68d1df858308cd53b2a66 Mon Sep 17 00:00:00 2001 From: Abseil Team Date: Thu, 26 Dec 2024 15:30:16 -0800 Subject: [PATCH] gtest: Output a canned test case for test suite setup / teardown failures in XML/JSON This surfaces useful information about the environment failure in a structured form. As we can see from the updated test, previously unsurfaced information is now present. PiperOrigin-RevId: 709892315 Change-Id: I2656294d50c33f995bef5c96195a66cff3c4b907 --- googletest/src/gtest.cc | 50 ++++++++-- .../test/googletest-json-output-unittest.py | 95 +++++++++++++++++-- googletest/test/gtest_xml_output_unittest.py | 65 ++++++++++--- googletest/test/gtest_xml_output_unittest_.cc | 16 ++++ 4 files changed, 198 insertions(+), 28 deletions(-) diff --git a/googletest/src/gtest.cc b/googletest/src/gtest.cc index c08ab419..3c1cac6e 100644 --- a/googletest/src/gtest.cc +++ b/googletest/src/gtest.cc @@ -3989,6 +3989,12 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener { static void OutputXmlTestSuiteForTestResult(::std::ostream* stream, const TestResult& result); + // Streams a test case XML stanza containing the given test result. + // + // Requires: result.Failed() + static void OutputXmlTestCaseForTestResult(::std::ostream* stream, + const TestResult& result); + // Streams an XML representation of a TestResult object. static void OutputXmlTestResult(::std::ostream* stream, const TestResult& result); @@ -4236,6 +4242,15 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult( FormatEpochTimeInMillisAsIso8601(result.start_timestamp())); *stream << ">"; + OutputXmlTestCaseForTestResult(stream, result); + + // Complete the test suite. + *stream << " \n"; +} + +// Streams a test case XML stanza containing the given test result. +void XmlUnitTestResultPrinter::OutputXmlTestCaseForTestResult( + ::std::ostream* stream, const TestResult& result) { // Output the boilerplate for a minimal test case with a single test. *stream << " \n"; } // Prints an XML representation of a TestInfo object. @@ -4379,6 +4391,10 @@ void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream, if (test_suite.GetTestInfo(i)->is_reportable()) OutputXmlTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i)); } + if (test_suite.ad_hoc_test_result().Failed()) { + OutputXmlTestCaseForTestResult(stream, test_suite.ad_hoc_test_result()); + } + *stream << " \n"; } @@ -4518,6 +4534,12 @@ class JsonUnitTestResultPrinter : public EmptyTestEventListener { static void OutputJsonTestSuiteForTestResult(::std::ostream* stream, const TestResult& result); + // Streams a test case JSON stanza containing the given test result. + // + // Requires: result.Failed() + static void OutputJsonTestCaseForTestResult(::std::ostream* stream, + const TestResult& result); + // Streams a JSON representation of a TestResult object. static void OutputJsonTestResult(::std::ostream* stream, const TestResult& result); @@ -4688,6 +4710,15 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult( } *stream << Indent(6) << "\"testsuite\": [\n"; + OutputJsonTestCaseForTestResult(stream, result); + + // Finish the test suite. + *stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}"; +} + +// Streams a test case JSON stanza containing the given test result. +void JsonUnitTestResultPrinter::OutputJsonTestCaseForTestResult( + ::std::ostream* stream, const TestResult& result) { // Output the boilerplate for a new test case. *stream << Indent(8) << "{\n"; OutputJsonKey(stream, "testcase", "name", "", Indent(10)); @@ -4704,9 +4735,6 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult( // Output the actual test result. OutputJsonTestResult(stream, result); - - // Finish the test suite. - *stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}"; } // Prints a JSON representation of a TestInfo object. @@ -4851,6 +4879,16 @@ void JsonUnitTestResultPrinter::PrintJsonTestSuite( OutputJsonTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i)); } } + + // If there was a failure in the test suite setup or teardown include that in + // the output. + if (test_suite.ad_hoc_test_result().Failed()) { + if (comma) { + *stream << ",\n"; + } + OutputJsonTestCaseForTestResult(stream, test_suite.ad_hoc_test_result()); + } + *stream << "\n" << kIndent << "]\n" << Indent(4) << "}"; } diff --git a/googletest/test/googletest-json-output-unittest.py b/googletest/test/googletest-json-output-unittest.py index d3338e3d..270de05f 100644 --- a/googletest/test/googletest-json-output-unittest.py +++ b/googletest/test/googletest-json-output-unittest.py @@ -57,7 +57,7 @@ else: STACK_TRACE_TEMPLATE = '\n' EXPECTED_NON_EMPTY = { - 'tests': 26, + 'tests': 28, 'failures': 5, 'disabled': 2, 'errors': 0, @@ -419,6 +419,83 @@ EXPECTED_NON_EMPTY = { }, ], }, + { + 'name': 'SetupFailTest', + 'tests': 1, + 'failures': 0, + 'disabled': 0, + 'errors': 0, + 'time': '*', + 'timestamp': '*', + 'testsuite': [ + { + 'name': 'NoopPassingTest', + 'file': 'gtest_xml_output_unittest_.cc', + 'line': 168, + 'status': 'RUN', + 'result': 'SKIPPED', + 'timestamp': '*', + 'time': '*', + 'classname': 'SetupFailTest', + 'skipped': [ + {'message': 'gtest_xml_output_unittest_.cc:*\n'} + ], + }, + { + 'name': '', + 'status': 'RUN', + 'result': 'COMPLETED', + 'timestamp': '*', + 'time': '*', + 'classname': '', + 'failures': [{ + 'failure': ( + 'gtest_xml_output_unittest_.cc:*\nExpected equality' + ' of these values:\n 1\n 2' + + STACK_TRACE_TEMPLATE + ), + 'type': '', + }], + }, + ], + }, + { + 'name': 'TearDownFailTest', + 'tests': 1, + 'failures': 0, + 'disabled': 0, + 'errors': 0, + 'timestamp': '*', + 'time': '*', + 'testsuite': [ + { + 'name': 'NoopPassingTest', + 'file': 'gtest_xml_output_unittest_.cc', + 'line': 175, + 'status': 'RUN', + 'result': 'COMPLETED', + 'timestamp': '*', + 'time': '*', + 'classname': 'TearDownFailTest', + }, + { + 'name': '', + 'status': 'RUN', + 'result': 'COMPLETED', + 'timestamp': '*', + 'time': '*', + 'classname': '', + 'failures': [{ + 'failure': ( + 'gtest_xml_output_unittest_.cc:*\nExpected equality' + ' of these values:\n 1\n 2' + + STACK_TRACE_TEMPLATE + ), + 'type': '', + }], + }, + ], + }, { 'name': 'TypedTest/0', 'tests': 1, @@ -431,7 +508,7 @@ EXPECTED_NON_EMPTY = { 'name': 'HasTypeParamAttribute', 'type_param': 'int', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 173, + 'line': 189, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', @@ -451,7 +528,7 @@ EXPECTED_NON_EMPTY = { 'name': 'HasTypeParamAttribute', 'type_param': 'long', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 173, + 'line': 189, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', @@ -471,7 +548,7 @@ EXPECTED_NON_EMPTY = { 'name': 'HasTypeParamAttribute', 'type_param': 'int', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 180, + 'line': 196, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', @@ -491,7 +568,7 @@ EXPECTED_NON_EMPTY = { 'name': 'HasTypeParamAttribute', 'type_param': 'long', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 180, + 'line': 196, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', @@ -512,7 +589,7 @@ EXPECTED_NON_EMPTY = { 'name': 'HasValueParamAttribute/0', 'value_param': '33', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 164, + 'line': 180, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', @@ -523,7 +600,7 @@ EXPECTED_NON_EMPTY = { 'name': 'HasValueParamAttribute/1', 'value_param': '42', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 164, + 'line': 180, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', @@ -534,7 +611,7 @@ EXPECTED_NON_EMPTY = { 'name': 'AnotherTestThatHasValueParamAttribute/0', 'value_param': '33', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 165, + 'line': 181, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', @@ -545,7 +622,7 @@ EXPECTED_NON_EMPTY = { 'name': 'AnotherTestThatHasValueParamAttribute/1', 'value_param': '42', 'file': 'gtest_xml_output_unittest_.cc', - 'line': 165, + 'line': 181, 'status': 'RUN', 'result': 'COMPLETED', 'time': '*', diff --git a/googletest/test/gtest_xml_output_unittest.py b/googletest/test/gtest_xml_output_unittest.py index c3fea2c0..f2fc9b94 100755 --- a/googletest/test/gtest_xml_output_unittest.py +++ b/googletest/test/gtest_xml_output_unittest.py @@ -29,14 +29,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Unit test for the gtest_xml_output module""" +"""Unit test for the gtest_xml_output module.""" import datetime import errno import os import re import sys -from xml.dom import minidom, Node +from xml.dom import minidom from googletest.test import gtest_test_utils from googletest.test import gtest_xml_test_utils @@ -67,7 +67,7 @@ else: sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG) EXPECTED_NON_EMPTY_XML = """ - + @@ -173,23 +173,44 @@ It is good practice to tell why you skip a test. + + + + + + + + + + + + + + - - - - + + + + - + - + - + - + """ % { 'stack': STACK_TRACE_TEMPLATE, @@ -205,6 +226,24 @@ EXPECTED_FILTERED_TEST_XML = """ """ +ACTUAL_OUTPUT = """ + + + + + + + + + + + + + + + +""" + EXPECTED_SHARDED_TEST_XML = """ @@ -217,8 +256,8 @@ EXPECTED_SHARDED_TEST_XML = """ - - + + """ diff --git a/googletest/test/gtest_xml_output_unittest_.cc b/googletest/test/gtest_xml_output_unittest_.cc index 0ab33022..4a391571 100644 --- a/googletest/test/gtest_xml_output_unittest_.cc +++ b/googletest/test/gtest_xml_output_unittest_.cc @@ -158,6 +158,22 @@ TEST(NoFixtureTest, ExternalUtilityThatCallsRecordStringValuedProperty) { ExternalUtilityThatCallsRecordProperty("key_for_utility_string", "1"); } +// Ensures that SetUpTestSuite and TearDownTestSuite failures are reported in +// the XML output. +class SetupFailTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { ASSERT_EQ(1, 2); } +}; + +TEST_F(SetupFailTest, NoopPassingTest) {} + +class TearDownFailTest : public ::testing::Test { + protected: + static void TearDownTestSuite() { ASSERT_EQ(1, 2); } +}; + +TEST_F(TearDownFailTest, NoopPassingTest) {} + // Verifies that the test parameter value is output in the 'value_param' // XML attribute for value-parameterized tests. class ValueParamTest : public TestWithParam {};