3party/googletest
0
0
Fork 0
mirror of https://github.com/google/googletest.git synced 2025-01-12 23:47:00 +08:00
Code Issues Projects Releases Wiki Activity

gtest: Output a canned test case for test suite setup / teardown failures in XML/JSON

Browse Source 
This surfaces useful information about the environment failure in a structured form.

As we can see from the updated test, previously unsurfaced information is now present.

PiperOrigin-RevId: 709892315
Change-Id: I2656294d50c33f995bef5c96195a66cff3c4b907
This commit is contained in:
Abseil Team 2024-12-26 15:30:16 -08:00 committed by Copybara-Service
parent e54519b094
commit 7d76a231b0
4 changed files with 198 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
googletest/src/gtest.cc
View File

@ -3989,6 +3989,12 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener {
static void OutputXmlTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams a test case XML stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputXmlTestCaseForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams an XML representation of a TestResult object.
static void OutputXmlTestResult(::std::ostream* stream,
const TestResult& result);
@ -4236,6 +4242,15 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
FormatEpochTimeInMillisAsIso8601(result.start_timestamp()));
*stream << ">";
OutputXmlTestCaseForTestResult(stream, result);
// Complete the test suite.
*stream << " </testsuite>\n";
}
// Streams a test case XML stanza containing the given test result.
void XmlUnitTestResultPrinter::OutputXmlTestCaseForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a minimal test case with a single test.
*stream << " <testcase";
OutputXmlAttribute(stream, "testcase", "name", "");
@ -4250,9 +4265,6 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
// Output the actual test result.
OutputXmlTestResult(stream, result);
// Complete the test suite.
*stream << " </testsuite>\n";
}
// Prints an XML representation of a TestInfo object.
@ -4379,6 +4391,10 @@ void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream,
if (test_suite.GetTestInfo(i)->is_reportable())
OutputXmlTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
}
if (test_suite.ad_hoc_test_result().Failed()) {
OutputXmlTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
}
*stream << " </" << kTestsuite << ">\n";
}
@ -4518,6 +4534,12 @@ class JsonUnitTestResultPrinter : public EmptyTestEventListener {
static void OutputJsonTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams a test case JSON stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputJsonTestCaseForTestResult(::std::ostream* stream,
const TestResult& result);
// Streams a JSON representation of a TestResult object.
static void OutputJsonTestResult(::std::ostream* stream,
const TestResult& result);
@ -4688,6 +4710,15 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
}
*stream << Indent(6) << "\"testsuite\": [\n";
OutputJsonTestCaseForTestResult(stream, result);
// Finish the test suite.
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
}
// Streams a test case JSON stanza containing the given test result.
void JsonUnitTestResultPrinter::OutputJsonTestCaseForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a new test case.
*stream << Indent(8) << "{\n";
OutputJsonKey(stream, "testcase", "name", "", Indent(10));
@ -4704,9 +4735,6 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
// Output the actual test result.
OutputJsonTestResult(stream, result);
// Finish the test suite.
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
}
// Prints a JSON representation of a TestInfo object.
@ -4851,6 +4879,16 @@ void JsonUnitTestResultPrinter::PrintJsonTestSuite(
OutputJsonTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
}
}
// If there was a failure in the test suite setup or teardown include that in
// the output.
if (test_suite.ad_hoc_test_result().Failed()) {
if (comma) {
*stream << ",\n";
}
OutputJsonTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
}
*stream << "\n" << kIndent << "]\n" << Indent(4) << "}";
}

95
googletest/test/googletest-json-output-unittest.py
View File

@ -57,7 +57,7 @@ else:
STACK_TRACE_TEMPLATE = '\n'
EXPECTED_NON_EMPTY = {
'tests': 26,
'tests': 28,
'failures': 5,
'disabled': 2,
'errors': 0,
@ -419,6 +419,83 @@ EXPECTED_NON_EMPTY = {
},
],
},
{
'name': 'SetupFailTest',
'tests': 1,
'failures': 0,
'disabled': 0,
'errors': 0,
'time': '*',
'timestamp': '*',
'testsuite': [
{
'name': 'NoopPassingTest',
'file': 'gtest_xml_output_unittest_.cc',
'line': 168,
'status': 'RUN',
'result': 'SKIPPED',
'timestamp': '*',
'time': '*',
'classname': 'SetupFailTest',
'skipped': [
{'message': 'gtest_xml_output_unittest_.cc:*\n'}
],
},
{
'name': '',
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': '',
'failures': [{
'failure': (
'gtest_xml_output_unittest_.cc:*\nExpected equality'
' of these values:\n 1\n 2'
+ STACK_TRACE_TEMPLATE
),
'type': '',
}],
},
],
},
{
'name': 'TearDownFailTest',
'tests': 1,
'failures': 0,
'disabled': 0,
'errors': 0,
'timestamp': '*',
'time': '*',
'testsuite': [
{
'name': 'NoopPassingTest',
'file': 'gtest_xml_output_unittest_.cc',
'line': 175,
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': 'TearDownFailTest',
},
{
'name': '',
'status': 'RUN',
'result': 'COMPLETED',
'timestamp': '*',
'time': '*',
'classname': '',
'failures': [{
'failure': (
'gtest_xml_output_unittest_.cc:*\nExpected equality'
' of these values:\n 1\n 2'
+ STACK_TRACE_TEMPLATE
),
'type': '',
}],
},
],
},
{
'name': 'TypedTest/0',
'tests': 1,
@ -431,7 +508,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'int',
'file': 'gtest_xml_output_unittest_.cc',
'line': 173,
'line': 189,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -451,7 +528,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'long',
'file': 'gtest_xml_output_unittest_.cc',
'line': 173,
'line': 189,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -471,7 +548,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'int',
'file': 'gtest_xml_output_unittest_.cc',
'line': 180,
'line': 196,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -491,7 +568,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasTypeParamAttribute',
'type_param': 'long',
'file': 'gtest_xml_output_unittest_.cc',
'line': 180,
'line': 196,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -512,7 +589,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasValueParamAttribute/0',
'value_param': '33',
'file': 'gtest_xml_output_unittest_.cc',
'line': 164,
'line': 180,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -523,7 +600,7 @@ EXPECTED_NON_EMPTY = {
'name': 'HasValueParamAttribute/1',
'value_param': '42',
'file': 'gtest_xml_output_unittest_.cc',
'line': 164,
'line': 180,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -534,7 +611,7 @@ EXPECTED_NON_EMPTY = {
'name': 'AnotherTestThatHasValueParamAttribute/0',
'value_param': '33',
'file': 'gtest_xml_output_unittest_.cc',
'line': 165,
'line': 181,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',
@ -545,7 +622,7 @@ EXPECTED_NON_EMPTY = {
'name': 'AnotherTestThatHasValueParamAttribute/1',
'value_param': '42',
'file': 'gtest_xml_output_unittest_.cc',
'line': 165,
'line': 181,
'status': 'RUN',
'result': 'COMPLETED',
'time': '*',

65
googletest/test/gtest_xml_output_unittest.py
View File

@ -29,14 +29,14 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Unit test for the gtest_xml_output module"""
"""Unit test for the gtest_xml_output module."""
import datetime
import errno
import os
import re
import sys
from xml.dom import minidom, Node
from xml.dom import minidom
from googletest.test import gtest_test_utils
from googletest.test import gtest_xml_test_utils
@ -67,7 +67,7 @@ else:
sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="26" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuites tests="28" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
@ -173,23 +173,44 @@ It is good practice to tell why you skip a test.
</properties>
</testcase>
</testsuite>
<testsuite name="SetupFailTest" tests="1" failures="0" disabled="0" skipped="1" errors="0" time="*" timestamp="*">
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="168" status="run" result="skipped" time="*" timestamp="*" classname="SetupFailTest">
<skipped message="gtest_xml_output_unittest_.cc:*&#x0A;"><![CDATA[gtest_xml_output_unittest_.cc:*
]]></skipped>
</testcase>
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="TearDownFailTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="175" status="run" result="completed" time="*" timestamp="*" classname="TearDownFailTest"/>
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="164" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="165" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="165" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="180" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="180" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="181" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="181" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
</testsuite>
<testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
</testsuite>
<testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
</testsuite>
</testsuites>""" % {
'stack': STACK_TRACE_TEMPLATE,
@ -205,6 +226,24 @@ EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
</testsuite>
</testsuites>"""
ACTUAL_OUTPUT = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" ad_hoc_property="42" name="AllTests">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
<testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
<testcase name="IntValuedProperty" file="gtest_xml_output_unittest_.cc" line="125" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
<properties>
<property name="key_int" value="1"/>
</properties>
</testcase>
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="int" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0"/>
</testsuite>
</testsuites>
"""
EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
@ -217,8 +256,8 @@ EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
</properties>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="*" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
</testsuite>
</testsuites>"""

16
googletest/test/gtest_xml_output_unittest_.cc
View File

@ -158,6 +158,22 @@ TEST(NoFixtureTest, ExternalUtilityThatCallsRecordStringValuedProperty) {
ExternalUtilityThatCallsRecordProperty("key_for_utility_string", "1");
}
// Ensures that SetUpTestSuite and TearDownTestSuite failures are reported in
// the XML output.
class SetupFailTest : public ::testing::Test {
protected:
static void SetUpTestSuite() { ASSERT_EQ(1, 2); }
};
TEST_F(SetupFailTest, NoopPassingTest) {}
class TearDownFailTest : public ::testing::Test {
protected:
static void TearDownTestSuite() { ASSERT_EQ(1, 2); }
};
TEST_F(TearDownFailTest, NoopPassingTest) {}
// Verifies that the test parameter value is output in the 'value_param'
// XML attribute for value-parameterized tests.
class ValueParamTest : public TestWithParam<int> {};