Googletest export

Add --gtest_fail_fast support to googletest.

  - Analogous functionality to to golang -test.failfast and python --failfast
  - Stops test execution upon first test failure.
  - Also add support Bazel equivalent env var (TESTBRIDGE_TEST_RUNNER_FAIL_FAST)

PiperOrigin-RevId: 302488880
This commit is contained in:
Abseil Team 2020-03-23 15:15:58 -04:00 committed by Xiaoyi Zhang
parent 749148f1ac
commit 1ced315a48
10 changed files with 784 additions and 54 deletions

View File

@ -2116,6 +2116,15 @@ For example:
everything in test suite `FooTest` except `FooTest.Bar` and everything in everything in test suite `FooTest` except `FooTest.Bar` and everything in
test suite `BarTest` except `BarTest.Foo`. test suite `BarTest` except `BarTest.Foo`.
#### Stop test execution upon first failure
By default, a googletest program runs all tests the user has defined. In some
cases (e.g. iterative test development & execution) it may be desirable stop
test execution upon first failure (trading improved latency for completeness).
If `GTEST_FAIL_FAST` environment variable or `--gtest_fail_fast` flag is set,
the test runner will stop execution as soon as the first test failure is
found.
#### Temporarily Disabling Tests #### Temporarily Disabling Tests
If you have a broken test that you cannot fix right away, you can add the If you have a broken test that you cannot fix right away, you can add the

View File

@ -101,6 +101,10 @@ GTEST_DECLARE_bool_(catch_exceptions);
// to let Google Test decide. // to let Google Test decide.
GTEST_DECLARE_string_(color); GTEST_DECLARE_string_(color);
// This flag controls whether the test runner should continue execution past
// first failure.
GTEST_DECLARE_bool_(fail_fast);
// This flag sets up the filter to select by name using a glob pattern // This flag sets up the filter to select by name using a glob pattern
// the tests to run. If the filter is not given all tests are executed. // the tests to run. If the filter is not given all tests are executed.
GTEST_DECLARE_string_(filter); GTEST_DECLARE_string_(filter);
@ -795,6 +799,9 @@ class GTEST_API_ TestInfo {
// deletes it. // deletes it.
void Run(); void Run();
// Skip and records the test result for this object.
void Skip();
static void ClearTestResult(TestInfo* test_info) { static void ClearTestResult(TestInfo* test_info) {
test_info->result_.Clear(); test_info->result_.Clear();
} }
@ -943,6 +950,9 @@ class GTEST_API_ TestSuite {
// Runs every test in this TestSuite. // Runs every test in this TestSuite.
void Run(); void Run();
// Skips the execution of tests under this TestSuite
void Skip();
// Runs SetUpTestSuite() for this TestSuite. This wrapper is needed // Runs SetUpTestSuite() for this TestSuite. This wrapper is needed
// for catching exceptions thrown from SetUpTestSuite(). // for catching exceptions thrown from SetUpTestSuite().
void RunSetUpTestSuite() { void RunSetUpTestSuite() {

View File

@ -84,6 +84,7 @@ const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests";
const char kBreakOnFailureFlag[] = "break_on_failure"; const char kBreakOnFailureFlag[] = "break_on_failure";
const char kCatchExceptionsFlag[] = "catch_exceptions"; const char kCatchExceptionsFlag[] = "catch_exceptions";
const char kColorFlag[] = "color"; const char kColorFlag[] = "color";
const char kFailFast[] = "fail_fast";
const char kFilterFlag[] = "filter"; const char kFilterFlag[] = "filter";
const char kListTestsFlag[] = "list_tests"; const char kListTestsFlag[] = "list_tests";
const char kOutputFlag[] = "output"; const char kOutputFlag[] = "output";
@ -164,6 +165,7 @@ class GTestFlagSaver {
color_ = GTEST_FLAG(color); color_ = GTEST_FLAG(color);
death_test_style_ = GTEST_FLAG(death_test_style); death_test_style_ = GTEST_FLAG(death_test_style);
death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); death_test_use_fork_ = GTEST_FLAG(death_test_use_fork);
fail_fast_ = GTEST_FLAG(fail_fast);
filter_ = GTEST_FLAG(filter); filter_ = GTEST_FLAG(filter);
internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); internal_run_death_test_ = GTEST_FLAG(internal_run_death_test);
list_tests_ = GTEST_FLAG(list_tests); list_tests_ = GTEST_FLAG(list_tests);
@ -187,6 +189,7 @@ class GTestFlagSaver {
GTEST_FLAG(death_test_style) = death_test_style_; GTEST_FLAG(death_test_style) = death_test_style_;
GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; GTEST_FLAG(death_test_use_fork) = death_test_use_fork_;
GTEST_FLAG(filter) = filter_; GTEST_FLAG(filter) = filter_;
GTEST_FLAG(fail_fast) = fail_fast_;
GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; GTEST_FLAG(internal_run_death_test) = internal_run_death_test_;
GTEST_FLAG(list_tests) = list_tests_; GTEST_FLAG(list_tests) = list_tests_;
GTEST_FLAG(output) = output_; GTEST_FLAG(output) = output_;
@ -208,6 +211,7 @@ class GTestFlagSaver {
std::string color_; std::string color_;
std::string death_test_style_; std::string death_test_style_;
bool death_test_use_fork_; bool death_test_use_fork_;
bool fail_fast_;
std::string filter_; std::string filter_;
std::string internal_run_death_test_; std::string internal_run_death_test_;
bool list_tests_; bool list_tests_;

View File

@ -213,6 +213,21 @@ static const char* GetDefaultFilter() {
return kUniversalFilter; return kUniversalFilter;
} }
// Bazel passes in the argument to '--test_runner_fail_fast' via the
// TESTBRIDGE_TEST_RUNNER_FAIL_FAST environment variable.
static bool GetDefaultFailFast() {
const char* const testbridge_test_runner_fail_fast =
internal::posix::GetEnv("TESTBRIDGE_TEST_RUNNER_FAIL_FAST");
if (testbridge_test_runner_fail_fast != nullptr) {
return strcmp(testbridge_test_runner_fail_fast, "1") == 0;
}
return false;
}
GTEST_DEFINE_bool_(
fail_fast, internal::BoolFromGTestEnv("fail_fast", GetDefaultFailFast()),
"True if and only if a test failure should stop further test execution.");
GTEST_DEFINE_bool_( GTEST_DEFINE_bool_(
also_run_disabled_tests, also_run_disabled_tests,
internal::BoolFromGTestEnv("also_run_disabled_tests", false), internal::BoolFromGTestEnv("also_run_disabled_tests", false),
@ -2863,6 +2878,28 @@ void TestInfo::Run() {
impl->set_current_test_info(nullptr); impl->set_current_test_info(nullptr);
} }
// Skip and records a skipped test result for this object.
void TestInfo::Skip() {
if (!should_run_) return;
internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
impl->set_current_test_info(this);
TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater();
// Notifies the unit test event listeners that a test is about to start.
repeater->OnTestStart(*this);
const TestPartResult test_part_result =
TestPartResult(TestPartResult::kSkip, this->file(), this->line(), "");
impl->GetTestPartResultReporterForCurrentThread()->ReportTestPartResult(
test_part_result);
// Notifies the unit test event listener that a test has just finished.
repeater->OnTestEnd(*this);
impl->set_current_test_info(nullptr);
}
// class TestSuite // class TestSuite
// Gets the number of successful tests in this test suite. // Gets the number of successful tests in this test suite.
@ -2975,6 +3012,12 @@ void TestSuite::Run() {
start_timestamp_ = internal::GetTimeInMillis(); start_timestamp_ = internal::GetTimeInMillis();
for (int i = 0; i < total_test_count(); i++) { for (int i = 0; i < total_test_count(); i++) {
GetMutableTestInfo(i)->Run(); GetMutableTestInfo(i)->Run();
if (GTEST_FLAG(fail_fast) && GetMutableTestInfo(i)->result()->Failed()) {
for (int j = i + 1; j < total_test_count(); j++) {
GetMutableTestInfo(j)->Skip();
}
break;
}
} }
elapsed_time_ = internal::GetTimeInMillis() - start_timestamp_; elapsed_time_ = internal::GetTimeInMillis() - start_timestamp_;
@ -2992,6 +3035,36 @@ void TestSuite::Run() {
impl->set_current_test_suite(nullptr); impl->set_current_test_suite(nullptr);
} }
// Skips all tests under this TestSuite.
void TestSuite::Skip() {
if (!should_run_) return;
internal::UnitTestImpl* const impl = internal::GetUnitTestImpl();
impl->set_current_test_suite(this);
TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater();
// Call both legacy and the new API
repeater->OnTestSuiteStart(*this);
// Legacy API is deprecated but still available
#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI
repeater->OnTestCaseStart(*this);
#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI
for (int i = 0; i < total_test_count(); i++) {
GetMutableTestInfo(i)->Skip();
}
// Call both legacy and the new API
repeater->OnTestSuiteEnd(*this);
// Legacy API is deprecated but still available
#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI
repeater->OnTestCaseEnd(*this);
#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI
impl->set_current_test_suite(nullptr);
}
// Clears the results of all tests in this test suite. // Clears the results of all tests in this test suite.
void TestSuite::ClearResult() { void TestSuite::ClearResult() {
ad_hoc_test_result_.Clear(); ad_hoc_test_result_.Clear();
@ -5523,6 +5596,13 @@ bool UnitTestImpl::RunAllTests() {
for (int test_index = 0; test_index < total_test_suite_count(); for (int test_index = 0; test_index < total_test_suite_count();
test_index++) { test_index++) {
GetMutableSuiteCase(test_index)->Run(); GetMutableSuiteCase(test_index)->Run();
if (GTEST_FLAG(fail_fast) &&
GetMutableSuiteCase(test_index)->Failed()) {
for (int j = test_index + 1; j < total_test_suite_count(); j++) {
GetMutableSuiteCase(j)->Skip();
}
break;
}
} }
} }
@ -6127,31 +6207,31 @@ static const char kColorEncodedHelpMessage[] =
static bool ParseGoogleTestFlag(const char* const arg) { static bool ParseGoogleTestFlag(const char* const arg) {
return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag,
&GTEST_FLAG(also_run_disabled_tests)) || &GTEST_FLAG(also_run_disabled_tests)) ||
ParseBoolFlag(arg, kBreakOnFailureFlag, ParseBoolFlag(arg, kBreakOnFailureFlag,
&GTEST_FLAG(break_on_failure)) || &GTEST_FLAG(break_on_failure)) ||
ParseBoolFlag(arg, kCatchExceptionsFlag, ParseBoolFlag(arg, kCatchExceptionsFlag,
&GTEST_FLAG(catch_exceptions)) || &GTEST_FLAG(catch_exceptions)) ||
ParseStringFlag(arg, kColorFlag, &GTEST_FLAG(color)) || ParseStringFlag(arg, kColorFlag, &GTEST_FLAG(color)) ||
ParseStringFlag(arg, kDeathTestStyleFlag, ParseStringFlag(arg, kDeathTestStyleFlag,
&GTEST_FLAG(death_test_style)) || &GTEST_FLAG(death_test_style)) ||
ParseBoolFlag(arg, kDeathTestUseFork, ParseBoolFlag(arg, kDeathTestUseFork,
&GTEST_FLAG(death_test_use_fork)) || &GTEST_FLAG(death_test_use_fork)) ||
ParseStringFlag(arg, kFilterFlag, &GTEST_FLAG(filter)) || ParseBoolFlag(arg, kFailFast, &GTEST_FLAG(fail_fast)) ||
ParseStringFlag(arg, kInternalRunDeathTestFlag, ParseStringFlag(arg, kFilterFlag, &GTEST_FLAG(filter)) ||
&GTEST_FLAG(internal_run_death_test)) || ParseStringFlag(arg, kInternalRunDeathTestFlag,
ParseBoolFlag(arg, kListTestsFlag, &GTEST_FLAG(list_tests)) || &GTEST_FLAG(internal_run_death_test)) ||
ParseStringFlag(arg, kOutputFlag, &GTEST_FLAG(output)) || ParseBoolFlag(arg, kListTestsFlag, &GTEST_FLAG(list_tests)) ||
ParseBoolFlag(arg, kPrintTimeFlag, &GTEST_FLAG(print_time)) || ParseStringFlag(arg, kOutputFlag, &GTEST_FLAG(output)) ||
ParseBoolFlag(arg, kPrintUTF8Flag, &GTEST_FLAG(print_utf8)) || ParseBoolFlag(arg, kPrintTimeFlag, &GTEST_FLAG(print_time)) ||
ParseInt32Flag(arg, kRandomSeedFlag, &GTEST_FLAG(random_seed)) || ParseBoolFlag(arg, kPrintUTF8Flag, &GTEST_FLAG(print_utf8)) ||
ParseInt32Flag(arg, kRepeatFlag, &GTEST_FLAG(repeat)) || ParseInt32Flag(arg, kRandomSeedFlag, &GTEST_FLAG(random_seed)) ||
ParseBoolFlag(arg, kShuffleFlag, &GTEST_FLAG(shuffle)) || ParseInt32Flag(arg, kRepeatFlag, &GTEST_FLAG(repeat)) ||
ParseInt32Flag(arg, kStackTraceDepthFlag, ParseBoolFlag(arg, kShuffleFlag, &GTEST_FLAG(shuffle)) ||
&GTEST_FLAG(stack_trace_depth)) || ParseInt32Flag(arg, kStackTraceDepthFlag,
ParseStringFlag(arg, kStreamResultToFlag, &GTEST_FLAG(stack_trace_depth)) ||
&GTEST_FLAG(stream_result_to)) || ParseStringFlag(arg, kStreamResultToFlag,
ParseBoolFlag(arg, kThrowOnFailureFlag, &GTEST_FLAG(stream_result_to)) ||
&GTEST_FLAG(throw_on_failure)); ParseBoolFlag(arg, kThrowOnFailureFlag, &GTEST_FLAG(throw_on_failure));
} }
#if GTEST_USE_OWN_FLAGFILE_FLAG_ #if GTEST_USE_OWN_FLAGFILE_FLAG_

View File

@ -58,6 +58,7 @@ cc_test(
"googletest-catch-exceptions-test_.cc", "googletest-catch-exceptions-test_.cc",
"googletest-color-test_.cc", "googletest-color-test_.cc",
"googletest-env-var-test_.cc", "googletest-env-var-test_.cc",
"googletest-failfast-unittest_.cc",
"googletest-filter-unittest_.cc", "googletest-filter-unittest_.cc",
"googletest-break-on-failure-unittest_.cc", "googletest-break-on-failure-unittest_.cc",
"googletest-listener-test.cc", "googletest-listener-test.cc",
@ -222,6 +223,21 @@ py_test(
deps = [":gtest_test_utils"], deps = [":gtest_test_utils"],
) )
cc_binary(
name = "googletest-failfast-unittest_",
testonly = 1,
srcs = ["googletest-failfast-unittest_.cc"],
deps = ["//:gtest"],
)
py_test(
name = "googletest-failfast-unittest",
size = "medium",
srcs = ["googletest-failfast-unittest.py"],
data = [":googletest-failfast-unittest_"],
deps = [":gtest_test_utils"],
)
cc_binary( cc_binary(
name = "googletest-filter-unittest_", name = "googletest-filter-unittest_",
testonly = 1, testonly = 1,

View File

@ -85,6 +85,8 @@ class GTestEnvVarTest(gtest_test_utils.TestCase):
TestFlag('break_on_failure', '1', '0') TestFlag('break_on_failure', '1', '0')
TestFlag('color', 'yes', 'auto') TestFlag('color', 'yes', 'auto')
SetEnvVar('TESTBRIDGE_TEST_RUNNER_FAIL_FAST', None) # For 'fail_fast' test
TestFlag('fail_fast', '1', '0')
TestFlag('filter', 'FooTest.Bar', '*') TestFlag('filter', 'FooTest.Bar', '*')
SetEnvVar('XML_OUTPUT_FILE', None) # For 'output' test SetEnvVar('XML_OUTPUT_FILE', None) # For 'output' test
TestFlag('output', 'xml:tmp/foo.xml', '') TestFlag('output', 'xml:tmp/foo.xml', '')

View File

@ -72,6 +72,11 @@ void PrintFlag(const char* flag) {
return; return;
} }
if (strcmp(flag, "fail_fast") == 0) {
cout << GTEST_FLAG(fail_fast);
return;
}
if (strcmp(flag, "filter") == 0) { if (strcmp(flag, "filter") == 0) {
cout << GTEST_FLAG(filter); cout << GTEST_FLAG(filter);
return; return;

View File

@ -0,0 +1,410 @@
#!/usr/bin/env python
#
# Copyright 2020 Google Inc. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Unit test for Google Test fail_fast.
A user can specify if a Google Test program should continue test execution
after a test failure via the GTEST_FAIL_FAST environment variable or the
--gtest_fail_fast flag. The default value of the flag can also be changed
by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST.
This script tests such functionality by invoking googletest-failfast-unittest_
(a program written with Google Test) with different environments and command
line flags.
"""
import os
import gtest_test_utils
# Constants.
# Bazel testbridge environment variable for fail fast
BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST'
# The environment variable for specifying fail fast.
FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST'
# The command line flag for specifying fail fast.
FAIL_FAST_FLAG = 'gtest_fail_fast'
# The command line flag to run disabled tests.
RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests'
# The command line flag for specifying a filter.
FILTER_FLAG = 'gtest_filter'
# Command to run the googletest-failfast-unittest_ program.
COMMAND = gtest_test_utils.GetTestExecutablePath(
'googletest-failfast-unittest_')
# The command line flag to tell Google Test to output the list of tests it
# will run.
LIST_TESTS_FLAG = '--gtest_list_tests'
# Indicates whether Google Test supports death tests.
SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess(
[COMMAND, LIST_TESTS_FLAG]).output
# Utilities.
environ = os.environ.copy()
def SetEnvVar(env_var, value):
"""Sets the env variable to 'value'; unsets it when 'value' is None."""
if value is not None:
environ[env_var] = value
elif env_var in environ:
del environ[env_var]
def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False):
"""Runs the test program and returns its output."""
args = []
xml_path = os.path.join(gtest_test_utils.GetTempDir(),
'.GTestFailFastUnitTest.xml')
args += ['--gtest_output=xml:' + xml_path]
if fail_fast is not None:
if isinstance(fail_fast, str):
args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)]
elif fail_fast:
args += ['--%s' % FAIL_FAST_FLAG]
else:
args += ['--no%s' % FAIL_FAST_FLAG]
if test_suite:
args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)]
if run_disabled:
args += ['--%s' % RUN_DISABLED_FLAG]
txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output
with open(xml_path) as xml_file:
return txt_out, xml_file.read()
# The unit test.
class GTestFailFastUnitTest(gtest_test_utils.TestCase):
"""Tests the env variable or the command line flag for fail_fast."""
def testDefaultBehavior(self):
"""Tests the behavior of not specifying the fail_fast."""
txt, _ = RunAndReturnOutput()
self.assertIn('22 FAILED TEST', txt)
def testGoogletestFlag(self):
txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True)
self.assertIn('1 FAILED TEST', txt)
self.assertIn('[ SKIPPED ] 3 tests', txt)
txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False)
self.assertIn('4 FAILED TEST', txt)
self.assertNotIn('[ SKIPPED ]', txt)
def testGoogletestEnvVar(self):
"""Tests the behavior of specifying fail_fast via Googletest env var."""
try:
SetEnvVar(FAIL_FAST_ENV_VAR, '1')
txt, _ = RunAndReturnOutput('HasSimpleTest')
self.assertIn('1 FAILED TEST', txt)
self.assertIn('[ SKIPPED ] 3 tests', txt)
SetEnvVar(FAIL_FAST_ENV_VAR, '0')
txt, _ = RunAndReturnOutput('HasSimpleTest')
self.assertIn('4 FAILED TEST', txt)
self.assertNotIn('[ SKIPPED ]', txt)
finally:
SetEnvVar(FAIL_FAST_ENV_VAR, None)
def testBazelEnvVar(self):
"""Tests the behavior of specifying fail_fast via Bazel testbridge."""
try:
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1')
txt, _ = RunAndReturnOutput('HasSimpleTest')
self.assertIn('1 FAILED TEST', txt)
self.assertIn('[ SKIPPED ] 3 tests', txt)
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
txt, _ = RunAndReturnOutput('HasSimpleTest')
self.assertIn('4 FAILED TEST', txt)
self.assertNotIn('[ SKIPPED ]', txt)
finally:
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
def testFlagOverridesEnvVar(self):
"""Tests precedence of flag over env var."""
try:
SetEnvVar(FAIL_FAST_ENV_VAR, '0')
txt, _ = RunAndReturnOutput('HasSimpleTest', True)
self.assertIn('1 FAILED TEST', txt)
self.assertIn('[ SKIPPED ] 3 tests', txt)
finally:
SetEnvVar(FAIL_FAST_ENV_VAR, None)
def testGoogletestEnvVarOverridesBazelEnvVar(self):
"""Tests that the Googletest native env var over Bazel testbridge."""
try:
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
SetEnvVar(FAIL_FAST_ENV_VAR, '1')
txt, _ = RunAndReturnOutput('HasSimpleTest')
self.assertIn('1 FAILED TEST', txt)
self.assertIn('[ SKIPPED ] 3 tests', txt)
finally:
SetEnvVar(FAIL_FAST_ENV_VAR, None)
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
def testEventListener(self):
txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True)
self.assertIn('1 FAILED TEST', txt)
self.assertIn('[ SKIPPED ] 3 tests', txt)
for expected_count, callback in [(1, 'OnTestSuiteStart'),
(5, 'OnTestStart'),
(5, 'OnTestEnd'),
(5, 'OnTestPartResult'),
(1, 'OnTestSuiteEnd')]:
self.assertEqual(
expected_count, txt.count(callback),
'Expected %d calls to callback %s match count on output: %s ' %
(expected_count, callback, txt))
txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False)
self.assertIn('3 FAILED TEST', txt)
self.assertIn('[ SKIPPED ] 1 test', txt)
for expected_count, callback in [(1, 'OnTestSuiteStart'),
(5, 'OnTestStart'),
(5, 'OnTestEnd'),
(5, 'OnTestPartResult'),
(1, 'OnTestSuiteEnd')]:
self.assertEqual(
expected_count, txt.count(callback),
'Expected %d calls to callback %s match count on output: %s ' %
(expected_count, callback, txt))
def assertXmlResultCount(self, result, count, xml):
self.assertEqual(
count, xml.count('result="%s"' % result),
'Expected \'result="%s"\' match count of %s: %s ' %
(result, count, xml))
def assertXmlStatusCount(self, status, count, xml):
self.assertEqual(
count, xml.count('status="%s"' % status),
'Expected \'status="%s"\' match count of %s: %s ' %
(status, count, xml))
def assertFailFastXmlAndTxtOutput(self,
fail_fast,
test_suite,
passed_count,
failure_count,
skipped_count,
suppressed_count,
run_disabled=False):
"""Assert XML and text output of a test execution."""
txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled)
if failure_count > 0:
self.assertIn('%s FAILED TEST' % failure_count, txt)
if suppressed_count > 0:
self.assertIn('%s DISABLED TEST' % suppressed_count, txt)
if skipped_count > 0:
self.assertIn('[ SKIPPED ] %s tests' % skipped_count, txt)
self.assertXmlStatusCount('run',
passed_count + failure_count + skipped_count, xml)
self.assertXmlStatusCount('notrun', suppressed_count, xml)
self.assertXmlResultCount('completed', passed_count + failure_count, xml)
self.assertXmlResultCount('skipped', skipped_count, xml)
self.assertXmlResultCount('suppressed', suppressed_count, xml)
def assertFailFastBehavior(self,
test_suite,
passed_count,
failure_count,
skipped_count,
suppressed_count,
run_disabled=False):
"""Assert --fail_fast via flag."""
for fail_fast in ('true', '1', 't', True):
self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,
failure_count, skipped_count,
suppressed_count, run_disabled)
def assertNotFailFastBehavior(self,
test_suite,
passed_count,
failure_count,
skipped_count,
suppressed_count,
run_disabled=False):
"""Assert --nofail_fast via flag."""
for fail_fast in ('false', '0', 'f', False):
self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,
failure_count, skipped_count,
suppressed_count, run_disabled)
def testFlag_HasFixtureTest(self):
"""Tests the behavior of fail_fast and TEST_F."""
self.assertFailFastBehavior(
test_suite='HasFixtureTest',
passed_count=1,
failure_count=1,
skipped_count=3,
suppressed_count=0)
self.assertNotFailFastBehavior(
test_suite='HasFixtureTest',
passed_count=1,
failure_count=4,
skipped_count=0,
suppressed_count=0)
def testFlag_HasSimpleTest(self):
"""Tests the behavior of fail_fast and TEST."""
self.assertFailFastBehavior(
test_suite='HasSimpleTest',
passed_count=1,
failure_count=1,
skipped_count=3,
suppressed_count=0)
self.assertNotFailFastBehavior(
test_suite='HasSimpleTest',
passed_count=1,
failure_count=4,
skipped_count=0,
suppressed_count=0)
def testFlag_HasParametersTest(self):
"""Tests the behavior of fail_fast and TEST_P."""
self.assertFailFastBehavior(
test_suite='HasParametersSuite/HasParametersTest',
passed_count=0,
failure_count=1,
skipped_count=3,
suppressed_count=0)
self.assertNotFailFastBehavior(
test_suite='HasParametersSuite/HasParametersTest',
passed_count=0,
failure_count=4,
skipped_count=0,
suppressed_count=0)
def testFlag_HasDisabledTest(self):
"""Tests the behavior of fail_fast and Disabled test cases."""
self.assertFailFastBehavior(
test_suite='HasDisabledTest',
passed_count=1,
failure_count=1,
skipped_count=2,
suppressed_count=1,
run_disabled=False)
self.assertNotFailFastBehavior(
test_suite='HasDisabledTest',
passed_count=1,
failure_count=3,
skipped_count=0,
suppressed_count=1,
run_disabled=False)
def testFlag_HasDisabledRunDisabledTest(self):
"""Tests the behavior of fail_fast and Disabled test cases enabled."""
self.assertFailFastBehavior(
test_suite='HasDisabledTest',
passed_count=1,
failure_count=1,
skipped_count=3,
suppressed_count=0,
run_disabled=True)
self.assertNotFailFastBehavior(
test_suite='HasDisabledTest',
passed_count=1,
failure_count=4,
skipped_count=0,
suppressed_count=0,
run_disabled=True)
def testFlag_HasDisabledSuiteTest(self):
"""Tests the behavior of fail_fast and Disabled test suites."""
self.assertFailFastBehavior(
test_suite='DISABLED_HasDisabledSuite',
passed_count=0,
failure_count=0,
skipped_count=0,
suppressed_count=5,
run_disabled=False)
self.assertNotFailFastBehavior(
test_suite='DISABLED_HasDisabledSuite',
passed_count=0,
failure_count=0,
skipped_count=0,
suppressed_count=5,
run_disabled=False)
def testFlag_HasDisabledSuiteRunDisabledTest(self):
"""Tests the behavior of fail_fast and Disabled test suites enabled."""
self.assertFailFastBehavior(
test_suite='DISABLED_HasDisabledSuite',
passed_count=1,
failure_count=1,
skipped_count=3,
suppressed_count=0,
run_disabled=True)
self.assertNotFailFastBehavior(
test_suite='DISABLED_HasDisabledSuite',
passed_count=1,
failure_count=4,
skipped_count=0,
suppressed_count=0,
run_disabled=True)
if SUPPORTS_DEATH_TESTS:
def testFlag_HasDeathTest(self):
"""Tests the behavior of fail_fast and death tests."""
self.assertFailFastBehavior(
test_suite='HasDeathTest',
passed_count=1,
failure_count=1,
skipped_count=3,
suppressed_count=0)
self.assertNotFailFastBehavior(
test_suite='HasDeathTest',
passed_count=1,
failure_count=4,
skipped_count=0,
suppressed_count=0)
if __name__ == '__main__':
gtest_test_utils.Main()

View File

@ -0,0 +1,167 @@
// Copyright 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Unit test for Google Test test filters.
//
// A user can specify which test(s) in a Google Test program to run via
// either the GTEST_FILTER environment variable or the --gtest_filter
// flag. This is used for testing such functionality.
//
// The program will be invoked from a Python unit test. Don't run it
// directly.
#include "gtest/gtest.h"
namespace {
// Test HasFixtureTest.
class HasFixtureTest : public testing::Test {};
TEST_F(HasFixtureTest, Test0) {}
TEST_F(HasFixtureTest, Test1) { FAIL() << "Expected failure."; }
TEST_F(HasFixtureTest, Test2) { FAIL() << "Expected failure."; }
TEST_F(HasFixtureTest, Test3) { FAIL() << "Expected failure."; }
TEST_F(HasFixtureTest, Test4) { FAIL() << "Expected failure."; }
// Test HasSimpleTest.
TEST(HasSimpleTest, Test0) {}
TEST(HasSimpleTest, Test1) { FAIL() << "Expected failure."; }
TEST(HasSimpleTest, Test2) { FAIL() << "Expected failure."; }
TEST(HasSimpleTest, Test3) { FAIL() << "Expected failure."; }
TEST(HasSimpleTest, Test4) { FAIL() << "Expected failure."; }
// Test HasDisabledTest.
TEST(HasDisabledTest, Test0) {}
TEST(HasDisabledTest, DISABLED_Test1) { FAIL() << "Expected failure."; }
TEST(HasDisabledTest, Test2) { FAIL() << "Expected failure."; }
TEST(HasDisabledTest, Test3) { FAIL() << "Expected failure."; }
TEST(HasDisabledTest, Test4) { FAIL() << "Expected failure."; }
// Test HasDeathTest
TEST(HasDeathTest, Test0) { EXPECT_DEATH_IF_SUPPORTED(exit(1), ".*"); }
TEST(HasDeathTest, Test1) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
TEST(HasDeathTest, Test2) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
TEST(HasDeathTest, Test3) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
TEST(HasDeathTest, Test4) {
EXPECT_DEATH_IF_SUPPORTED(FAIL() << "Expected failure.", ".*");
}
// Test DISABLED_HasDisabledSuite
TEST(DISABLED_HasDisabledSuite, Test0) {}
TEST(DISABLED_HasDisabledSuite, Test1) { FAIL() << "Expected failure."; }
TEST(DISABLED_HasDisabledSuite, Test2) { FAIL() << "Expected failure."; }
TEST(DISABLED_HasDisabledSuite, Test3) { FAIL() << "Expected failure."; }
TEST(DISABLED_HasDisabledSuite, Test4) { FAIL() << "Expected failure."; }
// Test HasParametersTest
class HasParametersTest : public testing::TestWithParam<int> {};
TEST_P(HasParametersTest, Test1) { FAIL() << "Expected failure."; }
TEST_P(HasParametersTest, Test2) { FAIL() << "Expected failure."; }
INSTANTIATE_TEST_SUITE_P(HasParametersSuite, HasParametersTest,
testing::Values(1, 2));
class MyTestListener : public ::testing::EmptyTestEventListener {
void OnTestSuiteStart(const ::testing::TestSuite& test_suite) override {
printf("We are in OnTestSuiteStart of %s.\n", test_suite.name());
}
void OnTestStart(const ::testing::TestInfo& test_info) override {
printf("We are in OnTestStart of %s.%s.\n", test_info.test_suite_name(),
test_info.name());
}
void OnTestPartResult(
const ::testing::TestPartResult& test_part_result) override {
printf("We are in OnTestPartResult %s:%d.\n", test_part_result.file_name(),
test_part_result.line_number());
}
void OnTestEnd(const ::testing::TestInfo& test_info) override {
printf("We are in OnTestEnd of %s.%s.\n", test_info.test_suite_name(),
test_info.name());
}
void OnTestSuiteEnd(const ::testing::TestSuite& test_suite) override {
printf("We are in OnTestSuiteEnd of %s.\n", test_suite.name());
}
};
TEST(HasSkipTest, Test0) { SUCCEED() << "Expected success."; }
TEST(HasSkipTest, Test1) { GTEST_SKIP() << "Expected skip."; }
TEST(HasSkipTest, Test2) { FAIL() << "Expected failure."; }
TEST(HasSkipTest, Test3) { FAIL() << "Expected failure."; }
TEST(HasSkipTest, Test4) { FAIL() << "Expected failure."; }
} // namespace
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
::testing::UnitTest::GetInstance()->listeners().Append(new MyTestListener());
return RUN_ALL_TESTS();
}

View File

@ -37,21 +37,22 @@
// code once "gtest.h" has been #included. // code once "gtest.h" has been #included.
// Do not move it after other gtest #includes. // Do not move it after other gtest #includes.
TEST(CommandLineFlagsTest, CanBeAccessedInCodeOnceGTestHIsIncluded) { TEST(CommandLineFlagsTest, CanBeAccessedInCodeOnceGTestHIsIncluded) {
bool dummy = testing::GTEST_FLAG(also_run_disabled_tests) bool dummy = testing::GTEST_FLAG(also_run_disabled_tests) ||
|| testing::GTEST_FLAG(break_on_failure) testing::GTEST_FLAG(break_on_failure) ||
|| testing::GTEST_FLAG(catch_exceptions) testing::GTEST_FLAG(catch_exceptions) ||
|| testing::GTEST_FLAG(color) != "unknown" testing::GTEST_FLAG(color) != "unknown" ||
|| testing::GTEST_FLAG(filter) != "unknown" testing::GTEST_FLAG(fail_fast) ||
|| testing::GTEST_FLAG(list_tests) testing::GTEST_FLAG(filter) != "unknown" ||
|| testing::GTEST_FLAG(output) != "unknown" testing::GTEST_FLAG(list_tests) ||
|| testing::GTEST_FLAG(print_time) testing::GTEST_FLAG(output) != "unknown" ||
|| testing::GTEST_FLAG(random_seed) testing::GTEST_FLAG(print_time) ||
|| testing::GTEST_FLAG(repeat) > 0 testing::GTEST_FLAG(random_seed) ||
|| testing::GTEST_FLAG(show_internal_stack_frames) testing::GTEST_FLAG(repeat) > 0 ||
|| testing::GTEST_FLAG(shuffle) testing::GTEST_FLAG(show_internal_stack_frames) ||
|| testing::GTEST_FLAG(stack_trace_depth) > 0 testing::GTEST_FLAG(shuffle) ||
|| testing::GTEST_FLAG(stream_result_to) != "unknown" testing::GTEST_FLAG(stack_trace_depth) > 0 ||
|| testing::GTEST_FLAG(throw_on_failure); testing::GTEST_FLAG(stream_result_to) != "unknown" ||
testing::GTEST_FLAG(throw_on_failure);
EXPECT_TRUE(dummy || !dummy); // Suppresses warning that dummy is unused. EXPECT_TRUE(dummy || !dummy); // Suppresses warning that dummy is unused.
} }
@ -202,6 +203,7 @@ using testing::GTEST_FLAG(break_on_failure);
using testing::GTEST_FLAG(catch_exceptions); using testing::GTEST_FLAG(catch_exceptions);
using testing::GTEST_FLAG(color); using testing::GTEST_FLAG(color);
using testing::GTEST_FLAG(death_test_use_fork); using testing::GTEST_FLAG(death_test_use_fork);
using testing::GTEST_FLAG(fail_fast);
using testing::GTEST_FLAG(filter); using testing::GTEST_FLAG(filter);
using testing::GTEST_FLAG(list_tests); using testing::GTEST_FLAG(list_tests);
using testing::GTEST_FLAG(output); using testing::GTEST_FLAG(output);
@ -1598,6 +1600,7 @@ class GTestFlagSaverTest : public Test {
GTEST_FLAG(catch_exceptions) = false; GTEST_FLAG(catch_exceptions) = false;
GTEST_FLAG(death_test_use_fork) = false; GTEST_FLAG(death_test_use_fork) = false;
GTEST_FLAG(color) = "auto"; GTEST_FLAG(color) = "auto";
GTEST_FLAG(fail_fast) = false;
GTEST_FLAG(filter) = ""; GTEST_FLAG(filter) = "";
GTEST_FLAG(list_tests) = false; GTEST_FLAG(list_tests) = false;
GTEST_FLAG(output) = ""; GTEST_FLAG(output) = "";
@ -1625,6 +1628,7 @@ class GTestFlagSaverTest : public Test {
EXPECT_FALSE(GTEST_FLAG(catch_exceptions)); EXPECT_FALSE(GTEST_FLAG(catch_exceptions));
EXPECT_STREQ("auto", GTEST_FLAG(color).c_str()); EXPECT_STREQ("auto", GTEST_FLAG(color).c_str());
EXPECT_FALSE(GTEST_FLAG(death_test_use_fork)); EXPECT_FALSE(GTEST_FLAG(death_test_use_fork));
EXPECT_FALSE(GTEST_FLAG(fail_fast));
EXPECT_STREQ("", GTEST_FLAG(filter).c_str()); EXPECT_STREQ("", GTEST_FLAG(filter).c_str());
EXPECT_FALSE(GTEST_FLAG(list_tests)); EXPECT_FALSE(GTEST_FLAG(list_tests));
EXPECT_STREQ("", GTEST_FLAG(output).c_str()); EXPECT_STREQ("", GTEST_FLAG(output).c_str());
@ -1641,6 +1645,7 @@ class GTestFlagSaverTest : public Test {
GTEST_FLAG(catch_exceptions) = true; GTEST_FLAG(catch_exceptions) = true;
GTEST_FLAG(color) = "no"; GTEST_FLAG(color) = "no";
GTEST_FLAG(death_test_use_fork) = true; GTEST_FLAG(death_test_use_fork) = true;
GTEST_FLAG(fail_fast) = true;
GTEST_FLAG(filter) = "abc"; GTEST_FLAG(filter) = "abc";
GTEST_FLAG(list_tests) = true; GTEST_FLAG(list_tests) = true;
GTEST_FLAG(output) = "xml:foo.xml"; GTEST_FLAG(output) = "xml:foo.xml";
@ -5495,20 +5500,22 @@ TEST_F(SetUpTestSuiteTest, TestSetupTestSuite2) {
// The Flags struct stores a copy of all Google Test flags. // The Flags struct stores a copy of all Google Test flags.
struct Flags { struct Flags {
// Constructs a Flags struct where each flag has its default value. // Constructs a Flags struct where each flag has its default value.
Flags() : also_run_disabled_tests(false), Flags()
break_on_failure(false), : also_run_disabled_tests(false),
catch_exceptions(false), break_on_failure(false),
death_test_use_fork(false), catch_exceptions(false),
filter(""), death_test_use_fork(false),
list_tests(false), fail_fast(false),
output(""), filter(""),
print_time(true), list_tests(false),
random_seed(0), output(""),
repeat(1), print_time(true),
shuffle(false), random_seed(0),
stack_trace_depth(kMaxStackTraceDepth), repeat(1),
stream_result_to(""), shuffle(false),
throw_on_failure(false) {} stack_trace_depth(kMaxStackTraceDepth),
stream_result_to(""),
throw_on_failure(false) {}
// Factory methods. // Factory methods.
@ -5544,6 +5551,14 @@ struct Flags {
return flags; return flags;
} }
// Creates a Flags struct where the gtest_fail_fast flag has
// the given value.
static Flags FailFast(bool fail_fast) {
Flags flags;
flags.fail_fast = fail_fast;
return flags;
}
// Creates a Flags struct where the gtest_filter flag has the given // Creates a Flags struct where the gtest_filter flag has the given
// value. // value.
static Flags Filter(const char* filter) { static Flags Filter(const char* filter) {
@ -5629,6 +5644,7 @@ struct Flags {
bool break_on_failure; bool break_on_failure;
bool catch_exceptions; bool catch_exceptions;
bool death_test_use_fork; bool death_test_use_fork;
bool fail_fast;
const char* filter; const char* filter;
bool list_tests; bool list_tests;
const char* output; const char* output;
@ -5650,6 +5666,7 @@ class ParseFlagsTest : public Test {
GTEST_FLAG(break_on_failure) = false; GTEST_FLAG(break_on_failure) = false;
GTEST_FLAG(catch_exceptions) = false; GTEST_FLAG(catch_exceptions) = false;
GTEST_FLAG(death_test_use_fork) = false; GTEST_FLAG(death_test_use_fork) = false;
GTEST_FLAG(fail_fast) = false;
GTEST_FLAG(filter) = ""; GTEST_FLAG(filter) = "";
GTEST_FLAG(list_tests) = false; GTEST_FLAG(list_tests) = false;
GTEST_FLAG(output) = ""; GTEST_FLAG(output) = "";
@ -5680,6 +5697,7 @@ class ParseFlagsTest : public Test {
EXPECT_EQ(expected.break_on_failure, GTEST_FLAG(break_on_failure)); EXPECT_EQ(expected.break_on_failure, GTEST_FLAG(break_on_failure));
EXPECT_EQ(expected.catch_exceptions, GTEST_FLAG(catch_exceptions)); EXPECT_EQ(expected.catch_exceptions, GTEST_FLAG(catch_exceptions));
EXPECT_EQ(expected.death_test_use_fork, GTEST_FLAG(death_test_use_fork)); EXPECT_EQ(expected.death_test_use_fork, GTEST_FLAG(death_test_use_fork));
EXPECT_EQ(expected.fail_fast, GTEST_FLAG(fail_fast));
EXPECT_STREQ(expected.filter, GTEST_FLAG(filter).c_str()); EXPECT_STREQ(expected.filter, GTEST_FLAG(filter).c_str());
EXPECT_EQ(expected.list_tests, GTEST_FLAG(list_tests)); EXPECT_EQ(expected.list_tests, GTEST_FLAG(list_tests));
EXPECT_STREQ(expected.output, GTEST_FLAG(output).c_str()); EXPECT_STREQ(expected.output, GTEST_FLAG(output).c_str());
@ -5766,6 +5784,15 @@ TEST_F(ParseFlagsTest, NoFlag) {
GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags(), false); GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags(), false);
} }
// Tests parsing --gtest_fail_fast.
TEST_F(ParseFlagsTest, FailFast) {
const char* argv[] = {"foo.exe", "--gtest_fail_fast", nullptr};
const char* argv2[] = {"foo.exe", nullptr};
GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::FailFast(true), false);
}
// Tests parsing a bad --gtest_filter flag. // Tests parsing a bad --gtest_filter flag.
TEST_F(ParseFlagsTest, FilterBad) { TEST_F(ParseFlagsTest, FilterBad) {
const char* argv[] = {"foo.exe", "--gtest_filter", nullptr}; const char* argv[] = {"foo.exe", "--gtest_filter", nullptr};