googletest: Add a flag to fail if no tests were selected to run.

There are two cases that prompt this behavior:

- All test cases in the binary are disabled.

- There are more shards defined than test cases, so some shards are empty.

The result in each case is the same: the test runner needs to spin up
additional processes that do nothing, which is wasteful, especially when tests
need expensive resources.

PiperOrigin-RevId: 769176856
Change-Id: Ifa399a0b7b68e4add5a94ca148b32b2938a8666d
This commit is contained in:
David Pizzuto
2025-06-09 09:30:39 -07:00
committed by Copybara-Service
parent 7e2c425db2
commit 28e9d1f267
4 changed files with 123 additions and 5 deletions

View File

@@ -1940,6 +1940,21 @@ test case is linked in.
Note that *any* test case linked in makes the program valid for the purpose of
this check. In particular, even a disabled test case suffices.
### Enforcing Running At Least One Test Case
In addition to enforcing that tests are defined in the binary with
`--gtest_fail_if_no_test_linked`, it is also possible to enforce that a test
case was actually executed to ensure that resources are not consumed by tests
that do nothing.
To catch such optimization opportunities, run the test program with the
`--gtest_fail_if_no_test_selected` flag or set the
`GTEST_FAIL_IF_NO_TEST_SELECTED` environment variable to a value other than `0`.
A test is considered selected if it begins to run, even if it is later skipped
via `GTEST_SKIP`. Thus, `DISABLED` tests do not count as selected and neither do
tests that are not matched by `--gtest_filter`.
### Repeating the Tests
Once in a while you'll run into a test whose result is hit-or-miss. Perhaps it

View File

@@ -269,6 +269,13 @@ GTEST_DEFINE_bool_(
"True if and only if the test should fail if no test case (including "
"disabled test cases) is linked.");
GTEST_DEFINE_bool_(
fail_if_no_test_selected,
testing::internal::BoolFromGTestEnv("fail_if_no_test_selected", false),
"True if and only if the test should fail if no test case is selected to "
"run. A test case is selected to run if it is not disabled and is matched "
"by the filter flag so that it starts executing.");
GTEST_DEFINE_bool_(
also_run_disabled_tests,
testing::internal::BoolFromGTestEnv("also_run_disabled_tests", false),
@@ -6079,6 +6086,14 @@ bool UnitTestImpl::RunAllTests() {
TearDownEnvironment);
repeater->OnEnvironmentsTearDownEnd(*parent_);
}
} else if (GTEST_FLAG_GET(fail_if_no_test_selected)) {
// If there were no tests to run, bail if we were requested to be strict.
constexpr char kNoTestsSelectedMessage[] =
"No tests were selected to run. Please make sure at least one test "
"exists and is not disabled! If the test is sharded, you may have "
"defined more shards than test cases, which is wasteful.";
ColoredPrintf(GTestColor::kRed, "%s\n", kNoTestsSelectedMessage);
return false;
}
elapsed_time_ = timer.Elapsed();
@@ -6763,6 +6778,7 @@ static bool ParseGoogleTestFlag(const char* const arg) {
GTEST_INTERNAL_PARSE_FLAG(death_test_use_fork);
GTEST_INTERNAL_PARSE_FLAG(fail_fast);
GTEST_INTERNAL_PARSE_FLAG(fail_if_no_test_linked);
GTEST_INTERNAL_PARSE_FLAG(fail_if_no_test_selected);
GTEST_INTERNAL_PARSE_FLAG(filter);
GTEST_INTERNAL_PARSE_FLAG(internal_run_death_test);
GTEST_INTERNAL_PARSE_FLAG(list_tests);

View File

@@ -80,11 +80,7 @@ class GTestFailIfNoTestLinkedTest(gtest_test_utils.TestCase):
)
)
warning_file_contents = open(warning_file, "r").read()
self.assertEqual(
warning_file_contents,
"This test program does NOT link in any test case. Please make sure"
" this is intended.\n",
)
self.assertIn("does NOT link", warning_file_contents)
def testFailsIfNoTestLinkedAndFlagSpecified(self):
"""Tests the behavior of no test linked and flag specified."""

View File

@@ -0,0 +1,91 @@
#!/usr/bin/env python3 # pylint: disable=g-interpreter-mismatch
#
# Copyright 2025, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for Google Test's --gtest_fail_if_no_test_selected flag."""
from googletest.test import gtest_test_utils
# The command line flag for enabling the fail-if-no-test-selected behavior.
FAIL_IF_NO_TEST_SELECTED_FLAG = "gtest_fail_if_no_test_selected"
# The environment variable for the test output warnings file.
TEST_WARNINGS_OUTPUT_FILE = "TEST_WARNINGS_OUTPUT_FILE"
class GTestFailIfNoTestSelectedTest(gtest_test_utils.TestCase):
"""Tests the --gtest_fail_if_no_test_selected flag."""
def Run(self, program_name, flags=None, env=None):
"""Run the given program with the given flag.
Args:
program_name: Name of the program to run.
flags: The command line flags to pass to the program, or None.
env: Dictionary with environment to pass to the subprocess.
Returns:
True if the program exits with code 0, false otherwise.
"""
exe_path = gtest_test_utils.GetTestExecutablePath(program_name)
args = [exe_path]
if flags is not None:
args.extend(flags)
process = gtest_test_utils.Subprocess(args, capture_stderr=False, env=env)
return process.exited and process.exit_code == 0
def testSucceedsWhenFlagIsNotSetAndOnlyDisabledTestsPresent(self):
"""Tests that no test selected results in success without the flag set."""
self.assertTrue(
self.Run("googletest-fail-if-no-test-linked-test-with-disabled-test_"),
)
def testFailsWhenFlagIsSetAndOnlyDisabledTestsPresent(self):
"""Tests that no test selected results in failure with the flag set."""
self.assertFalse(
self.Run(
"googletest-fail-if-no-test-linked-test-with-disabled-test_",
flags=[f"--{FAIL_IF_NO_TEST_SELECTED_FLAG}"],
),
)
def testSucceedsWhenFlagIsSetAndEnabledTestsPresent(self):
"""Tests that a test running still succeeds when the flag is set."""
self.assertTrue(
self.Run(
"googletest-fail-if-no-test-linked-test-with-enabled-test_",
flags=[f"--{FAIL_IF_NO_TEST_SELECTED_FLAG}"],
),
)
if __name__ == "__main__":
gtest_test_utils.Main()