From 28e9d1f26771c6517c3b4be10254887673c94018 Mon Sep 17 00:00:00 2001 From: David Pizzuto Date: Mon, 9 Jun 2025 09:30:39 -0700 Subject: [PATCH] googletest: Add a flag to fail if no tests were selected to run. There are two cases that prompt this behavior: - All test cases in the binary are disabled. - There are more shards defined than test cases, so some shards are empty. The result in each case is the same: the test runner needs to spin up additional processes that do nothing, which is wasteful, especially when tests need expensive resources. PiperOrigin-RevId: 769176856 Change-Id: Ifa399a0b7b68e4add5a94ca148b32b2938a8666d --- docs/advanced.md | 15 +++ googletest/src/gtest.cc | 16 ++++ .../googletest-fail-if-no-test-linked-test.py | 6 +- ...oogletest-fail-if-no-test-selected-test.py | 91 +++++++++++++++++++ 4 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 googletest/test/googletest-fail-if-no-test-selected-test.py diff --git a/docs/advanced.md b/docs/advanced.md index 35a237df..6b91129d 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -1940,6 +1940,21 @@ test case is linked in. Note that *any* test case linked in makes the program valid for the purpose of this check. In particular, even a disabled test case suffices. +### Enforcing Running At Least One Test Case + +In addition to enforcing that tests are defined in the binary with +`--gtest_fail_if_no_test_linked`, it is also possible to enforce that a test +case was actually executed to ensure that resources are not consumed by tests +that do nothing. + +To catch such optimization opportunities, run the test program with the +`--gtest_fail_if_no_test_selected` flag or set the +`GTEST_FAIL_IF_NO_TEST_SELECTED` environment variable to a value other than `0`. + +A test is considered selected if it begins to run, even if it is later skipped +via `GTEST_SKIP`. Thus, `DISABLED` tests do not count as selected and neither do +tests that are not matched by `--gtest_filter`. + ### Repeating the Tests Once in a while you'll run into a test whose result is hit-or-miss. Perhaps it diff --git a/googletest/src/gtest.cc b/googletest/src/gtest.cc index 6be9300e..9300e5c0 100644 --- a/googletest/src/gtest.cc +++ b/googletest/src/gtest.cc @@ -269,6 +269,13 @@ GTEST_DEFINE_bool_( "True if and only if the test should fail if no test case (including " "disabled test cases) is linked."); +GTEST_DEFINE_bool_( + fail_if_no_test_selected, + testing::internal::BoolFromGTestEnv("fail_if_no_test_selected", false), + "True if and only if the test should fail if no test case is selected to " + "run. A test case is selected to run if it is not disabled and is matched " + "by the filter flag so that it starts executing."); + GTEST_DEFINE_bool_( also_run_disabled_tests, testing::internal::BoolFromGTestEnv("also_run_disabled_tests", false), @@ -6079,6 +6086,14 @@ bool UnitTestImpl::RunAllTests() { TearDownEnvironment); repeater->OnEnvironmentsTearDownEnd(*parent_); } + } else if (GTEST_FLAG_GET(fail_if_no_test_selected)) { + // If there were no tests to run, bail if we were requested to be strict. + constexpr char kNoTestsSelectedMessage[] = + "No tests were selected to run. Please make sure at least one test " + "exists and is not disabled! If the test is sharded, you may have " + "defined more shards than test cases, which is wasteful."; + ColoredPrintf(GTestColor::kRed, "%s\n", kNoTestsSelectedMessage); + return false; } elapsed_time_ = timer.Elapsed(); @@ -6763,6 +6778,7 @@ static bool ParseGoogleTestFlag(const char* const arg) { GTEST_INTERNAL_PARSE_FLAG(death_test_use_fork); GTEST_INTERNAL_PARSE_FLAG(fail_fast); GTEST_INTERNAL_PARSE_FLAG(fail_if_no_test_linked); + GTEST_INTERNAL_PARSE_FLAG(fail_if_no_test_selected); GTEST_INTERNAL_PARSE_FLAG(filter); GTEST_INTERNAL_PARSE_FLAG(internal_run_death_test); GTEST_INTERNAL_PARSE_FLAG(list_tests); diff --git a/googletest/test/googletest-fail-if-no-test-linked-test.py b/googletest/test/googletest-fail-if-no-test-linked-test.py index f5854ba9..e62bfb29 100755 --- a/googletest/test/googletest-fail-if-no-test-linked-test.py +++ b/googletest/test/googletest-fail-if-no-test-linked-test.py @@ -80,11 +80,7 @@ class GTestFailIfNoTestLinkedTest(gtest_test_utils.TestCase): ) ) warning_file_contents = open(warning_file, "r").read() - self.assertEqual( - warning_file_contents, - "This test program does NOT link in any test case. Please make sure" - " this is intended.\n", - ) + self.assertIn("does NOT link", warning_file_contents) def testFailsIfNoTestLinkedAndFlagSpecified(self): """Tests the behavior of no test linked and flag specified.""" diff --git a/googletest/test/googletest-fail-if-no-test-selected-test.py b/googletest/test/googletest-fail-if-no-test-selected-test.py new file mode 100644 index 00000000..03a324cd --- /dev/null +++ b/googletest/test/googletest-fail-if-no-test-selected-test.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 # pylint: disable=g-interpreter-mismatch +# +# Copyright 2025, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Tests for Google Test's --gtest_fail_if_no_test_selected flag.""" + +from googletest.test import gtest_test_utils + + +# The command line flag for enabling the fail-if-no-test-selected behavior. +FAIL_IF_NO_TEST_SELECTED_FLAG = "gtest_fail_if_no_test_selected" + +# The environment variable for the test output warnings file. +TEST_WARNINGS_OUTPUT_FILE = "TEST_WARNINGS_OUTPUT_FILE" + + +class GTestFailIfNoTestSelectedTest(gtest_test_utils.TestCase): + """Tests the --gtest_fail_if_no_test_selected flag.""" + + def Run(self, program_name, flags=None, env=None): + """Run the given program with the given flag. + + Args: + program_name: Name of the program to run. + flags: The command line flags to pass to the program, or None. + env: Dictionary with environment to pass to the subprocess. + + Returns: + True if the program exits with code 0, false otherwise. + """ + + exe_path = gtest_test_utils.GetTestExecutablePath(program_name) + args = [exe_path] + if flags is not None: + args.extend(flags) + process = gtest_test_utils.Subprocess(args, capture_stderr=False, env=env) + return process.exited and process.exit_code == 0 + + def testSucceedsWhenFlagIsNotSetAndOnlyDisabledTestsPresent(self): + """Tests that no test selected results in success without the flag set.""" + self.assertTrue( + self.Run("googletest-fail-if-no-test-linked-test-with-disabled-test_"), + ) + + def testFailsWhenFlagIsSetAndOnlyDisabledTestsPresent(self): + """Tests that no test selected results in failure with the flag set.""" + self.assertFalse( + self.Run( + "googletest-fail-if-no-test-linked-test-with-disabled-test_", + flags=[f"--{FAIL_IF_NO_TEST_SELECTED_FLAG}"], + ), + ) + + def testSucceedsWhenFlagIsSetAndEnabledTestsPresent(self): + """Tests that a test running still succeeds when the flag is set.""" + self.assertTrue( + self.Run( + "googletest-fail-if-no-test-linked-test-with-enabled-test_", + flags=[f"--{FAIL_IF_NO_TEST_SELECTED_FLAG}"], + ), + ) + + +if __name__ == "__main__": + gtest_test_utils.Main()