Merge pull request #3484 from github/mbg/cli/force-nightly

Add feature for forcing the `nightly` bundle in `dynamic` workflows

.github/workflows/__bundle-from-nightly.yml

@@ -0,0 +1,69 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pr-checks/sync.sh
+# to regenerate this file.
+
+name: 'PR Check - Bundle: From nightly'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+      - main
+      - releases/v*
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+  schedule:
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+    inputs: {}
+  workflow_call:
+    inputs: {}
+defaults:
+  run:
+    shell: bash
+concurrency:
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || false }}
+  group: bundle-from-nightly-${{github.ref}}
+jobs:
+  bundle-from-nightly:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            version: linked
+    name: 'Bundle: From nightly'
+    if: github.triggering_actor != 'dependabot[bot]'
+    permissions:
+      contents: read
+      security-events: read
+    timeout-minutes: 45
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+      - name: Prepare test
+        id: prepare-test
+        uses: ./.github/actions/prepare-test
+        with:
+          version: ${{ matrix.version }}
+          use-all-platform-bundle: 'false'
+          setup-kotlin: 'true'
+      - id: init
+        uses: ./../action/init
+        env:
+          CODEQL_ACTION_FORCE_NIGHTLY: true
+        with:
+          tools: ${{ steps.prepare-test.outputs.tools-url }}
+          languages: javascript
+      - name: Fail if the CodeQL version is not a nightly
+        if: "!contains(steps.init.outputs.codeql-version, '+')"
+        run: exit 1
+    env:
+      CODEQL_ACTION_TEST_MODE: true

lib/analyze-action-post.js

@@ -161566,6 +161566,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

lib/analyze-action.js

@@ -107118,6 +107118,7 @@ function formatDuration(durationMs) {
 
 // src/diagnostics.ts
 var unwrittenDiagnostics = [];
+var unwrittenDefaultLanguageDiagnostics = [];
 function makeDiagnostic(id, name, data = void 0) {
   return {
     ...data,
@@ -107137,6 +107138,19 @@ function addDiagnostic(config, language, diagnostic) {
     unwrittenDiagnostics.push({ diagnostic, language });
   }
 }
+function addNoLanguageDiagnostic(config, diagnostic) {
+  if (config !== void 0) {
+    addDiagnostic(
+      config,
+      // Arbitrarily choose the first language. We could also choose all languages, but that
+      // increases the risk of misinterpreting the data.
+      config.languages[0],
+      diagnostic
+    );
+  } else {
+    unwrittenDefaultLanguageDiagnostics.push(diagnostic);
+  }
+}
 function writeDiagnostic(config, language, diagnostic) {
   const logger = getActionsLogger();
   const databasePath = language ? getCodeQLDatabasePath(config, language) : config.dbLocation;
@@ -107634,6 +107648,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",
@@ -108980,10 +108999,36 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
   let cliVersion2;
   let tagName;
   let url2;
-  if (toolsInput !== void 0 && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput)) {
-    logger.info(
-      `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`
-    );
+  const canForceNightlyWithFF = isDynamicWorkflow() || isInTestMode();
+  const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */);
+  const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
+  const nightlyRequestedByToolsInput = toolsInput !== void 0 && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
+  if (forceNightly || nightlyRequestedByToolsInput) {
+    if (forceNightly) {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as forced by the ${"force_nightly" /* ForceNightly */} feature flag.`
+      );
+      addNoLanguageDiagnostic(
+        void 0,
+        makeDiagnostic(
+          "codeql-action/forced-nightly-cli",
+          "A nightly release of CodeQL was used",
+          {
+            markdownMessage: "GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release.\n\nNightly releases do not undergo the same validation as regular releases and may lead to analysis instability.\n\nIf use of a nightly CodeQL release for this analysis is unexpected, please contact GitHub support.",
+            visibility: {
+              cliSummaryTable: true,
+              statusPage: true,
+              telemetry: true
+            },
+            severity: "note"
+          }
+        )
+      );
+    } else {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`
+      );
+    }
     toolsInput = await getNightlyToolsUrl(logger);
   }
   const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput);

lib/autobuild-action.js

@@ -103971,6 +103971,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

lib/init-action.js

@@ -104575,6 +104575,7 @@ function formatDuration(durationMs) {
 
 // src/diagnostics.ts
 var unwrittenDiagnostics = [];
+var unwrittenDefaultLanguageDiagnostics = [];
 function makeDiagnostic(id, name, data = void 0) {
   return {
     ...data,
@@ -104595,13 +104596,17 @@ function addDiagnostic(config, language, diagnostic) {
   }
 }
 function addNoLanguageDiagnostic(config, diagnostic) {
-  addDiagnostic(
-    config,
-    // Arbitrarily choose the first language. We could also choose all languages, but that
-    // increases the risk of misinterpreting the data.
-    config.languages[0],
-    diagnostic
-  );
+  if (config !== void 0) {
+    addDiagnostic(
+      config,
+      // Arbitrarily choose the first language. We could also choose all languages, but that
+      // increases the risk of misinterpreting the data.
+      config.languages[0],
+      diagnostic
+    );
+  } else {
+    unwrittenDefaultLanguageDiagnostics.push(diagnostic);
+  }
 }
 function writeDiagnostic(config, language, diagnostic) {
   const logger = getActionsLogger();
@@ -104638,13 +104643,16 @@ function logUnwrittenDiagnostics() {
 }
 function flushDiagnostics(config) {
   const logger = getActionsLogger();
-  logger.debug(
-    `Writing ${unwrittenDiagnostics.length} diagnostic(s) to database.`
-  );
+  const diagnosticsCount = unwrittenDiagnostics.length + unwrittenDefaultLanguageDiagnostics.length;
+  logger.debug(`Writing ${diagnosticsCount} diagnostic(s) to database.`);
   for (const unwritten of unwrittenDiagnostics) {
     writeDiagnostic(config, unwritten.language, unwritten.diagnostic);
   }
+  for (const unwritten of unwrittenDefaultLanguageDiagnostics) {
+    addNoLanguageDiagnostic(config, unwritten);
+  }
   unwrittenDiagnostics = [];
+  unwrittenDefaultLanguageDiagnostics = [];
 }
 function makeTelemetryDiagnostic(id, name, attributes) {
   return makeDiagnostic(id, name, {
@@ -105167,6 +105175,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",
@@ -107383,10 +107396,36 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, apiDetails, varian
   let cliVersion2;
   let tagName;
   let url;
-  if (toolsInput !== void 0 && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput)) {
-    logger.info(
-      `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`
-    );
+  const canForceNightlyWithFF = isDynamicWorkflow() || isInTestMode();
+  const forceNightlyValueFF = await features.getValue("force_nightly" /* ForceNightly */);
+  const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
+  const nightlyRequestedByToolsInput = toolsInput !== void 0 && CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
+  if (forceNightly || nightlyRequestedByToolsInput) {
+    if (forceNightly) {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as forced by the ${"force_nightly" /* ForceNightly */} feature flag.`
+      );
+      addNoLanguageDiagnostic(
+        void 0,
+        makeDiagnostic(
+          "codeql-action/forced-nightly-cli",
+          "A nightly release of CodeQL was used",
+          {
+            markdownMessage: "GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release.\n\nNightly releases do not undergo the same validation as regular releases and may lead to analysis instability.\n\nIf use of a nightly CodeQL release for this analysis is unexpected, please contact GitHub support.",
+            visibility: {
+              cliSummaryTable: true,
+              statusPage: true,
+              telemetry: true
+            },
+            severity: "note"
+          }
+        )
+      );
+    } else {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`
+      );
+    }
     toolsInput = await getNightlyToolsUrl(logger);
   }
   const forceShippedTools = toolsInput && CODEQL_BUNDLE_VERSION_ALIAS.includes(toolsInput);

lib/resolve-environment-action.js

@@ -103958,6 +103958,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

lib/start-proxy-action-post.js

@@ -160972,6 +160972,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

lib/start-proxy-action.js

@@ -120664,6 +120664,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

lib/upload-sarif-action-post.js

@@ -161134,6 +161134,11 @@ var featureConfig = {
     legacyApi: true,
     minimumVersion: void 0
   },
+  ["force_nightly" /* ForceNightly */]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: void 0
+  },
   ["ignore_generated_files" /* IgnoreGeneratedFiles */]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

pr-checks/checks/bundle-from-nightly.yml

@@ -0,0 +1,15 @@
+name: "Bundle: From nightly"
+description: "The nightly CodeQL bundle should be used when forced"
+versions:
+  - linked # overruled by the FF set below
+steps:
+  - id: init
+    uses: ./../action/init
+    env:
+      CODEQL_ACTION_FORCE_NIGHTLY: true
+    with:
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+      languages: javascript
+  - name: Fail if the CodeQL version is not a nightly
+    if: "!contains(steps.init.outputs.codeql-version, '+')"
+    run: exit 1

src/diagnostics.ts

@@ -66,6 +66,12 @@ interface UnwrittenDiagnostic {
 /** A list of diagnostics which have not yet been written to disk. */
 let unwrittenDiagnostics: UnwrittenDiagnostic[] = [];
 
+/**
+ * A list of diagnostics which have not yet been written to disk,
+ * and where the language does not matter.
+ */
+let unwrittenDefaultLanguageDiagnostics: DiagnosticMessage[] = [];
+
 /**
  * Constructs a new diagnostic message with the specified id and name, as well as optional additional data.
  *
@@ -119,16 +125,20 @@ export function addDiagnostic(
 
 /** Adds a diagnostic that is not specific to any language. */
 export function addNoLanguageDiagnostic(
-  config: Config,
+  config: Config | undefined,
   diagnostic: DiagnosticMessage,
 ) {
-  addDiagnostic(
-    config,
-    // Arbitrarily choose the first language. We could also choose all languages, but that
-    // increases the risk of misinterpreting the data.
-    config.languages[0],
-    diagnostic,
-  );
+  if (config !== undefined) {
+    addDiagnostic(
+      config,
+      // Arbitrarily choose the first language. We could also choose all languages, but that
+      // increases the risk of misinterpreting the data.
+      config.languages[0],
+      diagnostic,
+    );
+  } else {
+    unwrittenDefaultLanguageDiagnostics.push(diagnostic);
+  }
 }
 
 /**
@@ -188,16 +198,21 @@ export function logUnwrittenDiagnostics() {
 /** Writes all unwritten diagnostics to disk. */
 export function flushDiagnostics(config: Config) {
   const logger = getActionsLogger();
-  logger.debug(
-    `Writing ${unwrittenDiagnostics.length} diagnostic(s) to database.`,
-  );
+
+  const diagnosticsCount =
+    unwrittenDiagnostics.length + unwrittenDefaultLanguageDiagnostics.length;
+  logger.debug(`Writing ${diagnosticsCount} diagnostic(s) to database.`);
 
   for (const unwritten of unwrittenDiagnostics) {
     writeDiagnostic(config, unwritten.language, unwritten.diagnostic);
   }
+  for (const unwritten of unwrittenDefaultLanguageDiagnostics) {
+    addNoLanguageDiagnostic(config, unwritten);
+  }
 
-  // Reset the unwritten diagnostics array.
+  // Reset the unwritten diagnostics arrays.
   unwrittenDiagnostics = [];
+  unwrittenDefaultLanguageDiagnostics = [];
 }
 
 /**

src/feature-flags.ts

@@ -46,6 +46,7 @@ export enum Feature {
   DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
   DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
   ExportDiagnosticsEnabled = "export_diagnostics_enabled",
+  ForceNightly = "force_nightly",
   IgnoreGeneratedFiles = "ignore_generated_files",
   ImprovedProxyCertificates = "improved_proxy_certificates",
   OverlayAnalysis = "overlay_analysis",
@@ -164,6 +165,11 @@ export const featureConfig = {
     legacyApi: true,
     minimumVersion: undefined,
   },
+  [Feature.ForceNightly]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
+    minimumVersion: undefined,
+  },
   [Feature.IgnoreGeneratedFiles]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",

src/setup-codeql.test.ts

@@ -1,18 +1,22 @@
 import * as path from "path";
 
+import * as github from "@actions/github";
 import * as toolcache from "@actions/tool-cache";
 import test, { ExecutionContext } from "ava";
 import * as sinon from "sinon";
 
 import * as actionsUtil from "./actions-util";
+import * as api from "./api-client";
 import { Feature, FeatureEnablement } from "./feature-flags";
 import { getRunnerLogger } from "./logging";
 import * as setupCodeql from "./setup-codeql";
+import * as tar from "./tar";
 import {
   LINKED_CLI_VERSION,
   LoggedMessage,
   SAMPLE_DEFAULT_CLI_VERSION,
   SAMPLE_DOTCOM_API_DETAILS,
+  checkExpectedLogMessages,
   createFeatures,
   getRecordingLogger,
   initializeFeatures,
@@ -268,13 +272,127 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
   });
 });
 
+test("getCodeQLSource correctly returns nightly CLI version when tools == nightly", async (t) => {
+  const loggedMessages: LoggedMessage[] = [];
+  const logger = getRecordingLogger(loggedMessages);
+  const features = createFeatures([]);
+
+  const expectedDate = "30260213";
+  const expectedTag = `codeql-bundle-${expectedDate}`;
+
+  // Ensure that we consistently select "zstd" for the test.
+  sinon.stub(process, "platform").value("linux");
+  sinon.stub(tar, "isZstdAvailable").resolves({
+    available: true,
+    foundZstdBinary: true,
+  });
+
+  const client = github.getOctokit("123");
+  const listReleases = sinon.stub(client.rest.repos, "listReleases");
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+  listReleases.resolves({
+    data: [{ tag_name: expectedTag }],
+  } as any);
+  sinon.stub(api, "getApiClient").value(() => client);
+
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    const source = await setupCodeql.getCodeQLSource(
+      "nightly",
+      SAMPLE_DEFAULT_CLI_VERSION,
+      SAMPLE_DOTCOM_API_DETAILS,
+      GitHubVariant.DOTCOM,
+      false,
+      features,
+      logger,
+    );
+
+    // Check that the `CodeQLToolsSource` object matches our expectations.
+    const expectedVersion = `0.0.0-${expectedDate}`;
+    const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`;
+    t.deepEqual(source, {
+      bundleVersion: expectedDate,
+      cliVersion: undefined,
+      codeqlURL: expectedURL,
+      compressionMethod: "zstd",
+      sourceType: "download",
+      toolsVersion: expectedVersion,
+    } satisfies setupCodeql.CodeQLToolsSource);
+
+    // Afterwards, ensure that we see the expected messages in the log.
+    checkExpectedLogMessages(t, loggedMessages, [
+      "Using the latest CodeQL CLI nightly, as requested by 'tools: nightly'.",
+      `Bundle version ${expectedDate} is not in SemVer format. Will treat it as pre-release ${expectedVersion}.`,
+      `Attempting to obtain CodeQL tools. CLI version: unknown, bundle tag name: ${expectedTag}`,
+      `Using CodeQL CLI sourced from ${expectedURL}`,
+    ]);
+  });
+});
+
+test("getCodeQLSource correctly returns nightly CLI version when forced by FF", async (t) => {
+  const loggedMessages: LoggedMessage[] = [];
+  const logger = getRecordingLogger(loggedMessages);
+  const features = createFeatures([Feature.ForceNightly]);
+
+  const expectedDate = "30260213";
+  const expectedTag = `codeql-bundle-${expectedDate}`;
+
+  // Ensure that we consistently select "zstd" for the test.
+  sinon.stub(process, "platform").value("linux");
+  sinon.stub(tar, "isZstdAvailable").resolves({
+    available: true,
+    foundZstdBinary: true,
+  });
+
+  const client = github.getOctokit("123");
+  const listReleases = sinon.stub(client.rest.repos, "listReleases");
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+  listReleases.resolves({
+    data: [{ tag_name: expectedTag }],
+  } as any);
+  sinon.stub(api, "getApiClient").value(() => client);
+
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    process.env["GITHUB_EVENT_NAME"] = "dynamic";
+
+    const source = await setupCodeql.getCodeQLSource(
+      undefined,
+      SAMPLE_DEFAULT_CLI_VERSION,
+      SAMPLE_DOTCOM_API_DETAILS,
+      GitHubVariant.DOTCOM,
+      false,
+      features,
+      logger,
+    );
+
+    // Check that the `CodeQLToolsSource` object matches our expectations.
+    const expectedVersion = `0.0.0-${expectedDate}`;
+    const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`;
+    t.deepEqual(source, {
+      bundleVersion: expectedDate,
+      cliVersion: undefined,
+      codeqlURL: expectedURL,
+      compressionMethod: "zstd",
+      sourceType: "download",
+      toolsVersion: expectedVersion,
+    } satisfies setupCodeql.CodeQLToolsSource);
+
+    // Afterwards, ensure that we see the expected messages in the log.
+    checkExpectedLogMessages(t, loggedMessages, [
+      `Using the latest CodeQL CLI nightly, as forced by the ${Feature.ForceNightly} feature flag.`,
+      `Bundle version ${expectedDate} is not in SemVer format. Will treat it as pre-release ${expectedVersion}.`,
+      `Attempting to obtain CodeQL tools. CLI version: unknown, bundle tag name: ${expectedTag}`,
+      `Using CodeQL CLI sourced from ${expectedURL}`,
+    ]);
+  });
+});
+
 test("getCodeQLSource correctly returns latest version from toolcache when tools == toolcache", async (t) => {
   const loggedMessages: LoggedMessage[] = [];
   const logger = getRecordingLogger(loggedMessages);
   const features = createFeatures([Feature.AllowToolcacheInput]);
 
-  process.env["GITHUB_EVENT_NAME"] = "dynamic";
-
   const latestToolcacheVersion = "3.2.1";
   const latestVersionPath = "/path/to/latest";
   const testVersions = ["2.3.1", latestToolcacheVersion, "1.2.3"];
@@ -288,6 +406,8 @@ test("getCodeQLSource correctly returns latest version from toolcache when tools
 
   await withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
+    process.env["GITHUB_EVENT_NAME"] = "dynamic";
+
     const source = await setupCodeql.getCodeQLSource(
       "toolcache",
       SAMPLE_DEFAULT_CLI_VERSION,
@@ -343,16 +463,17 @@ const toolcacheInputFallbackMacro = test.macro({
     const logger = getRecordingLogger(loggedMessages);
     const features = createFeatures(featureList);
 
-    for (const [k, v] of Object.entries(environment)) {
-      process.env[k] = v;
-    }
-
     const findAllVersionsStub = sinon
       .stub(toolcache, "findAllVersions")
       .returns(testVersions);
 
     await withTmpDir(async (tmpDir) => {
       setupActionsVars(tmpDir, tmpDir);
+
+      for (const [k, v] of Object.entries(environment)) {
+        process.env[k] = v;
+      }
+
       const source = await setupCodeql.getCodeQLSource(
         "toolcache",
         SAMPLE_DEFAULT_CLI_VERSION,

src/setup-codeql.ts

@@ -10,6 +10,7 @@ import { v4 as uuidV4 } from "uuid";
 import { isDynamicWorkflow, isRunningLocalAction } from "./actions-util";
 import * as api from "./api-client";
 import * as defaults from "./defaults.json";
+import { addNoLanguageDiagnostic, makeDiagnostic } from "./diagnostics";
 import {
   CODEQL_VERSION_ZSTD_BUNDLE,
   CodeQLDefaultVersionInfo,
@@ -55,7 +56,9 @@ function getCodeQLBundleExtension(
   }
 }
 
-function getCodeQLBundleName(compressionMethod: tar.CompressionMethod): string {
+export function getCodeQLBundleName(
+  compressionMethod: tar.CompressionMethod,
+): string {
   const extension = getCodeQLBundleExtension(compressionMethod);
 
   let platform: string;
@@ -196,7 +199,7 @@ export function convertToSemVer(version: string, logger: Logger): string {
   return s;
 }
 
-type CodeQLToolsSource =
+export type CodeQLToolsSource =
   | {
       codeqlTarPath: string;
       compressionMethod: tar.CompressionMethod;
@@ -261,6 +264,20 @@ async function findOverridingToolsInCache(
   return undefined;
 }
 
+/**
+ * Determines where the CodeQL CLI we want to use comes from. This can be from a local file,
+ * the Actions toolcache, or a download.
+ *
+ * @param toolsInput The argument provided for the `tools` input, if any.
+ * @param defaultCliVersion The default CLI version that's linked to the CodeQL Action.
+ * @param apiDetails Information about the GitHub API.
+ * @param variant The GitHub variant we are running on.
+ * @param tarSupportsZstd Whether zstd is supported by `tar`.
+ * @param features Information about enabled features.
+ * @param logger The logger to use.
+ *
+ * @returns Information about where the CodeQL CLI we want to use comes from.
+ */
 export async function getCodeQLSource(
   toolsInput: string | undefined,
   defaultCliVersion: CodeQLDefaultVersionInfo,
@@ -270,6 +287,9 @@ export async function getCodeQLSource(
   features: FeatureEnablement,
   logger: Logger,
 ): Promise<CodeQLToolsSource> {
+  // If there is an explicit `tools` input, it's not one of the reserved values, and it doesn't appear
+  // to point to a URL, then we assume it is a local path and use the CLI from there.
+  // TODO: This appears to misclassify filenames that happen to start with `http` as URLs.
   if (
     toolsInput &&
     !isReservedToolsValue(toolsInput) &&
@@ -302,13 +322,47 @@ export async function getCodeQLSource(
    */
   let url: string | undefined;
 
-  if (
+  // We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the
+  // `tools` input cannot be adjusted to explicitly request it.
+  const canForceNightlyWithFF = isDynamicWorkflow() || util.isInTestMode();
+  const forceNightlyValueFF = await features.getValue(Feature.ForceNightly);
+  const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
+
+  // For advanced workflows, a value from `CODEQL_NIGHTLY_TOOLS_INPUTS` can be specified explicitly
+  // for the `tools` input in the workflow file.
+  const nightlyRequestedByToolsInput =
     toolsInput !== undefined &&
-    CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput)
-  ) {
-    logger.info(
-      `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
-    );
+    CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
+
+  if (forceNightly || nightlyRequestedByToolsInput) {
+    if (forceNightly) {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as forced by the ${Feature.ForceNightly} feature flag.`,
+      );
+      addNoLanguageDiagnostic(
+        undefined,
+        makeDiagnostic(
+          "codeql-action/forced-nightly-cli",
+          "A nightly release of CodeQL was used",
+          {
+            markdownMessage:
+              "GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release.\n\n" +
+              "Nightly releases do not undergo the same validation as regular releases and may lead to analysis instability.\n\n" +
+              "If use of a nightly CodeQL release for this analysis is unexpected, please contact GitHub support.",
+            visibility: {
+              cliSummaryTable: true,
+              statusPage: true,
+              telemetry: true,
+            },
+            severity: "note",
+          },
+        ),
+      );
+    } else {
+      logger.info(
+        `Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
+      );
+    }
     toolsInput = await getNightlyToolsUrl(logger);
   }
 

src/testing-utils.ts

@@ -145,6 +145,7 @@ export function setupActionsVars(tempDir: string, toolsDir: string) {
   process.env["RUNNER_TEMP"] = tempDir;
   process.env["RUNNER_TOOL_CACHE"] = toolsDir;
   process.env["GITHUB_WORKSPACE"] = tempDir;
+  process.env["GITHUB_EVENT_NAME"] = "push";
 }
 
 export interface LoggedMessage {