Add dry run mode so we can dark ship

src/feature-flags.ts

@@ -97,6 +97,12 @@ export enum Feature {
    * database exists in the cache.
    */
   OverlayAnalysisMatchCodeqlVersion = "overlay_analysis_match_codeql_version",
+  /**
+   * Like `OverlayAnalysisMatchCodeqlVersion`, but only logs a diagnostic with the version that
+   * would have been chosen instead of actually changing the default CodeQL CLI version.
+   * `OverlayAnalysisMatchCodeqlVersion` overrides this flag.
+   */
+  OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run",
   OverlayAnalysisPython = "overlay_analysis_python",
   /**
    * Controls whether lower disk space requirements are used for overlay hardware checks.
@@ -307,6 +313,11 @@ export const featureConfig = {
     envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION",
     minimumVersion: undefined,
   },
+  [Feature.OverlayAnalysisMatchCodeqlVersionDryRun]: {
+    defaultValue: false,
+    envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN",
+    minimumVersion: undefined,
+  },
   [Feature.OverlayAnalysisResourceChecksV2]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2",

src/setup-codeql.test.ts

@@ -611,16 +611,20 @@ test.serial(
   },
 );
 
-function makeOverlayMatchFeatures(
-  matchFlagEnabled: boolean,
-): FeatureEnablement {
+function makeOverlayMatchFeatures(opts: {
+  matchFlagEnabled?: boolean;
+  dryRunFlagEnabled?: boolean;
+}): FeatureEnablement {
   return {
     getEnabledDefaultCliVersions: async () => {
       throw new Error("not implemented");
     },
     getValue: async (feature) => {
       if (feature === Feature.OverlayAnalysisMatchCodeqlVersion) {
-        return matchFlagEnabled;
+        return opts.matchFlagEnabled ?? false;
+      }
+      if (feature === Feature.OverlayAnalysisMatchCodeqlVersionDryRun) {
+        return opts.dryRunFlagEnabled ?? false;
       }
       return false;
     },
@@ -657,7 +661,7 @@ test.serial(
     const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
       overlayMatchEnabledVersions,
       ["javascript"],
-      makeOverlayMatchFeatures(true),
+      makeOverlayMatchFeatures({ matchFlagEnabled: true }),
       getRunnerLogger(true),
     );
     t.deepEqual(result, [
@@ -680,32 +684,13 @@ test.serial(
     const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
       overlayMatchEnabledVersions,
       ["javascript"],
-      makeOverlayMatchFeatures(true),
+      makeOverlayMatchFeatures({ matchFlagEnabled: true }),
       getRunnerLogger(true),
     );
     t.deepEqual(result, []);
   },
 );
 
-test.serial(
-  "getEnabledVersionsWithOverlayBaseDatabases does not list caches when gate is off",
-  async (t) => {
-    const listStub = sinon.stub(api, "listActionsCaches").resolves([]);
-
-    const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
-      overlayMatchEnabledVersions,
-      ["javascript"],
-      makeOverlayMatchFeatures(false),
-      getRunnerLogger(true),
-    );
-    t.deepEqual(result, []);
-    t.assert(
-      listStub.notCalled,
-      "Should not list Actions caches when the gating feature flag is off.",
-    );
-  },
-);
-
 test.serial(
   "getEnabledVersionsWithOverlayBaseDatabases does not list caches when rawLanguages is empty",
   async (t) => {
@@ -714,7 +699,7 @@ test.serial(
     const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
       overlayMatchEnabledVersions,
       undefined,
-      makeOverlayMatchFeatures(true),
+      makeOverlayMatchFeatures({ matchFlagEnabled: true }),
       getRunnerLogger(true),
     );
     t.deepEqual(result, []);
@@ -734,7 +719,7 @@ test.serial(
     const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
       overlayMatchEnabledVersions,
       ["javascript"],
-      makeOverlayMatchFeatures(true),
+      makeOverlayMatchFeatures({ matchFlagEnabled: true }),
       getRunnerLogger(true),
     );
     t.deepEqual(result, []);
@@ -754,7 +739,7 @@ test.serial(
     const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
       overlayMatchEnabledVersions,
       ["javascript"],
-      makeOverlayMatchFeatures(true),
+      makeOverlayMatchFeatures({ matchFlagEnabled: true }),
       getRunnerLogger(true),
     );
     t.deepEqual(result, [
@@ -762,3 +747,75 @@ test.serial(
     ]);
   },
 );
+
+test.serial(
+  "getEnabledVersionsWithOverlayBaseDatabases does not list caches when both gates are off",
+  async (t) => {
+    const listStub = sinon.stub(api, "listActionsCaches").resolves([]);
+
+    const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
+      overlayMatchEnabledVersions,
+      ["javascript"],
+      makeOverlayMatchFeatures({}),
+      getRunnerLogger(true),
+    );
+    t.deepEqual(result, []);
+    t.assert(
+      listStub.notCalled,
+      "Should not list Actions caches when both gating feature flags are off.",
+    );
+  },
+);
+
+test.serial(
+  "getEnabledVersionsWithOverlayBaseDatabases dry-run returns empty but lists caches",
+  async (t) => {
+    sinon.stub(api, "getAutomationID").resolves("test/");
+    const listStub = sinon.stub(api, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-aaaaaaaaaaaaaaaa-javascript-2.20.1-abc-1-1",
+      },
+    ]);
+
+    const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
+      overlayMatchEnabledVersions,
+      ["javascript"],
+      makeOverlayMatchFeatures({ dryRunFlagEnabled: true }),
+      getRunnerLogger(true),
+    );
+    t.deepEqual(
+      result,
+      [],
+      "Dry-run should return an empty list so the caller falls back.",
+    );
+    t.assert(
+      listStub.calledOnce,
+      "Dry-run should still list Actions caches to populate the diagnostic.",
+    );
+  },
+);
+
+test.serial(
+  "getEnabledVersionsWithOverlayBaseDatabases match flag wins over dry-run",
+  async (t) => {
+    sinon.stub(api, "getAutomationID").resolves("test/");
+    sinon.stub(api, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-aaaaaaaaaaaaaaaa-javascript-2.20.1-abc-1-1",
+      },
+    ]);
+
+    const result = await setupCodeql.getEnabledVersionsWithOverlayBaseDatabases(
+      overlayMatchEnabledVersions,
+      ["javascript"],
+      makeOverlayMatchFeatures({
+        matchFlagEnabled: true,
+        dryRunFlagEnabled: true,
+      }),
+      getRunnerLogger(true),
+    );
+    t.deepEqual(result, [
+      { cliVersion: "2.20.1", tagName: "codeql-bundle-v2.20.1" },
+    ]);
+  },
+);

src/setup-codeql.ts

@@ -14,7 +14,11 @@ import {
 } from "./actions-util";
 import * as api from "./api-client";
 import * as defaults from "./defaults.json";
-import { addNoLanguageDiagnostic, makeDiagnostic } from "./diagnostics";
+import {
+  addNoLanguageDiagnostic,
+  makeDiagnostic,
+  makeTelemetryDiagnostic,
+} from "./diagnostics";
 import {
   CODEQL_VERSION_ZSTD_BUNDLE,
   CodeQLDefaultVersionInfo,
@@ -270,7 +274,13 @@ async function findOverridingToolsInCache(
   return undefined;
 }
 
-/** Returns the sorted set of enabled versions that have cached overlay-base databases. */
+/**
+ * Returns the sorted set of enabled versions that have cached overlay-base databases for the
+ * given languages, or an empty list if neither the `OverlayAnalysisMatchCodeqlVersion` nor the
+ * `OverlayAnalysisMatchCodeqlVersionDryRun` feature flag is enabled. When only the dry-run flag
+ * is enabled, this performs the lookup and emits a telemetry diagnostic with the version that
+ * would have been chosen, but still returns an empty list so the caller falls back.
+ */
 export async function getEnabledVersionsWithOverlayBaseDatabases(
   defaultCliVersion: CodeQLDefaultVersionInfo,
   rawLanguages: string[] | undefined,
@@ -280,7 +290,13 @@ export async function getEnabledVersionsWithOverlayBaseDatabases(
   if (rawLanguages === undefined || rawLanguages.length === 0) {
     return [];
   }
-  if (!(await features.getValue(Feature.OverlayAnalysisMatchCodeqlVersion))) {
+  const isEnabled = await features.getValue(
+    Feature.OverlayAnalysisMatchCodeqlVersion,
+  );
+  const isDryRun =
+    !isEnabled &&
+    (await features.getValue(Feature.OverlayAnalysisMatchCodeqlVersionDryRun));
+  if (!isEnabled && !isDryRun) {
     return [];
   }
 
@@ -296,14 +312,50 @@ export async function getEnabledVersionsWithOverlayBaseDatabases(
     );
     return [];
   }
+
   if (cachedVersions === undefined || cachedVersions.length === 0) {
     return [];
   }
 
   const cachedVersionsSet = new Set(cachedVersions);
-  return defaultCliVersion.enabledVersions.filter((v) =>
+  const overlayVersions = defaultCliVersion.enabledVersions.filter((v) =>
     cachedVersionsSet.has(v.cliVersion),
   );
+
+  if (overlayVersions.length === 0) {
+    return [];
+  }
+
+  const isCachedVersionDifferent =
+    overlayVersions[0].cliVersion !==
+    defaultCliVersion.enabledVersions[0].cliVersion;
+
+  if (isCachedVersionDifferent) {
+    addNoLanguageDiagnostic(
+      undefined,
+      makeTelemetryDiagnostic(
+        "codeql-action/overlay-aware-default-codeql-version",
+        "Overlay-aware default CodeQL version selection",
+        {
+          cachedVersions,
+          enabledVersions: defaultCliVersion.enabledVersions.map(
+            (v) => v.cliVersion,
+          ),
+          isDryRun,
+          overlayAwareVersion: overlayVersions[0].cliVersion,
+        },
+      ),
+    );
+  }
+
+  if (isDryRun) {
+    logger.debug(
+      `Overlay-aware default CodeQL version selection is running in dry-run mode. Would have used version ${overlayVersions[0].cliVersion}.`,
+    );
+    return [];
+  }
+
+  return overlayVersions;
 }
 
 /**
@@ -334,10 +386,6 @@ async function resolveDefaultCliVersion(
     );
     return overlayVersions[0];
   }
-  logger.info(
-    `Using CodeQL version ${defaultCliVersion.enabledVersions[0].cliVersion} since no enabled ` +
-      `versions with cached overlay-base databases were found.`,
-  );
   return defaultCliVersion.enabledVersions[0];
 }