Retrieve CodeQL versions associated with cached overlay base DBs

src/overlay/caching.test.ts

@@ -23,6 +23,7 @@ import {
   downloadOverlayBaseDatabaseFromCache,
   getCacheRestoreKeyPrefix,
   getCacheSaveKey,
+  getCodeQlVersionsForOverlayBaseDatabases,
 } from "./caching";
 import { OverlayDatabaseMode } from "./overlay-database-mode";
 
@@ -285,3 +286,107 @@ test.serial("overlay-base database cache keys remain stable", async (t) => {
     `Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`,
   );
 });
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns unique versions sorted latest first",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123-1-1",
+      },
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.24.1-def456-2-1",
+      },
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-ghi789-3-1",
+      },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["javascript", "python"],
+      logger,
+    );
+    t.deepEqual(result, ["2.24.1", "2.23.0"]);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns empty list when no caches exist",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["python"],
+      logger,
+    );
+    t.deepEqual(result, []);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns empty list when cache keys are unparseable",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-malformed",
+      },
+      { key: undefined },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["python"],
+      logger,
+    );
+    t.deepEqual(result, []);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns the single version when only one cache exists",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1",
+      },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["cpp"],
+      logger,
+    );
+    t.deepEqual(result, ["2.25.0"]);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases resolves language aliases",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+    // The alias `c++` should be resolved to "cpp" and match cache entries keyed with "cpp"
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1",
+      },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["c++"],
+      logger,
+    );
+    t.deepEqual(result, ["2.25.0"]);
+  },
+);

src/overlay/caching.ts

@@ -1,17 +1,19 @@
 import * as fs from "fs";
 
 import * as actionsCache from "@actions/cache";
+import * as semver from "semver";
 
 import {
   getRequiredInput,
   getWorkflowRunAttempt,
   getWorkflowRunID,
 } from "../actions-util";
-import { getAutomationID } from "../api-client";
+import { getAutomationID, listActionsCaches } from "../api-client";
 import { createCacheKeyHash } from "../caching-utils";
 import { type CodeQL } from "../codeql";
 import { type Config } from "../config-utils";
 import { getCommitOid } from "../git-utils";
+import { Language, parseBuiltInLanguage } from "../languages";
 import { Logger, withGroupAsync } from "../logging";
 import {
   CleanupLevel,
@@ -404,7 +406,17 @@ export async function getCacheRestoreKeyPrefix(
   config: Config,
   codeQlVersion: string,
 ): Promise<string> {
-  const languages = [...config.languages].sort().join("_");
+  return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`;
+}
+
+/**
+ * Computes the cache key prefix for overlay-base databases, excluding the
+ * CodeQL version.
+ */
+async function getCacheKeyPrefixBase(
+  parsedLanguages: Language[],
+): Promise<string> {
+  const languagesComponent = [...parsedLanguages].sort().join("_");
 
   const cacheKeyComponents = {
     automationID: await getAutomationID(),
@@ -412,17 +424,85 @@ export async function getCacheRestoreKeyPrefix(
   };
   const componentsHash = createCacheKeyHash(cacheKeyComponents);
 
-  // For a cached overlay-base database to be considered compatible for overlay
-  // analysis, all components in the cache restore key must match:
-  //
   // CACHE_PREFIX: distinguishes overlay-base databases from other cache objects
   // CACHE_VERSION: cache format version
   // componentsHash: hash of additional components (see above for details)
-  // languages: the languages included in the overlay-base database
-  // codeQlVersion: CodeQL bundle version
+  // languagesComponent: the languages included in the overlay-base database
   //
-  // Technically we can also include languages and codeQlVersion in the
-  // componentsHash, but including them explicitly in the cache key makes it
-  // easier to debug and understand the cache key structure.
-  return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
+  // Technically we can also include languages in the componentsHash, but
+  // including them explicitly in the cache key makes it easier to debug and
+  // understand the cache key structure.
+  return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`;
+}
+
+/**
+ * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and
+ * returns all CodeQL versions found across matching cache entries.
+ *
+ * @returns Unique CodeQL versions found in cached overlay-base databases, sorted from latest to
+ * earliest, or undefined if one of the languages is not a built-in language.
+ */
+export async function getCodeQlVersionsForOverlayBaseDatabases(
+  rawLanguages: string[],
+  logger: Logger,
+): Promise<string[] | undefined> {
+  const languages = rawLanguages.map(parseBuiltInLanguage);
+  if (languages.includes(undefined)) {
+    logger.warning(
+      "One or more provided languages are not recognized as built-in languages. " +
+        "Skipping searching for overlay-base databases in cache.",
+    );
+    return undefined;
+  }
+  const cacheKeyPrefix = await getCacheKeyPrefixBase(
+    languages.filter((l) => l !== undefined),
+  );
+
+  logger.debug(
+    `Searching for overlay-base databases in Actions cache with ` +
+      `prefix ${cacheKeyPrefix}`,
+  );
+
+  const caches = await listActionsCaches(cacheKeyPrefix);
+
+  if (caches.length === 0) {
+    logger.info("No overlay-base databases found in Actions cache.");
+    return [];
+  }
+
+  logger.info(
+    `Found ${caches.length} overlay-base ` +
+      `${caches.length === 1 ? "database" : "databases"} in the Actions cache.`,
+  );
+
+  // Parse CodeQL versions from cache keys.
+  // After the prefix, the remaining key format starts with
+  // `${codeQlVersion}-`.
+  const versionRegex = /^([\d.]+)-/;
+  const versionSet = new Set<string>();
+
+  for (const cache of caches) {
+    if (!cache.key) continue;
+    const suffix = cache.key.substring(cacheKeyPrefix.length);
+    const match = suffix.match(versionRegex);
+    if (match && semver.valid(match[1])) {
+      versionSet.add(match[1]);
+    }
+  }
+
+  if (versionSet.size === 0) {
+    logger.info(
+      "Could not parse any CodeQL versions from overlay-base database " +
+        "cache keys.",
+    );
+    return [];
+  }
+
+  const versions = [...versionSet].sort(semver.rcompare);
+
+  logger.info(
+    `Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
+  );
+
+  return versions;
 }