Merge branch 'main' into sam-robson/overlay-fallback

package-lock.json

@@ -410,15 +410,6 @@
         "undici": "^6.23.0"
       }
     },
-    "node_modules/@actions/github/node_modules/undici": {
-      "version": "6.23.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-6.23.0.tgz",
-      "integrity": "sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=18.17"
-      }
-    },
     "node_modules/@actions/glob": {
       "version": "0.5.1",
       "resolved": "https://registry.npmjs.org/@actions/glob/-/glob-0.5.1.tgz",
@@ -439,15 +430,6 @@
         "undici": "^6.23.0"
       }
     },
-    "node_modules/@actions/http-client/node_modules/undici": {
-      "version": "6.23.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-6.23.0.tgz",
-      "integrity": "sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=18.17"
-      }
-    },
     "node_modules/@actions/io": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@actions/io/-/io-2.0.0.tgz",
@@ -1494,14 +1476,6 @@
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
       }
     },
-    "node_modules/@fastify/busboy": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
-      "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/@github/browserslist-config": {
       "version": "1.0.0",
       "dev": true,
@@ -9854,14 +9828,12 @@
       }
     },
     "node_modules/undici": {
-      "version": "5.29.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz",
-      "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==",
-      "dependencies": {
-        "@fastify/busboy": "^2.0.0"
-      },
+      "version": "6.24.1",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-6.24.1.tgz",
+      "integrity": "sha512-sC+b0tB1whOCzbtlx20fx3WgCXwkW627p4EA9uM+/tNNPkSS+eSEld6pAs9nDv7WbY1UUljBMYPtu9BCOrCWKA==",
+      "license": "MIT",
       "engines": {
-        "node": ">=14.0"
+        "node": ">=18.17"
       }
     },
     "node_modules/undici-types": {

package.json

@@ -90,6 +90,7 @@
       "semver": ">=6.3.1"
     },
     "brace-expansion@2.0.1": "2.0.2",
-    "glob": "^11.1.0"
+    "glob": "^11.1.0",
+    "undici": "^6.24.0"
   }
 }

src/api-client.ts

@@ -128,6 +128,8 @@ export async function getGitHubVersionFromApi(
 
   // Doesn't strictly have to be the meta endpoint as we're only
   // using the response headers which are available on every request.
+  //
+  // See https://docs.github.com/en/rest/meta/meta#get-github-meta-information.
   // eslint-disable-next-line @typescript-eslint/no-unsafe-call
   const response = await apiClient.rest.meta.get();
 
@@ -164,6 +166,9 @@ export async function getGitHubVersion(): Promise<GitHubVersion> {
 
 /**
  * Get the path of the currently executing workflow relative to the repository root.
+ *
+ * See https://docs.github.com/en/rest/actions/workflow-runs#get-a-workflow-run
+ * and https://docs.github.com/en/rest/actions/workflows#get-a-workflow.
  */
 export async function getWorkflowRelativePath(): Promise<string> {
   const repo_nwo = getRepositoryNwo();
@@ -252,9 +257,13 @@ export interface ActionsCacheItem {
   size_in_bytes?: number;
 }
 
-/** List all Actions cache entries matching the provided key and ref. */
+/**
+ * List all Actions cache entries starting with the provided key prefix and matching the provided ref.
+ *
+ * See https://docs.github.com/en/rest/actions/cache#list-github-actions-caches-for-a-repository.
+ */
 export async function listActionsCaches(
-  key: string,
+  keyPrefix: string,
   ref?: string,
 ): Promise<ActionsCacheItem[]> {
   const repositoryNwo = getRepositoryNwo();
@@ -264,13 +273,17 @@ export async function listActionsCaches(
     {
       owner: repositoryNwo.owner,
       repo: repositoryNwo.repo,
-      key,
+      key: keyPrefix,
       ref,
     },
   );
 }
 
-/** Delete an Actions cache item by its ID. */
+/**
+ * Delete an Actions cache item by its ID.
+ *
+ * See https://docs.github.com/en/rest/actions/cache#delete-a-github-actions-cache-for-a-repository-using-a-cache-id.
+ */
 export async function deleteActionsCache(id: number) {
   const repositoryNwo = getRepositoryNwo();
 
@@ -281,7 +294,11 @@ export async function deleteActionsCache(id: number) {
   });
 }
 
-/** Retrieve all custom repository properties. */
+/**
+ * Retrieve all custom repository properties.
+ *
+ * See https://docs.github.com/en/rest/repos/custom-properties#get-all-custom-property-values-for-a-repository.
+ */
 export async function getRepositoryProperties(repositoryNwo: RepositoryNwo) {
   return getApiClient().request("GET /repos/:owner/:repo/properties/values", {
     owner: repositoryNwo.owner,

src/overlay/caching.test.ts

@@ -7,7 +7,7 @@ import * as sinon from "sinon";
 
 import * as actionsUtil from "../actions-util";
 import * as apiClient from "../api-client";
-import { ResolveDatabaseOutput } from "../codeql";
+import type { ResolveDatabaseOutput } from "../codeql";
 import * as gitUtils from "../git-utils";
 import { BuiltInLanguage } from "../languages";
 import { getRunnerLogger } from "../logging";
@@ -23,6 +23,7 @@ import {
   downloadOverlayBaseDatabaseFromCache,
   getCacheRestoreKeyPrefix,
   getCacheSaveKey,
+  getCodeQlVersionsForOverlayBaseDatabases,
 } from "./caching";
 import { OverlayDatabaseMode } from "./overlay-database-mode";
 
@@ -285,3 +286,134 @@ test.serial("overlay-base database cache keys remain stable", async (t) => {
     `Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`,
   );
 });
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns unique versions sorted latest first",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123-1-1",
+      },
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.24.1-def456-2-1",
+      },
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-ghi789-3-1",
+      },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["javascript", "python"],
+      logger,
+    );
+    t.deepEqual(result, ["2.24.1", "2.23.0"]);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns empty list when no caches exist",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["python"],
+      logger,
+    );
+    t.deepEqual(result, []);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns empty list when cache keys are unparseable",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-malformed",
+      },
+      { key: undefined },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["python"],
+      logger,
+    );
+    t.deepEqual(result, []);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases returns the single version when only one cache exists",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1",
+      },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["cpp"],
+      logger,
+    );
+    t.deepEqual(result, ["2.25.0"]);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases resolves language aliases",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+    // The alias `c++` should be resolved to "cpp" and match cache entries keyed with "cpp"
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-cpp-2.25.0-abc123-1-1",
+      },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["c++"],
+      logger,
+    );
+    t.deepEqual(result, ["2.25.0"]);
+  },
+);
+
+test.serial(
+  "getCodeQlVersionsForOverlayBaseDatabases ignores nightly versions with build metadata",
+  async (t) => {
+    const logger = getRunnerLogger(true);
+
+    sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+    sinon.stub(apiClient, "listActionsCaches").resolves([
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1",
+      },
+      {
+        // Nightly release with semver build metadata; should be ignored.
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0+202604211234-def456-2-1",
+      },
+      {
+        key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1",
+      },
+    ]);
+
+    const result = await getCodeQlVersionsForOverlayBaseDatabases(
+      ["python"],
+      logger,
+    );
+    t.deepEqual(result, ["2.25.0", "2.24.0"]);
+  },
+);

src/overlay/caching.ts

@@ -1,18 +1,20 @@
 import * as fs from "fs";
 
 import * as actionsCache from "@actions/cache";
+import * as semver from "semver";
 
 import {
   getRequiredInput,
   getWorkflowRunAttempt,
   getWorkflowRunID,
 } from "../actions-util";
-import { getAutomationID } from "../api-client";
+import { getAutomationID, listActionsCaches } from "../api-client";
 import { createCacheKeyHash } from "../caching-utils";
 import { type CodeQL } from "../codeql";
 import { type Config } from "../config-utils";
 import { getCommitOid } from "../git-utils";
-import { Logger, withGroupAsync } from "../logging";
+import { type Language, parseBuiltInLanguage } from "../languages";
+import { type Logger, withGroupAsync } from "../logging";
 import {
   CleanupLevel,
   getBaseDatabaseOidsFilePath,
@@ -404,7 +406,17 @@ export async function getCacheRestoreKeyPrefix(
   config: Config,
   codeQlVersion: string,
 ): Promise<string> {
-  const languages = [...config.languages].sort().join("_");
+  return `${await getCacheKeyPrefixBase(config.languages)}${codeQlVersion}-`;
+}
+
+/**
+ * Computes the cache key prefix for overlay-base databases, excluding the
+ * CodeQL version.
+ */
+async function getCacheKeyPrefixBase(
+  parsedLanguages: Language[],
+): Promise<string> {
+  const languagesComponent = [...parsedLanguages].sort().join("_");
 
   const cacheKeyComponents = {
     automationID: await getAutomationID(),
@@ -412,17 +424,97 @@ export async function getCacheRestoreKeyPrefix(
   };
   const componentsHash = createCacheKeyHash(cacheKeyComponents);
 
-  // For a cached overlay-base database to be considered compatible for overlay
-  // analysis, all components in the cache restore key must match:
-  //
   // CACHE_PREFIX: distinguishes overlay-base databases from other cache objects
   // CACHE_VERSION: cache format version
   // componentsHash: hash of additional components (see above for details)
-  // languages: the languages included in the overlay-base database
-  // codeQlVersion: CodeQL bundle version
+  // languagesComponent: the languages included in the overlay-base database
   //
-  // Technically we can also include languages and codeQlVersion in the
-  // componentsHash, but including them explicitly in the cache key makes it
-  // easier to debug and understand the cache key structure.
-  return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
+  // Technically we can also include languages in the componentsHash, but
+  // including them explicitly in the cache key makes it easier to debug and
+  // understand the cache key structure.
+  return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`;
+}
+
+/**
+ * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and
+ * returns all stable CodeQL versions found across matching cache entries.
+ *
+ * Note that we do not guarantee that the cache entry for these versions of CodeQL will still be
+ * present by the time we attempt to restore the cache. We could achieve that with a download retry
+ * loop, but we expect that if there is sufficient Actions cache contention that an overlay-base
+ * cache entry for a particular CodeQL version is evicted before we can use it, then it is likely
+ * that the same thing will happen to other overlay-base cache entries, and therefore we will not be
+ * able to use overlay.
+ *
+ * @returns Unique stable CodeQL versions found in cached overlay-base databases, sorted from latest to
+ * earliest, or undefined if one of the languages is not a built-in language.
+ */
+export async function getCodeQlVersionsForOverlayBaseDatabases(
+  rawLanguages: string[],
+  logger: Logger,
+): Promise<string[] | undefined> {
+  const languages = rawLanguages.map(parseBuiltInLanguage);
+  if (languages.includes(undefined)) {
+    logger.warning(
+      "One or more provided languages are not recognized as built-in languages. " +
+        "Skipping searching for overlay-base databases in cache.",
+    );
+    return undefined;
+  }
+  const cacheKeyPrefix = await getCacheKeyPrefixBase(
+    languages.filter((l) => l !== undefined),
+  );
+
+  logger.debug(
+    `Searching for overlay-base databases in Actions cache with ` +
+      `prefix ${cacheKeyPrefix}`,
+  );
+
+  const caches = await listActionsCaches(cacheKeyPrefix);
+
+  if (caches.length === 0) {
+    logger.info("No overlay-base databases found in Actions cache.");
+    return [];
+  }
+
+  logger.info(
+    `Found ${caches.length} overlay-base ` +
+      `${caches.length === 1 ? "database" : "databases"} in the Actions cache.`,
+  );
+
+  // Parse CodeQL versions from cache keys, matching only stable releases.
+  //
+  // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have
+  // a suffix like `+202604201548` that will break the match.
+  //
+  // Caveat: this relies on the fact that we haven't released any CodeQL bundles with the
+  // `x.y.z-<pre-release>` semver format which does not interact well with the current overlay base
+  // DB cache key format.
+  const versionRegex = /^([\d.]+)-/;
+  const versionSet = new Set<string>();
+
+  for (const cache of caches) {
+    if (!cache.key) continue;
+    const suffix = cache.key.substring(cacheKeyPrefix.length);
+    const match = suffix.match(versionRegex);
+    if (match && semver.valid(match[1])) {
+      versionSet.add(match[1]);
+    }
+  }
+
+  if (versionSet.size === 0) {
+    logger.info(
+      "Could not parse any CodeQL versions from overlay-base database " +
+        "cache keys.",
+    );
+    return [];
+  }
+
+  const versions = [...versionSet].sort(semver.rcompare);
+
+  logger.info(
+    `Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
+  );
+
+  return versions;
 }