Refactoring: Introduce overlay/caching.ts

src/analyze-action.ts

@@ -32,7 +32,7 @@ import { EnvVar } from "./environment";
 import { initFeatures } from "./feature-flags";
 import { KnownLanguage } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
-import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay";
+import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay/caching";
 import { getRepositoryNwo } from "./repository";
 import * as statusReport from "./status-report";
 import {

src/analyze.ts

@@ -23,7 +23,7 @@ import { EnvVar } from "./environment";
 import { FeatureEnablement, Feature } from "./feature-flags";
 import { KnownLanguage, Language } from "./languages";
 import { Logger, withGroupAsync } from "./logging";
-import { OverlayDatabaseMode } from "./overlay";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import type * as sarif from "./sarif";
 import { DatabaseCreationTimings, EventReport } from "./status-report";
 import { endTracingForCluster } from "./tracer-config";

src/codeql.ts

@@ -24,11 +24,8 @@ import {
 import { isAnalyzingDefaultBranch } from "./git-utils";
 import { Language } from "./languages";
 import { Logger } from "./logging";
-import {
-  OverlayDatabaseMode,
-  writeBaseDatabaseOidsFile,
-  writeOverlayChangesFile,
-} from "./overlay";
+import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from "./overlay";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import * as setupCodeql from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
 import { ToolsDownloadStatusReport } from "./tools-download";

src/config-utils.test.ts

@@ -20,8 +20,9 @@ import * as gitUtils from "./git-utils";
 import { GitVersionInfo } from "./git-utils";
 import { KnownLanguage, Language } from "./languages";
 import { getRunnerLogger } from "./logging";
-import { CODEQL_OVERLAY_MINIMUM_VERSION, OverlayDatabaseMode } from "./overlay";
+import { CODEQL_OVERLAY_MINIMUM_VERSION } from "./overlay";
 import { OverlayDisabledReason } from "./overlay/diagnostics";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import * as overlayStatus from "./overlay/status";
 import { parseRepositoryNwo } from "./repository";
 import {

src/config-utils.ts

@@ -50,11 +50,12 @@ import {
 } from "./git-utils";
 import { KnownLanguage, Language } from "./languages";
 import { Logger } from "./logging";
-import { CODEQL_OVERLAY_MINIMUM_VERSION, OverlayDatabaseMode } from "./overlay";
+import { CODEQL_OVERLAY_MINIMUM_VERSION } from "./overlay";
 import {
   addOverlayDisablementDiagnostics,
   OverlayDisabledReason,
 } from "./overlay/diagnostics";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import { shouldSkipOverlayAnalysis } from "./overlay/status";
 import { RepositoryNwo } from "./repository";
 import { ToolsFeature } from "./tools-features";

src/database-upload.ts

@@ -12,7 +12,7 @@ import { Config } from "./config-utils";
 import { Feature, FeatureEnablement } from "./feature-flags";
 import * as gitUtils from "./git-utils";
 import { Logger, withGroupAsync } from "./logging";
-import { OverlayDatabaseMode } from "./overlay";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import { RepositoryNwo } from "./repository";
 import * as util from "./util";
 import { asHTTPError, bundleDb, CleanupLevel, parseGitHubUrl } from "./util";

src/init-action-post-helper.test.ts

@@ -12,7 +12,7 @@ import { EnvVar } from "./environment";
 import { Feature } from "./feature-flags";
 import * as initActionPostHelper from "./init-action-post-helper";
 import { getRunnerLogger } from "./logging";
-import { OverlayDatabaseMode } from "./overlay";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import * as overlayStatus from "./overlay/status";
 import { parseRepositoryNwo } from "./repository";
 import {

src/init-action-post-helper.ts

@@ -21,7 +21,7 @@ import * as dependencyCaching from "./dependency-caching";
 import { EnvVar } from "./environment";
 import { Feature, FeatureEnablement } from "./feature-flags";
 import { Logger } from "./logging";
-import { OverlayDatabaseMode } from "./overlay";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import {
   createOverlayStatus,
   OverlayStatus,

src/init-action.ts

@@ -63,8 +63,8 @@ import { getActionsLogger, Logger, withGroupAsync } from "./logging";
 import {
   downloadOverlayBaseDatabaseFromCache,
   OverlayBaseDatabaseDownloadStats,
-  OverlayDatabaseMode,
-} from "./overlay";
+} from "./overlay/caching";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import { getRepositoryNwo, RepositoryNwo } from "./repository";
 import { ToolsSource } from "./setup-codeql";
 import {

src/overlay/caching.test.ts

@@ -0,0 +1,287 @@
+import * as fs from "fs";
+import * as path from "path";
+
+import * as actionsCache from "@actions/cache";
+import test from "ava";
+import * as sinon from "sinon";
+
+import * as actionsUtil from "../actions-util";
+import * as apiClient from "../api-client";
+import { ResolveDatabaseOutput } from "../codeql";
+import * as gitUtils from "../git-utils";
+import { KnownLanguage } from "../languages";
+import { getRunnerLogger } from "../logging";
+import {
+  createTestConfig,
+  mockCodeQLVersion,
+  setupTests,
+} from "../testing-utils";
+import * as utils from "../util";
+import { withTmpDir } from "../util";
+
+import {
+  downloadOverlayBaseDatabaseFromCache,
+  getCacheRestoreKeyPrefix,
+  getCacheSaveKey,
+} from "./caching";
+import { OverlayDatabaseMode } from "./overlay-database-mode";
+
+setupTests(test);
+
+interface DownloadOverlayBaseDatabaseTestCase {
+  overlayDatabaseMode: OverlayDatabaseMode;
+  useOverlayDatabaseCaching: boolean;
+  isInTestMode: boolean;
+  restoreCacheResult: string | undefined | Error;
+  hasBaseDatabaseOidsFile: boolean;
+  tryGetFolderBytesSucceeds: boolean;
+  codeQLVersion: string;
+  resolveDatabaseOutput: ResolveDatabaseOutput | Error;
+}
+
+const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
+  overlayDatabaseMode: OverlayDatabaseMode.Overlay,
+  useOverlayDatabaseCaching: true,
+  isInTestMode: false,
+  restoreCacheResult: "cache-key",
+  hasBaseDatabaseOidsFile: true,
+  tryGetFolderBytesSucceeds: true,
+  codeQLVersion: "2.20.5",
+  resolveDatabaseOutput: { overlayBaseSpecifier: "20250626:XXX" },
+};
+
+const testDownloadOverlayBaseDatabaseFromCache = test.macro({
+  exec: async (
+    t,
+    _title: string,
+    partialTestCase: Partial<DownloadOverlayBaseDatabaseTestCase>,
+    expectDownloadSuccess: boolean,
+  ) => {
+    await withTmpDir(async (tmpDir) => {
+      const dbLocation = path.join(tmpDir, "db");
+      await fs.promises.mkdir(dbLocation, { recursive: true });
+
+      const logger = getRunnerLogger(true);
+      const testCase = { ...defaultDownloadTestCase, ...partialTestCase };
+      const config = createTestConfig({
+        dbLocation,
+        languages: [KnownLanguage.java],
+      });
+
+      config.overlayDatabaseMode = testCase.overlayDatabaseMode;
+      config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching;
+
+      if (testCase.hasBaseDatabaseOidsFile) {
+        const baseDatabaseOidsFile = path.join(
+          dbLocation,
+          "base-database-oids.json",
+        );
+        await fs.promises.writeFile(baseDatabaseOidsFile, JSON.stringify({}));
+      }
+
+      const stubs: sinon.SinonStub[] = [];
+
+      const getAutomationIDStub = sinon
+        .stub(apiClient, "getAutomationID")
+        .resolves("test-automation-id/");
+      stubs.push(getAutomationIDStub);
+
+      const isInTestModeStub = sinon
+        .stub(utils, "isInTestMode")
+        .returns(testCase.isInTestMode);
+      stubs.push(isInTestModeStub);
+
+      if (testCase.restoreCacheResult instanceof Error) {
+        const restoreCacheStub = sinon
+          .stub(actionsCache, "restoreCache")
+          .rejects(testCase.restoreCacheResult);
+        stubs.push(restoreCacheStub);
+      } else {
+        const restoreCacheStub = sinon
+          .stub(actionsCache, "restoreCache")
+          .resolves(testCase.restoreCacheResult);
+        stubs.push(restoreCacheStub);
+      }
+
+      const tryGetFolderBytesStub = sinon
+        .stub(utils, "tryGetFolderBytes")
+        .resolves(testCase.tryGetFolderBytesSucceeds ? 1024 * 1024 : undefined);
+      stubs.push(tryGetFolderBytesStub);
+
+      const codeql = mockCodeQLVersion(testCase.codeQLVersion);
+
+      if (testCase.resolveDatabaseOutput instanceof Error) {
+        const resolveDatabaseStub = sinon
+          .stub(codeql, "resolveDatabase")
+          .rejects(testCase.resolveDatabaseOutput);
+        stubs.push(resolveDatabaseStub);
+      } else {
+        const resolveDatabaseStub = sinon
+          .stub(codeql, "resolveDatabase")
+          .resolves(testCase.resolveDatabaseOutput);
+        stubs.push(resolveDatabaseStub);
+      }
+
+      try {
+        const result = await downloadOverlayBaseDatabaseFromCache(
+          codeql,
+          config,
+          logger,
+        );
+
+        if (expectDownloadSuccess) {
+          t.truthy(result);
+        } else {
+          t.is(result, undefined);
+        }
+      } finally {
+        for (const stub of stubs) {
+          stub.restore();
+        }
+      }
+    });
+  },
+  title: (_, title) => `downloadOverlayBaseDatabaseFromCache: ${title}`,
+});
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns stats when successful",
+  {},
+  true,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when mode is OverlayDatabaseMode.OverlayBase",
+  {
+    overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when mode is OverlayDatabaseMode.None",
+  {
+    overlayDatabaseMode: OverlayDatabaseMode.None,
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when caching is disabled",
+  {
+    useOverlayDatabaseCaching: false,
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined in test mode",
+  {
+    isInTestMode: true,
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when cache miss",
+  {
+    restoreCacheResult: undefined,
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when download fails",
+  {
+    restoreCacheResult: new Error("Download failed"),
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when downloaded database is invalid",
+  {
+    hasBaseDatabaseOidsFile: false,
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when downloaded database doesn't have an overlayBaseSpecifier",
+  {
+    resolveDatabaseOutput: {},
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when resolving database metadata fails",
+  {
+    resolveDatabaseOutput: new Error("Failed to resolve database metadata"),
+  },
+  false,
+);
+
+test.serial(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when filesystem error occurs",
+  {
+    tryGetFolderBytesSucceeds: false,
+  },
+  false,
+);
+
+test.serial("overlay-base database cache keys remain stable", async (t) => {
+  const logger = getRunnerLogger(true);
+  const config = createTestConfig({ languages: ["python", "javascript"] });
+  const codeQlVersion = "2.23.0";
+  const commitOid = "abc123def456";
+
+  sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
+  sinon.stub(gitUtils, "getCommitOid").resolves(commitOid);
+  sinon.stub(actionsUtil, "getWorkflowRunID").returns(12345);
+  sinon.stub(actionsUtil, "getWorkflowRunAttempt").returns(1);
+
+  const saveKey = await getCacheSaveKey(
+    config,
+    codeQlVersion,
+    "checkout-path",
+    logger,
+  );
+  const expectedSaveKey =
+    "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456-12345-1";
+  t.is(
+    saveKey,
+    expectedSaveKey,
+    "Cache save key changed unexpectedly. " +
+      "This may indicate breaking changes in the cache key generation logic.",
+  );
+
+  const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
+    config,
+    codeQlVersion,
+  );
+  const expectedRestoreKeyPrefix =
+    "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-";
+  t.is(
+    restoreKeyPrefix,
+    expectedRestoreKeyPrefix,
+    "Cache restore key prefix changed unexpectedly. " +
+      "This may indicate breaking changes in the cache key generation logic.",
+  );
+
+  t.true(
+    saveKey.startsWith(restoreKeyPrefix),
+    `Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`,
+  );
+});

src/overlay/caching.ts

@@ -0,0 +1,428 @@
+import * as fs from "fs";
+
+import * as actionsCache from "@actions/cache";
+
+import {
+  getRequiredInput,
+  getWorkflowRunAttempt,
+  getWorkflowRunID,
+} from "../actions-util";
+import { getAutomationID } from "../api-client";
+import { createCacheKeyHash } from "../caching-utils";
+import { type CodeQL } from "../codeql";
+import { type Config } from "../config-utils";
+import { getCommitOid } from "../git-utils";
+import { Logger, withGroupAsync } from "../logging";
+import {
+  CleanupLevel,
+  getBaseDatabaseOidsFilePath,
+  getCodeQLDatabasePath,
+  getErrorMessage,
+  isInTestMode,
+  tryGetFolderBytes,
+  waitForResultWithTimeLimit,
+} from "../util";
+
+import { OverlayDatabaseMode } from "./overlay-database-mode";
+
+/**
+ * The maximum (uncompressed) size of the overlay base database that we will
+ * upload. By default, the Actions Cache has an overall capacity of 10 GB, and
+ * the Actions Cache client library uses zstd compression.
+ *
+ * Ideally we would apply a size limit to the compressed overlay-base database,
+ * but we cannot do so because compression is handled transparently by the
+ * Actions Cache client library. Instead we place a limit on the uncompressed
+ * size of the overlay-base database.
+ *
+ * Assuming 2.5:1 compression ratio, the 7.5 GB limit on uncompressed data would
+ * translate to a limit of around 3 GB after compression.
+ */
+const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
+const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
+  OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000;
+
+// Constants for database caching
+const CACHE_VERSION = 1;
+const CACHE_PREFIX = "codeql-overlay-base-database";
+
+// The purpose of this ten-minute limit is to guard against the possibility
+// that the cache service is unresponsive, which would otherwise cause the
+// entire action to hang.  Normally we expect cache operations to complete
+// within two minutes.
+const MAX_CACHE_OPERATION_MS = 600_000;
+
+/**
+ * Checks that the overlay-base database is valid by checking for the
+ * existence of the base database OIDs file.
+ *
+ * @param config The configuration object
+ * @param logger The logger instance
+ * @param warningPrefix Prefix for the check failure warning message
+ * @returns True if the verification succeeded, false otherwise
+ */
+async function checkOverlayBaseDatabase(
+  codeql: CodeQL,
+  config: Config,
+  logger: Logger,
+  warningPrefix: string,
+): Promise<boolean> {
+  // An overlay-base database should contain the base database OIDs file.
+  const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
+  if (!fs.existsSync(baseDatabaseOidsFilePath)) {
+    logger.warning(
+      `${warningPrefix}: ${baseDatabaseOidsFilePath} does not exist`,
+    );
+    return false;
+  }
+
+  for (const language of config.languages) {
+    const dbPath = getCodeQLDatabasePath(config, language);
+    try {
+      const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
+      if (
+        resolveDatabaseOutput === undefined ||
+        !("overlayBaseSpecifier" in resolveDatabaseOutput)
+      ) {
+        logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
+        return false;
+      } else {
+        logger.debug(
+          `Overlay base specifier for ${language} overlay-base database found: ` +
+            `${resolveDatabaseOutput.overlayBaseSpecifier}`,
+        );
+      }
+    } catch (e) {
+      logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/**
+ * Uploads the overlay-base database to the GitHub Actions cache. If conditions
+ * for uploading are not met, the function does nothing and returns false.
+ *
+ * This function uses the `checkout_path` input to determine the repository path
+ * and works only when called from `analyze` or `upload-sarif`.
+ *
+ * @param codeql The CodeQL instance
+ * @param config The configuration object
+ * @param logger The logger instance
+ * @returns A promise that resolves to true if the upload was performed and
+ * successfully completed, or false otherwise
+ */
+export async function cleanupAndUploadOverlayBaseDatabaseToCache(
+  codeql: CodeQL,
+  config: Config,
+  logger: Logger,
+): Promise<boolean> {
+  const overlayDatabaseMode = config.overlayDatabaseMode;
+  if (overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase) {
+    logger.debug(
+      `Overlay database mode is ${overlayDatabaseMode}. ` +
+        "Skip uploading overlay-base database to cache.",
+    );
+    return false;
+  }
+  if (!config.useOverlayDatabaseCaching) {
+    logger.debug(
+      "Overlay database caching is disabled. " +
+        "Skip uploading overlay-base database to cache.",
+    );
+    return false;
+  }
+  if (isInTestMode()) {
+    logger.debug(
+      "In test mode. Skip uploading overlay-base database to cache.",
+    );
+    return false;
+  }
+
+  const databaseIsValid = await checkOverlayBaseDatabase(
+    codeql,
+    config,
+    logger,
+    "Abort uploading overlay-base database to cache",
+  );
+  if (!databaseIsValid) {
+    return false;
+  }
+
+  // Clean up the database using the overlay cleanup level.
+  await withGroupAsync("Cleaning up databases", async () => {
+    await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
+  });
+
+  const dbLocation = config.dbLocation;
+
+  const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
+  if (databaseSizeBytes === undefined) {
+    logger.warning(
+      "Failed to determine database size. " +
+        "Skip uploading overlay-base database to cache.",
+    );
+    return false;
+  }
+
+  if (databaseSizeBytes > OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES) {
+    const databaseSizeMB = Math.round(databaseSizeBytes / 1_000_000);
+    logger.warning(
+      `Database size (${databaseSizeMB} MB) ` +
+        `exceeds maximum upload size (${OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB} MB). ` +
+        "Skip uploading overlay-base database to cache.",
+    );
+    return false;
+  }
+
+  const codeQlVersion = (await codeql.getVersion()).version;
+  const checkoutPath = getRequiredInput("checkout_path");
+  const cacheSaveKey = await getCacheSaveKey(
+    config,
+    codeQlVersion,
+    checkoutPath,
+    logger,
+  );
+  logger.info(
+    `Uploading overlay-base database to Actions cache with key ${cacheSaveKey}`,
+  );
+
+  try {
+    const cacheId = await waitForResultWithTimeLimit(
+      MAX_CACHE_OPERATION_MS,
+      actionsCache.saveCache([dbLocation], cacheSaveKey),
+      () => {},
+    );
+    if (cacheId === undefined) {
+      logger.warning("Timed out while uploading overlay-base database");
+      return false;
+    }
+  } catch (error) {
+    logger.warning(
+      "Failed to upload overlay-base database to cache: " +
+        `${error instanceof Error ? error.message : String(error)}`,
+    );
+    return false;
+  }
+  logger.info(`Successfully uploaded overlay-base database from ${dbLocation}`);
+  return true;
+}
+
+export interface OverlayBaseDatabaseDownloadStats {
+  databaseSizeBytes: number;
+  databaseDownloadDurationMs: number;
+}
+
+/**
+ * Downloads the overlay-base database from the GitHub Actions cache. If conditions
+ * for downloading are not met, the function does nothing and returns false.
+ *
+ * @param codeql The CodeQL instance
+ * @param config The configuration object
+ * @param logger The logger instance
+ * @returns A promise that resolves to download statistics if an overlay-base
+ * database was successfully downloaded, or undefined if the download was
+ * either not performed or failed.
+ */
+export async function downloadOverlayBaseDatabaseFromCache(
+  codeql: CodeQL,
+  config: Config,
+  logger: Logger,
+): Promise<OverlayBaseDatabaseDownloadStats | undefined> {
+  const overlayDatabaseMode = config.overlayDatabaseMode;
+  if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
+    logger.debug(
+      `Overlay database mode is ${overlayDatabaseMode}. ` +
+        "Skip downloading overlay-base database from cache.",
+    );
+    return undefined;
+  }
+  if (!config.useOverlayDatabaseCaching) {
+    logger.debug(
+      "Overlay database caching is disabled. " +
+        "Skip downloading overlay-base database from cache.",
+    );
+    return undefined;
+  }
+  if (isInTestMode()) {
+    logger.debug(
+      "In test mode. Skip downloading overlay-base database from cache.",
+    );
+    return undefined;
+  }
+
+  const dbLocation = config.dbLocation;
+  const codeQlVersion = (await codeql.getVersion()).version;
+  const cacheRestoreKeyPrefix = await getCacheRestoreKeyPrefix(
+    config,
+    codeQlVersion,
+  );
+
+  logger.info(
+    "Looking in Actions cache for overlay-base database with " +
+      `restore key ${cacheRestoreKeyPrefix}`,
+  );
+
+  let databaseDownloadDurationMs = 0;
+  try {
+    const databaseDownloadStart = performance.now();
+    const foundKey = await waitForResultWithTimeLimit(
+      // This ten-minute limit for the cache restore operation is mainly to
+      // guard against the possibility that the cache service is unresponsive
+      // and hangs outside the data download.
+      //
+      // Data download (which is normally the most time-consuming part of the
+      // restore operation) should not run long enough to hit this limit. Even
+      // for an extremely large 10GB database, at a download speed of 40MB/s
+      // (see below), the download should complete within five minutes. If we
+      // do hit this limit, there are likely more serious problems other than
+      // mere slow download speed.
+      //
+      // This is important because we don't want any ongoing file operations
+      // on the database directory when we do hit this limit. Hitting this
+      // time limit takes us to a fallback path where we re-initialize the
+      // database from scratch at dbLocation, and having the cache restore
+      // operation continue to write into dbLocation in the background would
+      // really mess things up. We want to hit this limit only in the case
+      // of a hung cache service, not just slow download speed.
+      MAX_CACHE_OPERATION_MS,
+      actionsCache.restoreCache(
+        [dbLocation],
+        cacheRestoreKeyPrefix,
+        undefined,
+        {
+          // Azure SDK download (which is the default) uses 128MB segments; see
+          // https://github.com/actions/toolkit/blob/main/packages/cache/README.md.
+          // Setting segmentTimeoutInMs to 3000 translates to segment download
+          // speed of about 40 MB/s, which should be achievable unless the
+          // download is unreliable (in which case we do want to abort).
+          segmentTimeoutInMs: 3000,
+        },
+      ),
+      () => {
+        logger.info("Timed out downloading overlay-base database from cache");
+      },
+    );
+    databaseDownloadDurationMs = Math.round(
+      performance.now() - databaseDownloadStart,
+    );
+
+    if (foundKey === undefined) {
+      logger.info("No overlay-base database found in Actions cache");
+      return undefined;
+    }
+
+    logger.info(
+      `Downloaded overlay-base database in cache with key ${foundKey}`,
+    );
+  } catch (error) {
+    logger.warning(
+      "Failed to download overlay-base database from cache: " +
+        `${error instanceof Error ? error.message : String(error)}`,
+    );
+    return undefined;
+  }
+
+  const databaseIsValid = await checkOverlayBaseDatabase(
+    codeql,
+    config,
+    logger,
+    "Downloaded overlay-base database is invalid",
+  );
+  if (!databaseIsValid) {
+    logger.warning("Downloaded overlay-base database failed validation");
+    return undefined;
+  }
+
+  const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
+  if (databaseSizeBytes === undefined) {
+    logger.info(
+      "Filesystem error while accessing downloaded overlay-base database",
+    );
+    // The problem that warrants reporting download failure is not that we are
+    // unable to determine the size of the database. Rather, it is that we
+    // encountered a filesystem error while accessing the database, which
+    // indicates that an overlay analysis will likely fail.
+    return undefined;
+  }
+
+  logger.info(`Successfully downloaded overlay-base database to ${dbLocation}`);
+  return {
+    databaseSizeBytes: Math.round(databaseSizeBytes),
+    databaseDownloadDurationMs,
+  };
+}
+
+/**
+ * Computes the cache key for saving the overlay-base database to the GitHub
+ * Actions cache.
+ *
+ * The key consists of the restore key prefix (which does not include the
+ * commit SHA) and the commit SHA of the current checkout.
+ */
+export async function getCacheSaveKey(
+  config: Config,
+  codeQlVersion: string,
+  checkoutPath: string,
+  logger: Logger,
+): Promise<string> {
+  let runId = 1;
+  let attemptId = 1;
+  try {
+    runId = getWorkflowRunID();
+    attemptId = getWorkflowRunAttempt();
+  } catch (e) {
+    logger.warning(
+      `Failed to get workflow run ID or attempt ID. Reason: ${getErrorMessage(e)}`,
+    );
+  }
+  const sha = await getCommitOid(checkoutPath);
+  const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
+    config,
+    codeQlVersion,
+  );
+  return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`;
+}
+
+/**
+ * Computes the cache key prefix for restoring the overlay-base database from
+ * the GitHub Actions cache.
+ *
+ * Actions cache supports using multiple restore keys to indicate preference,
+ * and this function could in principle take advantage of that feature by
+ * returning a list of restore key prefixes. However, since overlay-base
+ * databases are built from the default branch and used in PR analysis, it is
+ * exceedingly unlikely that the commit SHA will ever be the same.
+ *
+ * Therefore, this function returns only a single restore key prefix, which does
+ * not include the commit SHA. This allows us to restore the most recent
+ * compatible overlay-base database.
+ */
+export async function getCacheRestoreKeyPrefix(
+  config: Config,
+  codeQlVersion: string,
+): Promise<string> {
+  const languages = [...config.languages].sort().join("_");
+
+  const cacheKeyComponents = {
+    automationID: await getAutomationID(),
+    // Add more components here as needed in the future
+  };
+  const componentsHash = createCacheKeyHash(cacheKeyComponents);
+
+  // For a cached overlay-base database to be considered compatible for overlay
+  // analysis, all components in the cache restore key must match:
+  //
+  // CACHE_PREFIX: distinguishes overlay-base databases from other cache objects
+  // CACHE_VERSION: cache format version
+  // componentsHash: hash of additional components (see above for details)
+  // languages: the languages included in the overlay-base database
+  // codeQlVersion: CodeQL bundle version
+  //
+  // Technically we can also include languages and codeQlVersion in the
+  // componentsHash, but including them explicitly in the cache key makes it
+  // easier to debug and understand the cache key structure.
+  return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
+}

src/overlay/index.test.ts

@@ -1,32 +1,16 @@
 import * as fs from "fs";
 import * as path from "path";
 
-import * as actionsCache from "@actions/cache";
 import test from "ava";
 import * as sinon from "sinon";
 
 import * as actionsUtil from "../actions-util";
-import * as apiClient from "../api-client";
-import { ResolveDatabaseOutput } from "../codeql";
 import * as gitUtils from "../git-utils";
-import { KnownLanguage } from "../languages";
 import { getRunnerLogger } from "../logging";
-import {
-  createTestConfig,
-  mockCodeQLVersion,
-  setupTests,
-} from "../testing-utils";
-import * as utils from "../util";
+import { createTestConfig, setupTests } from "../testing-utils";
 import { withTmpDir } from "../util";
 
-import {
-  downloadOverlayBaseDatabaseFromCache,
-  getCacheRestoreKeyPrefix,
-  getCacheSaveKey,
-  OverlayDatabaseMode,
-  writeBaseDatabaseOidsFile,
-  writeOverlayChangesFile,
-} from ".";
+import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from ".";
 
 setupTests(test);
 
@@ -344,261 +328,3 @@ test.serial(
     });
   },
 );
-
-interface DownloadOverlayBaseDatabaseTestCase {
-  overlayDatabaseMode: OverlayDatabaseMode;
-  useOverlayDatabaseCaching: boolean;
-  isInTestMode: boolean;
-  restoreCacheResult: string | undefined | Error;
-  hasBaseDatabaseOidsFile: boolean;
-  tryGetFolderBytesSucceeds: boolean;
-  codeQLVersion: string;
-  resolveDatabaseOutput: ResolveDatabaseOutput | Error;
-}
-
-const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
-  overlayDatabaseMode: OverlayDatabaseMode.Overlay,
-  useOverlayDatabaseCaching: true,
-  isInTestMode: false,
-  restoreCacheResult: "cache-key",
-  hasBaseDatabaseOidsFile: true,
-  tryGetFolderBytesSucceeds: true,
-  codeQLVersion: "2.20.5",
-  resolveDatabaseOutput: { overlayBaseSpecifier: "20250626:XXX" },
-};
-
-const testDownloadOverlayBaseDatabaseFromCache = test.macro({
-  exec: async (
-    t,
-    _title: string,
-    partialTestCase: Partial<DownloadOverlayBaseDatabaseTestCase>,
-    expectDownloadSuccess: boolean,
-  ) => {
-    await withTmpDir(async (tmpDir) => {
-      const dbLocation = path.join(tmpDir, "db");
-      await fs.promises.mkdir(dbLocation, { recursive: true });
-
-      const logger = getRunnerLogger(true);
-      const testCase = { ...defaultDownloadTestCase, ...partialTestCase };
-      const config = createTestConfig({
-        dbLocation,
-        languages: [KnownLanguage.java],
-      });
-
-      config.overlayDatabaseMode = testCase.overlayDatabaseMode;
-      config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching;
-
-      if (testCase.hasBaseDatabaseOidsFile) {
-        const baseDatabaseOidsFile = path.join(
-          dbLocation,
-          "base-database-oids.json",
-        );
-        await fs.promises.writeFile(baseDatabaseOidsFile, JSON.stringify({}));
-      }
-
-      const stubs: sinon.SinonStub[] = [];
-
-      const getAutomationIDStub = sinon
-        .stub(apiClient, "getAutomationID")
-        .resolves("test-automation-id/");
-      stubs.push(getAutomationIDStub);
-
-      const isInTestModeStub = sinon
-        .stub(utils, "isInTestMode")
-        .returns(testCase.isInTestMode);
-      stubs.push(isInTestModeStub);
-
-      if (testCase.restoreCacheResult instanceof Error) {
-        const restoreCacheStub = sinon
-          .stub(actionsCache, "restoreCache")
-          .rejects(testCase.restoreCacheResult);
-        stubs.push(restoreCacheStub);
-      } else {
-        const restoreCacheStub = sinon
-          .stub(actionsCache, "restoreCache")
-          .resolves(testCase.restoreCacheResult);
-        stubs.push(restoreCacheStub);
-      }
-
-      const tryGetFolderBytesStub = sinon
-        .stub(utils, "tryGetFolderBytes")
-        .resolves(testCase.tryGetFolderBytesSucceeds ? 1024 * 1024 : undefined);
-      stubs.push(tryGetFolderBytesStub);
-
-      const codeql = mockCodeQLVersion(testCase.codeQLVersion);
-
-      if (testCase.resolveDatabaseOutput instanceof Error) {
-        const resolveDatabaseStub = sinon
-          .stub(codeql, "resolveDatabase")
-          .rejects(testCase.resolveDatabaseOutput);
-        stubs.push(resolveDatabaseStub);
-      } else {
-        const resolveDatabaseStub = sinon
-          .stub(codeql, "resolveDatabase")
-          .resolves(testCase.resolveDatabaseOutput);
-        stubs.push(resolveDatabaseStub);
-      }
-
-      try {
-        const result = await downloadOverlayBaseDatabaseFromCache(
-          codeql,
-          config,
-          logger,
-        );
-
-        if (expectDownloadSuccess) {
-          t.truthy(result);
-        } else {
-          t.is(result, undefined);
-        }
-      } finally {
-        for (const stub of stubs) {
-          stub.restore();
-        }
-      }
-    });
-  },
-  title: (_, title) => `downloadOverlayBaseDatabaseFromCache: ${title}`,
-});
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns stats when successful",
-  {},
-  true,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when mode is OverlayDatabaseMode.OverlayBase",
-  {
-    overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when mode is OverlayDatabaseMode.None",
-  {
-    overlayDatabaseMode: OverlayDatabaseMode.None,
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when caching is disabled",
-  {
-    useOverlayDatabaseCaching: false,
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined in test mode",
-  {
-    isInTestMode: true,
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when cache miss",
-  {
-    restoreCacheResult: undefined,
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when download fails",
-  {
-    restoreCacheResult: new Error("Download failed"),
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when downloaded database is invalid",
-  {
-    hasBaseDatabaseOidsFile: false,
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when downloaded database doesn't have an overlayBaseSpecifier",
-  {
-    resolveDatabaseOutput: {},
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when resolving database metadata fails",
-  {
-    resolveDatabaseOutput: new Error("Failed to resolve database metadata"),
-  },
-  false,
-);
-
-test.serial(
-  testDownloadOverlayBaseDatabaseFromCache,
-  "returns undefined when filesystem error occurs",
-  {
-    tryGetFolderBytesSucceeds: false,
-  },
-  false,
-);
-
-test.serial("overlay-base database cache keys remain stable", async (t) => {
-  const logger = getRunnerLogger(true);
-  const config = createTestConfig({ languages: ["python", "javascript"] });
-  const codeQlVersion = "2.23.0";
-  const commitOid = "abc123def456";
-
-  sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
-  sinon.stub(gitUtils, "getCommitOid").resolves(commitOid);
-  sinon.stub(actionsUtil, "getWorkflowRunID").returns(12345);
-  sinon.stub(actionsUtil, "getWorkflowRunAttempt").returns(1);
-
-  const saveKey = await getCacheSaveKey(
-    config,
-    codeQlVersion,
-    "checkout-path",
-    logger,
-  );
-  const expectedSaveKey =
-    "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456-12345-1";
-  t.is(
-    saveKey,
-    expectedSaveKey,
-    "Cache save key changed unexpectedly. " +
-      "This may indicate breaking changes in the cache key generation logic.",
-  );
-
-  const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
-    config,
-    codeQlVersion,
-  );
-  const expectedRestoreKeyPrefix =
-    "codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-";
-  t.is(
-    restoreKeyPrefix,
-    expectedRestoreKeyPrefix,
-    "Cache restore key prefix changed unexpectedly. " +
-      "This may indicate breaking changes in the cache key generation logic.",
-  );
-
-  t.true(
-    saveKey.startsWith(restoreKeyPrefix),
-    `Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`,
-  );
-});

src/overlay/index.ts

@@ -1,37 +1,12 @@
 import * as fs from "fs";
 import * as path from "path";
 
-import * as actionsCache from "@actions/cache";
-
 import * as actionsUtil from "../actions-util";
-import {
-  getOptionalInput,
-  getRequiredInput,
-  getTemporaryDirectory,
-  getWorkflowRunAttempt,
-  getWorkflowRunID,
-} from "../actions-util";
-import { getAutomationID } from "../api-client";
-import { createCacheKeyHash } from "../caching-utils";
-import { type CodeQL } from "../codeql";
+import { getOptionalInput, getTemporaryDirectory } from "../actions-util";
 import { type Config } from "../config-utils";
-import { getCommitOid, getFileOidsUnderPath, getGitRoot } from "../git-utils";
-import { Logger, withGroupAsync } from "../logging";
-import {
-  CleanupLevel,
-  getBaseDatabaseOidsFilePath,
-  getCodeQLDatabasePath,
-  getErrorMessage,
-  isInTestMode,
-  tryGetFolderBytes,
-  waitForResultWithTimeLimit,
-} from "../util";
-
-export enum OverlayDatabaseMode {
-  Overlay = "overlay",
-  OverlayBase = "overlay-base",
-  None = "none",
-}
+import { getFileOidsUnderPath, getGitRoot } from "../git-utils";
+import { Logger } from "../logging";
+import { getBaseDatabaseOidsFilePath } from "../util";
 
 export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.8";
 
@@ -45,23 +20,6 @@ export const CODEQL_OVERLAY_MINIMUM_VERSION_JAVASCRIPT = "2.23.9";
 export const CODEQL_OVERLAY_MINIMUM_VERSION_PYTHON = "2.23.9";
 export const CODEQL_OVERLAY_MINIMUM_VERSION_RUBY = "2.23.9";
 
-/**
- * The maximum (uncompressed) size of the overlay base database that we will
- * upload. By default, the Actions Cache has an overall capacity of 10 GB, and
- * the Actions Cache client library uses zstd compression.
- *
- * Ideally we would apply a size limit to the compressed overlay-base database,
- * but we cannot do so because compression is handled transparently by the
- * Actions Cache client library. Instead we place a limit on the uncompressed
- * size of the overlay-base database.
- *
- * Assuming 2.5:1 compression ratio, the 7.5 GB limit on uncompressed data would
- * translate to a limit of around 3 GB after compression.
- */
-const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
-const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
-  OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000;
-
 /**
  * Writes a JSON file containing Git OIDs for all tracked files (represented
  * by path relative to the source root) under the source root. The file is
@@ -235,388 +193,3 @@ async function getDiffRangeFilePaths(
     .filter((rel) => !rel.startsWith(".."));
   return [...new Set(relativePaths)];
 }
-
-// Constants for database caching
-const CACHE_VERSION = 1;
-const CACHE_PREFIX = "codeql-overlay-base-database";
-
-// The purpose of this ten-minute limit is to guard against the possibility
-// that the cache service is unresponsive, which would otherwise cause the
-// entire action to hang.  Normally we expect cache operations to complete
-// within two minutes.
-const MAX_CACHE_OPERATION_MS = 600_000;
-
-/**
- * Checks that the overlay-base database is valid by checking for the
- * existence of the base database OIDs file.
- *
- * @param config The configuration object
- * @param logger The logger instance
- * @param warningPrefix Prefix for the check failure warning message
- * @returns True if the verification succeeded, false otherwise
- */
-async function checkOverlayBaseDatabase(
-  codeql: CodeQL,
-  config: Config,
-  logger: Logger,
-  warningPrefix: string,
-): Promise<boolean> {
-  // An overlay-base database should contain the base database OIDs file.
-  const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
-  if (!fs.existsSync(baseDatabaseOidsFilePath)) {
-    logger.warning(
-      `${warningPrefix}: ${baseDatabaseOidsFilePath} does not exist`,
-    );
-    return false;
-  }
-
-  for (const language of config.languages) {
-    const dbPath = getCodeQLDatabasePath(config, language);
-    try {
-      const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
-      if (
-        resolveDatabaseOutput === undefined ||
-        !("overlayBaseSpecifier" in resolveDatabaseOutput)
-      ) {
-        logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
-        return false;
-      } else {
-        logger.debug(
-          `Overlay base specifier for ${language} overlay-base database found: ` +
-            `${resolveDatabaseOutput.overlayBaseSpecifier}`,
-        );
-      }
-    } catch (e) {
-      logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
-      return false;
-    }
-  }
-
-  return true;
-}
-
-/**
- * Uploads the overlay-base database to the GitHub Actions cache. If conditions
- * for uploading are not met, the function does nothing and returns false.
- *
- * This function uses the `checkout_path` input to determine the repository path
- * and works only when called from `analyze` or `upload-sarif`.
- *
- * @param codeql The CodeQL instance
- * @param config The configuration object
- * @param logger The logger instance
- * @returns A promise that resolves to true if the upload was performed and
- * successfully completed, or false otherwise
- */
-export async function cleanupAndUploadOverlayBaseDatabaseToCache(
-  codeql: CodeQL,
-  config: Config,
-  logger: Logger,
-): Promise<boolean> {
-  const overlayDatabaseMode = config.overlayDatabaseMode;
-  if (overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase) {
-    logger.debug(
-      `Overlay database mode is ${overlayDatabaseMode}. ` +
-        "Skip uploading overlay-base database to cache.",
-    );
-    return false;
-  }
-  if (!config.useOverlayDatabaseCaching) {
-    logger.debug(
-      "Overlay database caching is disabled. " +
-        "Skip uploading overlay-base database to cache.",
-    );
-    return false;
-  }
-  if (isInTestMode()) {
-    logger.debug(
-      "In test mode. Skip uploading overlay-base database to cache.",
-    );
-    return false;
-  }
-
-  const databaseIsValid = await checkOverlayBaseDatabase(
-    codeql,
-    config,
-    logger,
-    "Abort uploading overlay-base database to cache",
-  );
-  if (!databaseIsValid) {
-    return false;
-  }
-
-  // Clean up the database using the overlay cleanup level.
-  await withGroupAsync("Cleaning up databases", async () => {
-    await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
-  });
-
-  const dbLocation = config.dbLocation;
-
-  const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
-  if (databaseSizeBytes === undefined) {
-    logger.warning(
-      "Failed to determine database size. " +
-        "Skip uploading overlay-base database to cache.",
-    );
-    return false;
-  }
-
-  if (databaseSizeBytes > OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES) {
-    const databaseSizeMB = Math.round(databaseSizeBytes / 1_000_000);
-    logger.warning(
-      `Database size (${databaseSizeMB} MB) ` +
-        `exceeds maximum upload size (${OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB} MB). ` +
-        "Skip uploading overlay-base database to cache.",
-    );
-    return false;
-  }
-
-  const codeQlVersion = (await codeql.getVersion()).version;
-  const checkoutPath = getRequiredInput("checkout_path");
-  const cacheSaveKey = await getCacheSaveKey(
-    config,
-    codeQlVersion,
-    checkoutPath,
-    logger,
-  );
-  logger.info(
-    `Uploading overlay-base database to Actions cache with key ${cacheSaveKey}`,
-  );
-
-  try {
-    const cacheId = await waitForResultWithTimeLimit(
-      MAX_CACHE_OPERATION_MS,
-      actionsCache.saveCache([dbLocation], cacheSaveKey),
-      () => {},
-    );
-    if (cacheId === undefined) {
-      logger.warning("Timed out while uploading overlay-base database");
-      return false;
-    }
-  } catch (error) {
-    logger.warning(
-      "Failed to upload overlay-base database to cache: " +
-        `${error instanceof Error ? error.message : String(error)}`,
-    );
-    return false;
-  }
-  logger.info(`Successfully uploaded overlay-base database from ${dbLocation}`);
-  return true;
-}
-
-export interface OverlayBaseDatabaseDownloadStats {
-  databaseSizeBytes: number;
-  databaseDownloadDurationMs: number;
-}
-
-/**
- * Downloads the overlay-base database from the GitHub Actions cache. If conditions
- * for downloading are not met, the function does nothing and returns false.
- *
- * @param codeql The CodeQL instance
- * @param config The configuration object
- * @param logger The logger instance
- * @returns A promise that resolves to download statistics if an overlay-base
- * database was successfully downloaded, or undefined if the download was
- * either not performed or failed.
- */
-export async function downloadOverlayBaseDatabaseFromCache(
-  codeql: CodeQL,
-  config: Config,
-  logger: Logger,
-): Promise<OverlayBaseDatabaseDownloadStats | undefined> {
-  const overlayDatabaseMode = config.overlayDatabaseMode;
-  if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
-    logger.debug(
-      `Overlay database mode is ${overlayDatabaseMode}. ` +
-        "Skip downloading overlay-base database from cache.",
-    );
-    return undefined;
-  }
-  if (!config.useOverlayDatabaseCaching) {
-    logger.debug(
-      "Overlay database caching is disabled. " +
-        "Skip downloading overlay-base database from cache.",
-    );
-    return undefined;
-  }
-  if (isInTestMode()) {
-    logger.debug(
-      "In test mode. Skip downloading overlay-base database from cache.",
-    );
-    return undefined;
-  }
-
-  const dbLocation = config.dbLocation;
-  const codeQlVersion = (await codeql.getVersion()).version;
-  const cacheRestoreKeyPrefix = await getCacheRestoreKeyPrefix(
-    config,
-    codeQlVersion,
-  );
-
-  logger.info(
-    "Looking in Actions cache for overlay-base database with " +
-      `restore key ${cacheRestoreKeyPrefix}`,
-  );
-
-  let databaseDownloadDurationMs = 0;
-  try {
-    const databaseDownloadStart = performance.now();
-    const foundKey = await waitForResultWithTimeLimit(
-      // This ten-minute limit for the cache restore operation is mainly to
-      // guard against the possibility that the cache service is unresponsive
-      // and hangs outside the data download.
-      //
-      // Data download (which is normally the most time-consuming part of the
-      // restore operation) should not run long enough to hit this limit. Even
-      // for an extremely large 10GB database, at a download speed of 40MB/s
-      // (see below), the download should complete within five minutes. If we
-      // do hit this limit, there are likely more serious problems other than
-      // mere slow download speed.
-      //
-      // This is important because we don't want any ongoing file operations
-      // on the database directory when we do hit this limit. Hitting this
-      // time limit takes us to a fallback path where we re-initialize the
-      // database from scratch at dbLocation, and having the cache restore
-      // operation continue to write into dbLocation in the background would
-      // really mess things up. We want to hit this limit only in the case
-      // of a hung cache service, not just slow download speed.
-      MAX_CACHE_OPERATION_MS,
-      actionsCache.restoreCache(
-        [dbLocation],
-        cacheRestoreKeyPrefix,
-        undefined,
-        {
-          // Azure SDK download (which is the default) uses 128MB segments; see
-          // https://github.com/actions/toolkit/blob/main/packages/cache/README.md.
-          // Setting segmentTimeoutInMs to 3000 translates to segment download
-          // speed of about 40 MB/s, which should be achievable unless the
-          // download is unreliable (in which case we do want to abort).
-          segmentTimeoutInMs: 3000,
-        },
-      ),
-      () => {
-        logger.info("Timed out downloading overlay-base database from cache");
-      },
-    );
-    databaseDownloadDurationMs = Math.round(
-      performance.now() - databaseDownloadStart,
-    );
-
-    if (foundKey === undefined) {
-      logger.info("No overlay-base database found in Actions cache");
-      return undefined;
-    }
-
-    logger.info(
-      `Downloaded overlay-base database in cache with key ${foundKey}`,
-    );
-  } catch (error) {
-    logger.warning(
-      "Failed to download overlay-base database from cache: " +
-        `${error instanceof Error ? error.message : String(error)}`,
-    );
-    return undefined;
-  }
-
-  const databaseIsValid = await checkOverlayBaseDatabase(
-    codeql,
-    config,
-    logger,
-    "Downloaded overlay-base database is invalid",
-  );
-  if (!databaseIsValid) {
-    logger.warning("Downloaded overlay-base database failed validation");
-    return undefined;
-  }
-
-  const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
-  if (databaseSizeBytes === undefined) {
-    logger.info(
-      "Filesystem error while accessing downloaded overlay-base database",
-    );
-    // The problem that warrants reporting download failure is not that we are
-    // unable to determine the size of the database. Rather, it is that we
-    // encountered a filesystem error while accessing the database, which
-    // indicates that an overlay analysis will likely fail.
-    return undefined;
-  }
-
-  logger.info(`Successfully downloaded overlay-base database to ${dbLocation}`);
-  return {
-    databaseSizeBytes: Math.round(databaseSizeBytes),
-    databaseDownloadDurationMs,
-  };
-}
-
-/**
- * Computes the cache key for saving the overlay-base database to the GitHub
- * Actions cache.
- *
- * The key consists of the restore key prefix (which does not include the
- * commit SHA) and the commit SHA of the current checkout.
- */
-export async function getCacheSaveKey(
-  config: Config,
-  codeQlVersion: string,
-  checkoutPath: string,
-  logger: Logger,
-): Promise<string> {
-  let runId = 1;
-  let attemptId = 1;
-  try {
-    runId = getWorkflowRunID();
-    attemptId = getWorkflowRunAttempt();
-  } catch (e) {
-    logger.warning(
-      `Failed to get workflow run ID or attempt ID. Reason: ${getErrorMessage(e)}`,
-    );
-  }
-  const sha = await getCommitOid(checkoutPath);
-  const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
-    config,
-    codeQlVersion,
-  );
-  return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`;
-}
-
-/**
- * Computes the cache key prefix for restoring the overlay-base database from
- * the GitHub Actions cache.
- *
- * Actions cache supports using multiple restore keys to indicate preference,
- * and this function could in principle take advantage of that feature by
- * returning a list of restore key prefixes. However, since overlay-base
- * databases are built from the default branch and used in PR analysis, it is
- * exceedingly unlikely that the commit SHA will ever be the same.
- *
- * Therefore, this function returns only a single restore key prefix, which does
- * not include the commit SHA. This allows us to restore the most recent
- * compatible overlay-base database.
- */
-export async function getCacheRestoreKeyPrefix(
-  config: Config,
-  codeQlVersion: string,
-): Promise<string> {
-  const languages = [...config.languages].sort().join("_");
-
-  const cacheKeyComponents = {
-    automationID: await getAutomationID(),
-    // Add more components here as needed in the future
-  };
-  const componentsHash = createCacheKeyHash(cacheKeyComponents);
-
-  // For a cached overlay-base database to be considered compatible for overlay
-  // analysis, all components in the cache restore key must match:
-  //
-  // CACHE_PREFIX: distinguishes overlay-base databases from other cache objects
-  // CACHE_VERSION: cache format version
-  // componentsHash: hash of additional components (see above for details)
-  // languages: the languages included in the overlay-base database
-  // codeQlVersion: CodeQL bundle version
-  //
-  // Technically we can also include languages and codeQlVersion in the
-  // componentsHash, but including them explicitly in the cache key makes it
-  // easier to debug and understand the cache key structure.
-  return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
-}

src/overlay/overlay-database-mode.ts

@@ -0,0 +1,5 @@
+export enum OverlayDatabaseMode {
+  Overlay = "overlay",
+  OverlayBase = "overlay-base",
+  None = "none",
+}

src/status-report.ts

@@ -18,7 +18,7 @@ import { DocUrl } from "./doc-url";
 import { EnvVar } from "./environment";
 import { getRef } from "./git-utils";
 import { Logger } from "./logging";
-import { OverlayBaseDatabaseDownloadStats } from "./overlay";
+import { OverlayBaseDatabaseDownloadStats } from "./overlay/caching";
 import { getRepositoryNwo } from "./repository";
 import { ToolsSource } from "./setup-codeql";
 import {

src/testing-utils.ts

@@ -21,7 +21,7 @@ import {
   FeatureEnablement,
 } from "./feature-flags";
 import { Logger } from "./logging";
-import { OverlayDatabaseMode } from "./overlay";
+import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
 import {
   DEFAULT_DEBUG_ARTIFACT_NAME,
   DEFAULT_DEBUG_DATABASE_NAME,