From 1279e8d41c608bda4cf183c22c4860d3bfb99cb5 Mon Sep 17 00:00:00 2001 From: Henry Mercer Date: Wed, 22 Apr 2026 00:04:57 +0100 Subject: [PATCH] Mitigate caches being evicted before they can be downloaded --- lib/analyze-action.js | 2 ++ lib/init-action.js | 2 ++ src/api-client.ts | 1 + src/overlay/caching.test.ts | 34 ++++++++++++++++++++ src/overlay/caching.ts | 64 ++++++++++++++++++++++++++++++------- 5 files changed, 91 insertions(+), 12 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 4ed11db5b..4554cd183 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -111199,6 +111199,8 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; +var CACHE_ENTRY_MAX_AGE_DAYS = 6; +var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS2 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); diff --git a/lib/init-action.js b/lib/init-action.js index 1ad4f602f..499472f2b 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -109583,6 +109583,8 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; var CACHE_PREFIX = "codeql-overlay-base-database"; +var CACHE_ENTRY_MAX_AGE_DAYS = 6; +var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3; var MAX_CACHE_OPERATION_MS3 = 6e5; async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) { const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); diff --git a/src/api-client.ts b/src/api-client.ts index 4b8cb7b34..8de5058d1 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -249,6 +249,7 @@ export interface ActionsCacheItem { created_at?: string; id?: number; key?: string; + last_accessed_at?: string; size_in_bytes?: number; } diff --git a/src/overlay/caching.test.ts b/src/overlay/caching.test.ts index 18711fb46..2c60fd789 100644 --- a/src/overlay/caching.test.ts +++ b/src/overlay/caching.test.ts @@ -417,3 +417,37 @@ test.serial( t.deepEqual(result, ["2.25.0", "2.24.0"]); }, ); + +test.serial( + "getCodeQlVersionsForOverlayBaseDatabases ignores cache entries close to eviction", + async (t) => { + const logger = getRunnerLogger(true); + + const now = Date.now(); + const isoDaysAgo = (days: number) => + new Date(now - days * 24 * 60 * 60 * 1000).toISOString(); + + sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/"); + sinon.stub(apiClient, "listActionsCaches").resolves([ + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1", + last_accessed_at: isoDaysAgo(1), + }, + { + // Older than the 6-day threshold; close to the 7-day eviction window. + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0-def456-2-1", + last_accessed_at: isoDaysAgo(6.5), + }, + { + key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1", + last_accessed_at: isoDaysAgo(3), + }, + ]); + + const result = await getCodeQlVersionsForOverlayBaseDatabases( + ["python"], + logger, + ); + t.deepEqual(result, ["2.25.0", "2.24.0"]); + }, +); diff --git a/src/overlay/caching.ts b/src/overlay/caching.ts index 367725dfc..cdde47b75 100644 --- a/src/overlay/caching.ts +++ b/src/overlay/caching.ts @@ -8,7 +8,11 @@ import { getWorkflowRunAttempt, getWorkflowRunID, } from "../actions-util"; -import { getAutomationID, listActionsCaches } from "../api-client"; +import { + type ActionsCacheItem, + getAutomationID, + listActionsCaches, +} from "../api-client"; import { createCacheKeyHash } from "../caching-utils"; import { type CodeQL } from "../codeql"; import { type Config } from "../config-utils"; @@ -48,6 +52,12 @@ const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = const CACHE_VERSION = 1; const CACHE_PREFIX = "codeql-overlay-base-database"; +// The Actions cache evicts entries that have not been accessed in the past 7 +// days. We conservatively set a limit of 6 days to avoid using a cached base DB +// that may be evicted before we can download it. +const CACHE_ENTRY_MAX_AGE_DAYS = 6; +const CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1000; + // The purpose of this ten-minute limit is to guard against the possibility // that the cache service is unresponsive, which would otherwise cause the // entire action to hang. Normally we expect cache operations to complete @@ -435,6 +445,39 @@ async function getCacheKeyPrefixBase( return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`; } +/** + * Lists overlay-base database cache entries with the given key prefix, ignoring entries that are + * old enough that they may be evicted by the Actions cache before we attempt to download them. + */ +async function listRecentOverlayBaseDatabaseCaches( + cacheKeyPrefix: string, + logger: Logger, +): Promise { + const allCaches = await listActionsCaches(cacheKeyPrefix); + + if (allCaches.length === 0) { + logger.info("No overlay-base databases found in Actions cache."); + return []; + } + + const cutoffMs = Date.now() - CACHE_ENTRY_MAX_AGE_MS; + const recentCaches = allCaches.filter((cache) => { + if (!cache.last_accessed_at) return true; + const lastAccessedMs = Date.parse(cache.last_accessed_at); + return Number.isNaN(lastAccessedMs) || lastAccessedMs >= cutoffMs; + }); + const numTooOldDatabases = allCaches.length - recentCaches.length; + const tooOldSuffix = + numTooOldDatabases > 0 + ? ` (ignoring ${numTooOldDatabases} that may be evicted soon)` + : ""; + logger.info( + `Found ${allCaches.length} overlay-base ${allCaches.length === 1 ? "database" : "databases"} in the Actions cache${tooOldSuffix}.`, + ); + + return recentCaches; +} + /** * Searches the GitHub Actions cache for overlay-base databases matching the given languages, and * returns all stable CodeQL versions found across matching cache entries. @@ -448,7 +491,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( ): Promise { const languages = rawLanguages.map(parseBuiltInLanguage); if (languages.includes(undefined)) { - logger.warning( + logger.info( "One or more provided languages are not recognized as built-in languages. " + "Skipping searching for overlay-base databases in cache.", ); @@ -463,22 +506,19 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( `prefix ${cacheKeyPrefix}`, ); - const caches = await listActionsCaches(cacheKeyPrefix); + const caches = await listRecentOverlayBaseDatabaseCaches( + cacheKeyPrefix, + logger, + ); if (caches.length === 0) { - logger.info("No overlay-base databases found in Actions cache."); return []; } - logger.info( - `Found ${caches.length} overlay-base ` + - `${caches.length === 1 ? "database" : "databases"} in the Actions cache.`, - ); - // Parse CodeQL versions from cache keys, matching only stable releases. // - // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have - // a suffix like `+202604201548` that will break the match. + // After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies have a + // suffix like `+202604201548` that will prevent a match. // // Caveat: this relies on the fact that we haven't released any CodeQL bundles with the // `x.y.z-` semver format which does not interact well with the current overlay base @@ -506,7 +546,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases( const versions = [...versionSet].sort(semver.rcompare); logger.info( - `Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, + `Found overlay-base databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`, ); return versions;