Revert "Mitigate caches being evicted before they can be downloaded"

This reverts commit 1279e8d41c.
This commit is contained in:
Henry Mercer
2026-05-05 18:37:17 +01:00
parent 1279e8d41c
commit 7587714d0a
5 changed files with 12 additions and 91 deletions
-2
View File
@@ -111199,8 +111199,6 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6;
var CACHE_VERSION2 = 1;
var CACHE_PREFIX = "codeql-overlay-base-database";
var CACHE_ENTRY_MAX_AGE_DAYS = 6;
var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3;
var MAX_CACHE_OPERATION_MS2 = 6e5;
async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) {
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
-2
View File
@@ -109583,8 +109583,6 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6;
var CACHE_VERSION2 = 1;
var CACHE_PREFIX = "codeql-overlay-base-database";
var CACHE_ENTRY_MAX_AGE_DAYS = 6;
var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3;
var MAX_CACHE_OPERATION_MS3 = 6e5;
async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) {
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
-1
View File
@@ -249,7 +249,6 @@ export interface ActionsCacheItem {
created_at?: string;
id?: number;
key?: string;
last_accessed_at?: string;
size_in_bytes?: number;
}
-34
View File
@@ -417,37 +417,3 @@ test.serial(
t.deepEqual(result, ["2.25.0", "2.24.0"]);
},
);
test.serial(
"getCodeQlVersionsForOverlayBaseDatabases ignores cache entries close to eviction",
async (t) => {
const logger = getRunnerLogger(true);
const now = Date.now();
const isoDaysAgo = (days: number) =>
new Date(now - days * 24 * 60 * 60 * 1000).toISOString();
sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
sinon.stub(apiClient, "listActionsCaches").resolves([
{
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1",
last_accessed_at: isoDaysAgo(1),
},
{
// Older than the 6-day threshold; close to the 7-day eviction window.
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0-def456-2-1",
last_accessed_at: isoDaysAgo(6.5),
},
{
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1",
last_accessed_at: isoDaysAgo(3),
},
]);
const result = await getCodeQlVersionsForOverlayBaseDatabases(
["python"],
logger,
);
t.deepEqual(result, ["2.25.0", "2.24.0"]);
},
);
+12 -52
View File
@@ -8,11 +8,7 @@ import {
getWorkflowRunAttempt,
getWorkflowRunID,
} from "../actions-util";
import {
type ActionsCacheItem,
getAutomationID,
listActionsCaches,
} from "../api-client";
import { getAutomationID, listActionsCaches } from "../api-client";
import { createCacheKeyHash } from "../caching-utils";
import { type CodeQL } from "../codeql";
import { type Config } from "../config-utils";
@@ -52,12 +48,6 @@ const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
const CACHE_VERSION = 1;
const CACHE_PREFIX = "codeql-overlay-base-database";
// The Actions cache evicts entries that have not been accessed in the past 7
// days. We conservatively set a limit of 6 days to avoid using a cached base DB
// that may be evicted before we can download it.
const CACHE_ENTRY_MAX_AGE_DAYS = 6;
const CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1000;
// The purpose of this ten-minute limit is to guard against the possibility
// that the cache service is unresponsive, which would otherwise cause the
// entire action to hang. Normally we expect cache operations to complete
@@ -445,39 +435,6 @@ async function getCacheKeyPrefixBase(
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`;
}
/**
* Lists overlay-base database cache entries with the given key prefix, ignoring entries that are
* old enough that they may be evicted by the Actions cache before we attempt to download them.
*/
async function listRecentOverlayBaseDatabaseCaches(
cacheKeyPrefix: string,
logger: Logger,
): Promise<ActionsCacheItem[]> {
const allCaches = await listActionsCaches(cacheKeyPrefix);
if (allCaches.length === 0) {
logger.info("No overlay-base databases found in Actions cache.");
return [];
}
const cutoffMs = Date.now() - CACHE_ENTRY_MAX_AGE_MS;
const recentCaches = allCaches.filter((cache) => {
if (!cache.last_accessed_at) return true;
const lastAccessedMs = Date.parse(cache.last_accessed_at);
return Number.isNaN(lastAccessedMs) || lastAccessedMs >= cutoffMs;
});
const numTooOldDatabases = allCaches.length - recentCaches.length;
const tooOldSuffix =
numTooOldDatabases > 0
? ` (ignoring ${numTooOldDatabases} that may be evicted soon)`
: "";
logger.info(
`Found ${allCaches.length} overlay-base ${allCaches.length === 1 ? "database" : "databases"} in the Actions cache${tooOldSuffix}.`,
);
return recentCaches;
}
/**
* Searches the GitHub Actions cache for overlay-base databases matching the given languages, and
* returns all stable CodeQL versions found across matching cache entries.
@@ -491,7 +448,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
): Promise<string[] | undefined> {
const languages = rawLanguages.map(parseBuiltInLanguage);
if (languages.includes(undefined)) {
logger.info(
logger.warning(
"One or more provided languages are not recognized as built-in languages. " +
"Skipping searching for overlay-base databases in cache.",
);
@@ -506,19 +463,22 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
`prefix ${cacheKeyPrefix}`,
);
const caches = await listRecentOverlayBaseDatabaseCaches(
cacheKeyPrefix,
logger,
);
const caches = await listActionsCaches(cacheKeyPrefix);
if (caches.length === 0) {
logger.info("No overlay-base databases found in Actions cache.");
return [];
}
logger.info(
`Found ${caches.length} overlay-base ` +
`${caches.length === 1 ? "database" : "databases"} in the Actions cache.`,
);
// Parse CodeQL versions from cache keys, matching only stable releases.
//
// After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies have a
// suffix like `+202604201548` that will prevent a match.
// After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have
// a suffix like `+202604201548` that will break the match.
//
// Caveat: this relies on the fact that we haven't released any CodeQL bundles with the
// `x.y.z-<pre-release>` semver format which does not interact well with the current overlay base
@@ -546,7 +506,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
const versions = [...versionSet].sort(semver.rcompare);
logger.info(
`Found overlay-base databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
`Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
);
return versions;