mirror of
https://github.com/github/codeql-action.git
synced 2026-05-09 15:20:28 +00:00
Revert "Mitigate caches being evicted before they can be downloaded"
This reverts commit 1279e8d41c.
This commit is contained in:
Generated
-2
@@ -111199,8 +111199,6 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
|
||||
var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6;
|
||||
var CACHE_VERSION2 = 1;
|
||||
var CACHE_PREFIX = "codeql-overlay-base-database";
|
||||
var CACHE_ENTRY_MAX_AGE_DAYS = 6;
|
||||
var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3;
|
||||
var MAX_CACHE_OPERATION_MS2 = 6e5;
|
||||
async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) {
|
||||
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
|
||||
|
||||
Generated
-2
@@ -109583,8 +109583,6 @@ var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
|
||||
var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6;
|
||||
var CACHE_VERSION2 = 1;
|
||||
var CACHE_PREFIX = "codeql-overlay-base-database";
|
||||
var CACHE_ENTRY_MAX_AGE_DAYS = 6;
|
||||
var CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1e3;
|
||||
var MAX_CACHE_OPERATION_MS3 = 6e5;
|
||||
async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) {
|
||||
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
|
||||
|
||||
@@ -249,7 +249,6 @@ export interface ActionsCacheItem {
|
||||
created_at?: string;
|
||||
id?: number;
|
||||
key?: string;
|
||||
last_accessed_at?: string;
|
||||
size_in_bytes?: number;
|
||||
}
|
||||
|
||||
|
||||
@@ -417,37 +417,3 @@ test.serial(
|
||||
t.deepEqual(result, ["2.25.0", "2.24.0"]);
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getCodeQlVersionsForOverlayBaseDatabases ignores cache entries close to eviction",
|
||||
async (t) => {
|
||||
const logger = getRunnerLogger(true);
|
||||
|
||||
const now = Date.now();
|
||||
const isoDaysAgo = (days: number) =>
|
||||
new Date(now - days * 24 * 60 * 60 * 1000).toISOString();
|
||||
|
||||
sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
|
||||
sinon.stub(apiClient, "listActionsCaches").resolves([
|
||||
{
|
||||
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1",
|
||||
last_accessed_at: isoDaysAgo(1),
|
||||
},
|
||||
{
|
||||
// Older than the 6-day threshold; close to the 7-day eviction window.
|
||||
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0-def456-2-1",
|
||||
last_accessed_at: isoDaysAgo(6.5),
|
||||
},
|
||||
{
|
||||
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1",
|
||||
last_accessed_at: isoDaysAgo(3),
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await getCodeQlVersionsForOverlayBaseDatabases(
|
||||
["python"],
|
||||
logger,
|
||||
);
|
||||
t.deepEqual(result, ["2.25.0", "2.24.0"]);
|
||||
},
|
||||
);
|
||||
|
||||
+12
-52
@@ -8,11 +8,7 @@ import {
|
||||
getWorkflowRunAttempt,
|
||||
getWorkflowRunID,
|
||||
} from "../actions-util";
|
||||
import {
|
||||
type ActionsCacheItem,
|
||||
getAutomationID,
|
||||
listActionsCaches,
|
||||
} from "../api-client";
|
||||
import { getAutomationID, listActionsCaches } from "../api-client";
|
||||
import { createCacheKeyHash } from "../caching-utils";
|
||||
import { type CodeQL } from "../codeql";
|
||||
import { type Config } from "../config-utils";
|
||||
@@ -52,12 +48,6 @@ const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
|
||||
const CACHE_VERSION = 1;
|
||||
const CACHE_PREFIX = "codeql-overlay-base-database";
|
||||
|
||||
// The Actions cache evicts entries that have not been accessed in the past 7
|
||||
// days. We conservatively set a limit of 6 days to avoid using a cached base DB
|
||||
// that may be evicted before we can download it.
|
||||
const CACHE_ENTRY_MAX_AGE_DAYS = 6;
|
||||
const CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1000;
|
||||
|
||||
// The purpose of this ten-minute limit is to guard against the possibility
|
||||
// that the cache service is unresponsive, which would otherwise cause the
|
||||
// entire action to hang. Normally we expect cache operations to complete
|
||||
@@ -445,39 +435,6 @@ async function getCacheKeyPrefixBase(
|
||||
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists overlay-base database cache entries with the given key prefix, ignoring entries that are
|
||||
* old enough that they may be evicted by the Actions cache before we attempt to download them.
|
||||
*/
|
||||
async function listRecentOverlayBaseDatabaseCaches(
|
||||
cacheKeyPrefix: string,
|
||||
logger: Logger,
|
||||
): Promise<ActionsCacheItem[]> {
|
||||
const allCaches = await listActionsCaches(cacheKeyPrefix);
|
||||
|
||||
if (allCaches.length === 0) {
|
||||
logger.info("No overlay-base databases found in Actions cache.");
|
||||
return [];
|
||||
}
|
||||
|
||||
const cutoffMs = Date.now() - CACHE_ENTRY_MAX_AGE_MS;
|
||||
const recentCaches = allCaches.filter((cache) => {
|
||||
if (!cache.last_accessed_at) return true;
|
||||
const lastAccessedMs = Date.parse(cache.last_accessed_at);
|
||||
return Number.isNaN(lastAccessedMs) || lastAccessedMs >= cutoffMs;
|
||||
});
|
||||
const numTooOldDatabases = allCaches.length - recentCaches.length;
|
||||
const tooOldSuffix =
|
||||
numTooOldDatabases > 0
|
||||
? ` (ignoring ${numTooOldDatabases} that may be evicted soon)`
|
||||
: "";
|
||||
logger.info(
|
||||
`Found ${allCaches.length} overlay-base ${allCaches.length === 1 ? "database" : "databases"} in the Actions cache${tooOldSuffix}.`,
|
||||
);
|
||||
|
||||
return recentCaches;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches the GitHub Actions cache for overlay-base databases matching the given languages, and
|
||||
* returns all stable CodeQL versions found across matching cache entries.
|
||||
@@ -491,7 +448,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
|
||||
): Promise<string[] | undefined> {
|
||||
const languages = rawLanguages.map(parseBuiltInLanguage);
|
||||
if (languages.includes(undefined)) {
|
||||
logger.info(
|
||||
logger.warning(
|
||||
"One or more provided languages are not recognized as built-in languages. " +
|
||||
"Skipping searching for overlay-base databases in cache.",
|
||||
);
|
||||
@@ -506,19 +463,22 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
|
||||
`prefix ${cacheKeyPrefix}`,
|
||||
);
|
||||
|
||||
const caches = await listRecentOverlayBaseDatabaseCaches(
|
||||
cacheKeyPrefix,
|
||||
logger,
|
||||
);
|
||||
const caches = await listActionsCaches(cacheKeyPrefix);
|
||||
|
||||
if (caches.length === 0) {
|
||||
logger.info("No overlay-base databases found in Actions cache.");
|
||||
return [];
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Found ${caches.length} overlay-base ` +
|
||||
`${caches.length === 1 ? "database" : "databases"} in the Actions cache.`,
|
||||
);
|
||||
|
||||
// Parse CodeQL versions from cache keys, matching only stable releases.
|
||||
//
|
||||
// After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies have a
|
||||
// suffix like `+202604201548` that will prevent a match.
|
||||
// After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have
|
||||
// a suffix like `+202604201548` that will break the match.
|
||||
//
|
||||
// Caveat: this relies on the fact that we haven't released any CodeQL bundles with the
|
||||
// `x.y.z-<pre-release>` semver format which does not interact well with the current overlay base
|
||||
@@ -546,7 +506,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
|
||||
const versions = [...versionSet].sort(semver.rcompare);
|
||||
|
||||
logger.info(
|
||||
`Found overlay-base databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
|
||||
`Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
|
||||
);
|
||||
|
||||
return versions;
|
||||
|
||||
Reference in New Issue
Block a user