Return keys of restored caches from downloadDependencyCaches

This commit is contained in:
Michael B. Gale
2025-11-14 11:36:59 +00:00
parent 85f1517bb4
commit 11889c27fd
4 changed files with 95 additions and 30 deletions
+17 -7
View File
@@ -87317,6 +87317,7 @@ async function checkHashPatterns(codeql, features, language, cacheConfig, checkT
}
async function downloadDependencyCaches(codeql, features, languages, logger) {
const status = [];
const restoredKeys = [];
for (const language of languages) {
const cacheConfig = defaultCacheConfigs[language];
if (cacheConfig === void 0) {
@@ -87355,14 +87356,22 @@ async function downloadDependencyCaches(codeql, features, languages, logger) {
const download_duration_ms = Math.round(performance.now() - start);
if (hitKey !== void 0) {
logger.info(`Cache hit on key ${hitKey} for ${language}.`);
const hit_kind = hitKey === primaryKey ? "exact" /* Exact */ : "partial" /* Partial */;
status.push({ language, hit_kind, download_duration_ms });
let hit_kind = "partial" /* Partial */;
if (hitKey === primaryKey) {
hit_kind = "exact" /* Exact */;
}
status.push({
language,
hit_kind,
download_duration_ms
});
restoredKeys.push(hitKey);
} else {
status.push({ language, hit_kind: "miss" /* Miss */ });
logger.info(`No suitable cache found for ${language}.`);
}
}
return status;
return { statusReport: status, restoredKeys };
}
async function cacheKey2(codeql, features, language, patterns) {
const hash = await glob.hashFiles(patterns.join("\n"));
@@ -89994,7 +90003,7 @@ async function run() {
return;
}
let overlayBaseDatabaseStats;
let dependencyCachingResults;
let dependencyCachingStatus;
try {
if (config.overlayDatabaseMode === "overlay" /* Overlay */ && config.useOverlayDatabaseCaching) {
overlayBaseDatabaseStats = await downloadOverlayBaseDatabaseFromCache(
@@ -90135,12 +90144,13 @@ exec ${goBinaryPath} "$@"`
}
}
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
dependencyCachingResults = await downloadDependencyCaches(
const dependencyCachingResult = await downloadDependencyCaches(
codeql,
features,
config.languages,
logger
);
dependencyCachingStatus = dependencyCachingResult.statusReport;
}
if (await codeQlVersionAtLeast(codeql, "2.17.1")) {
} else {
@@ -90241,7 +90251,7 @@ exec ${goBinaryPath} "$@"`
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger,
error4
);
@@ -90259,7 +90269,7 @@ exec ${goBinaryPath} "$@"`
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger
);
}
+48 -14
View File
@@ -237,15 +237,17 @@ test("downloadDependencyCaches - does not restore caches with feature keys if no
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
const result = await downloadDependencyCaches(
codeql,
createFeatures([]),
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Miss);
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Miss);
t.deepEqual(result.restoredKeys, []);
t.assert(restoreCacheStub.calledOnce);
});
@@ -257,7 +259,8 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
sinon.stub(glob, "hashFiles").resolves("abcdef");
const mockHash = "abcdef";
sinon.stub(glob, "hashFiles").resolves(mockHash);
const keyWithFeature = await cacheKey(
codeql,
@@ -277,15 +280,28 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Exact);
// Check that the status report for telemetry indicates that one cache was restored with an exact match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Exact);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(mockHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
// `restoreCache` should have been called exactly once.
t.assert(restoreCacheStub.calledOnce);
});
@@ -297,8 +313,14 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
const logger = getRecordingLogger(messages);
const features = createFeatures([Feature.CsharpNewCacheKey]);
// We expect two calls to `hashFiles`: the first by the call to `cacheKey` below,
// and the second by `downloadDependencyCaches`. We use the result of the first
// call as part of the cache key that identifies a mock, existing cache. The result
// of the second call is for the primary restore key, which we don't want to match
// the first key so that we can test the restore keys logic.
const restoredHash = "abcdef";
const hashFilesStub = sinon.stub(glob, "hashFiles");
hashFilesStub.onFirstCall().resolves("abcdef");
hashFilesStub.onFirstCall().resolves(restoredHash);
hashFilesStub.onSecondCall().resolves("123456");
const keyWithFeature = await cacheKey(
@@ -319,15 +341,27 @@ test("downloadDependencyCaches - restores caches with feature keys if features a
.resolves(CSHARP_BASE_PATTERNS);
makePatternCheckStub.withArgs(CSHARP_EXTRA_PATTERNS).resolves(undefined);
const results = await downloadDependencyCaches(
const result = await downloadDependencyCaches(
codeql,
features,
[KnownLanguage.csharp],
logger,
);
t.is(results.length, 1);
t.is(results[0].language, KnownLanguage.csharp);
t.is(results[0].hit_kind, CacheHitKind.Partial);
// Check that the status report for telemetry indicates that one cache was restored with a partial match.
const statusReport = result.statusReport;
t.is(statusReport.length, 1);
t.is(statusReport[0].language, KnownLanguage.csharp);
t.is(statusReport[0].hit_kind, CacheHitKind.Partial);
// Check that the restored key has been returned.
const restoredKeys = result.restoredKeys;
t.is(restoredKeys.length, 1);
t.assert(
restoredKeys[0].endsWith(restoredHash),
"Expected restored key to end with hash returned by `hashFiles`",
);
t.assert(restoreCacheStub.calledOnce);
});
+25 -5
View File
@@ -193,6 +193,14 @@ export interface DependencyCacheRestoreStatus {
/** An array of `DependencyCacheRestoreStatus` objects for each analysed language with a caching configuration. */
export type DependencyCacheRestoreStatusReport = DependencyCacheRestoreStatus[];
/** Represents the results of `downloadDependencyCaches`. */
export interface DownloadDependencyCachesResult {
/** The status report for telemetry */
statusReport: DependencyCacheRestoreStatusReport;
/** An array of cache keys that we have restored and therefore know to exist. */
restoredKeys: string[];
}
/**
* A wrapper around `cacheConfig.getHashPatterns` which logs when there are no files to calculate
* a hash for the cache key from.
@@ -239,8 +247,9 @@ export async function downloadDependencyCaches(
features: FeatureEnablement,
languages: Language[],
logger: Logger,
): Promise<DependencyCacheRestoreStatusReport> {
): Promise<DownloadDependencyCachesResult> {
const status: DependencyCacheRestoreStatusReport = [];
const restoredKeys: string[] = [];
for (const language of languages) {
const cacheConfig = defaultCacheConfigs[language];
@@ -288,16 +297,27 @@ export async function downloadDependencyCaches(
if (hitKey !== undefined) {
logger.info(`Cache hit on key ${hitKey} for ${language}.`);
const hit_kind =
hitKey === primaryKey ? CacheHitKind.Exact : CacheHitKind.Partial;
status.push({ language, hit_kind, download_duration_ms });
// We have a partial cache hit, unless the key of the restored cache matches the
// primary restore key.
let hit_kind = CacheHitKind.Partial;
if (hitKey === primaryKey) {
hit_kind = CacheHitKind.Exact;
}
status.push({
language,
hit_kind,
download_duration_ms,
});
restoredKeys.push(hitKey);
} else {
status.push({ language, hit_kind: CacheHitKind.Miss });
logger.info(`No suitable cache found for ${language}.`);
}
}
return status;
return { statusReport: status, restoredKeys };
}
/** Enumerates possible outcomes for storing caches. */
+5 -4
View File
@@ -371,7 +371,7 @@ async function run() {
}
let overlayBaseDatabaseStats: OverlayBaseDatabaseDownloadStats | undefined;
let dependencyCachingResults: DependencyCacheRestoreStatusReport | undefined;
let dependencyCachingStatus: DependencyCacheRestoreStatusReport | undefined;
try {
if (
config.overlayDatabaseMode === OverlayDatabaseMode.Overlay &&
@@ -579,12 +579,13 @@ async function run() {
// Restore dependency cache(s), if they exist.
if (shouldRestoreCache(config.dependencyCachingEnabled)) {
dependencyCachingResults = await downloadDependencyCaches(
const dependencyCachingResult = await downloadDependencyCaches(
codeql,
features,
config.languages,
logger,
);
dependencyCachingStatus = dependencyCachingResult.statusReport;
}
// Suppress warnings about disabled Python library extraction.
@@ -732,7 +733,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger,
error,
);
@@ -755,7 +756,7 @@ async function run() {
toolsSource,
toolsVersion,
overlayBaseDatabaseStats,
dependencyCachingResults,
dependencyCachingStatus,
logger,
);
}