From 249a3cbb5c7c03cd2c5dd6872f30f562c2710753 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Tue, 23 Sep 2025 11:48:49 +0100 Subject: [PATCH] Add telemetry for storing dependency caches --- lib/analyze-action.js | 30 ++++++++++++++++++++++++--- src/analyze-action.ts | 27 +++++++++++++++++++++--- src/dependency-caching.ts | 43 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/lib/analyze-action.js b/lib/analyze-action.js index d4ea9eada..0b5e2b252 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -93292,6 +93292,7 @@ async function makeGlobber(patterns) { return glob.create(patterns.join("\n")); } async function uploadDependencyCaches(config, logger, minimizeJavaJars) { + const status = {}; for (const language of config.languages) { const cacheConfig = getDefaultCacheConfig()[language]; if (cacheConfig === void 0) { @@ -93302,6 +93303,7 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) { } const globber = await makeGlobber(cacheConfig.hash); if ((await globber.glob()).length === 0) { + status[language] = { result: "no-hash" /* NoHash */ }; logger.info( `Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.` ); @@ -93309,6 +93311,7 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) { } const size = await getTotalCacheSize(cacheConfig.paths, logger, true); if (size === 0) { + status[language] = { result: "empty" /* Empty */ }; logger.info( `Skipping upload of dependency cache for ${language} since it is empty.` ); @@ -93319,18 +93322,27 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) { `Uploading cache of size ${size} for ${language} with key ${key}...` ); try { + const start = performance.now(); await actionsCache3.saveCache(cacheConfig.paths, key); + const upload_duration_ms = Math.round(performance.now() - start); + status[language] = { + result: "stored" /* Stored */, + upload_size_bytes: Math.round(size), + upload_duration_ms + }; } catch (error2) { if (error2 instanceof actionsCache3.ReserveCacheError) { logger.info( `Not uploading cache for ${language}, because ${key} is already in use.` ); logger.debug(error2.message); + status[language] = { result: "duplicate" /* Duplicate */ }; } else { throw error2; } } } + return status; } async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) { const hash2 = await glob.hashFiles(cacheConfig.hash.join("\n")); @@ -95895,7 +95907,7 @@ function filterAlertsByDiffRange(logger, sarif) { } // src/analyze-action.ts -async function sendStatusReport2(startedAt, config, stats, error2, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, logger) { +async function sendStatusReport2(startedAt, config, stats, error2, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, dependencyCacheResults, logger) { const status = getActionsStatus(error2, stats?.analyze_failure_language); const statusReportBase = await createStatusReportBase( "finish" /* Analyze */, @@ -95912,7 +95924,10 @@ async function sendStatusReport2(startedAt, config, stats, error2, trapCacheUplo ...statusReportBase, ...stats || {}, ...dbCreationTimings || {}, - ...trapCacheCleanup || {} + ...trapCacheCleanup || {}, + dependency_caching_upload_results: JSON.stringify( + dependencyCacheResults ?? {} + ) }; if (config && didUploadTrapCaches) { const trapCacheUploadStatusReport = { @@ -95993,6 +96008,7 @@ async function run() { let trapCacheUploadTime = void 0; let dbCreationTimings = void 0; let didUploadTrapCaches = false; + let dependencyCacheResults; initializeEnvironment(getActionVersion()); persistInputs(); const logger = getActionsLogger(); @@ -96131,7 +96147,11 @@ async function run() { "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, codeql ); - await uploadDependencyCaches(config, logger, minimizeJavaJars); + dependencyCacheResults = await uploadDependencyCaches( + config, + logger, + minimizeJavaJars + ); } if (isInTestMode()) { logger.debug("In test mode. Waiting for processing is disabled."); @@ -96162,6 +96182,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger ); return; @@ -96179,6 +96200,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger ); } else if (runStats) { @@ -96191,6 +96213,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger ); } else { @@ -96203,6 +96226,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger ); } diff --git a/src/analyze-action.ts b/src/analyze-action.ts index 31251be48..6422981ce 100644 --- a/src/analyze-action.ts +++ b/src/analyze-action.ts @@ -26,7 +26,10 @@ import { isCodeScanningEnabled, } from "./config-utils"; import { uploadDatabases } from "./database-upload"; -import { uploadDependencyCaches } from "./dependency-caching"; +import { + DependencyCacheUploadStatusReport, + uploadDependencyCaches, +} from "./dependency-caching"; import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils"; import { EnvVar } from "./environment"; import { Feature, Features } from "./feature-flags"; @@ -55,10 +58,15 @@ interface AnalysisStatusReport extends uploadLib.UploadStatusReport, QueriesStatusReport {} +interface DependencyCachingUploadStatusReport { + dependency_caching_upload_results?: string; +} + interface FinishStatusReport extends StatusReportBase, DatabaseCreationTimings, - AnalysisStatusReport {} + AnalysisStatusReport, + DependencyCachingUploadStatusReport {} interface FinishWithTrapUploadStatusReport extends FinishStatusReport { /** Size of TRAP caches that we uploaded, in bytes. */ @@ -76,6 +84,7 @@ async function sendStatusReport( dbCreationTimings: DatabaseCreationTimings | undefined, didUploadTrapCaches: boolean, trapCacheCleanup: TrapCacheCleanupStatusReport | undefined, + dependencyCacheResults: DependencyCacheUploadStatusReport | undefined, logger: Logger, ) { const status = getActionsStatus(error, stats?.analyze_failure_language); @@ -95,6 +104,9 @@ async function sendStatusReport( ...(stats || {}), ...(dbCreationTimings || {}), ...(trapCacheCleanup || {}), + dependency_caching_upload_results: JSON.stringify( + dependencyCacheResults ?? {}, + ), }; if (config && didUploadTrapCaches) { const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = { @@ -209,6 +221,7 @@ async function run() { let trapCacheUploadTime: number | undefined = undefined; let dbCreationTimings: DatabaseCreationTimings | undefined = undefined; let didUploadTrapCaches = false; + let dependencyCacheResults: DependencyCacheUploadStatusReport | undefined; util.initializeEnvironment(actionsUtil.getActionVersion()); // Make inputs accessible in the `post` step, details at @@ -388,7 +401,11 @@ async function run() { Feature.JavaMinimizeDependencyJars, codeql, ); - await uploadDependencyCaches(config, logger, minimizeJavaJars); + dependencyCacheResults = await uploadDependencyCaches( + config, + logger, + minimizeJavaJars, + ); } // We don't upload results in test mode, so don't wait for processing @@ -431,6 +448,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger, ); return; @@ -449,6 +467,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger, ); } else if (runStats) { @@ -461,6 +480,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger, ); } else { @@ -473,6 +493,7 @@ async function run() { dbCreationTimings, didUploadTrapCaches, trapCacheCleanupTelemetry, + dependencyCacheResults, logger, ); } diff --git a/src/dependency-caching.ts b/src/dependency-caching.ts index 773e4de9e..f5a2ada8f 100644 --- a/src/dependency-caching.ts +++ b/src/dependency-caching.ts @@ -181,18 +181,45 @@ export async function downloadDependencyCaches( return status; } +/** Enumerates possible outcomes for cache hits. */ +export enum CacheStoreResult { + /** We were unable to calculate a hash for the key. */ + NoHash = "no-hash", + /** There is nothing to store in the cache. */ + Empty = "empty", + /** There already exists a cache with the key we are trying to store. */ + Duplicate = "duplicate", + /** The cache was stored successfully. */ + Stored = "stored", +} + +/** Represents results of trying to upload a dependency cache for a language. */ +export interface DependencyCacheUploadStatus { + result: CacheStoreResult; + upload_size_bytes?: number; + upload_duration_ms?: number; +} + +/** A partial mapping from languages to the results of uploading dependency caches for them. */ +export type DependencyCacheUploadStatusReport = Partial< + Record +>; + /** * Attempts to store caches for the languages that were analyzed. * * @param config The configuration for this workflow. * @param logger A logger to record some informational messages to. * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. + * + * @returns A partial mapping of languages to results of uploading dependency caches for them. */ export async function uploadDependencyCaches( config: Config, logger: Logger, minimizeJavaJars: boolean, -): Promise { +): Promise { + const status: DependencyCacheUploadStatusReport = {}; for (const language of config.languages) { const cacheConfig = getDefaultCacheConfig()[language]; @@ -208,6 +235,7 @@ export async function uploadDependencyCaches( const globber = await makeGlobber(cacheConfig.hash); if ((await globber.glob()).length === 0) { + status[language] = { result: CacheStoreResult.NoHash }; logger.info( `Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`, ); @@ -228,6 +256,7 @@ export async function uploadDependencyCaches( // Skip uploading an empty cache. if (size === 0) { + status[language] = { result: CacheStoreResult.Empty }; logger.info( `Skipping upload of dependency cache for ${language} since it is empty.`, ); @@ -241,7 +270,15 @@ export async function uploadDependencyCaches( ); try { + const start = performance.now(); await actionsCache.saveCache(cacheConfig.paths, key); + const upload_duration_ms = Math.round(performance.now() - start); + + status[language] = { + result: CacheStoreResult.Stored, + upload_size_bytes: Math.round(size), + upload_duration_ms, + }; } catch (error) { // `ReserveCacheError` indicates that the cache key is already in use, which means that a // cache with that key already exists or is in the process of being uploaded by another @@ -251,12 +288,16 @@ export async function uploadDependencyCaches( `Not uploading cache for ${language}, because ${key} is already in use.`, ); logger.debug(error.message); + + status[language] = { result: CacheStoreResult.Duplicate }; } else { // Propagate other errors upwards. throw error; } } } + + return status; } /**