/* * We perform enablement checks for overlay analysis to avoid using it on runners that are too small * to support it. However these checks cannot avoid every potential issue without being overly * conservative. Therefore, if our enablement checks enable overlay analysis for a runner that is * too small, we want to remember that, so that we will not try to use overlay analysis until * something changes (e.g. a larger runner is provisioned, or a new CodeQL version is released). * * We use the Actions cache as a lightweight way of providing this functionality. */ import * as fs from "fs"; import * as path from "path"; import * as actionsCache from "@actions/cache"; import { getTemporaryDirectory, getWorkflowRunAttempt, getWorkflowRunID, } from "../actions-util"; import { type CodeQL } from "../codeql"; import { Logger } from "../logging"; import { DiskUsage, getErrorMessage, getRequiredEnvParam, waitForResultWithTimeLimit, } from "../util"; /** The maximum time to wait for a cache operation to complete. */ const MAX_CACHE_OPERATION_MS = 30_000; /** File name for the serialized overlay status. */ const STATUS_FILE_NAME = "overlay-status.json"; /** Path to the local overlay status file. */ function getStatusFilePath(languages: string[]): string { return path.join( getTemporaryDirectory(), "overlay-status", [...languages].sort().join("+"), STATUS_FILE_NAME, ); } /** Details of the job that recorded an overlay status. */ interface JobInfo { /** The check run ID. This is optional since it is not always available. */ checkRunId?: number; /** The workflow run ID. */ workflowRunId: number; /** The workflow run attempt number. */ workflowRunAttempt: number; /** The name of the job (from GITHUB_JOB). */ name: string; } /** Status of an overlay analysis for a group of languages. */ export interface OverlayStatus { /** Whether the job attempted to build an overlay base database. */ attemptedToBuildOverlayBaseDatabase: boolean; /** Whether the job successfully built an overlay base database. */ builtOverlayBaseDatabase: boolean; /** Details of the job that recorded this status. */ job?: JobInfo; } /** Creates an `OverlayStatus` populated with the details of the current job. */ export function createOverlayStatus( attributes: Omit, checkRunId?: number, ): OverlayStatus { const job: JobInfo = { workflowRunId: getWorkflowRunID(), workflowRunAttempt: getWorkflowRunAttempt(), name: getRequiredEnvParam("GITHUB_JOB"), checkRunId, }; return { ...attributes, job, }; } /** * Whether overlay analysis should be skipped, based on the cached status for the given languages and disk usage. */ export async function shouldSkipOverlayAnalysis( codeql: CodeQL, languages: string[], diskUsage: DiskUsage, logger: Logger, ): Promise { const status = await getOverlayStatus(codeql, languages, diskUsage, logger); if (status === undefined) { return false; } if ( status.attemptedToBuildOverlayBaseDatabase && !status.builtOverlayBaseDatabase ) { logger.debug( "Cached overlay status indicates that building an overlay base database was unsuccessful.", ); return true; } logger.debug( "Cached overlay status does not indicate a previous unsuccessful attempt to build an overlay base database.", ); return false; } /** * Retrieve overlay status from the Actions cache, if available. * * @returns `undefined` if no status was found in the cache (e.g. first run with * this cache key) or if the cache operation fails. */ export async function getOverlayStatus( codeql: CodeQL, languages: string[], diskUsage: DiskUsage, logger: Logger, ): Promise { const cacheKey = await getCacheKey(codeql, languages, diskUsage); const statusFile = getStatusFilePath(languages); try { await fs.promises.mkdir(path.dirname(statusFile), { recursive: true }); const foundKey = await waitForResultWithTimeLimit( MAX_CACHE_OPERATION_MS, actionsCache.restoreCache([statusFile], cacheKey), () => { logger.warning("Timed out restoring overlay status from cache."); }, ); if (foundKey === undefined) { logger.debug("No overlay status found in Actions cache."); return undefined; } if (!fs.existsSync(statusFile)) { logger.debug( "Overlay status cache entry found but status file is missing.", ); return undefined; } const contents = await fs.promises.readFile(statusFile, "utf-8"); const parsed: unknown = JSON.parse(contents); if ( typeof parsed !== "object" || parsed === null || typeof parsed["attemptedToBuildOverlayBaseDatabase"] !== "boolean" || typeof parsed["builtOverlayBaseDatabase"] !== "boolean" ) { logger.debug( "Ignoring overlay status cache entry with unexpected format.", ); return undefined; } return parsed as OverlayStatus; } catch (error) { logger.warning( `Failed to restore overlay status from cache: ${getErrorMessage(error)}`, ); return undefined; } } /** * Save overlay status to the Actions cache. * * @returns `true` if the status was saved successfully, `false` otherwise. */ export async function saveOverlayStatus( codeql: CodeQL, languages: string[], diskUsage: DiskUsage, status: OverlayStatus, logger: Logger, ): Promise { const cacheKey = await getCacheKey(codeql, languages, diskUsage); const statusFile = getStatusFilePath(languages); try { await fs.promises.mkdir(path.dirname(statusFile), { recursive: true }); await fs.promises.writeFile(statusFile, JSON.stringify(status)); const cacheId = await waitForResultWithTimeLimit( MAX_CACHE_OPERATION_MS, actionsCache.saveCache([statusFile], cacheKey), () => { logger.warning("Timed out saving overlay status to cache."); }, ); if (cacheId === undefined) { return false; } logger.debug(`Saved overlay status to Actions cache with key ${cacheKey}`); return true; } catch (error) { logger.warning( `Failed to save overlay status to cache: ${getErrorMessage(error)}`, ); return false; } } export async function getCacheKey( codeql: CodeQL, languages: string[], diskUsage: DiskUsage, ): Promise { // Total disk space, rounded to the nearest 10 GB. This is included in the cache key so that if a // customer upgrades their runner, we will try again to use overlay analysis, even if the CodeQL // version has not changed. We round to the nearest 10 GB to work around small differences in disk // space. // // Limitation: this can still flip from "too small" to "large enough" and back again if the disk // space fluctuates above and below a multiple of 10 GB. const diskSpaceToNearest10Gb = `${10 * Math.floor(diskUsage.numTotalBytes / (10 * 1024 * 1024 * 1024))}GB`; // Include the CodeQL version in the cache key so we will try again to use overlay analysis when // new queries and libraries that may be more efficient are released. return `codeql-overlay-status-${[...languages].sort().join("+")}-${(await codeql.getVersion()).version}-runner-${diskSpaceToNearest10Gb}`; }