import * as fs from "fs"; import * as path from "path"; import { performance } from "perf_hooks"; import * as core from "@actions/core"; import * as yaml from "js-yaml"; import { getActionVersion, getOptionalInput, isAnalyzingPullRequest, isDynamicWorkflow, } from "./actions-util"; import { AnalysisConfig, AnalysisKind, codeQualityQueries, getAnalysisConfig, } from "./analyses"; import * as api from "./api-client"; import { CachingKind, getCachingKind } from "./caching-utils"; import { type CodeQL } from "./codeql"; import { calculateAugmentation, ExcludeQueryFilter, generateCodeScanningConfig, parseUserConfig, UserConfig, } from "./config/db-config"; import { addNoLanguageDiagnostic, makeTelemetryDiagnostic, } from "./diagnostics"; import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils"; import { EnvVar } from "./environment"; import * as errorMessages from "./error-messages"; import { Feature, FeatureEnablement } from "./feature-flags"; import { RepositoryProperties, RepositoryPropertyName, } from "./feature-flags/properties"; import { getGeneratedFiles, getGitRoot, getGitVersionOrThrow, GIT_MINIMUM_VERSION_FOR_OVERLAY_WITH_SUBMODULES, GitVersionInfo, hasSubmodules, isAnalyzingDefaultBranch, } from "./git-utils"; import { BuiltInLanguage, Language } from "./languages"; import { Logger } from "./logging"; import { CODEQL_OVERLAY_MINIMUM_VERSION } from "./overlay"; import { addOverlayDisablementDiagnostics, OverlayDisabledReason, } from "./overlay/diagnostics"; import { OverlayDatabaseMode } from "./overlay/overlay-database-mode"; import { shouldSkipOverlayAnalysis } from "./overlay/status"; import { RepositoryNwo } from "./repository"; import { ToolsFeature } from "./tools-features"; import { downloadTrapCaches } from "./trap-caching"; import { GitHubVersion, ConfigurationError, BuildMode, codeQlVersionAtLeast, cloneObject, isDefined, checkDiskUsage, getCodeQLMemoryLimit, getErrorMessage, isInTestMode, joinAtMost, DiskUsage, Result, Success, Failure, isHostedRunner, } from "./util"; /** * The minimum available disk space (in MB) required to perform overlay analysis. * If the available disk space on the runner is below the threshold when deciding * whether to perform overlay analysis, then the action will not perform overlay * analysis unless overlay analysis has been explicitly enabled via environment * variable. */ const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000; const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000; /** * The v2 minimum available disk space (in MB) required to perform overlay * analysis. This is a lower threshold than the v1 limit, allowing overlay * analysis to run on runners with less available disk space. */ const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14000; const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1_000_000; /** * The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If * CodeQL will be given less memory than this threshold, then the action will not perform overlay * analysis unless overlay analysis has been explicitly enabled via environment variable. * * This check is not performed for CodeQL >= `CODEQL_VERSION_REDUCED_OVERLAY_MEMORY_USAGE` since * improved memory usage in that version makes the check unnecessary. */ const OVERLAY_MINIMUM_MEMORY_MB = 5 * 1024; /** * Versions 2.24.3+ of CodeQL reduce overlay analysis's peak RAM usage. * * In particular, RAM usage with overlay analysis enabled should generally be no higher than it is * without overlay analysis for these versions. */ const CODEQL_VERSION_REDUCED_OVERLAY_MEMORY_USAGE = "2.24.3"; export type RegistryConfigWithCredentials = RegistryConfigNoCredentials & { // Token to use when downloading packs from this registry. token: string; }; /** * The list of registries and the associated pack globs that determine where each * pack can be downloaded from. */ export interface RegistryConfigNoCredentials { // URL of a package registry, eg- https://ghcr.io/v2/ url: string; // List of globs that determine which packs are associated with this registry. packages: string[] | string; // Kind of registry, either "github" or "docker". Default is "docker". // "docker" refers specifically to the GitHub Container Registry, which is the usual way of sharing CodeQL packs. // "github" refers to packs published as content in a GitHub repository. This kind of registry is used in scenarios // where GHCR is not available, such as certain GHES environments. kind?: "github" | "docker"; } /** * Format of the parsed config file. */ export interface Config { /** * The version of the CodeQL Action that the configuration is for. */ version: string; /** * Set of analysis kinds that are enabled. */ analysisKinds: AnalysisKind[]; /** * Set of languages to run analysis for. */ languages: Language[]; /** * Build mode, if set. Currently only a single build mode is supported per job. */ buildMode: BuildMode | undefined; /** * A unaltered copy of the original user input. * Mainly intended to be used for status reporting. * If any field is useful for the actual processing * of the action then consider pulling it out to a * top-level field above. */ originalUserInput: UserConfig; /** * Directory to use for temporary files that should be * deleted at the end of the job. */ tempDir: string; /** * Path of the CodeQL executable. */ codeQLCmd: string; /** * Version of GitHub we are talking to. */ gitHubVersion: GitHubVersion; /** * The location where CodeQL databases should be stored. */ dbLocation: string; /** * Specifies whether we are debugging mode and should try to produce extra * output for debugging purposes when possible. */ debugMode: boolean; /** * Specifies the name of the debugging artifact if we are in debug mode. */ debugArtifactName: string; /** * Specifies the name of the database in the debugging artifact. */ debugDatabaseName: string; /** * The configuration we computed by combining `originalUserInput` with `augmentationProperties`, * as well as adjustments made to it based on unsupported or required options. */ computedConfig: UserConfig; /** * Partial map from languages to locations of TRAP caches for that language. * If a key is omitted, then TRAP caching should not be used for that language. */ trapCaches: { [language: Language]: string }; /** * Time taken to download TRAP caches. Used for status reporting. */ trapCacheDownloadTime: number; /** A value indicating how dependency caching should be used. */ dependencyCachingEnabled: CachingKind; /** The keys of caches that we restored, if any. */ dependencyCachingRestoredKeys: string[]; /** * Extra query exclusions to append to the config. */ extraQueryExclusions: ExcludeQueryFilter[]; /** * The overlay database mode to use. */ overlayDatabaseMode: OverlayDatabaseMode; /** * Whether to use caching for overlay databases. If it is true, the action * will upload the created overlay-base database to the actions cache, and * download an overlay-base database from the actions cache before it creates * a new overlay database. If it is false, the action assumes that the * workflow will be responsible for managing database storage and retrieval. * * This property has no effect unless `overlayDatabaseMode` is `Overlay` or * `OverlayBase`. */ useOverlayDatabaseCaching: boolean; /** * A partial mapping from repository properties that affect us to their values. */ repositoryProperties: RepositoryProperties; /** * Whether to enable file coverage information. */ enableFileCoverageInformation: boolean; } async function getSupportedLanguageMap( codeql: CodeQL, logger: Logger, ): Promise> { const resolveSupportedLanguagesUsingCli = await codeql.supportsFeature( ToolsFeature.BuiltinExtractorsSpecifyDefaultQueries, ); const resolveResult = await codeql.betterResolveLanguages({ filterToLanguagesWithQueries: resolveSupportedLanguagesUsingCli, }); if (resolveSupportedLanguagesUsingCli) { logger.debug( `The CodeQL CLI supports the following languages: ${Object.keys(resolveResult.extractors).join(", ")}`, ); } const supportedLanguages: Record = {}; // Populate canonical language names for (const extractor of Object.keys(resolveResult.extractors)) { // If the CLI supports resolving languages with default queries, use these // as the set of supported languages. Otherwise, require the language to be // a built-in language. if ( resolveSupportedLanguagesUsingCli || BuiltInLanguage[extractor] !== undefined ) { supportedLanguages[extractor] = extractor; } } // Populate language aliases if (resolveResult.aliases) { for (const [alias, extractor] of Object.entries(resolveResult.aliases)) { supportedLanguages[alias] = extractor; } } return supportedLanguages; } const baseWorkflowsPath = ".github/workflows"; /** * Determines if there exists a `.github/workflows` directory with at least * one file in it, which we use as an indicator that there are Actions * workflows in the workspace. This doesn't perfectly detect whether there * are actually workflows, but should be a good approximation. * * Alternatively, we could check specifically for yaml files, or call the * API to check if it knows about workflows. * * @returns True if the non-empty directory exists, false if not. */ export function hasActionsWorkflows(sourceRoot: string): boolean { const workflowsPath = path.resolve(sourceRoot, baseWorkflowsPath); const stats = fs.lstatSync(workflowsPath, { throwIfNoEntry: false }); return ( stats !== undefined && stats.isDirectory() && fs.readdirSync(workflowsPath).length > 0 ); } /** * Gets the set of languages in the current repository. */ async function getRawLanguagesInRepo( repository: RepositoryNwo, sourceRoot: string, logger: Logger, ): Promise { logger.debug( `Automatically detecting languages (${repository.owner}/${repository.repo})`, ); const response = await api.getApiClient().rest.repos.listLanguages({ owner: repository.owner, repo: repository.repo, }); logger.debug(`Languages API response: ${JSON.stringify(response)}`); const result = Object.keys(response.data as Record).map( (language) => language.trim().toLowerCase(), ); if (hasActionsWorkflows(sourceRoot)) { logger.debug(`Found a .github/workflows directory`); result.push("actions"); } logger.debug(`Raw languages in repository: ${result.join(", ")}`); return result; } /** * Get the languages to analyse. * * The result is obtained from the action input parameter 'languages' if that * has been set, otherwise it is deduced as all languages in the repo that * can be analysed. * * If no languages could be detected from either the workflow or the repository * then throw an error. */ export async function getLanguages( codeql: CodeQL, languagesInput: string | undefined, repository: RepositoryNwo, sourceRoot: string, logger: Logger, ): Promise { // Obtain languages without filtering them. const { rawLanguages, autodetected } = await getRawLanguages( languagesInput, repository, sourceRoot, logger, ); const languageMap = await getSupportedLanguageMap(codeql, logger); const languagesSet = new Set(); const unknownLanguages: string[] = []; // Make sure they are supported for (const language of rawLanguages) { const extractorName = languageMap[language]; if (extractorName === undefined) { unknownLanguages.push(language); } else { languagesSet.add(extractorName); } } const languages = Array.from(languagesSet); if (!autodetected && unknownLanguages.length > 0) { throw new ConfigurationError( errorMessages.getUnknownLanguagesError(unknownLanguages), ); } // If the languages parameter was not given and no languages were // detected then fail here as this is a workflow configuration error. if (languages.length === 0) { throw new ConfigurationError(errorMessages.getNoLanguagesError()); } if (autodetected) { logger.info(`Autodetected languages: ${languages.join(", ")}`); } else { logger.info(`Languages from configuration: ${languages.join(", ")}`); } return languages; } export function getRawLanguagesNoAutodetect( languagesInput: string | undefined, ): string[] { return (languagesInput || "") .split(",") .map((x) => x.trim().toLowerCase()) .filter((x) => x.length > 0); } /** * Gets the set of languages in the current repository without checking to * see if these languages are actually supported by CodeQL. * * @param languagesInput The languages from the workflow input * @param repository the owner/name of the repository * @param logger a logger * @returns A tuple containing a list of languages in this repository that might be * analyzable and whether or not this list was determined automatically. */ async function getRawLanguages( languagesInput: string | undefined, repository: RepositoryNwo, sourceRoot: string, logger: Logger, ): Promise<{ rawLanguages: string[]; autodetected: boolean; }> { // If the user has specified languages, use those. const languagesFromInput = getRawLanguagesNoAutodetect(languagesInput); if (languagesFromInput.length > 0) { return { rawLanguages: languagesFromInput, autodetected: false }; } // Otherwise, autodetect languages in the repository. return { rawLanguages: await getRawLanguagesInRepo(repository, sourceRoot, logger), autodetected: true, }; } /** Inputs required to initialize a configuration. */ export interface InitConfigInputs { languagesInput: string | undefined; queriesInput: string | undefined; packsInput: string | undefined; configFile: string | undefined; dbLocation: string | undefined; configInput: string | undefined; buildModeInput: string | undefined; ramInput: string | undefined; dependencyCachingEnabled: string | undefined; debugMode: boolean; debugArtifactName: string; debugDatabaseName: string; repository: RepositoryNwo; tempDir: string; codeql: CodeQL; workspacePath: string; sourceRoot: string; githubVersion: GitHubVersion; apiDetails: api.GitHubApiCombinedDetails; features: FeatureEnablement; repositoryProperties: RepositoryProperties; enableFileCoverageInformation: boolean; analysisKinds: AnalysisKind[]; logger: Logger; } /** * Initialise the CodeQL Action state, which includes the base configuration for the Action * and computes the configuration for the CodeQL CLI. */ export async function initActionState( { languagesInput, queriesInput, packsInput, buildModeInput, dbLocation, dependencyCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeql, sourceRoot, githubVersion, features, repositoryProperties, analysisKinds, logger, enableFileCoverageInformation, }: InitConfigInputs, userConfig: UserConfig, ): Promise { const languages = await getLanguages( codeql, languagesInput, repository, sourceRoot, logger, ); const buildMode = await parseBuildModeInput( buildModeInput, languages, features, logger, ); const augmentationProperties = await calculateAugmentation( packsInput, queriesInput, repositoryProperties, languages, ); // If `code-quality` is the only enabled analysis kind, we don't support query customisation. // It would be a problem if queries that are configured in repository properties cause `code-quality`-only // analyses to break. We therefore ignore query customisations that are configured in repository properties // if `code-quality` is the only enabled analysis kind. if ( analysisKinds.length === 1 && analysisKinds.includes(AnalysisKind.CodeQuality) && augmentationProperties.repoPropertyQueries.input ) { logger.info( `Ignoring queries configured in the repository properties, because query customisations are not supported for Code Quality analyses.`, ); augmentationProperties.repoPropertyQueries = { combines: false, input: undefined, }; } // Compute the full Code Scanning configuration that combines the configuration from the // configuration file / `config` input with other inputs, such as `queries`. const computedConfig = generateCodeScanningConfig( logger, userConfig, augmentationProperties, ); return { version: getActionVersion(), analysisKinds, languages, buildMode, originalUserInput: userConfig, computedConfig, tempDir, codeQLCmd: codeql.getPath(), gitHubVersion: githubVersion, dbLocation: dbLocationOrDefault(dbLocation, tempDir), debugMode, debugArtifactName, debugDatabaseName, trapCaches: {}, trapCacheDownloadTime: 0, dependencyCachingEnabled: getCachingKind(dependencyCachingEnabled), dependencyCachingRestoredKeys: [], extraQueryExclusions: [], overlayDatabaseMode: OverlayDatabaseMode.None, useOverlayDatabaseCaching: false, repositoryProperties, enableFileCoverageInformation, }; } async function downloadCacheWithTime( codeQL: CodeQL, languages: Language[], logger: Logger, ): Promise<{ trapCaches: { [language: string]: string }; trapCacheDownloadTime: number; }> { const start = performance.now(); const trapCaches = await downloadTrapCaches(codeQL, languages, logger); const trapCacheDownloadTime = performance.now() - start; return { trapCaches, trapCacheDownloadTime }; } async function loadUserConfig( logger: Logger, configFile: string, workspacePath: string, apiDetails: api.GitHubApiCombinedDetails, tempDir: string, validateConfig: boolean, ): Promise { if (isLocal(configFile)) { if (configFile !== userConfigFromActionPath(tempDir)) { // If the config file is not generated by the Action, it should be relative to the workspace. configFile = path.resolve(workspacePath, configFile); // Error if the config file is now outside of the workspace if (!(configFile + path.sep).startsWith(workspacePath + path.sep)) { throw new ConfigurationError( errorMessages.getConfigFileOutsideWorkspaceErrorMessage(configFile), ); } } return getLocalConfig(logger, configFile, validateConfig); } else { return await getRemoteConfig( logger, configFile, apiDetails, validateConfig, ); } } /** * Maps languages to their overlay analysis feature flags. Only languages that * are GA or in staff-ship for overlay analysis are included here. Languages * without an entry will have overlay analysis disabled. */ const OVERLAY_ANALYSIS_FEATURES: Partial> = { cpp: Feature.OverlayAnalysisCpp, csharp: Feature.OverlayAnalysisCsharp, go: Feature.OverlayAnalysisGo, java: Feature.OverlayAnalysisJava, javascript: Feature.OverlayAnalysisJavascript, python: Feature.OverlayAnalysisPython, ruby: Feature.OverlayAnalysisRuby, }; const OVERLAY_ANALYSIS_CODE_SCANNING_FEATURES: Partial< Record > = { cpp: Feature.OverlayAnalysisCodeScanningCpp, csharp: Feature.OverlayAnalysisCodeScanningCsharp, go: Feature.OverlayAnalysisCodeScanningGo, java: Feature.OverlayAnalysisCodeScanningJava, javascript: Feature.OverlayAnalysisCodeScanningJavascript, python: Feature.OverlayAnalysisCodeScanningPython, ruby: Feature.OverlayAnalysisCodeScanningRuby, }; /** * Checks whether the overlay analysis feature is enabled for the given * languages and configuration. */ async function checkOverlayAnalysisFeatureEnabled( features: FeatureEnablement, codeql: CodeQL, languages: Language[], codeScanningConfig: UserConfig, ): Promise> { if (!(await features.getValue(Feature.OverlayAnalysis, codeql))) { return new Failure(OverlayDisabledReason.OverallFeatureNotEnabled); } let enableForCodeScanningOnly = false; for (const language of languages) { const feature = OVERLAY_ANALYSIS_FEATURES[language]; if (feature && (await features.getValue(feature, codeql))) { continue; } const codeScanningFeature = OVERLAY_ANALYSIS_CODE_SCANNING_FEATURES[language]; if ( codeScanningFeature && (await features.getValue(codeScanningFeature, codeql)) ) { enableForCodeScanningOnly = true; continue; } return new Failure(OverlayDisabledReason.LanguageNotEnabled); } if (enableForCodeScanningOnly) { // A code-scanning configuration runs only the (default) code-scanning suite // if the default queries are not disabled, and no packs, queries, or // query-filters are specified. const usesDefaultQueriesOnly = codeScanningConfig["disable-default-queries"] !== true && codeScanningConfig.packs === undefined && codeScanningConfig.queries === undefined && codeScanningConfig["query-filters"] === undefined; if (!usesDefaultQueriesOnly) { return new Failure(OverlayDisabledReason.NonDefaultQueries); } } return new Success(undefined); } /** Checks if the runner has enough disk space for overlay analysis. */ function runnerHasSufficientDiskSpace( diskUsage: DiskUsage, logger: Logger, useV2ResourceChecks: boolean, ): boolean { const minimumDiskSpaceBytes = useV2ResourceChecks ? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES : OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES; if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) { const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000); const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000); logger.info( `Setting overlay database mode to ${OverlayDatabaseMode.None} ` + `due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`, ); return false; } return true; } /** Checks if the runner has enough memory for overlay analysis. */ async function runnerHasSufficientMemory( codeql: CodeQL, ramInput: string | undefined, logger: Logger, ): Promise { if ( await codeQlVersionAtLeast( codeql, CODEQL_VERSION_REDUCED_OVERLAY_MEMORY_USAGE, ) ) { logger.debug( `Skipping memory check for overlay analysis because CodeQL version is at least ${CODEQL_VERSION_REDUCED_OVERLAY_MEMORY_USAGE}.`, ); return true; } const memoryFlagValue = getCodeQLMemoryLimit(ramInput, logger); if (memoryFlagValue < OVERLAY_MINIMUM_MEMORY_MB) { logger.info( `Setting overlay database mode to ${OverlayDatabaseMode.None} ` + `due to insufficient memory for CodeQL analysis (${memoryFlagValue} MB, needed ${OVERLAY_MINIMUM_MEMORY_MB} MB).`, ); return false; } logger.debug( `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`, ); return true; } /** * Checks if the runner has sufficient disk space and memory for overlay * analysis. */ async function checkRunnerResources( codeql: CodeQL, diskUsage: DiskUsage, ramInput: string | undefined, logger: Logger, useV2ResourceChecks: boolean, ): Promise> { if (!runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks)) { return new Failure(OverlayDisabledReason.InsufficientDiskSpace); } if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) { return new Failure(OverlayDisabledReason.InsufficientMemory); } return new Success(undefined); } interface EnabledOverlayConfig { overlayDatabaseMode: Exclude; useOverlayDatabaseCaching: boolean; } /** * Calculate and validate the overlay database mode and caching to use. * * - If the environment variable `CODEQL_OVERLAY_DATABASE_MODE` is set, use it. * In this case, the workflow is responsible for managing database storage and * retrieval, and the action will not perform overlay database caching. Think * of it as a "manual control" mode where the calling workflow is responsible * for making sure that everything is set up correctly. * - Otherwise, if `Feature.OverlayAnalysis` is enabled, calculate the mode * based on what we are analyzing. Think of it as a "automatic control" mode * where the action will do the right thing by itself. * - If we are analyzing a pull request, use `Overlay` with caching. * - If we are analyzing the default branch, use `OverlayBase` with caching. * - Otherwise, use `None`. * * For `Overlay` and `OverlayBase`, the function performs further checks and * reverts to `None` if any check should fail. * * @returns A `Success` containing the overlay database mode and whether the * action should perform overlay-base database caching, or a `Failure` * containing the reason why overlay analysis is disabled. */ export async function checkOverlayEnablement( codeql: CodeQL, features: FeatureEnablement, languages: Language[], sourceRoot: string, buildMode: BuildMode | undefined, ramInput: string | undefined, codeScanningConfig: UserConfig, repositoryProperties: RepositoryProperties, gitVersion: GitVersionInfo | undefined, logger: Logger, ): Promise> { const modeEnv = process.env.CODEQL_OVERLAY_DATABASE_MODE; // Any unrecognized CODEQL_OVERLAY_DATABASE_MODE value will be ignored and // treated as if the environment variable was not set. if ( modeEnv === OverlayDatabaseMode.Overlay || modeEnv === OverlayDatabaseMode.OverlayBase || modeEnv === OverlayDatabaseMode.None ) { logger.info( `Setting overlay database mode to ${modeEnv} ` + "from the CODEQL_OVERLAY_DATABASE_MODE environment variable.", ); if (modeEnv === OverlayDatabaseMode.None) { return new Failure(OverlayDisabledReason.DisabledByEnvironmentVariable); } return validateOverlayDatabaseMode( modeEnv, false, codeql, languages, sourceRoot, buildMode, gitVersion, logger, ); } if (repositoryProperties[RepositoryPropertyName.DISABLE_OVERLAY] === true) { logger.info( `Setting overlay database mode to ${OverlayDatabaseMode.None} ` + `because the ${RepositoryPropertyName.DISABLE_OVERLAY} repository property is set to true.`, ); return new Failure(OverlayDisabledReason.DisabledByRepositoryProperty); } const featureResult = await checkOverlayAnalysisFeatureEnabled( features, codeql, languages, codeScanningConfig, ); if (featureResult.isFailure()) { return featureResult; } const performResourceChecks = !(await features.getValue( Feature.OverlayAnalysisSkipResourceChecks, codeql, )); const useV2ResourceChecks = await features.getValue( Feature.OverlayAnalysisResourceChecksV2, ); const checkOverlayStatus = await features.getValue( Feature.OverlayAnalysisStatusCheck, ); const needDiskUsage = performResourceChecks || checkOverlayStatus; const diskUsage = needDiskUsage ? await checkDiskUsage(logger) : undefined; if (needDiskUsage && diskUsage === undefined) { logger.warning( `Unable to determine disk usage, therefore setting overlay database mode to ${OverlayDatabaseMode.None}.`, ); return new Failure(OverlayDisabledReason.UnableToDetermineDiskUsage); } const resourceResult = performResourceChecks && diskUsage !== undefined ? await checkRunnerResources( codeql, diskUsage, ramInput, logger, useV2ResourceChecks, ) : new Success(undefined); if (resourceResult.isFailure()) { return resourceResult; } if ( checkOverlayStatus && diskUsage !== undefined && (await shouldSkipOverlayAnalysis(codeql, languages, diskUsage, logger)) ) { logger.info( `Setting overlay database mode to ${OverlayDatabaseMode.None} ` + "because overlay analysis previously failed with this combination of languages, " + "disk space, and CodeQL version.", ); return new Failure(OverlayDisabledReason.SkippedDueToCachedStatus); } let overlayDatabaseMode: OverlayDatabaseMode; if (isAnalyzingPullRequest()) { overlayDatabaseMode = OverlayDatabaseMode.Overlay; logger.info( `Setting overlay database mode to ${overlayDatabaseMode} ` + "with caching because we are analyzing a pull request.", ); } else if (await isAnalyzingDefaultBranch()) { overlayDatabaseMode = OverlayDatabaseMode.OverlayBase; logger.info( `Setting overlay database mode to ${overlayDatabaseMode} ` + "with caching because we are analyzing the default branch.", ); } else { return new Failure(OverlayDisabledReason.NotPullRequestOrDefaultBranch); } return validateOverlayDatabaseMode( overlayDatabaseMode, true, codeql, languages, sourceRoot, buildMode, gitVersion, logger, ); } /** * Validates that the given overlay database mode is compatible with the current * configuration (build mode, CodeQL version, git repository, git version). Returns * the mode unchanged if all checks pass, or falls back to `None` with the * appropriate disabled reason. */ async function validateOverlayDatabaseMode( overlayDatabaseMode: Exclude, useOverlayDatabaseCaching: boolean, codeql: CodeQL, languages: Language[], sourceRoot: string, buildMode: BuildMode | undefined, gitVersion: GitVersionInfo | undefined, logger: Logger, ): Promise> { if ( buildMode !== BuildMode.None && ( await Promise.all( languages.map( async (l) => l !== BuiltInLanguage.go && // Workaround to allow overlay analysis for Go with any build // mode, since it does not yet support BMN. The Go autobuilder and/or extractor will // ensure that overlay-base databases are only created for supported Go build setups, // and that we'll fall back to full databases in other cases. (await codeql.isTracedLanguage(l)), ), ) ).some(Boolean) ) { logger.warning( `Cannot build an ${overlayDatabaseMode} database because ` + `build-mode is set to "${buildMode}" instead of "none". ` + "Falling back to creating a normal full database instead.", ); return new Failure(OverlayDisabledReason.IncompatibleBuildMode); } if (!(await codeQlVersionAtLeast(codeql, CODEQL_OVERLAY_MINIMUM_VERSION))) { logger.warning( `Cannot build an ${overlayDatabaseMode} database because ` + `the CodeQL CLI is older than ${CODEQL_OVERLAY_MINIMUM_VERSION}. ` + "Falling back to creating a normal full database instead.", ); return new Failure(OverlayDisabledReason.IncompatibleCodeQl); } const gitRoot = await getGitRoot(sourceRoot); if (gitRoot === undefined) { logger.warning( `Cannot build an ${overlayDatabaseMode} database because ` + `the source root "${sourceRoot}" is not inside a git repository. ` + "Falling back to creating a normal full database instead.", ); return new Failure(OverlayDisabledReason.NoGitRoot); } if (hasSubmodules(gitRoot)) { if (gitVersion === undefined) { logger.warning( `Cannot build an ${overlayDatabaseMode} database because ` + "the repository has submodules and the Git version could not be determined. " + "Falling back to creating a normal full database instead.", ); return new Failure(OverlayDisabledReason.IncompatibleGit); } if ( !gitVersion.isAtLeast(GIT_MINIMUM_VERSION_FOR_OVERLAY_WITH_SUBMODULES) ) { logger.warning( `Cannot build an ${overlayDatabaseMode} database because ` + "the repository has submodules and the installed Git version is older " + `than ${GIT_MINIMUM_VERSION_FOR_OVERLAY_WITH_SUBMODULES}. ` + "Falling back to creating a normal full database instead.", ); return new Failure(OverlayDisabledReason.IncompatibleGit); } } return new Success({ overlayDatabaseMode, useOverlayDatabaseCaching, }); } export async function isTrapCachingEnabled( features: FeatureEnablement, overlayDatabaseMode: OverlayDatabaseMode, ): Promise { // If the workflow specified something, always respect that. const trapCaching = getOptionalInput("trap-caching"); if (trapCaching !== undefined) return trapCaching === "true"; // On self-hosted runners which may have slow network access, disable TRAP caching by default. if (!isHostedRunner()) return false; // If overlay analysis is enabled, then disable TRAP caching since overlay analysis supersedes it. // This change is gated behind a feature flag. if ( overlayDatabaseMode !== OverlayDatabaseMode.None && (await features.getValue(Feature.OverlayAnalysisDisableTrapCaching)) ) { return false; } // Otherwise, enable TRAP caching. return true; } async function setCppTrapCachingEnvironmentVariables( config: Config, logger: Logger, ): Promise { if (config.languages.includes(BuiltInLanguage.cpp)) { const envVar = "CODEQL_EXTRACTOR_CPP_TRAP_CACHING"; if (process.env[envVar]) { logger.info( `Environment variable ${envVar} already set, leaving it unchanged.`, ); } else if (config.trapCaches[BuiltInLanguage.cpp]) { logger.info("Enabling TRAP caching for C/C++."); core.exportVariable(envVar, "true"); } else { logger.debug(`Disabling TRAP caching for C/C++.`); core.exportVariable(envVar, "false"); } } } function dbLocationOrDefault( dbLocation: string | undefined, tempDir: string, ): string { return dbLocation || path.resolve(tempDir, "codeql_databases"); } function userConfigFromActionPath(tempDir: string): string { return path.resolve(tempDir, "user-config-from-action.yml"); } /** * Checks whether the given `UserConfig` contains any query customisations. * * @returns Returns `true` if the `UserConfig` customises which queries are run. */ function hasQueryCustomisation(userConfig: UserConfig): boolean { return ( isDefined(userConfig["disable-default-queries"]) || isDefined(userConfig.queries) || isDefined(userConfig["query-filters"]) ); } /** * Load and return the config. * * This will parse the config from the user input if present, or generate * a default config. The parsed config is then stored to a known location. */ export async function initConfig( features: FeatureEnablement, inputs: InitConfigInputs, ): Promise { const { logger, tempDir } = inputs; // if configInput is set, it takes precedence over configFile if (inputs.configInput) { if (inputs.configFile) { logger.warning( `Both a config file and config input were provided. Ignoring config file.`, ); } inputs.configFile = userConfigFromActionPath(tempDir); fs.writeFileSync(inputs.configFile, inputs.configInput); logger.debug(`Using config from action input: ${inputs.configFile}`); } let userConfig: UserConfig = {}; if (!inputs.configFile) { logger.debug("No configuration file was provided"); } else { logger.debug(`Using configuration file: ${inputs.configFile}`); const validateConfig = await features.getValue(Feature.ValidateDbConfig); userConfig = await loadUserConfig( logger, inputs.configFile, inputs.workspacePath, inputs.apiDetails, tempDir, validateConfig, ); } const config = await initActionState(inputs, userConfig); // If Code Quality analysis is the only enabled analysis kind, then we will initialise // the database for Code Quality. That entails disabling the default queries and only // running quality queries. We do not currently support query customisations in that case. if (config.analysisKinds.length === 1 && isCodeQualityEnabled(config)) { // Warn if any query customisations are present in the computed configuration. if (hasQueryCustomisation(config.computedConfig)) { throw new ConfigurationError( "Query customizations are unsupported, because only `code-quality` analysis is enabled.", ); } const queries = codeQualityQueries.map((v) => ({ uses: v })); // Set the query customisation options for Code Quality only analysis. config.computedConfig["disable-default-queries"] = true; config.computedConfig.queries = queries; config.computedConfig["query-filters"] = []; } let gitVersion: GitVersionInfo | undefined = undefined; try { gitVersion = await getGitVersionOrThrow(); logger.info(`Using Git version ${gitVersion.fullVersion}`); await logGitVersionTelemetry(config, gitVersion); } catch (e) { logger.warning(`Could not determine Git version: ${getErrorMessage(e)}`); // Throw the error in test mode so it's more visible, unless the environment // variable is set to tolerate this, for example because we're running in a // Docker container where git may not be available. if ( isInTestMode() && process.env[EnvVar.TOLERATE_MISSING_GIT_VERSION] !== "true" ) { throw e; } } // If we are in a dynamic workflow or the corresponding FF is enabled, try to determine // which files in the repository are marked as generated and add them to // the `paths-ignore` configuration. if ( (await features.getValue(Feature.IgnoreGeneratedFiles)) && isDynamicWorkflow() ) { try { const generatedFilesCheckStartedAt = performance.now(); const generatedFiles = await getGeneratedFiles(inputs.sourceRoot); const generatedFilesDuration = Math.round( performance.now() - generatedFilesCheckStartedAt, ); if (generatedFiles.length > 0) { config.computedConfig["paths-ignore"] ??= []; config.computedConfig["paths-ignore"].push(...generatedFiles); logger.info( `Detected ${generatedFiles.length} generated file(s), which will be excluded from analysis: ${joinAtMost(generatedFiles, ", ", 10)}`, ); } else { logger.info(`Found no generated files.`); } await logGeneratedFilesTelemetry( config, generatedFilesDuration, generatedFiles.length, ); } catch (error) { logger.info(`Cannot ignore generated files: ${getErrorMessage(error)}`); } } else { logger.debug(`Skipping check for generated files.`); } // The choice of overlay database mode depends on the selection of languages // and queries, which in turn depends on the user config and the augmentation // properties. So we need to calculate the overlay database mode after the // rest of the config has been populated. const overlayDatabaseModeResult = await checkOverlayEnablement( inputs.codeql, inputs.features, config.languages, inputs.sourceRoot, config.buildMode, inputs.ramInput, config.computedConfig, config.repositoryProperties, gitVersion, logger, ); if (overlayDatabaseModeResult.isSuccess()) { const { overlayDatabaseMode, useOverlayDatabaseCaching } = overlayDatabaseModeResult.value; logger.info( `Using overlay database mode: ${overlayDatabaseMode} ` + `${useOverlayDatabaseCaching ? "with" : "without"} caching.`, ); config.overlayDatabaseMode = overlayDatabaseMode; config.useOverlayDatabaseCaching = useOverlayDatabaseCaching; } else { const overlayDisabledReason = overlayDatabaseModeResult.value; logger.info( `Using overlay database mode: ${OverlayDatabaseMode.None} without caching.`, ); config.overlayDatabaseMode = OverlayDatabaseMode.None; config.useOverlayDatabaseCaching = false; await addOverlayDisablementDiagnostics( config, inputs.codeql, overlayDisabledReason, ); } if ( config.overlayDatabaseMode === OverlayDatabaseMode.Overlay || (await shouldPerformDiffInformedAnalysis( inputs.codeql, inputs.features, logger, )) ) { config.extraQueryExclusions.push({ exclude: { tags: "exclude-from-incremental" }, }); } if (await isTrapCachingEnabled(features, config.overlayDatabaseMode)) { const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime( inputs.codeql, config.languages, logger, ); config.trapCaches = trapCaches; config.trapCacheDownloadTime = trapCacheDownloadTime; } await setCppTrapCachingEnvironmentVariables(config, logger); return config; } function parseRegistries( registriesInput: string | undefined, ): RegistryConfigWithCredentials[] | undefined { try { return registriesInput ? (yaml.load(registriesInput) as RegistryConfigWithCredentials[]) : undefined; } catch { throw new ConfigurationError( "Invalid registries input. Must be a YAML string.", ); } } export function parseRegistriesWithoutCredentials( registriesInput?: string, ): RegistryConfigNoCredentials[] | undefined { return parseRegistries(registriesInput)?.map((r) => { const { url, packages, kind } = r; return { url, packages, kind }; }); } function isLocal(configPath: string): boolean { // If the path starts with ./, look locally if (configPath.indexOf("./") === 0) { return true; } return configPath.indexOf("@") === -1; } function getLocalConfig( logger: Logger, configFile: string, validateConfig: boolean, ): UserConfig { // Error if the file does not exist if (!fs.existsSync(configFile)) { throw new ConfigurationError( errorMessages.getConfigFileDoesNotExistErrorMessage(configFile), ); } return parseUserConfig( logger, configFile, fs.readFileSync(configFile, "utf-8"), validateConfig, ); } async function getRemoteConfig( logger: Logger, configFile: string, apiDetails: api.GitHubApiCombinedDetails, validateConfig: boolean, ): Promise { // retrieve the various parts of the config location, and ensure they're present const format = new RegExp( "(?[^/]+)/(?[^/]+)/(?[^@]+)@(?.*)", ); const pieces = format.exec(configFile); // 5 = 4 groups + the whole expression if (pieces?.groups === undefined || pieces.length < 5) { throw new ConfigurationError( errorMessages.getConfigFileRepoFormatInvalidMessage(configFile), ); } const response = await api .getApiClientWithExternalAuth(apiDetails) .rest.repos.getContent({ owner: pieces.groups.owner, repo: pieces.groups.repo, path: pieces.groups.path, ref: pieces.groups.ref, }); let fileContents: string; if ("content" in response.data && response.data.content !== undefined) { fileContents = response.data.content; } else if (Array.isArray(response.data)) { throw new ConfigurationError( errorMessages.getConfigFileDirectoryGivenMessage(configFile), ); } else { throw new ConfigurationError( errorMessages.getConfigFileFormatInvalidMessage(configFile), ); } return parseUserConfig( logger, configFile, Buffer.from(fileContents, "base64").toString("binary"), validateConfig, ); } /** * Get the file path where the parsed config will be stored. */ export function getPathToParsedConfigFile(tempDir: string): string { return path.join(tempDir, "config"); } /** * Store the given config to the path returned from getPathToParsedConfigFile. */ export async function saveConfig(config: Config, logger: Logger) { const configString = JSON.stringify(config); const configFile = getPathToParsedConfigFile(config.tempDir); fs.mkdirSync(path.dirname(configFile), { recursive: true }); fs.writeFileSync(configFile, configString, "utf8"); logger.debug("Saved config:"); logger.debug(configString); } /** * Get the config that has been saved to the given temp dir. * If the config could not be found then returns undefined. */ export async function getConfig( tempDir: string, logger: Logger, ): Promise { const configFile = getPathToParsedConfigFile(tempDir); if (!fs.existsSync(configFile)) { return undefined; } const configString = fs.readFileSync(configFile, "utf8"); logger.debug("Loaded config:"); logger.debug(configString); const config = JSON.parse(configString) as Partial; if (config.version === undefined) { throw new ConfigurationError( `Loaded configuration file, but it does not contain the expected 'version' field.`, ); } if (config.version !== getActionVersion()) { throw new ConfigurationError( `Loaded a configuration file for version '${config.version}', but running version '${getActionVersion()}'`, ); } return config as Config; } /** * Generate a `qlconfig.yml` file from the `registries` input. * This file is used by the CodeQL CLI to list the registries to use for each * pack. * * @param registriesInput The value of the `registries` input. * @param tempDir a temporary directory to store the generated qlconfig.yml file. * @param logger a logger object. * @returns The path to the generated `qlconfig.yml` file and the auth tokens to * use for each registry. */ export async function generateRegistries( registriesInput: string | undefined, tempDir: string, logger: Logger, ) { const registries = parseRegistries(registriesInput); let registriesAuthTokens: string | undefined; let qlconfigFile: string | undefined; if (registries) { // generate a qlconfig.yml file to hold the registry configs. const qlconfig = createRegistriesBlock(registries); qlconfigFile = path.join(tempDir, "qlconfig.yml"); const qlconfigContents = yaml.dump(qlconfig); fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8"); logger.debug("Generated qlconfig.yml:"); logger.debug(qlconfigContents); registriesAuthTokens = registries .map((registry) => `${registry.url}=${registry.token}`) .join(","); } if (typeof process.env.CODEQL_REGISTRIES_AUTH === "string") { logger.debug( "Using CODEQL_REGISTRIES_AUTH environment variable to authenticate with registries.", ); } return { registriesAuthTokens: // if the user has explicitly set the CODEQL_REGISTRIES_AUTH env var then use that process.env.CODEQL_REGISTRIES_AUTH ?? registriesAuthTokens, qlconfigFile, }; } function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): { registries: RegistryConfigNoCredentials[]; } { if ( !Array.isArray(registries) || registries.some((r) => !r.url || !r.packages) ) { throw new ConfigurationError( "Invalid 'registries' input. Must be an array of objects with 'url' and 'packages' properties.", ); } // be sure to remove the `token` field from the registry before writing it to disk. const safeRegistries = registries.map((registry) => ({ // ensure the url ends with a slash to avoid a bug in the CLI 2.10.4 url: !registry?.url.endsWith("/") ? `${registry.url}/` : registry.url, packages: registry.packages, kind: registry.kind, })); const qlconfig = { registries: safeRegistries, }; return qlconfig; } /** * Create a temporary environment based on the existing environment and overridden * by the given environment variables that are passed in as arguments. * * Use this new environment in the context of the given operation. After completing * the operation, restore the original environment. * * This function does not support un-setting environment variables. * * @param env * @param operation */ export async function wrapEnvironment( env: Record, operation: () => Promise, ) { // Remember the original env const oldEnv = { ...process.env }; // Set the new env for (const [key, value] of Object.entries(env)) { // Ignore undefined keys if (value !== undefined) { process.env[key] = value; } } try { // Run the operation await operation(); } finally { // Restore the old env for (const [key, value] of Object.entries(oldEnv)) { process.env[key] = value; } } } // Exported for testing export async function parseBuildModeInput( input: string | undefined, languages: Language[], features: FeatureEnablement, logger: Logger, ): Promise { if (input === undefined) { return undefined; } if (!Object.values(BuildMode).includes(input as BuildMode)) { throw new ConfigurationError( `Invalid build mode: '${input}'. Supported build modes are: ${Object.values( BuildMode, ).join(", ")}.`, ); } if ( languages.includes(BuiltInLanguage.csharp) && (await features.getValue(Feature.DisableCsharpBuildless)) ) { logger.warning( "Scanning C# code without a build is temporarily unavailable. Falling back to 'autobuild' build mode.", ); return BuildMode.Autobuild; } if ( languages.includes(BuiltInLanguage.java) && (await features.getValue(Feature.DisableJavaBuildlessEnabled)) ) { logger.warning( "Scanning Java code without a build is temporarily unavailable. Falling back to 'autobuild' build mode.", ); return BuildMode.Autobuild; } return input as BuildMode; } /** * Appends `extraQueryExclusions` to `cliConfig`'s `query-filters`. * * @param extraQueryExclusions The extra query exclusions to append to the `query-filters`. * @param cliConfig The CodeQL CLI configuration to extend. * @returns Returns `cliConfig` if there are no extra query exclusions * or a copy of `cliConfig` where the extra query exclusions * have been appended to `query-filters`. */ export function appendExtraQueryExclusions( extraQueryExclusions: ExcludeQueryFilter[], cliConfig: UserConfig, ): Readonly { // make a copy so we can modify it and so that modifications to the input // object do not affect the result that is marked as `Readonly`. const augmentedConfig = cloneObject(cliConfig); if (extraQueryExclusions.length === 0) { return augmentedConfig; } augmentedConfig["query-filters"] = [ // Ordering matters. If the first filter is an inclusion, it implicitly // excludes all queries that are not included. If it is an exclusion, // it implicitly includes all queries that are not excluded. So user // filters (if any) should always be first to preserve intent. ...(augmentedConfig["query-filters"] || []), ...extraQueryExclusions, ]; if (augmentedConfig["query-filters"]?.length === 0) { delete augmentedConfig["query-filters"]; } return augmentedConfig; } /** * Returns `true` if Code Scanning analysis is enabled, or `false` if not. */ export function isCodeScanningEnabled(config: Config): boolean { return config.analysisKinds.includes(AnalysisKind.CodeScanning); } /** * Returns `true` if Code Quality analysis is enabled, or `false` if not. */ export function isCodeQualityEnabled(config: Config): boolean { return config.analysisKinds.includes(AnalysisKind.CodeQuality); } /** * Returns `true` if Code Scanning Risk Assessment analysis is enabled, or `false` if not. */ export function isRiskAssessmentEnabled(config: Config): boolean { return config.analysisKinds.includes(AnalysisKind.RiskAssessment); } /** * Returns the primary analysis kind that the Action is initialised with. If there is only * one analysis kind, then that is returned. * * The special case is Code Scanning + Code Quality, which can be enabled at the same time. * In that case, this function returns Code Scanning. */ function getPrimaryAnalysisKind(config: Config): AnalysisKind { if (config.analysisKinds.length === 1) { return config.analysisKinds[0]; } return isCodeScanningEnabled(config) ? AnalysisKind.CodeScanning : AnalysisKind.CodeQuality; } /** * Returns the primary analysis configuration that the Action is initialised with. */ export function getPrimaryAnalysisConfig(config: Config): AnalysisConfig { return getAnalysisConfig(getPrimaryAnalysisKind(config)); } /** Logs the Git version as a telemetry diagnostic. */ async function logGitVersionTelemetry( config: Config, gitVersion: GitVersionInfo, ): Promise { if (config.languages.length > 0) { addNoLanguageDiagnostic( config, makeTelemetryDiagnostic( "codeql-action/git-version-telemetry", "Git version telemetry", { fullVersion: gitVersion.fullVersion, truncatedVersion: gitVersion.truncatedVersion, }, ), ); } } /** * Logs the time it took to identify generated files and how many were discovered as * a telemetry diagnostic. * */ async function logGeneratedFilesTelemetry( config: Config, duration: number, generatedFilesCount: number, ): Promise { if (config.languages.length < 1) { return; } addNoLanguageDiagnostic( config, makeTelemetryDiagnostic( "codeql-action/generated-files-telemetry", "Generated files telemetry", { duration, generatedFilesCount, }, ), ); }