mirror of
https://github.com/github/codeql-action.git
synced 2026-05-08 14:50:21 +00:00
Introduce CODEQL_ACTION_SARIF_DUMP_DIR
Setting it will cause the SARIF files that would be uploaded to be dumped to the specified directory as `upload.sarif` or `upload.quality.sarif`. Crucially, this happens even if uploads are disabled, which is useful for testing.
This commit is contained in:
Paolo Tranquilli
+15
-7
@@ -330,22 +330,27 @@ async function run() {
|
||||
}
|
||||
core.setOutput("db-locations", dbLocations);
|
||||
core.setOutput("sarif-output", path.resolve(outputDir));
|
||||
const uploadInput = actionsUtil.getOptionalInput("upload");
|
||||
if (runStats && actionsUtil.getUploadValue(uploadInput) === "always") {
|
||||
const uploadInput = actionsUtil.getUploadValue(
|
||||
actionsUtil.getOptionalInput("upload"),
|
||||
);
|
||||
if (runStats) {
|
||||
if (isCodeScanningEnabled(config)) {
|
||||
uploadResult = await uploadLib.uploadFiles(
|
||||
uploadResult = await uploadLib.maybeUploadFiles(
|
||||
outputDir,
|
||||
actionsUtil.getRequiredInput("checkout_path"),
|
||||
actionsUtil.getOptionalInput("category"),
|
||||
features,
|
||||
logger,
|
||||
analyses.CodeScanning,
|
||||
uploadInput,
|
||||
);
|
||||
core.setOutput("sarif-id", uploadResult.sarifID);
|
||||
if (uploadResult) {
|
||||
core.setOutput("sarif-id", uploadResult.sarifID);
|
||||
}
|
||||
}
|
||||
|
||||
if (isCodeQualityEnabled(config)) {
|
||||
const qualityUploadResult = await uploadLib.uploadFiles(
|
||||
const qualityUploadResult = await uploadLib.maybeUploadFiles(
|
||||
outputDir,
|
||||
actionsUtil.getRequiredInput("checkout_path"),
|
||||
actionsUtil.fixCodeQualityCategory(
|
||||
@@ -355,11 +360,14 @@ async function run() {
|
||||
features,
|
||||
logger,
|
||||
analyses.CodeQuality,
|
||||
uploadInput,
|
||||
);
|
||||
core.setOutput("quality-sarif-id", qualityUploadResult.sarifID);
|
||||
if (qualityUploadResult) {
|
||||
core.setOutput("quality-sarif-id", qualityUploadResult.sarifID);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.info("Not uploading results");
|
||||
logger.info("No query status report, skipping upload");
|
||||
}
|
||||
|
||||
// Possibly upload the overlay-base database to actions cache.
|
||||
|
||||
@@ -119,4 +119,10 @@ export enum EnvVar {
|
||||
* Whether to enable experimental extractors for CodeQL.
|
||||
*/
|
||||
EXPERIMENTAL_FEATURES = "CODEQL_ENABLE_EXPERIMENTAL_FEATURES",
|
||||
|
||||
/**
|
||||
* Whether and where to dump the processed SARIF file that would be uploaded, regardless of
|
||||
* whether the upload is disabled. This is intended for testing and debugging purposes.
|
||||
*/
|
||||
SARIF_DUMP_DIR = "CODEQL_ACTION_SARIF_DUMP_DIR",
|
||||
}
|
||||
|
||||
+175
-87
@@ -623,18 +623,44 @@ export async function uploadFiles(
|
||||
logger: Logger,
|
||||
uploadTarget: analyses.AnalysisConfig,
|
||||
): Promise<UploadResult> {
|
||||
return maybeUploadFiles(
|
||||
inputSarifPath,
|
||||
checkoutPath,
|
||||
category,
|
||||
features,
|
||||
logger,
|
||||
uploadTarget,
|
||||
"always",
|
||||
) as Promise<UploadResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads a single SARIF file or a directory of SARIF files depending on what `inputSarifPath` refers
|
||||
* to. It will only upload if `uploadKind === "always"`, and return `undefined` otherwise. However
|
||||
* if `CODEQL_ACTION_SARIF_DUMP_DIR` is set, it will unconditionally process the input sarif files.
|
||||
*/
|
||||
export async function maybeUploadFiles(
|
||||
inputSarifPath: string,
|
||||
checkoutPath: string,
|
||||
category: string | undefined,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
uploadTarget: analyses.AnalysisConfig,
|
||||
uploadKind: actionsUtil.UploadKind,
|
||||
): Promise<UploadResult | undefined> {
|
||||
const sarifPaths = getSarifFilePaths(
|
||||
inputSarifPath,
|
||||
uploadTarget.sarifPredicate,
|
||||
);
|
||||
|
||||
return uploadSpecifiedFiles(
|
||||
return maybeUploadSpecifiedFiles(
|
||||
sarifPaths,
|
||||
checkoutPath,
|
||||
category,
|
||||
features,
|
||||
logger,
|
||||
uploadTarget,
|
||||
uploadKind,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -649,97 +675,159 @@ export async function uploadSpecifiedFiles(
|
||||
logger: Logger,
|
||||
uploadTarget: analyses.AnalysisConfig,
|
||||
): Promise<UploadResult> {
|
||||
logger.startGroup(`Uploading ${uploadTarget.name} results`);
|
||||
logger.info(`Processing sarif files: ${JSON.stringify(sarifPaths)}`);
|
||||
return maybeUploadSpecifiedFiles(
|
||||
sarifPaths,
|
||||
checkoutPath,
|
||||
category,
|
||||
features,
|
||||
logger,
|
||||
uploadTarget,
|
||||
"always",
|
||||
) as Promise<UploadResult>;
|
||||
}
|
||||
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
|
||||
let sarif: SarifFile;
|
||||
|
||||
if (sarifPaths.length > 1) {
|
||||
// Validate that the files we were asked to upload are all valid SARIF files
|
||||
for (const sarifPath of sarifPaths) {
|
||||
const parsedSarif = readSarifFile(sarifPath);
|
||||
validateSarifFileSchema(parsedSarif, sarifPath, logger);
|
||||
}
|
||||
|
||||
sarif = await combineSarifFilesUsingCLI(
|
||||
sarifPaths,
|
||||
gitHubVersion,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
} else {
|
||||
const sarifPath = sarifPaths[0];
|
||||
sarif = readSarifFile(sarifPath);
|
||||
validateSarifFileSchema(sarif, sarifPath, logger);
|
||||
|
||||
// Validate that there are no runs for the same category
|
||||
await throwIfCombineSarifFilesDisabled([sarif], gitHubVersion);
|
||||
async function maybeUploadSpecifiedFiles(
|
||||
sarifPaths: string[],
|
||||
checkoutPath: string,
|
||||
category: string | undefined,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
uploadTarget: analyses.AnalysisConfig,
|
||||
uploadKind: actionsUtil.UploadKind,
|
||||
): Promise<UploadResult | undefined> {
|
||||
const dumpDir = process.env[EnvVar.SARIF_DUMP_DIR];
|
||||
const upload = uploadKind === "always";
|
||||
if (!upload && !dumpDir) {
|
||||
logger.info(`Skipping upload of ${uploadTarget.name} results`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
sarif = filterAlertsByDiffRange(logger, sarif);
|
||||
sarif = await fingerprints.addFingerprints(sarif, checkoutPath, logger);
|
||||
logger.startGroup(`Processing ${uploadTarget.name} results`);
|
||||
try {
|
||||
logger.info(`Processing sarif files: ${JSON.stringify(sarifPaths)}`);
|
||||
|
||||
const analysisKey = await api.getAnalysisKey();
|
||||
const environment = actionsUtil.getRequiredInput("matrix");
|
||||
sarif = populateRunAutomationDetails(
|
||||
sarif,
|
||||
category,
|
||||
analysisKey,
|
||||
environment,
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
|
||||
let sarif: SarifFile;
|
||||
|
||||
if (sarifPaths.length > 1) {
|
||||
// Validate that the files we were asked to upload are all valid SARIF files
|
||||
for (const sarifPath of sarifPaths) {
|
||||
const parsedSarif = readSarifFile(sarifPath);
|
||||
validateSarifFileSchema(parsedSarif, sarifPath, logger);
|
||||
}
|
||||
|
||||
sarif = await combineSarifFilesUsingCLI(
|
||||
sarifPaths,
|
||||
gitHubVersion,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
} else {
|
||||
const sarifPath = sarifPaths[0];
|
||||
sarif = readSarifFile(sarifPath);
|
||||
validateSarifFileSchema(sarif, sarifPath, logger);
|
||||
|
||||
// Validate that there are no runs for the same category
|
||||
await throwIfCombineSarifFilesDisabled([sarif], gitHubVersion);
|
||||
}
|
||||
|
||||
sarif = filterAlertsByDiffRange(logger, sarif);
|
||||
sarif = await fingerprints.addFingerprints(sarif, checkoutPath, logger);
|
||||
|
||||
const analysisKey = await api.getAnalysisKey();
|
||||
const environment = actionsUtil.getRequiredInput("matrix");
|
||||
sarif = populateRunAutomationDetails(
|
||||
sarif,
|
||||
category,
|
||||
analysisKey,
|
||||
environment,
|
||||
);
|
||||
|
||||
const toolNames = util.getToolNames(sarif);
|
||||
|
||||
logger.debug(`Validating that each SARIF run has a unique category`);
|
||||
validateUniqueCategory(sarif, uploadTarget.sentinelPrefix);
|
||||
if (dumpDir) {
|
||||
dumpSarifFile(sarif, dumpDir, logger, uploadTarget);
|
||||
}
|
||||
if (!upload) {
|
||||
logger.info(
|
||||
`Skipping upload of ${uploadTarget.name} results because upload kind is "${uploadKind}"`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
logger.debug(`Serializing SARIF for upload`);
|
||||
const sarifPayload = JSON.stringify(sarif);
|
||||
logger.debug(`Compressing serialized SARIF`);
|
||||
const zippedSarif = zlib.gzipSync(sarifPayload).toString("base64");
|
||||
const checkoutURI = url.pathToFileURL(checkoutPath).href;
|
||||
|
||||
const payload = buildPayload(
|
||||
await gitUtils.getCommitOid(checkoutPath),
|
||||
await gitUtils.getRef(),
|
||||
analysisKey,
|
||||
util.getRequiredEnvParam("GITHUB_WORKFLOW"),
|
||||
zippedSarif,
|
||||
actionsUtil.getWorkflowRunID(),
|
||||
actionsUtil.getWorkflowRunAttempt(),
|
||||
checkoutURI,
|
||||
environment,
|
||||
toolNames,
|
||||
await gitUtils.determineBaseBranchHeadCommitOid(),
|
||||
);
|
||||
|
||||
// Log some useful debug info about the info
|
||||
const rawUploadSizeBytes = sarifPayload.length;
|
||||
logger.debug(`Raw upload size: ${rawUploadSizeBytes} bytes`);
|
||||
const zippedUploadSizeBytes = zippedSarif.length;
|
||||
logger.debug(`Base64 zipped upload size: ${zippedUploadSizeBytes} bytes`);
|
||||
const numResultInSarif = countResultsInSarif(sarifPayload);
|
||||
logger.debug(`Number of results in upload: ${numResultInSarif}`);
|
||||
|
||||
// Make the upload
|
||||
const sarifID = await uploadPayload(
|
||||
payload,
|
||||
getRepositoryNwo(),
|
||||
logger,
|
||||
uploadTarget.target,
|
||||
);
|
||||
|
||||
return {
|
||||
statusReport: {
|
||||
raw_upload_size_bytes: rawUploadSizeBytes,
|
||||
zipped_upload_size_bytes: zippedUploadSizeBytes,
|
||||
num_results_in_sarif: numResultInSarif,
|
||||
},
|
||||
sarifID,
|
||||
};
|
||||
} finally {
|
||||
logger.endGroup();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dumps the given processed SARIF file contents to `outputDir`.
|
||||
*/
|
||||
function dumpSarifFile(
|
||||
sarif: SarifFile,
|
||||
outputDir: string,
|
||||
logger: Logger,
|
||||
uploadTarget: analyses.AnalysisConfig,
|
||||
) {
|
||||
if (!fs.existsSync(outputDir)) {
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
} else if (!fs.lstatSync(outputDir).isDirectory()) {
|
||||
throw new ConfigurationError(
|
||||
`The path specified by the CODEQL_ACTION_SARIF_DUMP_DIR environment variable exists and is not a directory: ${outputDir}`,
|
||||
);
|
||||
}
|
||||
const outputFile = path.resolve(
|
||||
outputDir,
|
||||
`upload${uploadTarget.sarifExtension}`,
|
||||
);
|
||||
|
||||
const toolNames = util.getToolNames(sarif);
|
||||
|
||||
logger.debug(`Validating that each SARIF run has a unique category`);
|
||||
validateUniqueCategory(sarif, uploadTarget.sentinelPrefix);
|
||||
logger.debug(`Serializing SARIF for upload`);
|
||||
const sarifPayload = JSON.stringify(sarif);
|
||||
logger.debug(`Compressing serialized SARIF`);
|
||||
const zippedSarif = zlib.gzipSync(sarifPayload).toString("base64");
|
||||
const checkoutURI = url.pathToFileURL(checkoutPath).href;
|
||||
|
||||
const payload = buildPayload(
|
||||
await gitUtils.getCommitOid(checkoutPath),
|
||||
await gitUtils.getRef(),
|
||||
analysisKey,
|
||||
util.getRequiredEnvParam("GITHUB_WORKFLOW"),
|
||||
zippedSarif,
|
||||
actionsUtil.getWorkflowRunID(),
|
||||
actionsUtil.getWorkflowRunAttempt(),
|
||||
checkoutURI,
|
||||
environment,
|
||||
toolNames,
|
||||
await gitUtils.determineBaseBranchHeadCommitOid(),
|
||||
);
|
||||
|
||||
// Log some useful debug info about the info
|
||||
const rawUploadSizeBytes = sarifPayload.length;
|
||||
logger.debug(`Raw upload size: ${rawUploadSizeBytes} bytes`);
|
||||
const zippedUploadSizeBytes = zippedSarif.length;
|
||||
logger.debug(`Base64 zipped upload size: ${zippedUploadSizeBytes} bytes`);
|
||||
const numResultInSarif = countResultsInSarif(sarifPayload);
|
||||
logger.debug(`Number of results in upload: ${numResultInSarif}`);
|
||||
|
||||
// Make the upload
|
||||
const sarifID = await uploadPayload(
|
||||
payload,
|
||||
getRepositoryNwo(),
|
||||
logger,
|
||||
uploadTarget.target,
|
||||
);
|
||||
|
||||
logger.endGroup();
|
||||
|
||||
return {
|
||||
statusReport: {
|
||||
raw_upload_size_bytes: rawUploadSizeBytes,
|
||||
zipped_upload_size_bytes: zippedUploadSizeBytes,
|
||||
num_results_in_sarif: numResultInSarif,
|
||||
},
|
||||
sarifID,
|
||||
};
|
||||
logger.info(`Dumping processed SARIF file to ${outputFile}`);
|
||||
fs.writeFileSync(outputFile, JSON.stringify(sarif, null, 2));
|
||||
}
|
||||
|
||||
const STATUS_CHECK_FREQUENCY_MILLISECONDS = 5 * 1000;
|
||||
|
||||
Reference in New Issue
Block a user