feat: always include files from diff in overlay changed files

This commit is contained in:
Sam Robson
2026-03-06 16:03:16 +00:00
parent 72c0b0efb7
commit 0b55f65b55
14 changed files with 894 additions and 575 deletions
+17 -2
View File
@@ -162277,10 +162277,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path2.join(
getTemporaryDirectory(),
@@ -162306,6 +162308,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path2.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs2.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs2.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// src/tools-features.ts
var semver4 = __toESM(require_semver2());
+26 -172
View File
@@ -106859,29 +106859,6 @@ var persistInputs = function() {
);
core4.saveState(persistedInputsKey, JSON.stringify(inputEnvironmentVariables));
};
function getPullRequestBranches() {
const pullRequest = github.context.payload.pull_request;
if (pullRequest) {
return {
base: pullRequest.base.ref,
// We use the head label instead of the head ref here, because the head
// ref lacks owner information and by itself does not uniquely identify
// the head branch (which may be in a forked repository).
head: pullRequest.head.label
};
}
const codeScanningRef = process.env.CODE_SCANNING_REF;
const codeScanningBaseBranch = process.env.CODE_SCANNING_BASE_BRANCH;
if (codeScanningRef && codeScanningBaseBranch) {
return {
base: codeScanningBaseBranch,
// PR analysis under Default Setup analyzes the PR head commit instead of
// the merge commit, so we can use the provided ref directly.
head: codeScanningRef
};
}
return void 0;
}
var qualityCategoryMapping = {
"c#": "csharp",
cpp: "c-cpp",
@@ -107897,10 +107874,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path4.join(
getTemporaryDirectory(),
@@ -107926,6 +107905,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path4.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs3.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs3.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
var CACHE_VERSION = 1;
var CACHE_PREFIX = "codeql-overlay-base-database";
var MAX_CACHE_OPERATION_MS = 6e5;
@@ -108611,34 +108603,9 @@ function initFeatures(gitHubVersion, repositoryNwo, tempDir, logger) {
}
// src/diff-informed-analysis-utils.ts
async function getDiffInformedAnalysisBranches(codeql, features, logger) {
if (!await features.getValue("diff_informed_queries" /* DiffInformedQueries */, codeql)) {
return void 0;
}
const gitHubVersion = await getGitHubVersion();
if (gitHubVersion.type === "GitHub Enterprise Server" /* GHES */ && satisfiesGHESVersion(gitHubVersion.version, "<3.19", true)) {
return void 0;
}
const branches = getPullRequestBranches();
if (!branches) {
logger.info(
"Not performing diff-informed analysis because we are not analyzing a pull request."
);
}
return branches;
}
function getDiffRangesJsonFilePath() {
return path6.join(getTemporaryDirectory(), "pr-diff-range.json");
}
function writeDiffRangesJsonFile(logger, ranges) {
const jsonContents = JSON.stringify(ranges, null, 2);
const jsonFilePath = getDiffRangesJsonFilePath();
fs5.writeFileSync(jsonFilePath, jsonContents);
logger.debug(
`Wrote pr-diff-range JSON file to ${jsonFilePath}:
${jsonContents}`
);
}
function readDiffRangesJsonFile(logger) {
const jsonFilePath = getDiffRangesJsonFilePath();
if (!fs5.existsSync(jsonFilePath)) {
@@ -108652,116 +108619,6 @@ ${jsonContents}`
);
return JSON.parse(jsonContents);
}
async function getPullRequestEditedDiffRanges(branches, logger) {
const fileDiffs = await getFileDiffsWithBasehead(branches, logger);
if (fileDiffs === void 0) {
return void 0;
}
if (fileDiffs.length >= 300) {
logger.warning(
`Cannot retrieve the full diff because there are too many (${fileDiffs.length}) changed files in the pull request.`
);
return void 0;
}
const results = [];
for (const filediff of fileDiffs) {
const diffRanges = getDiffRanges(filediff, logger);
if (diffRanges === void 0) {
return void 0;
}
results.push(...diffRanges);
}
return results;
}
async function getFileDiffsWithBasehead(branches, logger) {
const repositoryNwo = getRepositoryNwoFromEnv(
"CODE_SCANNING_REPOSITORY",
"GITHUB_REPOSITORY"
);
const basehead = `${branches.base}...${branches.head}`;
try {
const response = await getApiClient().rest.repos.compareCommitsWithBasehead(
{
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
basehead,
per_page: 1
}
);
logger.debug(
`Response from compareCommitsWithBasehead(${basehead}):
${JSON.stringify(response, null, 2)}`
);
return response.data.files;
} catch (error3) {
if (error3.status) {
logger.warning(`Error retrieving diff ${basehead}: ${error3.message}`);
logger.debug(
`Error running compareCommitsWithBasehead(${basehead}):
Request: ${JSON.stringify(error3.request, null, 2)}
Error Response: ${JSON.stringify(error3.response, null, 2)}`
);
return void 0;
} else {
throw error3;
}
}
}
function getDiffRanges(fileDiff, logger) {
if (fileDiff.patch === void 0) {
if (fileDiff.changes === 0) {
return [];
}
return [
{
path: fileDiff.filename,
startLine: 0,
endLine: 0
}
];
}
let currentLine = 0;
let additionRangeStartLine = void 0;
const diffRanges = [];
const diffLines = fileDiff.patch.split("\n");
diffLines.push(" ");
for (const diffLine of diffLines) {
if (diffLine.startsWith("-")) {
continue;
}
if (diffLine.startsWith("+")) {
if (additionRangeStartLine === void 0) {
additionRangeStartLine = currentLine;
}
currentLine++;
continue;
}
if (additionRangeStartLine !== void 0) {
diffRanges.push({
path: fileDiff.filename,
startLine: additionRangeStartLine,
endLine: currentLine - 1
});
additionRangeStartLine = void 0;
}
if (diffLine.startsWith("@@ ")) {
const match = diffLine.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/);
if (match === null) {
logger.warning(
`Cannot parse diff hunk header for ${fileDiff.filename}: ${diffLine}`
);
return void 0;
}
currentLine = parseInt(match[1], 10);
continue;
}
if (diffLine.startsWith(" ")) {
currentLine++;
continue;
}
}
return diffRanges;
}
// src/overlay/status.ts
var actionsCache2 = __toESM(require_cache5());
@@ -110915,14 +110772,17 @@ async function finalizeDatabaseCreation(codeql, features, config, threadsFlag, m
trap_import_duration_ms: Math.round(trapImportTime)
};
}
async function setupDiffInformedQueryRun(branches, logger) {
async function setupDiffInformedQueryRun(logger) {
return await withGroupAsync(
"Generating diff range extension pack",
async () => {
logger.info(
`Calculating diff ranges for ${branches.base}...${branches.head}`
);
const diffRanges = await getPullRequestEditedDiffRanges(branches, logger);
const diffRanges = readDiffRangesJsonFile(logger);
if (diffRanges === void 0) {
logger.info(
"No precomputed diff ranges found; skipping diff-informed analysis stage."
);
return void 0;
}
const checkoutPath = getRequiredInput("checkout_path");
const packDir = writeDiffRangeDataExtensionPack(
logger,
@@ -110992,7 +110852,6 @@ dataExtensions:
`Wrote pr-diff-range extension pack to ${extensionFilePath}:
${extensionContents}`
);
writeDiffRangesJsonFile(logger, ranges);
return diffRangeDir;
}
var defaultSuites = /* @__PURE__ */ new Set([
@@ -113544,12 +113403,7 @@ async function run(startedAt2) {
getOptionalInput("ram") || process.env["CODEQL_RAM"],
logger
);
const branches = await getDiffInformedAnalysisBranches(
codeql,
features,
logger
);
const diffRangePackDir = branches ? await setupDiffInformedQueryRun(branches, logger) : void 0;
const diffRangePackDir = await setupDiffInformedQueryRun(logger);
await warnIfGoInstalledAfterInit(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await runFinalize(
+17 -2
View File
@@ -104331,10 +104331,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path2.join(
getTemporaryDirectory(),
@@ -104360,6 +104362,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path2.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs2.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs2.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// src/tools-features.ts
var semver4 = __toESM(require_semver2());
+17 -2
View File
@@ -165791,10 +165791,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path4.join(
getTemporaryDirectory(),
@@ -165820,6 +165822,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path4.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs3.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs3.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// src/tools-features.ts
var semver4 = __toESM(require_semver2());
+524 -355
View File
File diff suppressed because it is too large Load Diff
+17 -2
View File
@@ -104324,10 +104324,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path2.join(
getTemporaryDirectory(),
@@ -104353,6 +104355,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path2.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs2.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs2.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// src/tools-features.ts
var semver4 = __toESM(require_semver2());
+17 -2
View File
@@ -104216,10 +104216,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path3.join(
getTemporaryDirectory(),
@@ -104245,6 +104247,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path3.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs3.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs3.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// src/tools-features.ts
var semver3 = __toESM(require_semver2());
+17 -2
View File
@@ -107482,10 +107482,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path4.join(
getTemporaryDirectory(),
@@ -107511,6 +107513,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path4.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs3.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs3.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// src/tools-features.ts
var semver4 = __toESM(require_semver2());
+17 -2
View File
@@ -107191,10 +107191,12 @@ async function readBaseDatabaseOidsFile(config, logger) {
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`
);
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [.../* @__PURE__ */ new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path3.join(
getTemporaryDirectory(),
@@ -107220,6 +107222,19 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
}
return changes;
}
function getDiffRangeFilePaths(logger) {
const jsonFilePath = path3.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs3.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs3.readFileSync(jsonFilePath, "utf8")
);
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// src/tools-features.ts
var semver3 = __toESM(require_semver2());
+2 -9
View File
@@ -28,7 +28,6 @@ import {
DependencyCacheUploadStatusReport,
uploadDependencyCaches,
} from "./dependency-caching";
import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { initFeatures } from "./feature-flags";
import { KnownLanguage } from "./languages";
@@ -305,14 +304,8 @@ async function run(startedAt: Date) {
logger,
);
const branches = await getDiffInformedAnalysisBranches(
codeql,
features,
logger,
);
const diffRangePackDir = branches
? await setupDiffInformedQueryRun(branches, logger)
: undefined;
// Setup diff informed analysis if needed (based on whether init created the file)
const diffRangePackDir = await setupDiffInformedQueryRun(logger);
await warnIfGoInstalledAfterInit(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
+10 -16
View File
@@ -5,11 +5,7 @@ import { performance } from "perf_hooks";
import * as io from "@actions/io";
import * as yaml from "js-yaml";
import {
getTemporaryDirectory,
getRequiredInput,
PullRequestBranches,
} from "./actions-util";
import { getTemporaryDirectory, getRequiredInput } from "./actions-util";
import * as analyses from "./analyses";
import { setupCppAutobuild } from "./autobuild";
import { type CodeQL } from "./codeql";
@@ -21,8 +17,7 @@ import {
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
import {
DiffThunkRange,
writeDiffRangesJsonFile,
getPullRequestEditedDiffRanges,
readDiffRangesJsonFile,
} from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { FeatureEnablement, Feature } from "./feature-flags";
@@ -237,16 +232,19 @@ async function finalizeDatabaseCreation(
* the diff range information, or `undefined` if the feature is disabled.
*/
export async function setupDiffInformedQueryRun(
branches: PullRequestBranches,
logger: Logger,
): Promise<string | undefined> {
return await withGroupAsync(
"Generating diff range extension pack",
async () => {
logger.info(
`Calculating diff ranges for ${branches.base}...${branches.head}`,
);
const diffRanges = await getPullRequestEditedDiffRanges(branches, logger);
const diffRanges = readDiffRangesJsonFile(logger);
if (diffRanges === undefined) {
logger.info(
"No precomputed diff ranges found; skipping diff-informed analysis stage.",
);
return undefined;
}
const checkoutPath = getRequiredInput("checkout_path");
const packDir = writeDiffRangeDataExtensionPack(
logger,
@@ -368,10 +366,6 @@ dataExtensions:
`Wrote pr-diff-range extension pack to ${extensionFilePath}:\n${extensionContents}`,
);
// Write the diff ranges to a JSON file, for action-side alert filtering by the
// upload-lib module.
writeDiffRangesJsonFile(logger, ranges);
return diffRangeDir;
}
+43 -1
View File
@@ -37,6 +37,11 @@ import {
makeDiagnostic,
makeTelemetryDiagnostic,
} from "./diagnostics";
import {
getDiffInformedAnalysisBranches,
getPullRequestEditedDiffRanges,
writeDiffRangesJsonFile,
} from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { Feature, FeatureEnablement, initFeatures } from "./feature-flags";
import {
@@ -54,7 +59,7 @@ import {
runDatabaseInitCluster,
} from "./init";
import { JavaEnvVars, KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { getActionsLogger, Logger, withGroupAsync } from "./logging";
import {
downloadOverlayBaseDatabaseFromCache,
OverlayBaseDatabaseDownloadStats,
@@ -413,6 +418,7 @@ async function run(startedAt: Date) {
}
await checkInstallPython311(config.languages, codeql);
await computeAndPersistDiffRanges(codeql, features, logger);
} catch (unwrappedError) {
const error = wrapError(unwrappedError);
core.setFailed(error.message);
@@ -833,6 +839,42 @@ async function loadRepositoryProperties(
}
}
/**
* Compute and persist diff ranges when diff-informed analysis is enabled
* (feature flag + PR context). This writes the standard pr-diff-range.json
* file for later reuse in the analyze step. Failures are logged but non-fatal.
*/
async function computeAndPersistDiffRanges(
codeql: CodeQL,
features: FeatureEnablement,
logger: Logger,
): Promise<void> {
try {
await withGroupAsync("Compute PR diff ranges", async () => {
const branches = await getDiffInformedAnalysisBranches(
codeql,
features,
logger,
);
if (!branches) {
return;
}
const ranges = await getPullRequestEditedDiffRanges(branches, logger);
if (ranges === undefined) {
return;
}
writeDiffRangesJsonFile(logger, ranges);
const distinctFiles = new Set(ranges.map((r) => r.path)).size;
logger.info(
`Persisted ${ranges.length} diff range(s) across ${distinctFiles} file(s).`,
);
});
} catch (e) {
logger.warning(
`Failed to compute and persist PR diff ranges: ${getErrorMessage(e)}`,
);
}
}
async function recordZstdAvailability(
config: configUtils.Config,
zstdAvailability: ZstdAvailability,
+148 -6
View File
@@ -34,12 +34,14 @@ test.serial(
"writeOverlayChangesFile generates correct changes file",
async (t) => {
await withTmpDir(async (tmpDir) => {
const dbLocation = path.join(tmpDir, "db");
await fs.promises.mkdir(dbLocation, { recursive: true });
const sourceRoot = path.join(tmpDir, "src");
await fs.promises.mkdir(sourceRoot, { recursive: true });
const tempDir = path.join(tmpDir, "temp");
await fs.promises.mkdir(tempDir, { recursive: true });
const [dbLocation, sourceRoot, tempDir] = ["db", "src", "temp"].map((d) =>
path.join(tmpDir, d),
);
await Promise.all(
[dbLocation, sourceRoot, tempDir].map((d) =>
fs.promises.mkdir(d, { recursive: true }),
),
);
const logger = getRunnerLogger(true);
const config = createTestConfig({ dbLocation });
@@ -93,6 +95,146 @@ test.serial(
},
);
test.serial(
"writeOverlayChangesFile merges additional diff files into overlay changes",
async (t) => {
await withTmpDir(async (tmpDir) => {
const [dbLocation, sourceRoot, tempDir] = ["db", "src", "temp"].map((d) =>
path.join(tmpDir, d),
);
await Promise.all(
[dbLocation, sourceRoot, tempDir].map((d) =>
fs.promises.mkdir(d, { recursive: true }),
),
);
const logger = getRunnerLogger(true);
const config = createTestConfig({ dbLocation });
// Mock the getFileOidsUnderPath function to return base OIDs
// "reverted.js" has the same OID in both base and current, simulating
// a revert PR where the file content matches the overlay-base
const baseOids = {
"unchanged.js": "aaa111",
"modified.js": "bbb222",
"reverted.js": "eee555",
};
const getFileOidsStubForBase = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(baseOids);
// Write the base database OIDs file
await writeBaseDatabaseOidsFile(config, sourceRoot);
getFileOidsStubForBase.restore();
// Mock the getFileOidsUnderPath function to return overlay OIDs
// "reverted.js" has the same OID as the base -- OID comparison alone
// would NOT include it, only additionalChangedFiles causes it to appear
const currentOids = {
"unchanged.js": "aaa111",
"modified.js": "ddd444", // Changed OID
"reverted.js": "eee555", // Same OID as base -- not detected by OID comparison
};
const getFileOidsStubForOverlay = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(currentOids);
const getTempDirStub = sinon
.stub(actionsUtil, "getTemporaryDirectory")
.returns(tempDir);
// Write a pr-diff-range.json file with diff ranges including
// "reverted.js" (unchanged OIDs) and "modified.js" (already in OID changes)
await fs.promises.writeFile(
path.join(tempDir, "pr-diff-range.json"),
JSON.stringify([
{ path: "reverted.js", startLine: 1, endLine: 10 },
{ path: "modified.js", startLine: 1, endLine: 5 },
{ path: "diff-only.js", startLine: 1, endLine: 3 },
]),
);
const changesFilePath = await writeOverlayChangesFile(
config,
sourceRoot,
logger,
);
getFileOidsStubForOverlay.restore();
getTempDirStub.restore();
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
t.deepEqual(
parsedContent.changes.sort(),
["diff-only.js", "modified.js", "reverted.js"],
"Should include OID-changed files, diff-only files, and deduplicate overlapping files",
);
});
},
);
test.serial(
"writeOverlayChangesFile works without additional diff files",
async (t) => {
await withTmpDir(async (tmpDir) => {
const [dbLocation, sourceRoot, tempDir] = ["db", "src", "temp"].map((d) =>
path.join(tmpDir, d),
);
await Promise.all(
[dbLocation, sourceRoot, tempDir].map((d) =>
fs.promises.mkdir(d, { recursive: true }),
),
);
const logger = getRunnerLogger(true);
const config = createTestConfig({ dbLocation });
// Mock the getFileOidsUnderPath function to return base OIDs
const baseOids = {
"unchanged.js": "aaa111",
"modified.js": "bbb222",
};
const getFileOidsStubForBase = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(baseOids);
await writeBaseDatabaseOidsFile(config, sourceRoot);
getFileOidsStubForBase.restore();
const currentOids = {
"unchanged.js": "aaa111",
"modified.js": "ddd444",
};
const getFileOidsStubForOverlay = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(currentOids);
const getTempDirStub = sinon
.stub(actionsUtil, "getTemporaryDirectory")
.returns(tempDir);
// No pr-diff-range.json file exists - should work the same as before
const changesFilePath = await writeOverlayChangesFile(
config,
sourceRoot,
logger,
);
getFileOidsStubForOverlay.restore();
getTempDirStub.restore();
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
t.deepEqual(
parsedContent.changes.sort(),
["modified.js"],
"Should only include OID-changed files when no additional files provided",
);
});
},
);
interface DownloadOverlayBaseDatabaseTestCase {
overlayDatabaseMode: OverlayDatabaseMode;
useOverlayDatabaseCaching: boolean;
+22 -2
View File
@@ -130,11 +130,17 @@ export async function writeOverlayChangesFile(
): Promise<string> {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`,
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`,
);
// Merge in any file paths from precomputed PR diff ranges to ensure the
// overlay always includes all files from the PR diff, even in edge cases
// like revert PRs where OID comparison shows no change.
const diffRangeFiles = getDiffRangeFilePaths(logger);
const changedFiles = [...new Set([...oidChangedFiles, ...diffRangeFiles])];
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path.join(
getTemporaryDirectory(),
@@ -165,6 +171,20 @@ function computeChangedFiles(
return changes;
}
function getDiffRangeFilePaths(logger: Logger): string[] {
const jsonFilePath = path.join(getTemporaryDirectory(), "pr-diff-range.json");
if (!fs.existsSync(jsonFilePath)) {
return [];
}
const diffRanges = JSON.parse(
fs.readFileSync(jsonFilePath, "utf8"),
) as Array<{ path: string }>;
logger.debug(
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`,
);
return [...new Set(diffRanges.map((r) => r.path))];
}
// Constants for database caching
const CACHE_VERSION = 1;
const CACHE_PREFIX = "codeql-overlay-base-database";