Compare commits

...

1 Commits

Author SHA1 Message Date
Henry Mercer 699d8f2cc5 Cache CLI version and extractor metadata 2026-06-01 19:02:26 +01:00
12 changed files with 322 additions and 37 deletions
+58 -15
View File
@@ -151640,6 +151640,7 @@ async function initActionState({
computedConfig,
tempDir,
codeQLCmd: codeql.getPath(),
codeQLMetadata: codeql.getCliMetadata(),
gitHubVersion: githubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
debugMode,
@@ -153793,19 +153794,29 @@ Details: ${e.stack}` : ""}`
);
}
}
async function getCodeQL(cmd) {
async function getCodeQL(cmd, cliMetadata) {
if (cachedCodeQL === void 0) {
cachedCodeQL = await getCodeQLForCmd(cmd, true);
cachedCodeQL = await getCodeQLForCmd(cmd, true, cliMetadata);
} else {
cachedCodeQL.hydrateCliMetadata(cliMetadata);
}
return cachedCodeQL;
}
async function getCodeQLForCmd(cmd, checkVersion) {
function cacheCodeQlVersionForStatusReports(versionInfo) {
if (getCachedCodeQlVersion() === void 0) {
cacheCodeQlVersion(versionInfo);
}
}
async function getCodeQLForCmd(cmd, checkVersion, initialCliMetadata) {
const cliMetadata = { codeQLCmd: cmd };
let cachedVersion;
let cachedUnfilteredBetterResolveLanguages;
const codeql = {
getPath() {
return cmd;
},
async getVersion() {
let result = getCachedCodeQlVersion();
let result = cachedVersion;
if (result === void 0) {
const output = await runCli(cmd, ["version", "--format=json"], {
noStreamStdout: true
@@ -153817,12 +153828,17 @@ async function getCodeQLForCmd(cmd, checkVersion) {
`Invalid JSON output from \`version --format=json\`: ${output}`
);
}
cacheCodeQlVersion(result);
cachedVersion = result;
}
cacheCodeQlVersionForStatusReports(result);
return result;
},
async printVersion() {
await runCli(cmd, ["version", "--format=json"]);
const version = await this.getVersion();
process.stdout.write(`[command]${cmd} version --format=json
`);
process.stdout.write(`${JSON.stringify(version)}
`);
},
async supportsFeature(feature) {
return isSupportedToolsFeature(await this.getVersion(), feature);
@@ -153992,6 +154008,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
async betterResolveLanguages({
filterToLanguagesWithQueries
} = { filterToLanguagesWithQueries: false }) {
if (!filterToLanguagesWithQueries && cachedUnfilteredBetterResolveLanguages) {
return cachedUnfilteredBetterResolveLanguages;
}
const codeqlArgs = [
"resolve",
"languages",
@@ -154003,7 +154022,11 @@ async function getCodeQLForCmd(cmd, checkVersion) {
];
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output);
const result = JSON.parse(output);
if (!filterToLanguagesWithQueries) {
cachedUnfilteredBetterResolveLanguages = result;
}
return result;
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`
@@ -154162,6 +154185,10 @@ ${output}`
await new toolrunner3.ToolRunner(cmd, args).exec();
},
async resolveExtractor(language) {
const cachedExtractorPath = cliMetadata.extractorPaths?.[language];
if (cachedExtractorPath !== void 0) {
return cachedExtractorPath;
}
let extractorPath = "";
await new toolrunner3.ToolRunner(
cmd,
@@ -154185,7 +154212,10 @@ ${output}`
}
}
).exec();
return JSON.parse(extractorPath);
const resolvedExtractorPath = JSON.parse(extractorPath);
cliMetadata.extractorPaths ??= {};
cliMetadata.extractorPaths[language] = resolvedExtractorPath;
return resolvedExtractorPath;
},
async resolveQueriesStartingPacks(queries) {
const codeqlArgs = [
@@ -154238,8 +154268,21 @@ ${output}`
args.push("--sarif-merge-runs-from-equal-category");
}
await runCli(cmd, args);
},
getCliMetadata() {
return cliMetadata;
},
hydrateCliMetadata(metadata) {
if (metadata?.codeQLCmd !== cliMetadata.codeQLCmd) {
return;
}
cliMetadata.extractorPaths = {
...metadata.extractorPaths,
...cliMetadata.extractorPaths
};
}
};
codeql.hydrateCliMetadata(initialCliMetadata);
if (checkVersion && !await codeQlVersionAtLeast(codeql, CODEQL_MINIMUM_VERSION)) {
throw new ConfigurationError(
`Expected a CodeQL CLI with version at least ${CODEQL_MINIMUM_VERSION} but got version ${(await codeql.getVersion()).version}`
@@ -154424,7 +154467,7 @@ async function setupCppAutobuild(codeql, logger) {
}
async function runAutobuild(config, language, logger) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = await getCodeQL(config.codeQLCmd);
const codeQL = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
if (language === "cpp" /* cpp */) {
await setupCppAutobuild(codeQL, logger);
}
@@ -157008,7 +157051,7 @@ async function combineSarifFilesUsingCLI(sarifFiles, gitHubVersion, features, lo
let tempDir = getTemporaryDirectory();
const config = await getConfig(tempDir, logger);
if (config !== void 0) {
codeQL = await getCodeQL(config.codeQLCmd);
codeQL = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
tempDir = config.tempDir;
} else {
logger.info(
@@ -157742,7 +157785,7 @@ async function run(startedAt) {
"Config file could not be found at expected location. Has the 'init' action been called?"
);
}
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
if (hasBadExpectErrorInput()) {
throw new ConfigurationError(
"`expect-error` input parameter is for internal use only. It should only be set by codeql-action or a fork."
@@ -158511,7 +158554,7 @@ async function runWrapper2() {
logger
);
if (config !== void 0) {
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
const version = await codeql.getVersion();
await uploadCombinedSarifArtifacts(
logger,
@@ -158592,7 +158635,7 @@ async function run2(startedAt) {
"Config file could not be found at expected location. Has the 'init' action been called?"
);
}
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
languages = await determineAutobuildLanguages(codeql, config, logger);
if (languages !== void 0) {
const workingDirectory = getOptionalInput("working-directory");
@@ -159524,7 +159567,7 @@ async function prepareFailedSarif(logger, features, config) {
}
async function generateFailedSarif(features, config, category, checkoutPath, sarifFile) {
const databasePath = config.dbLocation;
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
if (sarifFile === void 0) {
sarifFile = "../codeql-failed-run.sarif";
}
@@ -159790,7 +159833,7 @@ async function run4(startedAt) {
"Debugging artifacts are unavailable since the 'init' Action failed before it could produce any."
);
} else {
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
uploadFailedSarifResult = await uploadFailureInfo(
tryUploadAllAvailableDebugArtifacts,
printDebugLogs,
+1 -1
View File
@@ -38,7 +38,7 @@ export async function runWrapper() {
logger,
);
if (config !== undefined) {
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
const version = await codeql.getVersion();
await debugArtifacts.uploadCombinedSarifArtifacts(
logger,
+1 -1
View File
@@ -256,7 +256,7 @@ async function run(startedAt: Date) {
);
}
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
if (hasBadExpectErrorInput()) {
throw new util.ConfigurationError(
+1 -1
View File
@@ -101,7 +101,7 @@ async function run(startedAt: Date) {
);
}
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
languages = await determineAutobuildLanguages(codeql, config, logger);
if (languages !== undefined) {
+1 -1
View File
@@ -155,7 +155,7 @@ export async function runAutobuild(
logger: Logger,
) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = await getCodeQL(config.codeQLCmd);
const codeQL = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
if (language === BuiltInLanguage.cpp) {
await setupCppAutobuild(codeQL, logger);
}
+161
View File
@@ -1,4 +1,5 @@
import * as fs from "fs";
import * as path from "path";
import { ExecOptions } from "@actions/exec";
import * as toolrunner from "@actions/exec/lib/toolrunner";
@@ -123,6 +124,166 @@ async function stubCodeql(): Promise<codeql.CodeQL> {
return codeqlObject;
}
function stubSuccessfulToolRunner(
stdoutForArgs: (args: string[]) => string | undefined,
): sinon.SinonStub<any[], toolrunner.ToolRunner> {
const runnerConstructorStub = sinon.stub(
toolrunner,
"ToolRunner",
) as sinon.SinonStub<any[], toolrunner.ToolRunner>;
runnerConstructorStub.callsFake((_cmd, args, options: ExecOptions) => {
return {
exec: async () => {
const stdout = stdoutForArgs(args as string[]);
if (stdout !== undefined) {
options.listeners?.stdout?.(Buffer.from(stdout));
}
return 0;
},
} as toolrunner.ToolRunner;
});
return runnerConstructorStub;
}
test.serial("getVersion and printVersion share cached version", async (t) => {
const version = { version: "2.30.0" };
let versionCalls = 0;
stubSuccessfulToolRunner((args) => {
if (args.join(" ") === "version --format=json") {
versionCalls++;
return JSON.stringify(version);
}
return undefined;
});
const codeqlObject = await codeql.getCodeQLForTesting();
t.deepEqual(await codeqlObject.getVersion(), version);
await codeqlObject.printVersion();
t.is(versionCalls, 1);
});
test.serial(
"betterResolveLanguages caches only the unfiltered result",
async (t) => {
const unfilteredLanguages = {
aliases: { typescript: BuiltInLanguage.javascript },
extractors: {
html: [{ extractor_root: "/html" }],
javascript: [{ extractor_root: "/javascript" }],
},
};
const filteredLanguages = {
aliases: { typescript: BuiltInLanguage.javascript },
extractors: {
javascript: [{ extractor_root: "/javascript" }],
},
};
let unfilteredCalls = 0;
let filteredCalls = 0;
stubSuccessfulToolRunner((args) => {
if (args[0] === "resolve" && args[1] === "languages") {
if (args.includes("--filter-to-languages-with-queries")) {
filteredCalls++;
return JSON.stringify(filteredLanguages);
}
unfilteredCalls++;
return JSON.stringify(unfilteredLanguages);
}
return undefined;
});
const codeqlObject = await codeql.getCodeQLForTesting();
t.deepEqual(
await codeqlObject.betterResolveLanguages(),
unfilteredLanguages,
);
t.deepEqual(
await codeqlObject.betterResolveLanguages(),
unfilteredLanguages,
);
t.deepEqual(
await codeqlObject.betterResolveLanguages({
filterToLanguagesWithQueries: true,
}),
filteredLanguages,
);
t.deepEqual(
await codeqlObject.betterResolveLanguages({
filterToLanguagesWithQueries: true,
}),
filteredLanguages,
);
// The unfiltered result is cached after the first call; the filtered
// variant is not cached because nothing reuses it.
t.is(unfilteredCalls, 1);
t.is(filteredCalls, 2);
},
);
test.serial("resolveExtractor caches its result per language", async (t) => {
await util.withTmpDir(async (tempDir) => {
const extractorRoot = path.join(tempDir, "javascript");
fs.mkdirSync(path.join(extractorRoot, "tools"), { recursive: true });
fs.writeFileSync(
path.join(extractorRoot, "tools", "tracing-config.lua"),
"",
);
let resolveExtractorCalls = 0;
stubSuccessfulToolRunner((args) => {
if (args[0] === "resolve" && args[1] === "extractor") {
resolveExtractorCalls++;
return JSON.stringify(extractorRoot);
}
return undefined;
});
const codeqlObject = await codeql.getCodeQLForTesting();
t.is(
await codeqlObject.resolveExtractor(BuiltInLanguage.javascript),
extractorRoot,
);
t.is(
await codeqlObject.resolveExtractor(BuiltInLanguage.javascript),
extractorRoot,
);
t.true(await codeqlObject.isTracedLanguage(BuiltInLanguage.javascript));
t.is(resolveExtractorCalls, 1);
});
});
test.serial(
"hydrateCliMetadata seeds the cache from persisted metadata",
async (t) => {
let cliCalls = 0;
stubSuccessfulToolRunner((_args) => {
cliCalls++;
return "{}";
});
const codeqlObject = await codeql.getCodeQLForTesting();
codeqlObject.hydrateCliMetadata({
codeQLCmd: "codeql-for-testing",
extractorPaths: { javascript: "/javascript" },
});
t.is(
await codeqlObject.resolveExtractor(BuiltInLanguage.javascript),
"/javascript",
);
t.is(cliCalls, 0);
},
);
test.serial(
"downloads and caches explicitly requested bundles that aren't in the toolcache",
async (t) => {
+87 -13
View File
@@ -218,6 +218,10 @@ export interface CodeQL {
outputFile: string,
options: { mergeRunsFromEqualCategory?: boolean },
): Promise<void>;
/** Return cacheable metadata gathered from the CodeQL CLI. */
getCliMetadata(): CodeQLCliMetadata;
/** Hydrate the CodeQL wrapper with cacheable metadata gathered earlier in the job. */
hydrateCliMetadata(metadata: CodeQLCliMetadata | undefined): void;
}
export interface VersionInfo {
@@ -247,12 +251,10 @@ export interface BetterResolveLanguagesOutput {
[alias: string]: string;
};
extractors: {
[language: string]: [
{
extractor_root: string;
extractor_options?: any;
},
];
[language: string]: Array<{
extractor_root: string;
extractor_options?: any;
}>;
};
}
@@ -264,6 +266,11 @@ export interface ResolveBuildEnvironmentOutput {
};
}
export interface CodeQLCliMetadata {
codeQLCmd: string;
extractorPaths?: { [language: string]: string };
}
/**
* Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
*/
@@ -392,9 +399,14 @@ export async function setupCodeQL(
/**
* Use the CodeQL executable located at the given path.
*/
export async function getCodeQL(cmd: string): Promise<CodeQL> {
export async function getCodeQL(
cmd: string,
cliMetadata?: CodeQLCliMetadata,
): Promise<CodeQL> {
if (cachedCodeQL === undefined) {
cachedCodeQL = await getCodeQLForCmd(cmd, true);
cachedCodeQL = await getCodeQLForCmd(cmd, true, cliMetadata);
} else {
cachedCodeQL.hydrateCliMetadata(cliMetadata);
}
return cachedCodeQL;
}
@@ -492,6 +504,14 @@ export function createStubCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
),
resolveDatabase: resolveFunction(partialCodeql, "resolveDatabase"),
mergeResults: resolveFunction(partialCodeql, "mergeResults"),
getCliMetadata: resolveFunction(partialCodeql, "getCliMetadata", () => ({
codeQLCmd: partialCodeql.getPath?.() ?? "/tmp/dummy-path",
})),
hydrateCliMetadata: resolveFunction(
partialCodeql,
"hydrateCliMetadata",
() => {},
),
};
}
@@ -506,6 +526,12 @@ export async function getCodeQLForTesting(
return getCodeQLForCmd(cmd, false);
}
function cacheCodeQlVersionForStatusReports(versionInfo: VersionInfo): void {
if (util.getCachedCodeQlVersion() === undefined) {
util.cacheCodeQlVersion(versionInfo);
}
}
/**
* Return a CodeQL object for CodeQL CLI access.
*
@@ -517,13 +543,24 @@ export async function getCodeQLForTesting(
async function getCodeQLForCmd(
cmd: string,
checkVersion: boolean,
initialCliMetadata?: CodeQLCliMetadata,
): Promise<CodeQL> {
// Metadata persisted across the init/autobuild/analyze steps. Only extractor
// paths are reused by a later step, so that's all this holds.
const cliMetadata: CodeQLCliMetadata = { codeQLCmd: cmd };
// In-process-only caches. These aren't persisted because no later step reuses
// them: the CLI version always matches across steps, and `resolve languages`
// is only re-read within a single step.
let cachedVersion: VersionInfo | undefined;
let cachedUnfilteredBetterResolveLanguages:
| BetterResolveLanguagesOutput
| undefined;
const codeql: CodeQL = {
getPath() {
return cmd;
},
async getVersion() {
let result = util.getCachedCodeQlVersion();
let result = cachedVersion;
if (result === undefined) {
const output = await runCli(cmd, ["version", "--format=json"], {
noStreamStdout: true,
@@ -535,12 +572,15 @@ async function getCodeQLForCmd(
`Invalid JSON output from \`version --format=json\`: ${output}`,
);
}
util.cacheCodeQlVersion(result);
cachedVersion = result;
}
cacheCodeQlVersionForStatusReports(result);
return result;
},
async printVersion() {
await runCli(cmd, ["version", "--format=json"]);
const version = await this.getVersion();
process.stdout.write(`[command]${cmd} version --format=json\n`);
process.stdout.write(`${JSON.stringify(version)}\n`);
},
async supportsFeature(feature: ToolsFeature) {
return isSupportedToolsFeature(await this.getVersion(), feature);
@@ -758,6 +798,13 @@ async function getCodeQLForCmd(
filterToLanguagesWithQueries: boolean;
} = { filterToLanguagesWithQueries: false },
) {
if (
!filterToLanguagesWithQueries &&
cachedUnfilteredBetterResolveLanguages
) {
return cachedUnfilteredBetterResolveLanguages;
}
const codeqlArgs = [
"resolve",
"languages",
@@ -772,7 +819,11 @@ async function getCodeQLForCmd(
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as BetterResolveLanguagesOutput;
const result = JSON.parse(output) as BetterResolveLanguagesOutput;
if (!filterToLanguagesWithQueries) {
cachedUnfilteredBetterResolveLanguages = result;
}
return result;
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`,
@@ -968,6 +1019,11 @@ async function getCodeQLForCmd(
await new toolrunner.ToolRunner(cmd, args).exec();
},
async resolveExtractor(language: Language): Promise<string> {
const cachedExtractorPath = cliMetadata.extractorPaths?.[language];
if (cachedExtractorPath !== undefined) {
return cachedExtractorPath;
}
// Request it using `format=json` so we don't need to strip the trailing new line generated by
// the CLI.
let extractorPath = "";
@@ -993,7 +1049,10 @@ async function getCodeQLForCmd(
},
},
).exec();
return JSON.parse(extractorPath) as string;
const resolvedExtractorPath = JSON.parse(extractorPath) as string;
cliMetadata.extractorPaths ??= {};
cliMetadata.extractorPaths[language] = resolvedExtractorPath;
return resolvedExtractorPath;
},
async resolveQueriesStartingPacks(queries: string[]): Promise<string[]> {
const codeqlArgs = [
@@ -1058,7 +1117,22 @@ async function getCodeQLForCmd(
await runCli(cmd, args);
},
getCliMetadata() {
return cliMetadata;
},
hydrateCliMetadata(metadata: CodeQLCliMetadata | undefined): void {
if (metadata?.codeQLCmd !== cliMetadata.codeQLCmd) {
return;
}
cliMetadata.extractorPaths = {
...metadata.extractorPaths,
...cliMetadata.extractorPaths,
};
},
};
// Seed the cache with any metadata persisted by an earlier step.
codeql.hydrateCliMetadata(initialCliMetadata);
// To ensure that status reports include the CodeQL CLI version wherever
// possible, we want to call getVersion(), which populates the version value
// used by status reporting, at the earliest opportunity. But invoking
+6 -1
View File
@@ -19,7 +19,7 @@ import {
} from "./analyses";
import * as api from "./api-client";
import { CachingKind, getCachingKind } from "./caching-utils";
import { type CodeQL } from "./codeql";
import { type CodeQL, type CodeQLCliMetadata } from "./codeql";
import {
calculateAugmentation,
ExcludeQueryFilter,
@@ -177,6 +177,10 @@ export interface Config {
* Path of the CodeQL executable.
*/
codeQLCmd: string;
/**
* Cacheable metadata gathered from the CodeQL CLI while initializing the workflow.
*/
codeQLMetadata?: CodeQLCliMetadata;
/**
* Version of GitHub we are talking to.
*/
@@ -561,6 +565,7 @@ export async function initActionState(
computedConfig,
tempDir,
codeQLCmd: codeql.getPath(),
codeQLMetadata: codeql.getCliMetadata(),
gitHubVersion: githubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
debugMode,
+1 -1
View File
@@ -163,7 +163,7 @@ async function generateFailedSarif(
sarifFile?: string,
) {
const databasePath = config.dbLocation;
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
// Set the filename for the SARIF file if not already set.
if (sarifFile === undefined) {
+1 -1
View File
@@ -75,7 +75,7 @@ async function run(startedAt: Date) {
"Debugging artifacts are unavailable since the 'init' Action failed before it could produce any.",
);
} else {
const codeql = await getCodeQL(config.codeQLCmd);
const codeql = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
uploadFailedSarifResult = await initActionPostHelper.uploadFailureInfo(
debugArtifacts.tryUploadAllAvailableDebugArtifacts,
+3 -1
View File
@@ -560,7 +560,7 @@ export function mockBundleDownloadApi({
}
export function createTestConfig(overrides: Partial<Config>): Config {
return Object.assign(
const config = Object.assign(
{},
{
version: getActionVersion(),
@@ -590,6 +590,8 @@ export function createTestConfig(overrides: Partial<Config>): Config {
} satisfies Config,
overrides,
);
config.codeQLMetadata ??= { codeQLCmd: config.codeQLCmd };
return config;
}
export function makeTestToken(length: number = 36) {
+1 -1
View File
@@ -140,7 +140,7 @@ async function combineSarifFilesUsingCLI(
const config = await getConfig(tempDir, logger);
if (config !== undefined) {
codeQL = await getCodeQL(config.codeQLCmd);
codeQL = await getCodeQL(config.codeQLCmd, config.codeQLMetadata);
tempDir = config.tempDir;
} else {
logger.info(