github/codeql-action
0
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-04-25 16:28:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into update-bundle/codeql-bundle-v2.22.3

Browse Source
This commit is contained in:
Chuan-kai Lin
2025-08-06 10:44:20 -07:00
committed by GitHub
parent fff9bbe33f b1228d060c
commit ec8d9c637a
425 changed files with 49446 additions and 71470 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/analyze-action-env.test.ts
-1
View File
@@ -49,7 +49,6 @@ test("analyze action with RAM & threads from environment variables", async (t) =
requiredInputStub.withArgs("upload-database").returns("false");
requiredInputStub.withArgs("output").returns("out");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
optionalInputStub.withArgs("expect-error").returns("false");
sinon.stub(api, "getGitHubVersion").resolves(gitHubVersion);
setupActionsVars(tmpDir, tmpDir);
src/analyze-action-input.test.ts
-1
View File
@@ -47,7 +47,6 @@ test("analyze action with RAM & threads from action inputs", async (t) => {
requiredInputStub.withArgs("upload-database").returns("false");
requiredInputStub.withArgs("output").returns("out");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
optionalInputStub.withArgs("expect-error").returns("false");
sinon.stub(api, "getGitHubVersion").resolves(gitHubVersion);
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
src/analyze-action.ts
+19 -8
View File
@@ -25,7 +25,7 @@ import { uploadDependencyCaches } from "./dependency-caching";
import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import {
OverlayDatabaseMode,
@@ -122,8 +122,11 @@ function hasBadExpectErrorInput(): boolean {
* indicating whether Go extraction has extracted at least one file.
*/
function doesGoExtractionOutputExist(config: Config): boolean {
const golangDbDirectory = util.getCodeQLDatabasePath(config, Language.go);
const trapDirectory = path.join(golangDbDirectory, "trap", Language.go);
const golangDbDirectory = util.getCodeQLDatabasePath(
config,
KnownLanguage.go,
);
const trapDirectory = path.join(golangDbDirectory, "trap", KnownLanguage.go);
return (
fs.existsSync(trapDirectory) &&
fs
@@ -155,7 +158,7 @@ function doesGoExtractionOutputExist(config: Config): boolean {
* whether any extraction output already exists for Go.
*/
async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
if (!config.languages.includes(Language.go)) {
if (!config.languages.includes(KnownLanguage.go)) {
return;
}
if (config.buildMode) {
@@ -168,7 +171,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
logger.debug("Won't run Go autobuild since it has already been run.");
return;
}
if (dbIsFinalized(config, Language.go, logger)) {
if (dbIsFinalized(config, KnownLanguage.go, logger)) {
logger.debug(
"Won't run Go autobuild since there is already a finalized database for Go.",
);
@@ -191,7 +194,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
logger.debug(
"Running Go autobuild because extraction output (TRAP files) for Go code has not been found.",
);
await runAutobuild(config, Language.go, logger);
await runAutobuild(config, KnownLanguage.go, logger);
}
async function run() {
@@ -295,15 +298,23 @@ async function run() {
logger,
);
if (actionsUtil.getOptionalInput("cleanup-level") !== "") {
logger.info(
"The 'cleanup-level' input is ignored since the CodeQL Action no longer writes intermediate results to the database. This input can safely be removed from your workflow.",
);
}
// An overlay-base database should always use the 'overlay' cleanup level
// to preserve the cached intermediate results.
//
// Note that we may be overriding the 'cleanup-level' input parameter.
// Otherwise, use cleanup level 'none'. We are already discarding
// intermediate results during evaluation with '--expect-discarded-cache',
// so there is nothing to clean up.
const cleanupLevel =
config.augmentationProperties.overlayDatabaseMode ===
OverlayDatabaseMode.OverlayBase
? "overlay"
: actionsUtil.getOptionalInput("cleanup-level") || "brutal";
: "none";
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await runQueries(
src/analyze.test.ts
+5 -5
View File
@@ -13,7 +13,7 @@ import {
} from "./analyze";
import { setCodeQL } from "./codeql";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { KnownLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import {
setupTests,
@@ -41,7 +41,7 @@ test("status report fields", async (t) => {
const threadsFlag = "";
sinon.stub(uploadLib, "validateSarifFileSchema");
for (const language of Object.values(Language)) {
for (const language of Object.values(KnownLanguage)) {
setCodeQL({
databaseRunQueries: async () => {},
packDownload: async () => ({ packs: [] }),
@@ -331,13 +331,13 @@ test("getDiffRanges: malformed thunk header", async (t) => {
test("resolveQuerySuiteAlias", (t) => {
// default query suite names should resolve to something language-specific ending in `.qls`.
for (const suite of defaultSuites) {
const resolved = resolveQuerySuiteAlias(Language.go, suite);
const resolved = resolveQuerySuiteAlias(KnownLanguage.go, suite);
t.assert(
resolved.endsWith(".qls"),
"Resolved default suite doesn't end in .qls",
);
t.assert(
resolved.indexOf(Language.go) >= 0,
resolved.indexOf(KnownLanguage.go) >= 0,
"Resolved default suite doesn't contain language name",
);
}
@@ -346,6 +346,6 @@ test("resolveQuerySuiteAlias", (t) => {
const names = ["foo", "bar", "codeql/go-queries@1.0"];
for (const name of names) {
t.deepEqual(resolveQuerySuiteAlias(Language.go, name), name);
t.deepEqual(resolveQuerySuiteAlias(KnownLanguage.go, name), name);
}
});
src/analyze.ts
+13 -9
View File
@@ -23,7 +23,7 @@ import {
} from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import { FeatureEnablement, Feature } from "./feature-flags";
import { isScannedLanguage, Language } from "./languages";
import { KnownLanguage, Language } from "./languages";
import { Logger, withGroupAsync } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { getRepositoryNwoFromEnv } from "./repository";
@@ -177,14 +177,14 @@ export async function runExtraction(
continue;
}
if (shouldExtractLanguage(config, language)) {
if (await shouldExtractLanguage(codeql, config, language)) {
logger.startGroup(`Extracting ${language}`);
if (language === Language.python) {
if (language === KnownLanguage.python) {
await setupPythonExtractor(logger);
}
if (config.buildMode) {
if (
language === Language.cpp &&
language === KnownLanguage.cpp &&
config.buildMode === BuildMode.Autobuild
) {
await setupCppAutobuild(codeql, logger);
@@ -194,7 +194,10 @@ export async function runExtraction(
// database scratch directory by default. For dependency caching purposes, we want
// a stable path that caches can be restored into and that we can cache at the
// end of the workflow (i.e. that does not get removed when the scratch directory is).
if (language === Language.java && config.buildMode === BuildMode.None) {
if (
language === KnownLanguage.java &&
config.buildMode === BuildMode.None
) {
process.env["CODEQL_EXTRACTOR_JAVA_OPTION_BUILDLESS_DEPENDENCY_DIR"] =
getJavaTempDependencyDir();
}
@@ -208,15 +211,16 @@ export async function runExtraction(
}
}
function shouldExtractLanguage(
async function shouldExtractLanguage(
codeql: CodeQL,
config: configUtils.Config,
language: Language,
): boolean {
): Promise<boolean> {
return (
config.buildMode === BuildMode.None ||
(config.buildMode === BuildMode.Autobuild &&
process.env[EnvVar.AUTOBUILD_DID_COMPLETE_SUCCESSFULLY] !== "true") ||
(!config.buildMode && isScannedLanguage(language))
(!config.buildMode && (await codeql.isScannedLanguage(language)))
);
}
@@ -850,7 +854,7 @@ export async function warnIfGoInstalledAfterInit(
addDiagnostic(
config,
Language.go,
KnownLanguage.go,
makeDiagnostic(
"go/workflow/go-installed-after-codeql-init",
"Go was installed after the `codeql-action/init` Action was run",
src/autobuild.ts
+11 -10
View File
@@ -7,13 +7,13 @@ import * as configUtils from "./config-utils";
import { DocUrl } from "./doc-url";
import { EnvVar } from "./environment";
import { Feature, featureConfig, Features } from "./feature-flags";
import { isTracedLanguage, Language } from "./languages";
import { KnownLanguage, Language } from "./languages";
import { Logger } from "./logging";
import { getRepositoryNwo } from "./repository";
import { BuildMode } from "./util";
import { asyncFilter, BuildMode } from "./util";
export async function determineAutobuildLanguages(
_codeql: CodeQL,
codeql: CodeQL,
config: configUtils.Config,
logger: Logger,
): Promise<Language[] | undefined> {
@@ -32,11 +32,12 @@ export async function determineAutobuildLanguages(
// We want pick the dominant language in the repo from the ones we're able to build
// The languages are sorted in order specified by user or by lines of code if we got
// them from the GitHub API, so try to build the first language on the list.
const autobuildLanguages = config.languages.filter((l) =>
isTracedLanguage(l),
const autobuildLanguages = await asyncFilter(
config.languages,
async (language) => await codeql.isTracedLanguage(language),
);
if (!autobuildLanguages) {
if (autobuildLanguages.length === 0) {
logger.info(
"None of the languages in this project require extra build steps",
);
@@ -71,7 +72,7 @@ export async function determineAutobuildLanguages(
* version of the CodeQL Action.
*/
const autobuildLanguagesWithoutGo = autobuildLanguages.filter(
(l) => l !== Language.go,
(l) => l !== KnownLanguage.go,
);
const languages: Language[] = [];
@@ -83,7 +84,7 @@ export async function determineAutobuildLanguages(
// If Go is requested, run the Go autobuilder last to ensure it doesn't
// interfere with the other autobuilder.
if (autobuildLanguages.length !== autobuildLanguagesWithoutGo.length) {
languages.push(Language.go);
languages.push(KnownLanguage.go);
}
logger.debug(`Will autobuild ${languages.join(" and ")}.`);
@@ -155,7 +156,7 @@ export async function runAutobuild(
) {
logger.startGroup(`Attempting to automatically build ${language} code`);
const codeQL = await getCodeQL(config.codeQLCmd);
if (language === Language.cpp) {
if (language === KnownLanguage.cpp) {
await setupCppAutobuild(codeQL, logger);
}
if (config.buildMode) {
@@ -163,7 +164,7 @@ export async function runAutobuild(
} else {
await codeQL.runAutobuild(config, language);
}
if (language === Language.go) {
if (language === KnownLanguage.go) {
core.exportVariable(EnvVar.DID_AUTOBUILD_GOLANG, "true");
}
logger.endGroup();
src/cli-errors.ts
+6
View File
@@ -133,6 +133,7 @@ export enum CliConfigErrorCategory {
NoSourceCodeSeen = "NoSourceCodeSeen",
NoSupportedBuildCommandSucceeded = "NoSupportedBuildCommandSucceeded",
NoSupportedBuildSystemDetected = "NoSupportedBuildSystemDetected",
NotFoundInRegistry = "NotFoundInRegistry",
OutOfMemoryOrDisk = "OutOfMemoryOrDisk",
PackCannotBeFound = "PackCannotBeFound",
PackMissingAuth = "PackMissingAuth",
@@ -280,6 +281,11 @@ export const cliErrorsConfig: Record<
),
],
},
[CliConfigErrorCategory.NotFoundInRegistry]: {
cliErrorMessageCandidates: [
new RegExp("'.*' not found in the registry '.*'"),
],
},
};
/**
src/codeql.test.ts
+21 -19
View File
@@ -22,7 +22,7 @@ import {
import * as defaults from "./defaults.json";
import { DocUrl } from "./doc-url";
import { FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { KnownLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import { ToolsSource } from "./setup-codeql";
import {
@@ -50,7 +50,7 @@ test.beforeEach(() => {
initializeEnvironment("1.2.3");
stubConfig = createTestConfig({
languages: [Language.cpp],
languages: [KnownLanguage.cpp],
});
});
@@ -114,6 +114,16 @@ function mockApiDetails(apiDetails: GitHubApiDetails) {
process.env["GITHUB_API_URL"] = apiDetails.apiURL || "";
}
async function stubCodeql(): Promise<codeql.CodeQL> {
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
sinon
.stub(codeqlObject, "isTracedLanguage")
.withArgs(KnownLanguage.cpp)
.resolves(true);
return codeqlObject;
}
test("downloads and caches explicitly requested bundles that aren't in the toolcache", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@@ -499,8 +509,7 @@ const injectedConfigMacro = test.macro({
) => {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("1.0.0"));
const codeqlObject = await stubCodeql();
const thisStubConfig: Config = {
...stubConfig,
@@ -716,9 +725,7 @@ test(
test("passes a code scanning config AND qlconfig to the CLI", async (t: ExecutionContext<unknown>) => {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
const codeqlObject = await stubCodeql();
await codeqlObject.databaseInitCluster(
{ ...stubConfig, tempDir },
"",
@@ -745,8 +752,7 @@ test("passes a code scanning config AND qlconfig to the CLI", async (t: Executio
test("does not pass a qlconfig to the CLI when it is undefined", async (t: ExecutionContext<unknown>) => {
await util.withTmpDir(async (tempDir) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
const codeqlObject = await stubCodeql();
await codeqlObject.databaseInitCluster(
{ ...stubConfig, tempDir },
@@ -851,8 +857,7 @@ test("runTool summarizes several fatal errors", async (t) => {
`Running TRAP import for CodeQL database at /home/runner/work/_temp/codeql_databases/javascript...\n` +
`${heapError}\n${datasetImportError}.`;
stubToolRunnerConstructor(32, cliStderr);
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
const codeqlObject = await stubCodeql();
// io throws because of the test CodeQL object.
sinon.stub(io, "which").resolves("");
@@ -899,7 +904,7 @@ test("runTool summarizes autobuilder errors", async (t) => {
sinon.stub(io, "which").resolves("");
await t.throwsAsync(
async () => await codeqlObject.runAutobuild(stubConfig, Language.java),
async () => await codeqlObject.runAutobuild(stubConfig, KnownLanguage.java),
{
instanceOf: util.ConfigurationError,
message:
@@ -919,14 +924,13 @@ test("runTool truncates long autobuilder errors", async (t) => {
(_, i) => `[2019-09-18 12:00:00] [autobuild] [ERROR] line${i + 1}`,
).join("\n");
stubToolRunnerConstructor(1, stderr);
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
const codeqlObject = await stubCodeql();
sinon.stub(codeqlObject, "resolveExtractor").resolves("/path/to/extractor");
// io throws because of the test CodeQL object.
sinon.stub(io, "which").resolves("");
await t.throwsAsync(
async () => await codeqlObject.runAutobuild(stubConfig, Language.java),
async () => await codeqlObject.runAutobuild(stubConfig, KnownLanguage.java),
{
instanceOf: util.ConfigurationError,
message:
@@ -968,8 +972,7 @@ test("runTool recognizes fatal internal errors", async (t) => {
test("runTool outputs last line of stderr if fatal error could not be found", async (t) => {
const cliStderr = "line1\nline2\nline3\nline4\nline5";
stubToolRunnerConstructor(32, cliStderr);
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
const codeqlObject = await stubCodeql();
// io throws because of the test CodeQL object.
sinon.stub(io, "which").resolves("");
@@ -993,8 +996,7 @@ test("runTool outputs last line of stderr if fatal error could not be found", as
test("Avoids duplicating --overwrite flag if specified in CODEQL_ACTION_EXTRA_OPTIONS", async (t) => {
const runnerConstructorStub = stubToolRunnerConstructor();
const codeqlObject = await codeql.getCodeQLForTesting();
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
const codeqlObject = await stubCodeql();
// io throws because of the test CodeQL object.
sinon.stub(io, "which").resolves("");
src/codeql.ts
+22
View File
@@ -79,6 +79,14 @@ export interface CodeQL {
* Returns whether the CodeQL executable supports the specified feature.
*/
supportsFeature(feature: ToolsFeature): Promise<boolean>;
/**
* Returns whether the provided language is traced.
*/
isTracedLanguage(language: Language): Promise<boolean>;
/**
* Returns whether the provided language is scanned.
*/
isScannedLanguage(language: Language): Promise<boolean>;
/**
* Run 'codeql database init --db-cluster'.
*/
@@ -449,6 +457,8 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
!!partialCodeql.getVersion &&
isSupportedToolsFeature(await partialCodeql.getVersion(), feature),
),
isTracedLanguage: resolveFunction(partialCodeql, "isTracedLanguage"),
isScannedLanguage: resolveFunction(partialCodeql, "isScannedLanguage"),
databaseInitCluster: resolveFunction(partialCodeql, "databaseInitCluster"),
runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
extractScannedLanguage: resolveFunction(
@@ -558,6 +568,18 @@ export async function getCodeQLForCmd(
async supportsFeature(feature: ToolsFeature) {
return isSupportedToolsFeature(await this.getVersion(), feature);
},
async isTracedLanguage(language: Language) {
const extractorPath = await this.resolveExtractor(language);
const tracingConfigPath = path.join(
extractorPath,
"tools",
"tracing-config.lua",
);
return fs.existsSync(tracingConfigPath);
},
async isScannedLanguage(language: Language) {
return !(await this.isTracedLanguage(language));
},
async databaseInitCluster(
config: Config,
sourceRoot: string,
src/config-utils.test.ts
+116 -67
View File
@@ -18,7 +18,7 @@ import {
import * as configUtils from "./config-utils";
import { Feature } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Language } from "./languages";
import { KnownLanguage, Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import { parseRepositoryNwo } from "./repository";
@@ -128,6 +128,14 @@ test("load empty config", async (t) => {
const languages = "javascript,python";
const codeql = setCodeQL({
async betterResolveLanguages() {
return {
extractors: {
javascript: [{ extractor_root: "" }],
python: [{ extractor_root: "" }],
},
};
},
async resolveQueries() {
return {
byLanguage: {
@@ -172,6 +180,14 @@ test("loading config saves config", async (t) => {
const logger = getRunnerLogger(true);
const codeql = setCodeQL({
async betterResolveLanguages() {
return {
extractors: {
javascript: [{ extractor_root: "" }],
python: [{ extractor_root: "" }],
},
};
},
async resolveQueries() {
return {
byLanguage: {
@@ -303,6 +319,13 @@ test("load non-existent input", async (t) => {
test("load non-empty input", async (t) => {
return await withTmpDir(async (tempDir) => {
const codeql = setCodeQL({
async betterResolveLanguages() {
return {
extractors: {
javascript: [{ extractor_root: "" }],
},
};
},
async resolveQueries() {
return {
byLanguage: {
@@ -336,7 +359,7 @@ test("load non-empty input", async (t) => {
// And the config we expect it to parse to
const expectedConfig: configUtils.Config = {
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
buildMode: BuildMode.None,
originalUserInput: {
name: "my config",
@@ -427,6 +450,14 @@ test("Using config input and file together, config input should be used.", async
extraSearchPath: string | undefined;
}> = [];
const codeql = setCodeQL({
async betterResolveLanguages() {
return {
extractors: {
javascript: [{ extractor_root: "" }],
python: [{ extractor_root: "" }],
},
};
},
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined,
@@ -460,6 +491,13 @@ test("Using config input and file together, config input should be used.", async
test("API client used when reading remote config", async (t) => {
return await withTmpDir(async (tempDir) => {
const codeql = setCodeQL({
async betterResolveLanguages() {
return {
extractors: {
javascript: [{ extractor_root: "" }],
},
};
},
async resolveQueries() {
return {
byLanguage: {
@@ -668,7 +706,7 @@ const invalidPackNameMacro = test.macro({
parsePacksErrorMacro.exec(
t,
name,
[Language.cpp],
[KnownLanguage.cpp],
new RegExp(`^"${name}" is not a valid pack$`),
),
title: (_providedTitle: string | undefined, arg: string | undefined) =>
@@ -676,23 +714,23 @@ const invalidPackNameMacro = test.macro({
});
test("no packs", parsePacksMacro, "", [], undefined);
test("two packs", parsePacksMacro, "a/b,c/d@1.2.3", [Language.cpp], {
[Language.cpp]: ["a/b", "c/d@1.2.3"],
test("two packs", parsePacksMacro, "a/b,c/d@1.2.3", [KnownLanguage.cpp], {
[KnownLanguage.cpp]: ["a/b", "c/d@1.2.3"],
});
test(
"two packs with spaces",
parsePacksMacro,
" a/b , c/d@1.2.3 ",
[Language.cpp],
[KnownLanguage.cpp],
{
[Language.cpp]: ["a/b", "c/d@1.2.3"],
[KnownLanguage.cpp]: ["a/b", "c/d@1.2.3"],
},
);
test(
"two packs with language",
parsePacksErrorMacro,
"a/b,c/d@1.2.3",
[Language.cpp, Language.java],
[KnownLanguage.cpp, KnownLanguage.java],
new RegExp(
"Cannot specify a 'packs' input in a multi-language analysis. " +
"Use a codeql-config.yml file instead and specify packs by language.",
@@ -720,9 +758,9 @@ test(
// (globbing is not done)
"c/d@1.2.3:+*)_(",
].join(","),
[Language.cpp],
[KnownLanguage.cpp],
{
[Language.cpp]: [
[KnownLanguage.cpp]: [
"c/d@1.0",
"c/d@~1.0.0",
"c/d@~1.0.0:a/b",
@@ -834,7 +872,7 @@ test(
undefined,
undefined,
undefined,
[Language.javascript],
[KnownLanguage.javascript],
{
...configUtils.defaultAugmentationProperties,
},
@@ -846,7 +884,7 @@ test(
undefined,
" a, b , c, d",
undefined,
[Language.javascript],
[KnownLanguage.javascript],
{
...configUtils.defaultAugmentationProperties,
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
@@ -859,7 +897,7 @@ test(
undefined,
" + a, b , c, d ",
undefined,
[Language.javascript],
[KnownLanguage.javascript],
{
...configUtils.defaultAugmentationProperties,
queriesInputCombines: true,
@@ -873,7 +911,7 @@ test(
undefined,
undefined,
" a, b , c, d",
[Language.javascript],
[KnownLanguage.javascript],
{
...configUtils.defaultAugmentationProperties,
qualityQueriesInput: [
@@ -891,7 +929,7 @@ test(
undefined,
" a, b , c, d",
"e, f , g,h",
[Language.javascript],
[KnownLanguage.javascript],
{
...configUtils.defaultAugmentationProperties,
queriesInput: [{ uses: "a" }, { uses: "b" }, { uses: "c" }, { uses: "d" }],
@@ -910,7 +948,7 @@ test(
" codeql/a , codeql/b , codeql/c , codeql/d ",
undefined,
undefined,
[Language.javascript],
[KnownLanguage.javascript],
{
...configUtils.defaultAugmentationProperties,
packsInput: ["codeql/a", "codeql/b", "codeql/c", "codeql/d"],
@@ -923,7 +961,7 @@ test(
" + codeql/a, codeql/b, codeql/c, codeql/d",
undefined,
undefined,
[Language.javascript],
[KnownLanguage.javascript],
{
...configUtils.defaultAugmentationProperties,
packsInputCombines: true,
@@ -961,7 +999,7 @@ test(
undefined,
" + ",
undefined,
[Language.javascript],
[KnownLanguage.javascript],
/The workflow property "queries" is invalid/,
);
@@ -971,7 +1009,7 @@ test(
" + ",
undefined,
undefined,
[Language.javascript],
[KnownLanguage.javascript],
/The workflow property "packs" is invalid/,
);
@@ -981,7 +1019,7 @@ test(
" + a/b, c/d ",
undefined,
undefined,
[Language.javascript, Language.java],
[KnownLanguage.javascript, KnownLanguage.java],
/Cannot specify a 'packs' input in a multi-language analysis/,
);
@@ -1001,7 +1039,7 @@ test(
" a-pack-without-a-scope ",
undefined,
undefined,
[Language.javascript],
[KnownLanguage.javascript],
/"a-pack-without-a-scope" is not a valid pack/,
);
@@ -1043,7 +1081,6 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
[
{
name: "languages from input",
codeqlResolvedLanguages: ["javascript", "java", "python"],
languagesInput: "jAvAscript, \n jaVa",
languagesInRepository: ["SwiFt", "other"],
expectedLanguages: ["javascript", "java"],
@@ -1051,7 +1088,6 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
},
{
name: "languages from github api",
codeqlResolvedLanguages: ["javascript", "java", "python"],
languagesInput: "",
languagesInRepository: [" jAvAscript\n \t", " jaVa", "SwiFt", "other"],
expectedLanguages: ["javascript", "java"],
@@ -1059,7 +1095,6 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
},
{
name: "aliases from input",
codeqlResolvedLanguages: ["javascript", "csharp", "cpp", "java", "python"],
languagesInput: " typEscript\n \t, C#, c , KoTlin",
languagesInRepository: ["SwiFt", "other"],
expectedLanguages: ["javascript", "csharp", "cpp", "java"],
@@ -1067,7 +1102,6 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
},
{
name: "duplicate languages from input",
codeqlResolvedLanguages: ["javascript", "java", "python"],
languagesInput: "jAvAscript, \n jaVa, kotlin, typescript",
languagesInRepository: ["SwiFt", "other"],
expectedLanguages: ["javascript", "java"],
@@ -1075,7 +1109,6 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
},
{
name: "aliases from github api",
codeqlResolvedLanguages: ["javascript", "csharp", "cpp", "java", "python"],
languagesInput: "",
languagesInRepository: [" typEscript\n \t", " C#", "c", "other"],
expectedLanguages: ["javascript", "csharp", "cpp"],
@@ -1083,7 +1116,6 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
},
{
name: "no languages",
codeqlResolvedLanguages: ["javascript", "java", "python"],
languagesInput: "",
languagesInRepository: [],
expectedApiCall: true,
@@ -1091,7 +1123,6 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
},
{
name: "unrecognized languages from input",
codeqlResolvedLanguages: ["javascript", "java", "python"],
languagesInput: "a, b, c, javascript",
languagesInRepository: [],
expectedApiCall: false,
@@ -1100,15 +1131,26 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
].forEach((args) => {
test(`getLanguages: ${args.name}`, async (t) => {
const mockRequest = mockLanguagesInRepo(args.languagesInRepository);
const languages = args.codeqlResolvedLanguages.reduce(
(acc, lang) => ({
...acc,
[lang]: true,
}),
{},
);
const stubExtractorEntry = {
extractor_root: "",
};
const codeQL = setCodeQL({
resolveLanguages: () => Promise.resolve(languages),
betterResolveLanguages: () =>
Promise.resolve({
aliases: {
"c#": KnownLanguage.csharp,
c: KnownLanguage.cpp,
kotlin: KnownLanguage.java,
typescript: KnownLanguage.javascript,
},
extractors: {
cpp: [stubExtractorEntry],
csharp: [stubExtractorEntry],
java: [stubExtractorEntry],
javascript: [stubExtractorEntry],
python: [stubExtractorEntry],
},
}),
});
if (args.expectedLanguages) {
@@ -1141,12 +1183,12 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
for (const { displayName, language, feature } of [
{
displayName: "Java",
language: Language.java,
language: KnownLanguage.java,
feature: Feature.DisableJavaBuildlessEnabled,
},
{
displayName: "C#",
language: Language.csharp,
language: KnownLanguage.csharp,
feature: Feature.DisableCsharpBuildless,
},
]) {
@@ -1166,7 +1208,7 @@ for (const { displayName, language, feature } of [
const messages: LoggedMessage[] = [];
const buildMode = await configUtils.parseBuildModeInput(
"none",
[Language.python],
[KnownLanguage.python],
createFeatures([feature]),
getRecordingLogger(messages),
);
@@ -1212,7 +1254,7 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
isDefaultBranch: false,
repositoryOwner: "github",
buildMode: BuildMode.None,
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
codeqlVersion: "2.21.0",
gitRoot: "/some/git/root",
codeScanningConfig: {},
@@ -1265,6 +1307,13 @@ const getOverlayDatabaseModeMacro = test.macro({
// Set up CodeQL mock
const codeql = mockCodeQLVersion(setup.codeqlVersion);
// Mock traced languages
sinon
.stub(codeql, "isTracedLanguage")
.callsFake(async (lang: Language) => {
return [KnownLanguage.java].includes(lang as KnownLanguage);
});
// Mock git root detection
if (setup.gitRoot !== undefined) {
sinon.stub(gitUtils, "getGitRoot").resolves(setup.gitRoot);
@@ -1348,7 +1397,7 @@ test(
getOverlayDatabaseModeMacro,
"Ignore feature flag when analyzing non-default branch",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
},
{
@@ -1361,7 +1410,7 @@ test(
getOverlayDatabaseModeMacro,
"Overlay-base database on default branch when feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isDefaultBranch: true,
},
@@ -1375,7 +1424,7 @@ test(
getOverlayDatabaseModeMacro,
"Overlay-base database on default branch when feature enabled with custom analysis",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
codeScanningConfig: {
packs: ["some-custom-pack@1.0.0"],
@@ -1392,7 +1441,7 @@ test(
getOverlayDatabaseModeMacro,
"Overlay-base database on default branch when code-scanning feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1409,7 +1458,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1429,7 +1478,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with packs",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1449,7 +1498,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with queries",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1469,7 +1518,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when code-scanning feature enabled with query-filters",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1489,7 +1538,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when only language-specific feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysisJavascript],
isDefaultBranch: true,
},
@@ -1503,7 +1552,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when only code-scanning feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysisCodeScanningJavascript],
isDefaultBranch: true,
},
@@ -1517,7 +1566,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay-base database on default branch when language-specific feature disabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis],
isDefaultBranch: true,
},
@@ -1531,7 +1580,7 @@ test(
getOverlayDatabaseModeMacro,
"Overlay analysis on PR when feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
},
@@ -1545,7 +1594,7 @@ test(
getOverlayDatabaseModeMacro,
"Overlay analysis on PR when feature enabled with custom analysis",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
codeScanningConfig: {
packs: ["some-custom-pack@1.0.0"],
@@ -1562,7 +1611,7 @@ test(
getOverlayDatabaseModeMacro,
"Overlay analysis on PR when code-scanning feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1579,7 +1628,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1599,7 +1648,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with packs",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1619,7 +1668,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with queries",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1639,7 +1688,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when code-scanning feature enabled with query-filters",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
@@ -1659,7 +1708,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when only language-specific feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysisJavascript],
isPullRequest: true,
},
@@ -1673,7 +1722,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when only code-scanning feature enabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysisCodeScanningJavascript],
isPullRequest: true,
},
@@ -1687,7 +1736,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay analysis on PR when language-specific feature disabled",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis],
isPullRequest: true,
},
@@ -1727,7 +1776,7 @@ test(
getOverlayDatabaseModeMacro,
"Overlay PR analysis by feature flag for dsp-testing",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "dsp-testing",
@@ -1742,7 +1791,7 @@ test(
getOverlayDatabaseModeMacro,
"No overlay PR analysis by feature flag for other-org",
{
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
isPullRequest: true,
repositoryOwner: "other-org",
@@ -1759,7 +1808,7 @@ test(
{
overlayDatabaseEnvVar: "overlay",
buildMode: BuildMode.Autobuild,
languages: [Language.java],
languages: [KnownLanguage.java],
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
@@ -1773,7 +1822,7 @@ test(
{
overlayDatabaseEnvVar: "overlay",
buildMode: undefined,
languages: [Language.java],
languages: [KnownLanguage.java],
},
{
overlayDatabaseMode: OverlayDatabaseMode.None,
@@ -1808,12 +1857,12 @@ test(
);
// Exercise language-specific overlay analysis features code paths
for (const language in Language) {
for (const language in KnownLanguage) {
test(
getOverlayDatabaseModeMacro,
`Check default overlay analysis feature for ${language}`,
{
languages: [language as Language],
languages: [language],
features: [Feature.OverlayAnalysis],
isPullRequest: true,
},
src/config-utils.ts
+80 -70
View File
@@ -12,7 +12,7 @@ import { type CodeQL } from "./codeql";
import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import { Feature, FeatureEnablement } from "./feature-flags";
import { getGitRoot, isAnalyzingDefaultBranch } from "./git-utils";
import { isTracedLanguage, Language, parseLanguage } from "./languages";
import { KnownLanguage, Language } from "./languages";
import { Logger } from "./logging";
import {
CODEQL_OVERLAY_MINIMUM_VERSION,
@@ -146,7 +146,7 @@ export interface Config {
* Partial map from languages to locations of TRAP caches for that language.
* If a key is omitted, then TRAP caching should not be used for that language.
*/
trapCaches: Partial<Record<Language, string>>;
trapCaches: { [language: Language]: string };
/**
* Time taken to download TRAP caches. Used for status reporting.
@@ -303,14 +303,31 @@ export function getUnknownLanguagesError(languages: string[]): string {
return `Did not recognize the following languages: ${languages.join(", ")}`;
}
export async function getSupportedLanguageMap(
codeql: CodeQL,
): Promise<Record<string, string>> {
const resolveResult = await codeql.betterResolveLanguages();
const supportedLanguages: Record<string, string> = {};
// Populate canonical language names
for (const extractor of Object.keys(resolveResult.extractors)) {
supportedLanguages[extractor] = extractor;
}
// Populate language aliases
if (resolveResult.aliases) {
for (const [alias, extractor] of Object.entries(resolveResult.aliases)) {
supportedLanguages[alias] = extractor;
}
}
return supportedLanguages;
}
/**
* Gets the set of languages in the current repository that are
* scannable by CodeQL.
* Gets the set of languages in the current repository.
*/
export async function getLanguagesInRepo(
export async function getRawLanguagesInRepo(
repository: RepositoryNwo,
logger: Logger,
): Promise<Language[]> {
): Promise<string[]> {
logger.debug(`GitHub repo ${repository.owner} ${repository.repo}`);
const response = await api.getApiClient().rest.repos.listLanguages({
owner: repository.owner,
@@ -318,19 +335,9 @@ export async function getLanguagesInRepo(
});
logger.debug(`Languages API response: ${JSON.stringify(response)}`);
// The GitHub API is going to return languages in order of popularity,
// When we pick a language to autobuild we want to pick the most popular traced language
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
const languages: Set<Language> = new Set();
for (const lang of Object.keys(response.data as Record<string, number>)) {
const parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
}
}
return [...languages];
return Object.keys(response.data as Record<string, number>).map((language) =>
language.trim().toLowerCase(),
);
}
/**
@@ -344,7 +351,7 @@ export async function getLanguagesInRepo(
* then throw an error.
*/
export async function getLanguages(
codeQL: CodeQL,
codeql: CodeQL,
languagesInput: string | undefined,
repository: RepositoryNwo,
logger: Logger,
@@ -356,23 +363,24 @@ export async function getLanguages(
logger,
);
let languages = rawLanguages;
if (autodetected) {
const supportedLanguages = Object.keys(await codeQL.resolveLanguages());
const languageMap = await getSupportedLanguageMap(codeql);
const languagesSet = new Set<Language>();
const unknownLanguages: string[] = [];
languages = languages
.map(parseLanguage)
.filter((value) => value && supportedLanguages.includes(value))
.map((value) => value as Language);
logger.info(`Automatically detected languages: ${languages.join(", ")}`);
} else {
const aliases = (await codeQL.betterResolveLanguages()).aliases;
if (aliases) {
languages = languages.map((lang) => aliases[lang] || lang);
// Make sure they are supported
for (const language of rawLanguages) {
const extractorName = languageMap[language];
if (extractorName === undefined) {
unknownLanguages.push(language);
} else {
languagesSet.add(extractorName);
}
}
logger.info(`Languages from configuration: ${languages.join(", ")}`);
const languages = Array.from(languagesSet);
if (!autodetected && unknownLanguages.length > 0) {
throw new ConfigurationError(getUnknownLanguagesError(unknownLanguages));
}
// If the languages parameter was not given and no languages were
@@ -381,25 +389,22 @@ export async function getLanguages(
throw new ConfigurationError(getNoLanguagesError());
}
// Make sure they are supported
const parsedLanguages: Language[] = [];
const unknownLanguages: string[] = [];
for (const language of languages) {
const parsedLanguage = parseLanguage(language) as Language;
if (parsedLanguage === undefined) {
unknownLanguages.push(language);
} else if (!parsedLanguages.includes(parsedLanguage)) {
parsedLanguages.push(parsedLanguage);
}
if (autodetected) {
logger.info(`Autodetected languages: ${languages.join(", ")}`);
} else {
logger.info(`Languages from configuration: ${languages.join(", ")}`);
}
// Any unknown languages here would have come directly from the input
// since we filter unknown languages coming from the GitHub API.
if (unknownLanguages.length > 0) {
throw new ConfigurationError(getUnknownLanguagesError(unknownLanguages));
}
return languages;
}
return parsedLanguages;
export function getRawLanguagesNoAutodetect(
languagesInput: string | undefined,
): string[] {
return (languagesInput || "")
.split(",")
.map((x) => x.trim().toLowerCase())
.filter((x) => x.length > 0);
}
/**
@@ -416,22 +421,20 @@ export async function getRawLanguages(
languagesInput: string | undefined,
repository: RepositoryNwo,
logger: Logger,
) {
// Obtain from action input 'languages' if set
let rawLanguages = (languagesInput || "")
.split(",")
.map((x) => x.trim().toLowerCase())
.filter((x) => x.length > 0);
let autodetected: boolean;
if (rawLanguages.length) {
autodetected = false;
} else {
autodetected = true;
// Obtain all languages in the repo that can be analysed
rawLanguages = (await getLanguagesInRepo(repository, logger)) as string[];
): Promise<{
rawLanguages: string[];
autodetected: boolean;
}> {
// If the user has specified languages, use those.
const languagesFromInput = getRawLanguagesNoAutodetect(languagesInput);
if (languagesFromInput.length > 0) {
return { rawLanguages: languagesFromInput, autodetected: false };
}
return { rawLanguages, autodetected };
// Otherwise, autodetect languages in the repository.
return {
rawLanguages: await getRawLanguagesInRepo(repository, logger),
autodetected: true,
};
}
/** Inputs required to initialize a configuration. */
@@ -534,10 +537,10 @@ async function downloadCacheWithTime(
languages: Language[],
logger: Logger,
): Promise<{
trapCaches: Partial<Record<Language, string>>;
trapCaches: { [language: string]: string };
trapCacheDownloadTime: number;
}> {
let trapCaches = {};
let trapCaches: { [language: string]: string } = {};
let trapCacheDownloadTime = 0;
if (trapCachingEnabled) {
const start = performance.now();
@@ -804,7 +807,14 @@ export async function getOverlayDatabaseMode(
return nonOverlayAnalysis;
}
if (buildMode !== BuildMode.None && languages.some(isTracedLanguage)) {
if (
buildMode !== BuildMode.None &&
(
await Promise.all(
languages.map(async (l) => await codeql.isTracedLanguage(l)),
)
).some(Boolean)
) {
logger.warning(
`Cannot build an ${overlayDatabaseMode} database because ` +
`build-mode is set to "${buildMode}" instead of "none". ` +
@@ -1331,7 +1341,7 @@ export async function parseBuildModeInput(
}
if (
languages.includes(Language.csharp) &&
languages.includes(KnownLanguage.csharp) &&
(await features.getValue(Feature.DisableCsharpBuildless))
) {
logger.warning(
@@ -1341,7 +1351,7 @@ export async function parseBuildModeInput(
}
if (
languages.includes(Language.java) &&
languages.includes(KnownLanguage.java) &&
(await features.getValue(Feature.DisableJavaBuildlessEnabled))
) {
logger.warning(
src/database-upload.test.ts
+2 -2
View File
@@ -11,7 +11,7 @@ import { setCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import * as gitUtils from "./git-utils";
import { Language } from "./languages";
import { KnownLanguage } from "./languages";
import { RepositoryNwo } from "./repository";
import {
createTestConfig,
@@ -42,7 +42,7 @@ const testApiDetails: GitHubApiDetails = {
function getTestConfig(tmpDir: string): Config {
return createTestConfig({
languages: [Language.javascript],
languages: [KnownLanguage.javascript],
dbLocation: tmpDir,
});
}
src/init-action.ts
+40 -7
View File
@@ -40,7 +40,7 @@ import {
initConfig,
runInit,
} from "./init";
import { Language } from "./languages";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import {
downloadOverlayBaseDatabaseFromCache,
@@ -359,6 +359,39 @@ async function run() {
}
core.endGroup();
// Set CODEQL_ENABLE_EXPERIMENTAL_FEATURES for Rust. We need to set this environment
// variable before initializing the config, otherwise Rust analysis will not be
// enabled.
if (
// Only enable Rust analysis if the user has explicitly requested it - don't
// enable it via language autodetection.
configUtils
.getRawLanguagesNoAutodetect(getOptionalInput("languages"))
.includes(KnownLanguage.rust)
) {
const feat = Feature.RustAnalysis;
const minVer = featureConfig[feat].minimumVersion as string;
const envVar = "CODEQL_ENABLE_EXPERIMENTAL_FEATURES";
// if in default setup, it means the feature flag was on when rust was enabled
// if the feature flag gets turned off, let's not have rust analysis throwing a configuration error
// in that case rust analysis will be disabled only when default setup is refreshed
if (isDefaultSetup() || (await features.getValue(feat, codeql))) {
core.exportVariable(envVar, "true");
}
if (process.env[envVar] !== "true") {
throw new ConfigurationError(
`Experimental and not officially supported Rust analysis requires setting ${envVar}=true in the environment`,
);
}
const actualVer = (await codeql.getVersion()).version;
if (semver.lt(actualVer, minVer)) {
throw new ConfigurationError(
`Experimental rust analysis is supported by CodeQL CLI version ${minVer} or higher, but found version ${actualVer}`,
);
}
logger.info("Experimental rust analysis enabled");
}
config = await initConfig({
languagesInput: getOptionalInput("languages"),
queriesInput: getOptionalInput("queries"),
@@ -486,7 +519,7 @@ async function run() {
}
if (
config.languages.includes(Language.swift) &&
config.languages.includes(KnownLanguage.swift) &&
process.platform === "linux"
) {
logger.warning(
@@ -495,7 +528,7 @@ async function run() {
}
if (
config.languages.includes(Language.go) &&
config.languages.includes(KnownLanguage.go) &&
process.platform === "linux"
) {
try {
@@ -553,7 +586,7 @@ async function run() {
if (e instanceof FileCmdNotFoundError) {
addDiagnostic(
config,
Language.go,
KnownLanguage.go,
makeDiagnostic(
"go/workflow/file-program-unavailable",
"The `file` program is required on Linux, but does not appear to be installed",
@@ -603,7 +636,7 @@ async function run() {
core.exportVariable(kotlinLimitVar, "2.1.20");
}
if (config.languages.includes(Language.cpp)) {
if (config.languages.includes(KnownLanguage.cpp)) {
const envVar = "CODEQL_EXTRACTOR_CPP_TRAP_CACHING";
if (process.env[envVar]) {
logger.info(
@@ -622,7 +655,7 @@ async function run() {
}
// Set CODEQL_EXTRACTOR_CPP_BUILD_MODE_NONE
if (config.languages.includes(Language.cpp)) {
if (config.languages.includes(KnownLanguage.cpp)) {
const bmnVar = "CODEQL_EXTRACTOR_CPP_BUILD_MODE_NONE";
const value =
process.env[bmnVar] ||
@@ -633,7 +666,7 @@ async function run() {
// For rust: set CODEQL_ENABLE_EXPERIMENTAL_FEATURES, unless codeql already supports rust without it
if (
config.languages.includes(Language.rust) &&
config.languages.includes(KnownLanguage.rust) &&
!(await codeql.resolveLanguages()).rust
) {
const feat = Feature.RustAnalysis;
src/init.ts
+3 -3
View File
@@ -9,7 +9,7 @@ import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { CodeQL, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
import { Language } from "./languages";
import { KnownLanguage, Language } from "./languages";
import { Logger, withGroupAsync } from "./logging";
import { ToolsSource } from "./setup-codeql";
import { ZstdAvailability } from "./tar";
@@ -113,7 +113,7 @@ export async function checkInstallPython311(
codeql: CodeQL,
) {
if (
languages.includes(Language.python) &&
languages.includes(KnownLanguage.python) &&
process.platform === "win32" &&
!(await codeql.getVersion()).features?.supportsPython312
) {
@@ -138,7 +138,7 @@ export function cleanupDatabaseClusterDirectory(
if (
fs.existsSync(config.dbLocation) &&
(fs.statSync(config.dbLocation).isFile() ||
fs.readdirSync(config.dbLocation).length)
fs.readdirSync(config.dbLocation).length > 0)
) {
logger.warning(
`The database cluster directory ${config.dbLocation} must be empty. Attempting to clean it up.`,
src/languages.test.ts
-64
View File
@@ -1,64 +0,0 @@
import test from "ava";
import {
Language,
isScannedLanguage,
isTracedLanguage,
parseLanguage,
} from "./languages";
import { setupTests } from "./testing-utils";
setupTests(test);
test("parseLanguage", async (t) => {
// Exact matches
t.deepEqual(parseLanguage("csharp"), Language.csharp);
t.deepEqual(parseLanguage("cpp"), Language.cpp);
t.deepEqual(parseLanguage("go"), Language.go);
t.deepEqual(parseLanguage("java"), Language.java);
t.deepEqual(parseLanguage("javascript"), Language.javascript);
t.deepEqual(parseLanguage("python"), Language.python);
t.deepEqual(parseLanguage("rust"), Language.rust);
// Aliases
t.deepEqual(parseLanguage("c"), Language.cpp);
t.deepEqual(parseLanguage("c++"), Language.cpp);
t.deepEqual(parseLanguage("c#"), Language.csharp);
t.deepEqual(parseLanguage("kotlin"), Language.java);
t.deepEqual(parseLanguage("typescript"), Language.javascript);
// spaces and case-insensitivity
t.deepEqual(parseLanguage(" \t\nCsHaRp\t\t"), Language.csharp);
t.deepEqual(parseLanguage(" \t\nkOtLin\t\t"), Language.java);
// Not matches
t.deepEqual(parseLanguage("foo"), undefined);
t.deepEqual(parseLanguage(" "), undefined);
t.deepEqual(parseLanguage(""), undefined);
});
test("isTracedLanguage", async (t) => {
t.true(isTracedLanguage(Language.cpp));
t.true(isTracedLanguage(Language.csharp));
t.true(isTracedLanguage(Language.go));
t.true(isTracedLanguage(Language.java));
t.true(isTracedLanguage(Language.swift));
t.false(isTracedLanguage(Language.javascript));
t.false(isTracedLanguage(Language.python));
t.false(isTracedLanguage(Language.ruby));
t.false(isTracedLanguage(Language.rust));
});
test("isScannedLanguage", async (t) => {
t.false(isScannedLanguage(Language.cpp));
t.false(isScannedLanguage(Language.csharp));
t.false(isScannedLanguage(Language.go));
t.false(isScannedLanguage(Language.java));
t.false(isScannedLanguage(Language.swift));
t.true(isScannedLanguage(Language.javascript));
t.true(isScannedLanguage(Language.python));
t.true(isScannedLanguage(Language.ruby));
t.true(isScannedLanguage(Language.rust));
});
src/languages.ts
+10 -55
View File
@@ -1,6 +1,13 @@
// All the languages supported by CodeQL
export enum Language {
actions = "actions",
/** A language to analyze with CodeQL. */
export type Language = string;
/**
* A language supported by CodeQL that is treated specially by the Action.
*
* This is not an exhaustive list of languages supported by CodeQL and new
* languages do not need to be added here.
*/
export enum KnownLanguage {
csharp = "csharp",
cpp = "cpp",
go = "go",
@@ -11,55 +18,3 @@ export enum Language {
rust = "rust",
swift = "swift",
}
// Additional names for languages
export const LANGUAGE_ALIASES: { [lang: string]: Language } = {
c: Language.cpp,
"c++": Language.cpp,
"c#": Language.csharp,
kotlin: Language.java,
typescript: Language.javascript,
"javascript-typescript": Language.javascript,
"java-kotlin": Language.java,
};
/**
* Translate from user input or GitHub's API names for languages to CodeQL's
* names for languages.
*
* @param language The language to translate.
* @returns A language supported by CodeQL, an alias for a language, or
* `undefined` if the input language cannot be parsed into a language supported
* by CodeQL.
*/
export function parseLanguage(language: string): Language | undefined {
// Normalise to lower case
language = language.trim().toLowerCase();
// See if it's an exact match
if (language in Language) {
return language as Language;
}
// Check language aliases, but return the original language name,
// the alias will be resolved later.
if (language in LANGUAGE_ALIASES) {
return LANGUAGE_ALIASES[language];
}
return undefined;
}
export function isTracedLanguage(language: Language): boolean {
return [
Language.cpp,
Language.csharp,
Language.go,
Language.java,
Language.swift,
].includes(language);
}
export function isScannedLanguage(language: Language): boolean {
return !isTracedLanguage(language);
}
src/setup-codeql.ts
+1 -1
View File
@@ -155,7 +155,7 @@ export function tryGetTagNameFromUrl(
logger: Logger,
): string | undefined {
const matches = [...url.matchAll(/\/(codeql-bundle-[^/]*)\//g)];
if (!matches.length) {
if (matches.length === 0) {
logger.debug(`Could not determine tag name for URL ${url}.`);
return undefined;
}
src/start-proxy.test.ts
+29
View File
@@ -1,7 +1,9 @@
import test from "ava";
import { KnownLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import * as startProxyExports from "./start-proxy";
import { parseLanguage } from "./start-proxy";
import { setupTests } from "./testing-utils";
setupTests(test);
@@ -159,3 +161,30 @@ test("getCredentials throws an error when non-printable characters are used", as
);
}
});
test("parseLanguage", async (t) => {
// Exact matches
t.deepEqual(parseLanguage("csharp"), KnownLanguage.csharp);
t.deepEqual(parseLanguage("cpp"), KnownLanguage.cpp);
t.deepEqual(parseLanguage("go"), KnownLanguage.go);
t.deepEqual(parseLanguage("java"), KnownLanguage.java);
t.deepEqual(parseLanguage("javascript"), KnownLanguage.javascript);
t.deepEqual(parseLanguage("python"), KnownLanguage.python);
t.deepEqual(parseLanguage("rust"), KnownLanguage.rust);
// Aliases
t.deepEqual(parseLanguage("c"), KnownLanguage.cpp);
t.deepEqual(parseLanguage("c++"), KnownLanguage.cpp);
t.deepEqual(parseLanguage("c#"), KnownLanguage.csharp);
t.deepEqual(parseLanguage("kotlin"), KnownLanguage.java);
t.deepEqual(parseLanguage("typescript"), KnownLanguage.javascript);
// spaces and case-insensitivity
t.deepEqual(parseLanguage(" \t\nCsHaRp\t\t"), KnownLanguage.csharp);
t.deepEqual(parseLanguage(" \t\nkOtLin\t\t"), KnownLanguage.java);
// Not matches
t.deepEqual(parseLanguage("foo"), undefined);
t.deepEqual(parseLanguage(" "), undefined);
t.deepEqual(parseLanguage(""), undefined);
});
src/start-proxy.ts
+44 -6
View File
@@ -1,6 +1,6 @@
import * as core from "@actions/core";
import { parseLanguage, Language } from "./languages";
import { KnownLanguage } from "./languages";
import { Logger } from "./logging";
import { ConfigurationError } from "./util";
@@ -13,7 +13,49 @@ export type Credential = {
token?: string;
};
const LANGUAGE_TO_REGISTRY_TYPE: Record<Language, string> = {
/*
* Language aliases supported by the start-proxy Action.
*
* In general, the CodeQL CLI is the source of truth for language aliases, and to
* allow us to more easily support new languages, we want to avoid hardcoding these
* aliases in the Action itself. However this is difficult to do in the start-proxy
* Action since this Action does not use CodeQL, so we're accepting some hardcoding
* for this Action.
*/
const LANGUAGE_ALIASES: { [lang: string]: KnownLanguage } = {
c: KnownLanguage.cpp,
"c++": KnownLanguage.cpp,
"c#": KnownLanguage.csharp,
kotlin: KnownLanguage.java,
typescript: KnownLanguage.javascript,
"javascript-typescript": KnownLanguage.javascript,
"java-kotlin": KnownLanguage.java,
};
/**
* Parse the start-proxy language input into its canonical CodeQL language name.
*
* Exported for testing. Do not use this outside of the start-proxy Action
* to avoid complicating the process of adding new CodeQL languages.
*/
export function parseLanguage(language: string): KnownLanguage | undefined {
// Normalize to lower case
language = language.trim().toLowerCase();
// See if it's an exact match
if (language in KnownLanguage) {
return language as KnownLanguage;
}
// Check language aliases
if (language in LANGUAGE_ALIASES) {
return LANGUAGE_ALIASES[language];
}
return undefined;
}
const LANGUAGE_TO_REGISTRY_TYPE: Partial<Record<KnownLanguage, string>> = {
java: "maven_repository",
csharp: "nuget_feed",
javascript: "npm_registry",
@@ -21,10 +63,6 @@ const LANGUAGE_TO_REGISTRY_TYPE: Record<Language, string> = {
ruby: "rubygems_server",
rust: "cargo_registry",
go: "goproxy_server",
// We do not have an established proxy type for these languages, thus leaving empty.
actions: "",
cpp: "",
swift: "",
} as const;
/**
src/status-report.test.ts
+2 -2
View File
@@ -3,7 +3,7 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import { EnvVar } from "./environment";
import { Language } from "./languages";
import { KnownLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import {
ActionName,
@@ -47,7 +47,7 @@ test("createStatusReportBase", async (t) => {
new Date("May 19, 2023 05:19:00"),
createTestConfig({
buildMode: BuildMode.None,
languages: [Language.java, Language.swift],
languages: [KnownLanguage.java, KnownLanguage.swift],
}),
{ numAvailableBytes: 100, numTotalBytes: 500 },
getRunnerLogger(false),
src/tracer-config.test.ts
+28 -13
View File
@@ -2,14 +2,13 @@ import * as fs from "fs";
import * as path from "path";
import test from "ava";
import * as sinon from "sinon";
import { CodeQL, getCodeQLForTesting } from "./codeql";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import {
createTestConfig,
mockCodeQLVersion,
setupTests,
} from "./testing-utils";
import { KnownLanguage } from "./languages";
import { createTestConfig, makeVersionInfo, setupTests } from "./testing-utils";
import { ToolsFeature } from "./tools-features";
import { getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
@@ -17,19 +16,38 @@ setupTests(test);
function getTestConfig(tempDir: string): configUtils.Config {
return createTestConfig({
languages: [Language.java],
languages: [KnownLanguage.java],
tempDir,
dbLocation: path.resolve(tempDir, "codeql_databases"),
});
}
async function stubCodeql(
enabledFeatures: ToolsFeature[] = [],
): Promise<CodeQL> {
const codeqlObject = await getCodeQLForTesting();
sinon
.stub(codeqlObject, "getVersion")
.resolves(
makeVersionInfo(
"1.0.0",
Object.fromEntries(enabledFeatures.map((f) => [f, true])),
),
);
sinon
.stub(codeqlObject, "isTracedLanguage")
.withArgs(KnownLanguage.java)
.resolves(true);
return codeqlObject;
}
test("getCombinedTracerConfig - return undefined when no languages are traced languages", async (t) => {
await util.withTmpDir(async (tmpDir) => {
const config = getTestConfig(tmpDir);
// No traced languages
config.languages = [Language.javascript, Language.python];
config.languages = [KnownLanguage.javascript, KnownLanguage.python];
t.deepEqual(
await getCombinedTracerConfig(mockCodeQLVersion("1.0.0"), config),
await getCombinedTracerConfig(await stubCodeql(), config),
undefined,
);
});
@@ -64,10 +82,7 @@ test("getCombinedTracerConfig", async (t) => {
);
fs.writeFileSync(startTracingJson, JSON.stringify(startTracingEnv));
const result = await getCombinedTracerConfig(
mockCodeQLVersion("1.0.0"),
config,
);
const result = await getCombinedTracerConfig(await stubCodeql(), config);
t.notDeepEqual(result, undefined);
t.false(Object.prototype.hasOwnProperty.call(result?.env, "CODEQL_RUNNER"));
src/tracer-config.ts
+3 -4
View File
@@ -3,16 +3,15 @@ import * as path from "path";
import { type CodeQL } from "./codeql";
import { type Config } from "./config-utils";
import { isTracedLanguage } from "./languages";
import { Logger } from "./logging";
import { BuildMode } from "./util";
import { asyncSome, BuildMode } from "./util";
export type TracerConfig = {
env: { [key: string]: string };
};
export async function shouldEnableIndirectTracing(
_codeql: CodeQL,
codeql: CodeQL,
config: Config,
): Promise<boolean> {
// We don't need to trace build mode none, or languages which unconditionally don't need tracing.
@@ -27,7 +26,7 @@ export async function shouldEnableIndirectTracing(
}
// Otherwise, use direct tracing if any of the languages need to be traced.
return config.languages.some((l) => isTracedLanguage(l));
return asyncSome(config.languages, (l) => codeql.isTracedLanguage(l));
}
/**
src/trap-caching.test.ts
+9 -9
View File
@@ -15,7 +15,7 @@ import {
import * as configUtils from "./config-utils";
import { Feature } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { Language } from "./languages";
import { KnownLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import {
createFeatures,
@@ -41,7 +41,7 @@ const stubCodeql = setCodeQL({
async betterResolveLanguages() {
return {
extractors: {
[Language.javascript]: [
[KnownLanguage.javascript]: [
{
extractor_root: "some_root",
extractor_options: {
@@ -65,7 +65,7 @@ const stubCodeql = setCodeQL({
},
},
],
[Language.cpp]: [
[KnownLanguage.cpp]: [
{
extractor_root: "other_root",
},
@@ -76,7 +76,7 @@ const stubCodeql = setCodeQL({
});
const testConfigWithoutTmpDir = createTestConfig({
languages: [Language.javascript, Language.cpp],
languages: [KnownLanguage.javascript, KnownLanguage.cpp],
trapCaches: {
javascript: "/some/cache/dir",
},
@@ -84,7 +84,7 @@ const testConfigWithoutTmpDir = createTestConfig({
function getTestConfigWithTempDir(tempDir: string): configUtils.Config {
return createTestConfig({
languages: [Language.javascript, Language.ruby],
languages: [KnownLanguage.javascript, KnownLanguage.ruby],
tempDir,
dbLocation: path.resolve(tempDir, "codeql_databases"),
trapCaches: {
@@ -100,7 +100,7 @@ test("check flags for JS, analyzing default branch", async (t) => {
sinon.stub(gitUtils, "isAnalyzingDefaultBranch").resolves(true);
const result = await getTrapCachingExtractorConfigArgsForLang(
config,
Language.javascript,
KnownLanguage.javascript,
);
t.deepEqual(result, [
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
@@ -131,10 +131,10 @@ test("get languages that support TRAP caching", async (t) => {
const logger = getRecordingLogger(loggedMessages);
const languagesSupportingCaching = await getLanguagesSupportingCaching(
stubCodeql,
[Language.javascript, Language.cpp],
[KnownLanguage.javascript, KnownLanguage.cpp],
logger,
);
t.deepEqual(languagesSupportingCaching, [Language.javascript]);
t.deepEqual(languagesSupportingCaching, [KnownLanguage.javascript]);
});
test("upload cache key contains right fields", async (t) => {
@@ -178,7 +178,7 @@ test("download cache looks for the right key and creates dir", async (t) => {
);
await downloadTrapCaches(
stubCodeql,
[Language.javascript, Language.cpp],
[KnownLanguage.javascript, KnownLanguage.cpp],
logger,
);
t.assert(
src/trap-caching.ts
+2 -2
View File
@@ -50,8 +50,8 @@ export async function downloadTrapCaches(
codeql: CodeQL,
languages: Language[],
logger: Logger,
): Promise<Partial<Record<Language, string>>> {
const result: Partial<Record<Language, string>> = {};
): Promise<{ [language: string]: string }> {
const result: { [language: string]: string } = {};
const languagesSupportingCaching = await getLanguagesSupportingCaching(
codeql,
languages,
src/upload-lib.ts
+1 -1
View File
@@ -541,7 +541,7 @@ export function validateSarifFileSchema(
);
}
if (errors.length) {
if (errors.length > 0) {
// Output the more verbose error messages in groups as these may be very large.
for (const error of errors) {
logger.startGroup(`Error details: ${error.stack}`);
src/util.ts
+21 -5
View File
@@ -644,11 +644,11 @@ export function assertNever(value: never): never {
* knowing what version of CodeQL we're running.
*/
export function initializeEnvironment(version: string) {
core.exportVariable(String(EnvVar.FEATURE_MULTI_LANGUAGE), "false");
core.exportVariable(String(EnvVar.FEATURE_SANDWICH), "false");
core.exportVariable(String(EnvVar.FEATURE_SARIF_COMBINE), "true");
core.exportVariable(String(EnvVar.FEATURE_WILL_UPLOAD), "true");
core.exportVariable(String(EnvVar.VERSION), version);
core.exportVariable(EnvVar.FEATURE_MULTI_LANGUAGE, "false");
core.exportVariable(EnvVar.FEATURE_SANDWICH, "false");
core.exportVariable(EnvVar.FEATURE_SARIF_COMBINE, "true");
core.exportVariable(EnvVar.FEATURE_WILL_UPLOAD, "true");
core.exportVariable(EnvVar.VERSION, version);
}
/**
@@ -1250,3 +1250,19 @@ export async function isBinaryAccessible(
return false;
}
}
export async function asyncFilter<T>(
array: T[],
predicate: (value: T) => Promise<boolean>,
): Promise<T[]> {
const results = await Promise.all(array.map(predicate));
return array.filter((_, index) => results[index]);
}
export async function asyncSome<T>(
array: T[],
predicate: (value: T) => Promise<boolean>,
): Promise<boolean> {
const results = await Promise.all(array.map(predicate));
return results.some((result) => result);
}