Merge remote-tracking branch 'origin/releases/v3' into backport-v2.24.0-e8893c57a

This commit is contained in:
github-actions[bot]
2024-02-02 18:32:36 +00:00
87 changed files with 1293 additions and 1139 deletions
+8 -18
View File
@@ -6,11 +6,15 @@ import * as sinon from "sinon";
import { runQueries } from "./analyze";
import { setCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
import {
setupTests,
setupActionsVars,
createFeatures,
createTestConfig,
} from "./testing-utils";
import * as uploadLib from "./upload-lib";
import * as util from "./util";
@@ -82,25 +86,11 @@ test("status report fields", async (t) => {
databasePrintBaseline: async () => "",
});
const config: Config = {
const config = createTestConfig({
languages: [language],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
});
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
});
+3 -18
View File
@@ -26,6 +26,7 @@ import {
SAMPLE_DEFAULT_CLI_VERSION,
mockBundleDownloadApi,
makeVersionInfo,
createTestConfig,
} from "./testing-utils";
import * as util from "./util";
import { initializeEnvironment } from "./util";
@@ -37,25 +38,9 @@ let stubConfig: Config;
test.beforeEach(() => {
initializeEnvironment("1.2.3");
stubConfig = {
stubConfig = createTestConfig({
languages: [Language.cpp],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
});
});
async function installIntoToolcache({
+24 -10
View File
@@ -23,6 +23,7 @@ import {
import { isTracedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as setupCodeql from "./setup-codeql";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import * as util from "./util";
import { wrapError } from "./util";
@@ -90,6 +91,10 @@ export interface CodeQL {
* Print version information about CodeQL.
*/
printVersion(): Promise<void>;
/**
* Returns whether the CodeQL executable supports the specified feature.
*/
supportsFeature(feature: ToolsFeature): Promise<boolean>;
/**
* Run 'codeql database init --db-cluster'.
*/
@@ -452,17 +457,17 @@ function resolveFunction<T>(
export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
cachedCodeQL = {
getPath: resolveFunction(partialCodeql, "getPath", () => "/tmp/dummy-path"),
getVersion: resolveFunction(
partialCodeql,
"getVersion",
() =>
new Promise((resolve) =>
resolve({
version: "1.0.0",
}),
),
),
getVersion: resolveFunction(partialCodeql, "getVersion", async () => ({
version: "1.0.0",
})),
printVersion: resolveFunction(partialCodeql, "printVersion"),
supportsFeature: resolveFunction(
partialCodeql,
"supportsFeature",
async (feature) =>
!!partialCodeql.getVersion &&
isSupportedToolsFeature(await partialCodeql.getVersion(), feature),
),
databaseInitCluster: resolveFunction(partialCodeql, "databaseInitCluster"),
runAutobuild: resolveFunction(partialCodeql, "runAutobuild"),
extractScannedLanguage: resolveFunction(
@@ -561,6 +566,9 @@ export async function getCodeQLForCmd(
async printVersion() {
await runTool(cmd, ["version", "--format=json"]);
},
async supportsFeature(feature: ToolsFeature) {
return isSupportedToolsFeature(await this.getVersion(), feature);
},
async databaseInitCluster(
config: Config,
sourceRoot: string,
@@ -589,6 +597,12 @@ export async function getCodeQLForCmd(
extraArgs.push("--external-repository-token-stdin");
}
if (
config.buildMode !== undefined &&
(await this.supportsFeature(ToolsFeature.BuildModeOption))
) {
extraArgs.push(`--build-mode=${config.buildMode}`);
}
if (
qlconfigFile !== undefined &&
(await util.codeQlVersionAbove(this, CODEQL_VERSION_INIT_WITH_QLCONFIG))
+170 -267
View File
@@ -7,7 +7,12 @@ import * as yaml from "js-yaml";
import * as sinon from "sinon";
import * as api from "./api-client";
import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
import {
CodeQL,
getCachedCodeQL,
PackDownloadOutput,
setCodeQL,
} from "./codeql";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
@@ -26,15 +31,42 @@ import {
setupTests(test);
const sampleApiDetails = {
auth: "token",
externalRepoAuth: "token",
url: "https://github.example.com",
apiURL: undefined,
registriesAuthTokens: undefined,
};
const githubVersion = { type: GitHubVariant.DOTCOM } as GitHubVersion;
const gitHubVersion = { type: GitHubVariant.DOTCOM } as GitHubVersion;
function createTestInitConfigInputs(
overrides: Partial<configUtils.InitConfigInputs>,
): configUtils.InitConfigInputs {
return Object.assign(
{},
{
languagesInput: undefined,
queriesInput: undefined,
packsInput: undefined,
configFile: undefined,
dbLocation: undefined,
configInput: undefined,
buildModeInput: undefined,
trapCachingEnabled: false,
debugMode: false,
debugArtifactName: "",
debugDatabaseName: "",
repository: { owner: "github", repo: "example" },
tempDir: "",
codeql: {} as CodeQL,
workspacePath: "",
githubVersion,
apiDetails: {
auth: "token",
externalRepoAuth: "token",
url: "https://github.example.com",
apiURL: undefined,
registriesAuthTokens: undefined,
},
logger: getRunnerLogger(true),
},
overrides,
);
}
// Returns the filepath of the newly-created file
function createConfigFile(inputFileContents: string, tmpDir: string): string {
@@ -77,11 +109,11 @@ function mockListLanguages(languages: string[]) {
}
test("load empty config", async (t) => {
return await withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tempDir) => {
const logger = getRunnerLogger(true);
const languages = "javascript,python";
const codeQL = setCodeQL({
const codeql = setCodeQL({
async resolveQueries() {
return {
byLanguage: {
@@ -98,51 +130,34 @@ test("load empty config", async (t) => {
});
const config = await configUtils.initConfig(
languages,
undefined,
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
logger,
createTestInitConfigInputs({
languagesInput: languages,
repository: { owner: "github", repo: "example" },
tempDir,
codeql,
logger,
}),
);
t.deepEqual(
config,
await configUtils.getDefaultConfig(
languages,
undefined,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
codeQL,
gitHubVersion,
logger,
createTestInitConfigInputs({
languagesInput: languages,
tempDir,
codeql,
logger,
}),
),
);
});
});
test("loading config saves config", async (t) => {
return await withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tempDir) => {
const logger = getRunnerLogger(true);
const codeQL = setCodeQL({
const codeql = setCodeQL({
async resolveQueries() {
return {
byLanguage: {
@@ -159,36 +174,26 @@ test("loading config saves config", async (t) => {
});
// Sanity check the saved config file does not already exist
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tempDir)));
// Sanity check that getConfig returns undefined before we have called initConfig
t.deepEqual(await configUtils.getConfig(tmpDir, logger), undefined);
t.deepEqual(await configUtils.getConfig(tempDir, logger), undefined);
const config1 = await configUtils.initConfig(
"javascript,python",
undefined,
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
logger,
createTestInitConfigInputs({
languagesInput: "javascript,python",
tempDir,
codeql,
workspacePath: tempDir,
logger,
}),
);
// The saved config file should now exist
t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tempDir)));
// And that same newly-initialised config should now be returned by getConfig
const config2 = await configUtils.getConfig(tmpDir, logger);
const config2 = await configUtils.getConfig(tempDir, logger);
t.not(config2, undefined);
if (config2 !== undefined) {
// removes properties assigned to undefined.
@@ -199,26 +204,15 @@ test("loading config saves config", async (t) => {
});
test("load input outside of workspace", async (t) => {
return await withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tempDir) => {
try {
await configUtils.initConfig(
undefined,
undefined,
undefined,
"../input",
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
configFile: "../input",
tempDir,
codeql: getCachedCodeQL(),
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
@@ -226,7 +220,7 @@ test("load input outside of workspace", async (t) => {
err,
new UserError(
configUtils.getConfigFileOutsideWorkspaceErrorMessage(
path.join(tmpDir, "../input"),
path.join(tempDir, "../input"),
),
),
);
@@ -235,29 +229,18 @@ test("load input outside of workspace", async (t) => {
});
test("load non-local input with invalid repo syntax", async (t) => {
return await withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tempDir) => {
// no filename given, just a repo
const configFile = "octo-org/codeql-config@main";
try {
await configUtils.initConfig(
undefined,
undefined,
undefined,
configFile,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
configFile,
tempDir,
codeql: getCachedCodeQL(),
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
@@ -274,30 +257,20 @@ test("load non-local input with invalid repo syntax", async (t) => {
});
test("load non-existent input", async (t) => {
return await withTmpDir(async (tmpDir) => {
const languages = "javascript";
return await withTmpDir(async (tempDir) => {
const languagesInput = "javascript";
const configFile = "input";
t.false(fs.existsSync(path.join(tmpDir, configFile)));
t.false(fs.existsSync(path.join(tempDir, configFile)));
try {
await configUtils.initConfig(
languages,
undefined,
undefined,
configFile,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
languagesInput,
configFile,
tempDir,
codeql: getCachedCodeQL(),
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
@@ -305,7 +278,7 @@ test("load non-existent input", async (t) => {
err,
new UserError(
configUtils.getConfigFileDoesNotExistErrorMessage(
path.join(tmpDir, "input"),
path.join(tempDir, "input"),
),
),
);
@@ -314,8 +287,8 @@ test("load non-existent input", async (t) => {
});
test("load non-empty input", async (t) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
return await withTmpDir(async (tempDir) => {
const codeql = setCodeQL({
async resolveQueries() {
return {
byLanguage: {
@@ -345,11 +318,12 @@ test("load non-empty input", async (t) => {
paths:
- c/d`;
fs.mkdirSync(path.join(tmpDir, "foo"));
fs.mkdirSync(path.join(tempDir, "foo"));
// And the config we expect it to parse to
const expectedConfig: configUtils.Config = {
languages: [Language.javascript],
buildMode: "none",
originalUserInput: {
name: "my config",
"disable-default-queries": true,
@@ -357,10 +331,10 @@ test("load non-empty input", async (t) => {
"paths-ignore": ["a", "b"],
paths: ["c/d"],
},
tempDir: tmpDir,
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
tempDir,
codeQLCmd: codeql.getPath(),
gitHubVersion: githubVersion,
dbLocation: path.resolve(tempDir, "codeql_databases"),
debugMode: false,
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
@@ -369,27 +343,20 @@ test("load non-empty input", async (t) => {
trapCacheDownloadTime: 0,
};
const languages = "javascript";
const configFilePath = createConfigFile(inputFileContents, tmpDir);
const languagesInput = "javascript";
const configFilePath = createConfigFile(inputFileContents, tempDir);
const actualConfig = await configUtils.initConfig(
languages,
undefined,
undefined,
configFilePath,
undefined,
undefined,
false,
false,
"my-artifact",
"my-db",
{ owner: "github", repo: "example" },
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
languagesInput,
buildModeInput: "none",
configFile: configFilePath,
debugArtifactName: "my-artifact",
debugDatabaseName: "my-db",
tempDir,
codeql,
workspacePath: tempDir,
}),
);
// Should exactly equal the object we constructed earlier
@@ -417,15 +384,15 @@ function queriesToResolvedQueryForm(queries: string[]) {
}
test("Using config input and file together, config input should be used.", async (t) => {
return await withTmpDir(async (tmpDir) => {
process.env["RUNNER_TEMP"] = tmpDir;
process.env["GITHUB_WORKSPACE"] = tmpDir;
return await withTmpDir(async (tempDir) => {
process.env["RUNNER_TEMP"] = tempDir;
process.env["GITHUB_WORKSPACE"] = tempDir;
const inputFileContents = `
name: my config
queries:
- uses: ./foo_file`;
const configFilePath = createConfigFile(inputFileContents, tmpDir);
const configFilePath = createConfigFile(inputFileContents, tempDir);
const configInput = `
name: my config
@@ -438,13 +405,13 @@ test("Using config input and file together, config input should be used.", async
- c/d@1.2.3
`;
fs.mkdirSync(path.join(tmpDir, "foo"));
fs.mkdirSync(path.join(tempDir, "foo"));
const resolveQueriesArgs: Array<{
queries: string[];
extraSearchPath: string | undefined;
}> = [];
const codeQL = setCodeQL({
const codeql = setCodeQL({
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined,
@@ -458,26 +425,17 @@ test("Using config input and file together, config input should be used.", async
});
// Only JS, python packs will be ignored
const languages = "javascript";
const languagesInput = "javascript";
const config = await configUtils.initConfig(
languages,
undefined,
undefined,
undefined,
configFilePath,
configInput,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
languagesInput,
configFile: configFilePath,
configInput,
tempDir,
codeql,
workspacePath: tempDir,
}),
);
t.deepEqual(config.originalUserInput, yaml.load(configInput));
@@ -485,8 +443,8 @@ test("Using config input and file together, config input should be used.", async
});
test("API client used when reading remote config", async (t) => {
return await withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
return await withTmpDir(async (tempDir) => {
const codeql = setCodeQL({
async resolveQueries() {
return {
byLanguage: {
@@ -521,59 +479,38 @@ test("API client used when reading remote config", async (t) => {
const spyGetContents = mockGetContents(dummyResponse);
// Create checkout directory for remote queries repository
fs.mkdirSync(path.join(tmpDir, "foo/bar/dev"), { recursive: true });
fs.mkdirSync(path.join(tempDir, "foo/bar/dev"), { recursive: true });
const configFile = "octo-org/codeql-config/config.yaml@main";
const languages = "javascript";
const languagesInput = "javascript";
await configUtils.initConfig(
languages,
undefined,
undefined,
configFile,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
languagesInput,
configFile,
tempDir,
codeql,
workspacePath: tempDir,
}),
);
t.assert(spyGetContents.called);
});
});
test("Remote config handles the case where a directory is provided", async (t) => {
return await withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tempDir) => {
const dummyResponse = []; // directories are returned as arrays
mockGetContents(dummyResponse);
const repoReference = "octo-org/codeql-config/config.yaml@main";
try {
await configUtils.initConfig(
undefined,
undefined,
undefined,
repoReference,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
configFile: repoReference,
tempDir,
codeql: getCachedCodeQL(),
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
@@ -588,7 +525,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
});
test("Invalid format of remote config handled correctly", async (t) => {
return await withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tempDir) => {
const dummyResponse = {
// note no "content" property here
};
@@ -597,23 +534,12 @@ test("Invalid format of remote config handled correctly", async (t) => {
const repoReference = "octo-org/codeql-config/config.yaml@main";
try {
await configUtils.initConfig(
undefined,
undefined,
undefined,
repoReference,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
configFile: repoReference,
tempDir,
codeql: getCachedCodeQL(),
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
@@ -628,9 +554,9 @@ test("Invalid format of remote config handled correctly", async (t) => {
});
test("No detected languages", async (t) => {
return await withTmpDir(async (tmpDir) => {
return await withTmpDir(async (tempDir) => {
mockListLanguages([]);
const codeQL = setCodeQL({
const codeql = setCodeQL({
async resolveLanguages() {
return {};
},
@@ -641,23 +567,11 @@ test("No detected languages", async (t) => {
try {
await configUtils.initConfig(
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
tempDir,
codeql,
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
@@ -667,28 +581,17 @@ test("No detected languages", async (t) => {
});
test("Unknown languages", async (t) => {
return await withTmpDir(async (tmpDir) => {
const languages = "rubbish,english";
return await withTmpDir(async (tempDir) => {
const languagesInput = "rubbish,english";
try {
await configUtils.initConfig(
languages,
undefined,
undefined,
undefined,
undefined,
undefined,
false,
false,
"",
"",
{ owner: "github", repo: "example" },
tmpDir,
getCachedCodeQL(),
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true),
createTestInitConfigInputs({
languagesInput,
tempDir,
codeql: getCachedCodeQL(),
workspacePath: tempDir,
}),
);
throw new Error("initConfig did not throw error");
} catch (err) {
+99 -103
View File
@@ -80,6 +80,10 @@ export interface Config {
* Set of languages to run analysis for.
*/
languages: Language[];
/**
* Build mode, if set. Currently only a single build mode is supported per job.
*/
buildMode: string | undefined;
/**
* A unaltered copy of the original user input.
* Mainly intended to be used for status reporting.
@@ -391,49 +395,82 @@ export async function getRawLanguages(
return { rawLanguages, autodetected };
}
/** Inputs required to initialize a configuration. */
export interface InitConfigInputs {
languagesInput: string | undefined;
queriesInput: string | undefined;
packsInput: string | undefined;
configFile: string | undefined;
dbLocation: string | undefined;
configInput: string | undefined;
buildModeInput: string | undefined;
trapCachingEnabled: boolean;
debugMode: boolean;
debugArtifactName: string;
debugDatabaseName: string;
repository: RepositoryNwo;
tempDir: string;
codeql: CodeQL;
workspacePath: string;
githubVersion: GitHubVersion;
apiDetails: api.GitHubApiCombinedDetails;
logger: Logger;
}
type GetDefaultConfigInputs = Omit<
InitConfigInputs,
"configFile" | "configInput"
>;
type LoadConfigInputs = Omit<InitConfigInputs, "configInput"> & {
configFile: string;
};
/**
* Get the default config for when the user has not supplied one.
*/
export async function getDefaultConfig(
languagesInput: string | undefined,
rawQueriesInput: string | undefined,
rawPacksInput: string | undefined,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
codeQL: CodeQL,
gitHubVersion: GitHubVersion,
logger: Logger,
): Promise<Config> {
export async function getDefaultConfig({
languagesInput,
queriesInput,
packsInput,
buildModeInput,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
codeql,
githubVersion,
logger,
}: GetDefaultConfigInputs): Promise<Config> {
const languages = await getLanguages(
codeQL,
codeql,
languagesInput,
repository,
logger,
);
const augmentationProperties = calculateAugmentation(
rawPacksInput,
rawQueriesInput,
packsInput,
queriesInput,
languages,
);
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
codeQL,
codeql,
languages,
logger,
);
return {
languages,
buildMode: buildModeInput,
originalUserInput: {},
tempDir,
codeQLCmd: codeQL.getPath(),
gitHubVersion,
codeQLCmd: codeql.getPath(),
gitHubVersion: githubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
debugMode,
debugArtifactName,
@@ -466,24 +503,25 @@ async function downloadCacheWithTime(
/**
* Load the config from the given file.
*/
async function loadConfig(
languagesInput: string | undefined,
rawQueriesInput: string | undefined,
rawPacksInput: string | undefined,
configFile: string,
dbLocation: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: GitHubVersion,
apiDetails: api.GitHubApiCombinedDetails,
logger: Logger,
): Promise<Config> {
async function loadConfig({
languagesInput,
queriesInput,
packsInput,
buildModeInput,
configFile,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
codeql,
workspacePath,
githubVersion,
apiDetails,
logger,
}: LoadConfigInputs): Promise<Config> {
let parsedYAML: UserConfig;
if (isLocal(configFile)) {
@@ -495,31 +533,32 @@ async function loadConfig(
}
const languages = await getLanguages(
codeQL,
codeql,
languagesInput,
repository,
logger,
);
const augmentationProperties = calculateAugmentation(
rawPacksInput,
rawQueriesInput,
packsInput,
queriesInput,
languages,
);
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
codeQL,
codeql,
languages,
logger,
);
return {
languages,
buildMode: buildModeInput,
originalUserInput: parsedYAML,
tempDir,
codeQLCmd: codeQL.getPath(),
gitHubVersion,
codeQLCmd: codeql.getPath(),
gitHubVersion: githubVersion,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
debugMode,
debugArtifactName,
@@ -765,76 +804,33 @@ function dbLocationOrDefault(
* This will parse the config from the user input if present, or generate
* a default config. The parsed config is then stored to a known location.
*/
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
configInput: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: GitHubVersion,
apiDetails: api.GitHubApiCombinedDetails,
logger: Logger,
): Promise<Config> {
export async function initConfig(inputs: InitConfigInputs): Promise<Config> {
let config: Config;
const { logger, workspacePath } = inputs;
// if configInput is set, it takes precedence over configFile
if (configInput) {
if (configFile) {
if (inputs.configInput) {
if (inputs.configFile) {
logger.warning(
`Both a config file and config input were provided. Ignoring config file.`,
);
}
configFile = path.resolve(workspacePath, "user-config-from-action.yml");
fs.writeFileSync(configFile, configInput);
logger.debug(`Using config from action input: ${configFile}`);
inputs.configFile = path.resolve(
workspacePath,
"user-config-from-action.yml",
);
fs.writeFileSync(inputs.configFile, inputs.configInput);
logger.debug(`Using config from action input: ${inputs.configFile}`);
}
// If no config file was provided create an empty one
if (!configFile) {
if (!inputs.configFile) {
logger.debug("No configuration file was provided");
config = await getDefaultConfig(
languagesInput,
queriesInput,
packsInput,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
codeQL,
gitHubVersion,
logger,
);
config = await getDefaultConfig(inputs);
} else {
config = await loadConfig(
languagesInput,
queriesInput,
packsInput,
configFile,
dbLocation,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
codeQL,
workspacePath,
gitHubVersion,
apiDetails,
logger,
);
// Convince the type checker that inputs.configFile is defined.
config = await loadConfig({ ...inputs, configFile: inputs.configFile });
}
// Save the config so we can easily access it again in the future
+4 -15
View File
@@ -8,19 +8,18 @@ import * as actionsUtil from "./actions-util";
import { GitHubApiDetails } from "./api-client";
import * as apiClient from "./api-client";
import { setCodeQL } from "./codeql";
import { Config, defaultAugmentationProperties } from "./config-utils";
import { Config } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { Language } from "./languages";
import { RepositoryNwo } from "./repository";
import {
createTestConfig,
getRecordingLogger,
LoggedMessage,
setupActionsVars,
setupTests,
} from "./testing-utils";
import {
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
GitHubVariant,
HTTPError,
initializeEnvironment,
@@ -41,20 +40,10 @@ const testApiDetails: GitHubApiDetails = {
};
function getTestConfig(tmpDir: string): Config {
return {
return createTestConfig({
languages: [Language.javascript],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "foo",
gitHubVersion: { type: GitHubVariant.DOTCOM },
dbLocation: tmpDir,
debugMode: false,
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: defaultAugmentationProperties,
trapCaches: {},
trapCacheDownloadTime: 0,
};
});
}
async function mockHttpRequests(databaseUploadStatusCode: number) {
+1 -1
View File
@@ -105,7 +105,7 @@ export const featureConfig: Record<
// here!
envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
minimumVersion: "2.16.0",
defaultValue: false,
defaultValue: true,
},
};
+22 -23
View File
@@ -38,7 +38,7 @@ import {
getActionsStatus,
sendStatusReport,
} from "./status-report";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { ToolsFeature } from "./tools-features";
import { getTotalCacheSize } from "./trap-caching";
import {
checkDiskUsage,
@@ -265,29 +265,32 @@ async function run() {
}
core.endGroup();
config = await initConfig(
getOptionalInput("languages"),
getOptionalInput("queries"),
getOptionalInput("packs"),
getOptionalInput("config-file"),
getOptionalInput("db-location"),
getOptionalInput("config"),
getTrapCachingEnabled(),
config = await initConfig({
languagesInput: getOptionalInput("languages"),
queriesInput: getOptionalInput("queries"),
packsInput: getOptionalInput("packs"),
buildModeInput: getOptionalInput("build-mode"),
configFile: getOptionalInput("config-file"),
dbLocation: getOptionalInput("db-location"),
configInput: getOptionalInput("config"),
trapCachingEnabled: getTrapCachingEnabled(),
// Debug mode is enabled if:
// - The `init` Action is passed `debug: true`.
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
// or by setting the `ACTIONS_STEP_DEBUG` secret to `true`).
getOptionalInput("debug") === "true" || core.isDebug(),
getOptionalInput("debug-artifact-name") || DEFAULT_DEBUG_ARTIFACT_NAME,
getOptionalInput("debug-database-name") || DEFAULT_DEBUG_DATABASE_NAME,
repositoryNwo,
getTemporaryDirectory(),
debugMode: getOptionalInput("debug") === "true" || core.isDebug(),
debugArtifactName:
getOptionalInput("debug-artifact-name") || DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName:
getOptionalInput("debug-database-name") || DEFAULT_DEBUG_DATABASE_NAME,
repository: repositoryNwo,
tempDir: getTemporaryDirectory(),
codeql,
getRequiredEnvParam("GITHUB_WORKSPACE"),
gitHubVersion,
workspacePath: getRequiredEnvParam("GITHUB_WORKSPACE"),
githubVersion: gitHubVersion,
apiDetails,
logger,
);
});
await checkInstallPython311(config.languages, codeql);
@@ -325,9 +328,6 @@ async function run() {
}
try {
// Query CLI for supported features
const versionInfo = await codeql.getVersion();
// Forward Go flags
const goFlags = process.env["GOFLAGS"];
if (goFlags) {
@@ -351,10 +351,9 @@ async function run() {
// typically dynamically linked, this provides a suitable entry point for the CodeQL tracer.
if (
fileOutput.includes("statically linked") &&
!isSupportedToolsFeature(
versionInfo,
!(await codeql.supportsFeature(
ToolsFeature.IndirectTracingSupportsStaticBinaries,
)
))
) {
try {
logger.debug(`Applying static binary workaround for Go`);
+4 -38
View File
@@ -10,7 +10,6 @@ import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo } from "./feature-flags";
import { Language, isScannedLanguage } from "./languages";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import { ToolsSource } from "./setup-codeql";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
@@ -45,44 +44,11 @@ export async function initCodeQL(
}
export async function initConfig(
languagesInput: string | undefined,
queriesInput: string | undefined,
packsInput: string | undefined,
configFile: string | undefined,
dbLocation: string | undefined,
configInput: string | undefined,
trapCachingEnabled: boolean,
debugMode: boolean,
debugArtifactName: string,
debugDatabaseName: string,
repository: RepositoryNwo,
tempDir: string,
codeQL: CodeQL,
workspacePath: string,
gitHubVersion: util.GitHubVersion,
apiDetails: GitHubApiCombinedDetails,
logger: Logger,
inputs: configUtils.InitConfigInputs,
): Promise<configUtils.Config> {
const logger = inputs.logger;
logger.startGroup("Load language configuration");
const config = await configUtils.initConfig(
languagesInput,
queriesInput,
packsInput,
configFile,
dbLocation,
configInput,
trapCachingEnabled,
debugMode,
debugArtifactName,
debugDatabaseName,
repository,
tempDir,
codeQL,
workspacePath,
gitHubVersion,
apiDetails,
logger,
);
const config = await configUtils.initConfig(inputs);
printPathFiltersWarning(config, logger);
logger.endGroup();
return config;
@@ -124,7 +90,7 @@ export async function runInit(
} catch (e) {
throw processError(e);
}
return await getCombinedTracerConfig(await codeql.getVersion(), config);
return await getCombinedTracerConfig(codeql, config);
}
export function printPathFiltersWarning(
+8 -5
View File
@@ -96,7 +96,7 @@ export interface StatusReportBase {
/** Action runner hardware architecture (context runner.arch). */
runner_arch?: string;
/** Available disk space on the runner, in bytes. */
runner_available_disk_space_bytes: number;
runner_available_disk_space_bytes?: number;
/**
* Version of the runner image, for workflows running on GitHub-hosted runners. Absent otherwise.
*/
@@ -106,7 +106,7 @@ export interface StatusReportBase {
/** Action runner operating system release (x.y.z from os.release()). */
runner_os_release?: string;
/** Total disk space on the runner, in bytes. */
runner_total_disk_space_bytes: number;
runner_total_disk_space_bytes?: number;
/** Time the first action started. Normally the init action. */
started_at: string;
/** State this action is currently in. */
@@ -192,7 +192,7 @@ export async function createStatusReportBase(
actionName: ActionName,
status: ActionStatus,
actionStartedAt: Date,
diskInfo: DiskUsage,
diskInfo: DiskUsage | undefined,
cause?: string,
exception?: string,
): Promise<StatusReportBase> {
@@ -230,9 +230,7 @@ export async function createStatusReportBase(
job_name: jobName,
job_run_uuid: jobRunUUID,
ref,
runner_available_disk_space_bytes: diskInfo.numAvailableBytes,
runner_os: runnerOs,
runner_total_disk_space_bytes: diskInfo.numTotalBytes,
started_at: workflowStartedAt,
status,
testing_environment: testingEnvironment,
@@ -241,6 +239,11 @@ export async function createStatusReportBase(
workflow_run_id: workflowRunID,
};
if (diskInfo) {
statusReport.runner_available_disk_space_bytes = diskInfo.numAvailableBytes;
statusReport.runner_total_disk_space_bytes = diskInfo.numTotalBytes;
}
// Add optional parameters
if (cause) {
statusReport.cause = cause;
+40 -6
View File
@@ -8,14 +8,21 @@ import * as sinon from "sinon";
import * as apiClient from "./api-client";
import { GitHubApiDetails } from "./api-client";
import * as CodeQL from "./codeql";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
import {
CodeQLDefaultVersionInfo,
Feature,
FeatureEnablement,
} from "./feature-flags";
import { Logger } from "./logging";
import { HTTPError } from "./util";
import {
DEFAULT_DEBUG_ARTIFACT_NAME,
DEFAULT_DEBUG_DATABASE_NAME,
GitHubVariant,
GitHubVersion,
HTTPError,
} from "./util";
export const SAMPLE_DOTCOM_API_DETAILS = {
auth: "token",
@@ -73,7 +80,7 @@ export function setupTests(test: TestFn<any>) {
typedTest.beforeEach((t) => {
// Set an empty CodeQL object so that all method calls will fail
// unless the test explicitly sets one up.
CodeQL.setCodeQL({});
codeql.setCodeQL({});
// Replace stdout and stderr so we can record output during tests
t.context.testOutput = "";
@@ -214,7 +221,7 @@ export function mockLanguagesInRepo(languages: string[]) {
export const makeVersionInfo = (
version: string,
features?: { [name: string]: boolean },
): CodeQL.VersionInfo => ({
): codeql.VersionInfo => ({
version,
features,
});
@@ -223,11 +230,11 @@ export function mockCodeQLVersion(
version: string,
features?: { [name: string]: boolean },
) {
return {
return codeql.setCodeQL({
async getVersion() {
return makeVersionInfo(version, features);
},
} as CodeQL.CodeQL;
});
}
/**
@@ -291,3 +298,30 @@ export function mockBundleDownloadApi({
return `${baseUrl}${relativeUrl}`;
}
export function createTestConfig(overrides: Partial<Config>): Config {
return Object.assign(
{},
{
languages: [],
buildMode: undefined,
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: GitHubVariant.DOTCOM,
} as GitHubVersion,
dbLocation: "",
debugMode: false,
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
},
overrides,
);
}
+2 -1
View File
@@ -1,6 +1,7 @@
import { VersionInfo } from "./codeql";
import type { VersionInfo } from "./codeql";
export enum ToolsFeature {
BuildModeOption = "buildModeOption",
IndirectTracingSupportsStaticBinaries = "indirectTracingSupportsStaticBinaries",
SetsCodeqlRunnerEnvVar = "setsCodeqlRunnerEnvVar",
}
+13 -18
View File
@@ -5,27 +5,22 @@ import test from "ava";
import * as configUtils from "./config-utils";
import { Language } from "./languages";
import { makeVersionInfo, setupTests } from "./testing-utils";
import {
createTestConfig,
mockCodeQLVersion,
setupTests,
} from "./testing-utils";
import { getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
setupTests(test);
function getTestConfig(tmpDir: string): configUtils.Config {
return {
function getTestConfig(tempDir: string): configUtils.Config {
return createTestConfig({
languages: [Language.java],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: configUtils.defaultAugmentationProperties,
trapCaches: {},
trapCacheDownloadTime: 0,
};
tempDir,
dbLocation: path.resolve(tempDir, "codeql_databases"),
});
}
test("getCombinedTracerConfig - return undefined when no languages are traced languages", async (t) => {
@@ -34,7 +29,7 @@ test("getCombinedTracerConfig - return undefined when no languages are traced la
// No traced languages
config.languages = [Language.javascript, Language.python];
t.deepEqual(
await getCombinedTracerConfig(makeVersionInfo("1.0.0"), config),
await getCombinedTracerConfig(mockCodeQLVersion("1.0.0"), config),
undefined,
);
});
@@ -70,7 +65,7 @@ test("getCombinedTracerConfig - with start-tracing.json environment file", async
fs.writeFileSync(startTracingJson, JSON.stringify(startTracingEnv));
const result = await getCombinedTracerConfig(
makeVersionInfo("1.0.0"),
mockCodeQLVersion("1.0.0"),
config,
);
t.notDeepEqual(result, undefined);
@@ -130,7 +125,7 @@ test("getCombinedTracerConfig - with SetsCodeqlRunnerEnvVar feature enabled in C
fs.writeFileSync(startTracingJson, JSON.stringify(startTracingEnv));
const result = await getCombinedTracerConfig(
makeVersionInfo("1.0.0", { setsCodeqlRunnerEnvVar: true }),
mockCodeQLVersion("1.0.0", { setsCodeqlRunnerEnvVar: true }),
config,
);
t.notDeepEqual(result, undefined);
+4 -6
View File
@@ -1,10 +1,10 @@
import * as fs from "fs";
import * as path from "path";
import { VersionInfo } from "./codeql";
import { CodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { isTracedLanguage } from "./languages";
import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
import { ToolsFeature } from "./tools-features";
export type TracerConfig = {
env: { [key: string]: string };
@@ -61,7 +61,7 @@ export async function getTracerConfigForCluster(
}
export async function getCombinedTracerConfig(
versionInfo: VersionInfo,
codeql: CodeQL,
config: configUtils.Config,
): Promise<TracerConfig | undefined> {
// Abort if there are no traced languages as there's nothing to do
@@ -74,9 +74,7 @@ export async function getCombinedTracerConfig(
// If the CLI doesn't yet support setting the CODEQL_RUNNER environment variable to
// the runner executable path, we set it here in the Action.
if (
!isSupportedToolsFeature(versionInfo, ToolsFeature.SetsCodeqlRunnerEnvVar)
) {
if (!(await codeql.supportsFeature(ToolsFeature.SetsCodeqlRunnerEnvVar))) {
// On MacOS when System Integrity Protection is enabled, it's necessary to prefix
// the build command with the runner executable for indirect tracing, so we expose
// it here via the CODEQL_RUNNER environment variable.
+10 -36
View File
@@ -12,9 +12,9 @@ import {
getTrapCachingExtractorConfigArgsForLang,
} from "./codeql";
import * as configUtils from "./config-utils";
import { Config } from "./config-utils";
import { Language } from "./languages";
import {
createTestConfig,
getRecordingLogger,
makeVersionInfo,
setupTests,
@@ -69,49 +69,23 @@ const stubCodeql = setCodeQL({
},
});
const testConfigWithoutTmpDir: Config = {
const testConfigWithoutTmpDir = createTestConfig({
languages: [Language.javascript, Language.cpp],
originalUserInput: {},
tempDir: "",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: "",
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {
javascript: "/some/cache/dir",
},
trapCacheDownloadTime: 0,
};
});
function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
return {
function getTestConfigWithTempDir(tempDir: string): configUtils.Config {
return createTestConfig({
languages: [Language.javascript, Language.ruby],
originalUserInput: {},
tempDir: tmpDir,
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
packsInputCombines: false,
queriesInputCombines: false,
},
tempDir,
dbLocation: path.resolve(tempDir, "codeql_databases"),
trapCaches: {
javascript: path.resolve(tmpDir, "jsCache"),
ruby: path.resolve(tmpDir, "rubyCache"),
javascript: path.resolve(tempDir, "jsCache"),
ruby: path.resolve(tempDir, "rubyCache"),
},
trapCacheDownloadTime: 0,
};
});
}
test("check flags for JS, analyzing default branch", async (t) => {
+29 -18
View File
@@ -1011,26 +1011,37 @@ export interface DiskUsage {
numTotalBytes: number;
}
export async function checkDiskUsage(logger?: Logger): Promise<DiskUsage> {
const diskUsage = await checkDiskSpace(
getRequiredEnvParam("GITHUB_WORKSPACE"),
);
const gbInBytes = 1024 * 1024 * 1024;
if (logger && diskUsage.free < 2 * gbInBytes) {
const message =
"The Actions runner is running low on disk space " +
`(${(diskUsage.free / gbInBytes).toPrecision(4)} GB available).`;
if (process.env[EnvVar.HAS_WARNED_ABOUT_DISK_SPACE] !== "true") {
logger.warning(message);
} else {
logger.debug(message);
export async function checkDiskUsage(
logger?: Logger,
): Promise<DiskUsage | undefined> {
try {
const diskUsage = await checkDiskSpace(
getRequiredEnvParam("GITHUB_WORKSPACE"),
);
const gbInBytes = 1024 * 1024 * 1024;
if (logger && diskUsage.free < 2 * gbInBytes) {
const message =
"The Actions runner is running low on disk space " +
`(${(diskUsage.free / gbInBytes).toPrecision(4)} GB available).`;
if (process.env[EnvVar.HAS_WARNED_ABOUT_DISK_SPACE] !== "true") {
logger.warning(message);
} else {
logger.debug(message);
}
core.exportVariable(EnvVar.HAS_WARNED_ABOUT_DISK_SPACE, "true");
}
core.exportVariable(EnvVar.HAS_WARNED_ABOUT_DISK_SPACE, "true");
return {
numAvailableBytes: diskUsage.free,
numTotalBytes: diskUsage.size,
};
} catch (error) {
if (logger) {
logger.warning(
`Failed to check available disk space: ${getErrorMessage(error)}`,
);
}
return undefined;
}
return {
numAvailableBytes: diskUsage.free,
numTotalBytes: diskUsage.size,
};
}
/**