github/codeql-action
0
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-04-02 09:42:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2993 from github/cklin/overlay-pack-check

Browse Source 
Overlay: check query packs for compatibility
This commit is contained in:
Chuan-kai Lin
2025-08-11 07:42:07 -07:00
committed by GitHub
parent f45dfa6abd 821d3bd162
commit c6dcdfa33a
31 changed files with 1041 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
.github/workflows/__overlay-init-fallback.yml generated vendored Normal file
View File

@@ -0,0 +1,69 @@
# Warning: This file is generated automatically, and should not be modified.
# Instead, please modify the template in the pr-checks directory and run:
# (cd pr-checks; pip install ruamel.yaml@0.17.31 && python3 sync.py)
# to regenerate this file.
name: PR Check - Overlay database init fallback
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GO111MODULE: auto
on:
push:
branches:
- main
- releases/v*
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
schedule:
- cron: '0 5 * * *'
workflow_dispatch: {}
jobs:
overlay-init-fallback:
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
version: linked
- os: ubuntu-latest
version: nightly-latest
name: Overlay database init fallback
permissions:
contents: read
security-events: read
timeout-minutes: 45
runs-on: ${{ matrix.os }}
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Prepare test
id: prepare-test
uses: ./.github/actions/prepare-test
with:
version: ${{ matrix.version }}
use-all-platform-bundle: 'false'
setup-kotlin: 'true'
- uses: ./../action/init
with:
languages: actions # Any language without overlay support will do
tools: ${{ steps.prepare-test.outputs.tools-url }}
env:
CODEQL_OVERLAY_DATABASE_MODE: overlay-base
- uses: ./../action/analyze
id: analysis
with:
upload-database: false
- name: Check database
shell: bash
run: |
cd "$RUNNER_TEMP/codeql_databases/actions"
if ! grep -q 'overlayBaseDatabase: false' codeql-database.yml ; then
echo "This test needs to be updated to use a non-overlay language."
exit 1
fi
env:
CODEQL_ACTION_TEST_MODE: true

2
lib/analyze.js generated
View File

@@ -442,7 +442,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const queries = [];
if (config.augmentationProperties.qualityQueriesInput !== undefined) {
queries.push(path.join(util.getCodeQLDatabasePath(config, language), "temp", "config-queries.qls"));
queries.push(util.getGeneratedSuitePath(config, language));
for (const qualityQuery of config.augmentationProperties
.qualityQueriesInput) {
queries.push(resolveQuerySuiteAlias(language, qualityQuery.uses));

2
lib/analyze.js.map
View File

File diff suppressed because one or more lines are too long

1
lib/analyze.test.js generated
View File

@@ -66,7 +66,6 @@ const util = __importStar(require("./util"));
for (const language of Object.values(languages_1.KnownLanguage)) {
const codeql = (0, codeql_1.createStubCodeQL)({
databaseRunQueries: async () => { },
packDownload: async () => ({ packs: [] }),
databaseInterpretResults: async (_db, _queriesRun, sarifFile) => {
fs.writeFileSync(sarifFile, JSON.stringify({
runs: [

2
lib/analyze.test.js.map
View File

File diff suppressed because one or more lines are too long

82
lib/codeql.js generated
View File

@@ -195,9 +195,7 @@ function createStubCodeQL(partialCodeql) {
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
betterResolveLanguages: resolveFunction(partialCodeql, "betterResolveLanguages", async () => ({ aliases: {}, extractors: {} })),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
resolveBuildEnvironment: resolveFunction(partialCodeql, "resolveBuildEnvironment"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanupCluster: resolveFunction(partialCodeql, "databaseCleanupCluster"),
databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
@@ -206,6 +204,7 @@ function createStubCodeQL(partialCodeql) {
databaseExportDiagnostics: resolveFunction(partialCodeql, "databaseExportDiagnostics"),
diagnosticsExport: resolveFunction(partialCodeql, "diagnosticsExport"),
resolveExtractor: resolveFunction(partialCodeql, "resolveExtractor"),
resolveQueriesStartingPacks: resolveFunction(partialCodeql, "resolveQueriesStartingPacks"),
mergeResults: resolveFunction(partialCodeql, "mergeResults"),
};
}
@@ -419,25 +418,6 @@ async function getCodeQLForCmd(cmd, checkVersion) {
throw new Error(`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`);
}
},
async resolveQueries(queries, extraSearchPath) {
const codeqlArgs = [
"resolve",
"queries",
...queries,
"--format=bylanguage",
...getExtraOptionsFromEnv(["resolve", "queries"]),
];
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output);
}
catch (e) {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async resolveBuildEnvironment(workingDir, language) {
const codeqlArgs = [
"resolve",
@@ -527,50 +507,6 @@ async function getCodeQLForCmd(cmd, checkVersion) {
];
return await runCli(cmd, codeqlArgs);
},
/**
* Download specified packs into the package cache. If the specified
* package and version already exists (e.g., from a previous analysis run),
* then it is not downloaded again (unless the extra option `--force` is
* specified).
*
* If no version is specified, then the latest version is
* downloaded. The check to determine what the latest version is is done
* each time this package is requested.
*
* Optionally, a `qlconfigFile` is included. If used, then this file
* is used to determine which registry each pack is downloaded from.
*/
async packDownload(packs, qlconfigFile) {
const qlconfigArg = qlconfigFile
? [`--qlconfig-file=${qlconfigFile}`]
: [];
const codeqlArgs = [
"pack",
"download",
...qlconfigArg,
"--format=json",
"--resolve-query-specs",
...getExtraOptionsFromEnv(["pack", "download"]),
...packs,
];
const output = await runCli(cmd, codeqlArgs);
try {
const parsedOutput = JSON.parse(output);
if (Array.isArray(parsedOutput.packs) &&
// TODO PackDownloadOutput will not include the version if it is not specified
// in the input. The version is always the latest version available.
// It should be added to the output, but this requires a CLI change
parsedOutput.packs.every((p) => p.name /* && p.version */)) {
return parsedOutput;
}
else {
throw new Error("Unexpected output from pack download");
}
}
catch (e) {
throw new Error(`Attempted to download specified packs but got an error:\n${output}\n${e}`);
}
},
async databaseCleanupCluster(config, cleanupLevel) {
const cacheCleanupFlag = (await util.codeQlVersionAtLeast(this, CODEQL_VERSION_CACHE_CLEANUP))
? "--cache-cleanup"
@@ -653,6 +589,22 @@ async function getCodeQLForCmd(cmd, checkVersion) {
}).exec();
return JSON.parse(extractorPath);
},
async resolveQueriesStartingPacks(queries) {
const codeqlArgs = [
"resolve",
"queries",
"--format=startingpacks",
...getExtraOptionsFromEnv(["resolve", "queries"]),
...queries,
];
const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
try {
return JSON.parse(output);
}
catch (e) {
throw new Error(`Unexpected output from codeql resolve queries --format=startingpacks: ${e}`);
}
},
async mergeResults(sarifFiles, outputFile, { mergeRunsFromEqualCategory = false, }) {
const args = [
"github",

2
lib/codeql.js.map
View File

File diff suppressed because one or more lines are too long

86
lib/config-utils.test.js generated
View File

@@ -144,19 +144,6 @@ function mockListLanguages(languages) {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: { queries: ["query1.ql"] },
python: { queries: ["query2.ql"] },
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const config = await configUtils.initConfig(createTestInitConfigInputs({
languagesInput: languages,
@@ -185,19 +172,6 @@ function mockListLanguages(languages) {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: { queries: ["query1.ql"] },
python: { queries: ["query2.ql"] },
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
// Sanity check the saved config file does not already exist
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tempDir)));
@@ -283,21 +257,6 @@ function mockListLanguages(languages) {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: {
"/foo/a.ql": {},
"/bar/b.ql": {},
},
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
// Just create a generic config object with non-default values for all fields
const inputFileContents = `
@@ -350,24 +309,6 @@ function mockListLanguages(languages) {
t.deepEqual(actualConfig, expectedConfig);
});
});
/**
* Returns the provided queries, just in the right format for a resolved query
* This way we can test by seeing which returned items are in the final
* configuration.
*/
function queriesToResolvedQueryForm(queries) {
const dummyResolvedQueries = {};
for (const q of queries) {
dummyResolvedQueries[q] = {};
}
return {
byLanguage: {
javascript: dummyResolvedQueries,
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
}
(0, ava_1.default)("Using config input and file together, config input should be used.", async (t) => {
return await (0, util_1.withTmpDir)(async (tempDir) => {
process.env["RUNNER_TEMP"] = tempDir;
@@ -388,7 +329,6 @@ function queriesToResolvedQueryForm(queries) {
- c/d@1.2.3
`;
fs.mkdirSync(path.join(tempDir, "foo"));
const resolveQueriesArgs = [];
const codeql = (0, codeql_1.createStubCodeQL)({
async betterResolveLanguages() {
return {
@@ -398,13 +338,6 @@ function queriesToResolvedQueryForm(queries) {
},
};
},
async resolveQueries(queries, extraSearchPath) {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload() {
return { packs: [] };
},
});
// Only JS, python packs will be ignored
const languagesInput = "javascript";
@@ -429,20 +362,6 @@ function queriesToResolvedQueryForm(queries) {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: {
"foo.ql": {},
},
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload() {
return { packs: [] };
},
});
const inputFileContents = `
name: my config
@@ -519,9 +438,6 @@ function queriesToResolvedQueryForm(queries) {
async resolveLanguages() {
return {};
},
async packDownload() {
return { packs: [] };
},
});
try {
await configUtils.initConfig(createTestInitConfigInputs({
@@ -897,7 +813,7 @@ const defaultOverlayDatabaseModeTestSetup = {
repositoryOwner: "github",
buildMode: util_1.BuildMode.None,
languages: [languages_1.KnownLanguage.javascript],
codeqlVersion: "2.21.0",
codeqlVersion: overlay_database_utils_1.CODEQL_OVERLAY_MINIMUM_VERSION,
gitRoot: "/some/git/root",
codeScanningConfig: {},
};

2
lib/config-utils.test.js.map
View File

File diff suppressed because one or more lines are too long

28
lib/init-action.js generated
View File

@@ -55,6 +55,7 @@ const repository_1 = require("./repository");
const setup_codeql_1 = require("./setup-codeql");
const status_report_1 = require("./status-report");
const tools_features_1 = require("./tools-features");
const tracer_config_1 = require("./tracer-config");
const util_1 = require("./util");
const workflow_1 = require("./workflow");
async function sendCompletedStatusReport(startedAt, config, configFile, toolsDownloadStatusReport, toolsFeatureFlagsValid, toolsSource, toolsVersion, overlayBaseDatabaseStats, logger, error) {
@@ -457,7 +458,32 @@ async function run() {
core.exportVariable("CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB", "true");
}
}
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", (0, actions_util_1.getOptionalInput)("registries"), apiDetails, logger);
const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries((0, actions_util_1.getOptionalInput)("registries"), config.tempDir, logger);
const databaseInitEnvironment = {
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
};
await (0, init_1.runDatabaseInitCluster)(databaseInitEnvironment, codeql, config, sourceRoot, "Runner.Worker.exe", qlconfigFile, logger);
// To check custom query packs for compatibility with overlay analysis, we
// need to first initialize the database cluster, which downloads the
// user-specified custom query packs. But we also want to check custom query
// pack compatibility first, because database cluster initialization depends
// on the overlay database mode. The solution is to initialize the database
// cluster first, check custom query pack compatibility, and if we need to
// revert to `OverlayDatabaseMode.None`, re-initialize the database cluster
// with the new overlay database mode.
if (config.augmentationProperties.overlayDatabaseMode !==
overlay_database_utils_1.OverlayDatabaseMode.None &&
!(await (0, init_1.checkPacksForOverlayCompatibility)(codeql, config, logger))) {
logger.info("Reverting overlay database mode to None due to incompatible packs.");
config.augmentationProperties.overlayDatabaseMode =
overlay_database_utils_1.OverlayDatabaseMode.None;
(0, init_1.cleanupDatabaseClusterDirectory)(config, logger, {
disableExistingDirectoryWarning: true,
});
await (0, init_1.runDatabaseInitCluster)(databaseInitEnvironment, codeql, config, sourceRoot, "Runner.Worker.exe", qlconfigFile, logger);
}
const tracerConfig = await (0, tracer_config_1.getCombinedTracerConfig)(codeql, config);
if (tracerConfig !== undefined) {
for (const [key, value] of Object.entries(tracerConfig.env)) {
core.exportVariable(key, value);

2
lib/init-action.js.map
View File

File diff suppressed because one or more lines are too long

99
lib/init.js generated
View File

@@ -35,19 +35,20 @@ var __importStar = (this && this.__importStar) || (function () {
Object.defineProperty(exports, "__esModule", { value: true });
exports.initCodeQL = initCodeQL;
exports.initConfig = initConfig;
exports.runInit = runInit;
exports.runDatabaseInitCluster = runDatabaseInitCluster;
exports.checkPacksForOverlayCompatibility = checkPacksForOverlayCompatibility;
exports.checkInstallPython311 = checkInstallPython311;
exports.cleanupDatabaseClusterDirectory = cleanupDatabaseClusterDirectory;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
const io = __importStar(require("@actions/io"));
const yaml = __importStar(require("js-yaml"));
const actions_util_1 = require("./actions-util");
const codeql_1 = require("./codeql");
const configUtils = __importStar(require("./config-utils"));
const languages_1 = require("./languages");
const logging_1 = require("./logging");
const tracer_config_1 = require("./tracer-config");
const util = __importStar(require("./util"));
async function initCodeQL(toolsInput, apiDetails, tempDir, variant, defaultCliVersion, logger) {
logger.startGroup("Setup CodeQL tools");
@@ -67,16 +68,86 @@ async function initConfig(inputs) {
return await configUtils.initConfig(inputs);
});
}
async function runInit(codeql, config, sourceRoot, processName, registriesInput, apiDetails, logger) {
async function runDatabaseInitCluster(databaseInitEnvironment, codeql, config, sourceRoot, processName, qlconfigFile, logger) {
fs.mkdirSync(config.dbLocation, { recursive: true });
const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries(registriesInput, config.tempDir, logger);
await configUtils.wrapEnvironment({
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
async () => await codeql.databaseInitCluster(config, sourceRoot, processName, qlconfigFile, logger));
return await (0, tracer_config_1.getCombinedTracerConfig)(codeql, config);
await configUtils.wrapEnvironment(databaseInitEnvironment, async () => await codeql.databaseInitCluster(config, sourceRoot, processName, qlconfigFile, logger));
}
/**
* Check whether all query packs are compatible with the overlay analysis
* support in the CodeQL CLI. If the check fails, this function will log a
* warning and returns false.
*
* @param codeql A CodeQL instance.
* @param logger A logger.
* @returns `true` if all query packs are compatible with overlay analysis,
* `false` otherwise.
*/
async function checkPacksForOverlayCompatibility(codeql, config, logger) {
const codeQlOverlayVersion = (await codeql.getVersion()).overlayVersion;
if (codeQlOverlayVersion === undefined) {
logger.warning("The CodeQL CLI does not support overlay analysis.");
return false;
}
for (const language of config.languages) {
const suitePath = util.getGeneratedSuitePath(config, language);
const packDirs = await codeql.resolveQueriesStartingPacks([suitePath]);
if (packDirs.some((packDir) => !checkPackForOverlayCompatibility(packDir, codeQlOverlayVersion, logger))) {
return false;
}
}
return true;
}
/**
* Check a single pack for its overlay compatibility. If the check fails, this
* function will log a warning and returns false.
*
* @param packDir Path to the directory containing the pack.
* @param codeQlOverlayVersion The overlay version of the CodeQL CLI.
* @param logger A logger.
* @returns `true` if the pack is compatible with overlay analysis, `false`
* otherwise.
*/
function checkPackForOverlayCompatibility(packDir, codeQlOverlayVersion, logger) {
try {
let qlpackPath = path.join(packDir, "qlpack.yml");
if (!fs.existsSync(qlpackPath)) {
qlpackPath = path.join(packDir, "codeql-pack.yml");
}
const qlpackContents = yaml.load(fs.readFileSync(qlpackPath, "utf8"));
if (!qlpackContents.buildMetadata) {
// This is a source-only pack, and overlay compatibility checks apply only
// to precompiled packs.
return true;
}
const packInfoPath = path.join(packDir, ".packinfo");
if (!fs.existsSync(packInfoPath)) {
logger.warning(`The query pack at ${packDir} does not have a .packinfo file, ` +
"so it cannot support overlay analysis. Recompiling the query pack " +
"with the latest CodeQL CLI should solve this problem.");
return false;
}
const packInfoFileContents = JSON.parse(fs.readFileSync(packInfoPath, "utf8"));
const packOverlayVersion = packInfoFileContents.overlayVersion;
if (typeof packOverlayVersion !== "number") {
logger.warning(`The .packinfo file for the query pack at ${packDir} ` +
"does not have the overlayVersion field, which indicates that " +
"the pack is not compatible with overlay analysis.");
return false;
}
if (packOverlayVersion !== codeQlOverlayVersion) {
logger.warning(`The query pack at ${packDir} was compiled with ` +
`overlay version ${packOverlayVersion}, but the CodeQL CLI ` +
`supports overlay version ${codeQlOverlayVersion}. The ` +
"query pack needs to be recompiled to support overlay analysis.");
return false;
}
}
catch (e) {
logger.warning(`Error while checking pack at ${packDir} ` +
`for overlay compatibility: ${util.getErrorMessage(e)}`);
return false;
}
return true;
}
/**
* If we are running python 3.12+ on windows, we need to switch to python 3.11.
@@ -92,14 +163,16 @@ async function checkInstallPython311(languages, codeql) {
]).exec();
}
}
function cleanupDatabaseClusterDirectory(config, logger,
function cleanupDatabaseClusterDirectory(config, logger, options = {},
// We can't stub the fs module in tests, so we allow the caller to override the rmSync function
// for testing.
rmSync = fs.rmSync) {
if (fs.existsSync(config.dbLocation) &&
(fs.statSync(config.dbLocation).isFile() ||
fs.readdirSync(config.dbLocation).length > 0)) {
logger.warning(`The database cluster directory ${config.dbLocation} must be empty. Attempting to clean it up.`);
if (!options.disableExistingDirectoryWarning) {
logger.warning(`The database cluster directory ${config.dbLocation} must be empty. Attempting to clean it up.`);
}
try {
rmSync(config.dbLocation, {
force: true,

2
lib/init.js.map
View File

@@ -1 +1 @@
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmBA,gCAuCC;AAED,gCAMC;AAED,0BAkCC;AAMD,sDAkBC;AAED,0EAkDC;AAlLD,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,gDAAkC;AAElC,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAAsD;AACtD,uCAAmD;AAInD,mDAAwE;AACxE,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC;IAEpC,OAAO,MAAM,IAAA,wBAAc,EAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QACpE,OAAO,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,eAAmC,EACnC,UAAoC,EACpC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,MAAM,EAAE,oBAAoB,EAAE,YAAY,EAAE,GAC1C,MAAM,WAAW,CAAC,kBAAkB,CAClC,eAAe,EACf,MAAM,CAAC,OAAO,EACd,MAAM,CACP,CAAC;IACJ,MAAM,WAAW,CAAC,eAAe,CAC/B;QACE,YAAY,EAAE,UAAU,CAAC,IAAI;QAC7B,sBAAsB,EAAE,oBAAoB;KAC7C;IAED,0BAA0B;IAC1B,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;IACF,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,yBAAa,CAAC,MAAM,CAAC;QACxC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,EAAE;YAClE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc;AACd,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC/C,CAAC;QACD,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmBA,gCAuCC;AAED,gCAMC;AAED,wDAqBC;AAYD,8EA6BC;AAmFD,sDAkBC;AAED,0EAqDC;AA9RD,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,gDAAkC;AAClC,8CAAgC;AAEhC,iDAAsE;AAEtE,qCAA+C;AAC/C,4DAA8C;AAE9C,2CAAsD;AACtD,uCAAmD;AAInD,6CAA+B;AAExB,KAAK,UAAU,UAAU,CAC9B,UAA8B,EAC9B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,iBAA2C,EAC3C,MAAc;IAQd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EACJ,MAAM,EACN,yBAAyB,EACzB,WAAW,EACX,YAAY,EACZ,gBAAgB,GACjB,GAAG,MAAM,IAAA,oBAAW,EACnB,UAAU,EACV,UAAU,EACV,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO;QACL,MAAM;QACN,yBAAyB;QACzB,WAAW;QACX,YAAY;QACZ,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,MAAoC;IAEpC,OAAO,MAAM,IAAA,wBAAc,EAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QACpE,OAAO,MAAM,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,sBAAsB,CAC1C,uBAA2D,EAC3D,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,YAAgC,EAChC,MAAc;IAEd,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,WAAW,CAAC,eAAe,CAC/B,uBAAuB,EACvB,KAAK,IAAI,EAAE,CACT,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,MAAM,CACP,CACJ,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,iCAAiC,CACrD,MAAc,EACd,MAA0B,EAC1B,MAAc;IAEd,MAAM,oBAAoB,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,cAAc,CAAC;IACxE,IAAI,oBAAoB,KAAK,SAAS,EAAE,CAAC;QACvC,MAAM,CAAC,OAAO,CAAC,mDAAmD,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QACvE,IACE,QAAQ,CAAC,IAAI,CACX,CAAC,OAAO,EAAE,EAAE,CACV,CAAC,gCAAgC,CAC/B,OAAO,EACP,oBAAoB,EACpB,MAAM,CACP,CACJ,EACD,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAOD;;;;;;;;;GASG;AACH,SAAS,gCAAgC,CACvC,OAAe,EACf,oBAA4B,EAC5B,MAAc;IAEd,IAAI,CAAC;QACH,IAAI,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAClD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACrD,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAC9B,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAC1B,CAAC;QACZ,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YAClC,0EAA0E;YAC1E,wBAAwB;YACxB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,OAAO,CACZ,qBAAqB,OAAO,mCAAmC;gBAC7D,oEAAoE;gBACpE,uDAAuD,CAC1D,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CACrC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CACtC,CAAC;QACF,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,cAAc,CAAC;QAC/D,IAAI,OAAO,kBAAkB,KAAK,QAAQ,EAAE,CAAC;YAC3C,MAAM,CAAC,OAAO,CACZ,4CAA4C,OAAO,GAAG;gBACpD,+DAA+D;gBAC/D,mDAAmD,CACtD,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,kBAAkB,KAAK,oBAAoB,EAAE,CAAC;YAChD,MAAM,CAAC,OAAO,CACZ,qBAAqB,OAAO,qBAAqB;gBAC/C,mBAAmB,kBAAkB,uBAAuB;gBAC5D,4BAA4B,oBAAoB,QAAQ;gBACxD,gEAAgE,CACnE,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,OAAO,CACZ,gCAAgC,OAAO,GAAG;YACxC,8BAA8B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAC1D,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,MAAc;IAEd,IACE,SAAS,CAAC,QAAQ,CAAC,yBAAa,CAAC,MAAM,CAAC;QACxC,OAAO,CAAC,QAAQ,KAAK,OAAO;QAC5B,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EACxD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CACzB,SAAS,EACT,iBAAiB,EACjB,oBAAoB,CACrB,CAAC;QACF,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,EAAE;YAClE,MAAM;SACP,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAgB,+BAA+B,CAC7C,MAA0B,EAC1B,MAAc,EACd,UAAyD,EAAE;AAC3D,+FAA+F;AAC/F,eAAe;AACf,MAAM,GAAG,EAAE,CAAC,MAAM;IAElB,IACE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAChC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;YACtC,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAC/C,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,+BAA+B,EAAE,CAAC;YAC7C,MAAM,CAAC,OAAO,CACZ,kCAAkC,MAAM,CAAC,UAAU,4CAA4C,CAChG,CAAC;QACJ,CAAC;QACD,IAAI,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;gBACxB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,CAAC;gBACb,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CACT,yCAAyC,MAAM,CAAC,UAAU,GAAG,CAC9D,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,mEACZ,IAAA,+BAAgB,EAAC,aAAa,CAAC;gBAC7B,CAAC,CAAC,sCAAsC,MAAM,CAAC,UAAU,IAAI;gBAC7D,CAAC,CAAC,kCAAkC,MAAM,CAAC,UAAU,IAAI;oBACvD,yEACN,iEAAiE,CAAC;YAElE,kGAAkG;YAClG,IAAI,IAAA,iCAAkB,GAAE,EAAE,CAAC;gBACzB,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAC/B,GAAG,KAAK,4GAA4G;oBAClH,sEAAsE,IAAI,CAAC,eAAe,CACxF,CAAC,CACF,EAAE,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,sDAAsD;oBAC5D,+EAA+E;oBAC/E,yCAAyC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

218
lib/init.test.js generated
View File

@@ -39,7 +39,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path_1 = __importDefault(require("path"));
const ava_1 = __importDefault(require("ava"));
const codeql_1 = require("./codeql");
const init_1 = require("./init");
const languages_1 = require("./languages");
const testing_utils_1 = require("./testing-utils");
const util_1 = require("./util");
(0, testing_utils_1.setupTests)(ava_1.default);
@@ -88,7 +90,7 @@ for (const { runnerEnv, ErrorConstructor, message } of [
fs.writeFileSync(fileToCleanUp, "");
const rmSyncError = `Failed to clean up file ${fileToCleanUp}`;
const messages = [];
t.throws(() => (0, init_1.cleanupDatabaseClusterDirectory)((0, testing_utils_1.createTestConfig)({ dbLocation }), (0, testing_utils_1.getRecordingLogger)(messages), () => {
t.throws(() => (0, init_1.cleanupDatabaseClusterDirectory)((0, testing_utils_1.createTestConfig)({ dbLocation }), (0, testing_utils_1.getRecordingLogger)(messages), {}, () => {
throw new Error(rmSyncError);
}), {
instanceOf: ErrorConstructor,
@@ -100,4 +102,218 @@ for (const { runnerEnv, ErrorConstructor, message } of [
});
});
}
(0, ava_1.default)("cleanupDatabaseClusterDirectory can disable warning with options", async (t) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
const dbLocation = path_1.default.resolve(tmpDir, "dbs");
fs.mkdirSync(dbLocation, { recursive: true });
const fileToCleanUp = path_1.default.resolve(dbLocation, "something-to-cleanup.txt");
fs.writeFileSync(fileToCleanUp, "");
const messages = [];
(0, init_1.cleanupDatabaseClusterDirectory)((0, testing_utils_1.createTestConfig)({ dbLocation }), (0, testing_utils_1.getRecordingLogger)(messages), { disableExistingDirectoryWarning: true });
// Should only have the info message, not the warning
t.is(messages.length, 1);
t.is(messages[0].type, "info");
t.is(messages[0].message, `Cleaned up database cluster directory ${dbLocation}.`);
t.false(fs.existsSync(fileToCleanUp));
});
});
const testCheckPacksForOverlayCompatibility = ava_1.default.macro({
exec: async (t, _title, { cliOverlayVersion, languages, packs, expectedResult, }) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
const packDirsByLanguage = new Map();
for (const [packName, packInfo] of Object.entries(packs)) {
const packPath = path_1.default.join(tmpDir, packName);
fs.mkdirSync(packPath, { recursive: true });
if (packInfo.packinfoContents) {
fs.writeFileSync(path_1.default.join(packPath, ".packinfo"), packInfo.packinfoContents);
}
const qlpackFileName = packInfo.qlpackFileName || "qlpack.yml";
fs.writeFileSync(path_1.default.join(packPath, qlpackFileName), packInfo.sourceOnlyPack
? `name: ${packName}\nversion: 1.0.0\n`
: `name: ${packName}\nversion: 1.0.0\nbuildMetadata:\n sha: 123abc\n`);
if (!packDirsByLanguage.has(packInfo.language)) {
packDirsByLanguage.set(packInfo.language, []);
}
packDirsByLanguage.get(packInfo.language).push(packPath);
}
const codeql = (0, codeql_1.createStubCodeQL)({
getVersion: async () => ({
version: "2.22.2",
overlayVersion: cliOverlayVersion,
}),
resolveQueriesStartingPacks: async (suitePaths) => {
for (const language of packDirsByLanguage.keys()) {
const suiteForLanguage = path_1.default.join(language, "temp", "config-queries.qls");
if (suitePaths[0].endsWith(suiteForLanguage)) {
return packDirsByLanguage.get(language) || [];
}
}
return [];
},
});
const messages = [];
const result = await (0, init_1.checkPacksForOverlayCompatibility)(codeql, (0, testing_utils_1.createTestConfig)({ dbLocation: tmpDir, languages }), (0, testing_utils_1.getRecordingLogger)(messages));
t.is(result, expectedResult);
t.deepEqual(messages.length, expectedResult ? 0 : 1, "Expected log messages");
});
},
title: (_, title) => `checkPacksForOverlayCompatibility: ${title}`,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns false when CLI does not support overlay", {
cliOverlayVersion: undefined,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: false,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns true when there are no query packs", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {},
expectedResult: true,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns true when query pack has not been compiled", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: undefined,
sourceOnlyPack: true,
},
},
expectedResult: true,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns true when query pack has expected overlay version", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: true,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns true when query packs for all languages to analyze are compatible", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.cpp, languages_1.KnownLanguage.java],
packs: {
"codeql/cpp-queries": {
language: languages_1.KnownLanguage.cpp,
packinfoContents: '{"overlayVersion":2}',
},
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: true,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns true when query pack for a language not analyzed is incompatible", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/cpp-queries": {
language: languages_1.KnownLanguage.cpp,
packinfoContents: undefined,
},
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: true,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns false when query pack for a language to analyze is incompatible", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.cpp, languages_1.KnownLanguage.java],
packs: {
"codeql/cpp-queries": {
language: languages_1.KnownLanguage.cpp,
packinfoContents: '{"overlayVersion":1}',
},
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: false,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns false when query pack is missing .packinfo", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: undefined,
},
},
expectedResult: false,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns false when query pack has different overlay version", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":1}',
},
},
expectedResult: false,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns false when query pack is missing overlayVersion in .packinfo", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: "{}",
},
},
expectedResult: false,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns false when .packinfo is not valid JSON", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: "this_is_not_valid_json",
},
},
expectedResult: false,
});
(0, ava_1.default)(testCheckPacksForOverlayCompatibility, "returns true when query pack uses codeql-pack.yml filename", {
cliOverlayVersion: 2,
languages: [languages_1.KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: languages_1.KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
qlpackFileName: "codeql-pack.yml",
},
},
expectedResult: true,
});
//# sourceMappingURL=init.test.js.map

2
lib/init.test.js.map
View File

File diff suppressed because one or more lines are too long

2
lib/overlay-database-utils.js generated
View File

@@ -52,7 +52,7 @@ var OverlayDatabaseMode;
OverlayDatabaseMode["OverlayBase"] = "overlay-base";
OverlayDatabaseMode["None"] = "none";
})(OverlayDatabaseMode || (exports.OverlayDatabaseMode = OverlayDatabaseMode = {}));
exports.CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5";
exports.CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3";
/**
* Writes a JSON file containing Git OIDs for all tracked files (represented
* by path relative to the source root) under the source root. The file is

7
lib/testing-utils.js generated
View File

@@ -227,15 +227,16 @@ function mockLanguagesInRepo(languages) {
/**
* Constructs a `VersionInfo` object for testing purposes only.
*/
const makeVersionInfo = (version, features) => ({
const makeVersionInfo = (version, features, overlayVersion) => ({
version,
features,
overlayVersion,
});
exports.makeVersionInfo = makeVersionInfo;
function mockCodeQLVersion(version, features) {
function mockCodeQLVersion(version, features, overlayVersion) {
return codeql.createStubCodeQL({
async getVersion() {
return (0, exports.makeVersionInfo)(version, features);
return (0, exports.makeVersionInfo)(version, features, overlayVersion);
},
});
}

2
lib/testing-utils.js.map
View File

File diff suppressed because one or more lines are too long

7
lib/util.js generated
View File

@@ -48,6 +48,7 @@ exports.getThreadsFlagValue = getThreadsFlagValue;
exports.getCgroupCpuCountFromCpus = getCgroupCpuCountFromCpus;
exports.getThreadsFlag = getThreadsFlag;
exports.getCodeQLDatabasePath = getCodeQLDatabasePath;
exports.getGeneratedSuitePath = getGeneratedSuitePath;
exports.parseGitHubUrl = parseGitHubUrl;
exports.checkGitHubVersionInRange = checkGitHubVersionInRange;
exports.apiVersionInRange = apiVersionInRange;
@@ -403,6 +404,12 @@ function getThreadsFlag(userInput, logger) {
function getCodeQLDatabasePath(config, language) {
return path.resolve(config.dbLocation, language);
}
/**
* Get the path where the generated query suite for the given language lives.
*/
function getGeneratedSuitePath(config, language) {
return path.resolve(config.dbLocation, language, "temp", "config-queries.qls");
}
/**
* Parses user input of a github.com or GHES URL to a canonical form.
* Removes any API prefix or suffix if one is present.

2
lib/util.js.map
View File

File diff suppressed because one or more lines are too long

23
pr-checks/checks/overlay-init-fallback.yml Normal file
View File

@@ -0,0 +1,23 @@
name: "Overlay database init fallback"
description: "Tests that overlay init action succeeds with non-overlay packs"
versions: ["linked", "nightly-latest"]
operatingSystems: ["ubuntu"]
steps:
- uses: ./../action/init
with:
languages: actions # Any language without overlay support will do
tools: ${{ steps.prepare-test.outputs.tools-url }}
env:
CODEQL_OVERLAY_DATABASE_MODE: overlay-base
- uses: ./../action/analyze
id: analysis
with:
upload-database: false
- name: Check database
shell: bash
run: |
cd "$RUNNER_TEMP/codeql_databases/actions"
if ! grep -q 'overlayBaseDatabase: false' codeql-database.yml ; then
echo "This test needs to be updated to use a non-overlay language."
exit 1
fi

1
src/analyze.test.ts
View File

@@ -44,7 +44,6 @@ test("status report fields", async (t) => {
for (const language of Object.values(KnownLanguage)) {
const codeql = createStubCodeQL({
databaseRunQueries: async () => {},
packDownload: async () => ({ packs: [] }),
databaseInterpretResults: async (
_db: string,
_queriesRun: string[],

9
src/analyze.ts
View File

@@ -663,14 +663,7 @@ export async function runQueries(
const queries: string[] = [];
if (config.augmentationProperties.qualityQueriesInput !== undefined) {
queries.push(
path.join(
util.getCodeQLDatabasePath(config, language),
"temp",
"config-queries.qls",
),
);
queries.push(util.getGeneratedSuitePath(config, language));
for (const qualityQuery of config.augmentationProperties
.qualityQueriesInput) {
queries.push(resolveQuerySuiteAlias(language, qualityQuery.uses));

152
src/codeql.ts
View File

@@ -128,13 +128,6 @@ export interface CodeQL {
* Run 'codeql resolve languages' with '--format=betterjson'.
*/
betterResolveLanguages(): Promise<BetterResolveLanguagesOutput>;
/**
* Run 'codeql resolve queries'.
*/
resolveQueries(
queries: string[],
extraSearchPath: string | undefined,
): Promise<ResolveQueriesOutput>;
/**
* Run 'codeql resolve build-environment'
*/
@@ -143,14 +136,6 @@ export interface CodeQL {
language: string,
): Promise<ResolveBuildEnvironmentOutput>;
/**
* Run 'codeql pack download'.
*/
packDownload(
packs: string[],
qlconfigFile: string | undefined,
): Promise<PackDownloadOutput>;
/**
* Clean up all the databases within a database cluster.
*/
@@ -213,6 +198,10 @@ export interface CodeQL {
): Promise<void>;
/** Get the location of an extractor for the specified language. */
resolveExtractor(language: Language): Promise<string>;
/**
* Run 'codeql resolve queries --format=startingpacks'.
*/
resolveQueriesStartingPacks(queries: string[]): Promise<string[]>;
/**
* Run 'codeql github merge-results'.
*/
@@ -226,6 +215,15 @@ export interface CodeQL {
export interface VersionInfo {
version: string;
features?: { [name: string]: boolean };
/**
* The overlay version helps deal with backward incompatible changes for
* overlay analysis. When a precompiled query pack reports the same overlay
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
* analysis with that pack. Otherwise, if the overlay versions are different,
* or if either the pack or the CLI does not report an overlay version,
* we need to revert to non-overlay analysis.
*/
overlayVersion?: number;
}
export interface ResolveLanguagesOutput {
@@ -246,20 +244,6 @@ export interface BetterResolveLanguagesOutput {
};
}
export interface ResolveQueriesOutput {
byLanguage: {
[language: string]: {
[queryPath: string]: object;
};
};
noDeclaredLanguage: {
[queryPath: string]: object;
};
multipleDeclaredLanguages: {
[queryPath: string]: object;
};
}
export interface ResolveBuildEnvironmentOutput {
configuration?: {
[language: string]: {
@@ -268,17 +252,6 @@ export interface ResolveBuildEnvironmentOutput {
};
}
export interface PackDownloadOutput {
packs: PackDownloadItem[];
}
interface PackDownloadItem {
name: string;
version: string;
packDir: string;
installResult: string;
}
/**
* Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
*/
@@ -484,12 +457,10 @@ export function createStubCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
"betterResolveLanguages",
async () => ({ aliases: {}, extractors: {} }),
),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
resolveBuildEnvironment: resolveFunction(
partialCodeql,
"resolveBuildEnvironment",
),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanupCluster: resolveFunction(
partialCodeql,
"databaseCleanupCluster",
@@ -510,6 +481,10 @@ export function createStubCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
),
diagnosticsExport: resolveFunction(partialCodeql, "diagnosticsExport"),
resolveExtractor: resolveFunction(partialCodeql, "resolveExtractor"),
resolveQueriesStartingPacks: resolveFunction(
partialCodeql,
"resolveQueriesStartingPacks",
),
mergeResults: resolveFunction(partialCodeql, "mergeResults"),
};
}
@@ -781,28 +756,6 @@ export async function getCodeQLForCmd(
);
}
},
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined,
) {
const codeqlArgs = [
"resolve",
"queries",
...queries,
"--format=bylanguage",
...getExtraOptionsFromEnv(["resolve", "queries"]),
];
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveQueriesOutput;
} catch (e) {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async resolveBuildEnvironment(
workingDir: string | undefined,
language: string,
@@ -921,59 +874,6 @@ export async function getCodeQLForCmd(
];
return await runCli(cmd, codeqlArgs);
},
/**
* Download specified packs into the package cache. If the specified
* package and version already exists (e.g., from a previous analysis run),
* then it is not downloaded again (unless the extra option `--force` is
* specified).
*
* If no version is specified, then the latest version is
* downloaded. The check to determine what the latest version is is done
* each time this package is requested.
*
* Optionally, a `qlconfigFile` is included. If used, then this file
* is used to determine which registry each pack is downloaded from.
*/
async packDownload(
packs: string[],
qlconfigFile: string | undefined,
): Promise<PackDownloadOutput> {
const qlconfigArg = qlconfigFile
? [`--qlconfig-file=${qlconfigFile}`]
: ([] as string[]);
const codeqlArgs = [
"pack",
"download",
...qlconfigArg,
"--format=json",
"--resolve-query-specs",
...getExtraOptionsFromEnv(["pack", "download"]),
...packs,
];
const output = await runCli(cmd, codeqlArgs);
try {
const parsedOutput: PackDownloadOutput = JSON.parse(output);
if (
Array.isArray(parsedOutput.packs) &&
// TODO PackDownloadOutput will not include the version if it is not specified
// in the input. The version is always the latest version available.
// It should be added to the output, but this requires a CLI change
parsedOutput.packs.every((p) => p.name /* && p.version */)
) {
return parsedOutput;
} else {
throw new Error("Unexpected output from pack download");
}
} catch (e) {
throw new Error(
`Attempted to download specified packs but got an error:\n${output}\n${e}`,
);
}
},
async databaseCleanupCluster(
config: Config,
cleanupLevel: string,
@@ -1080,6 +980,24 @@ export async function getCodeQLForCmd(
).exec();
return JSON.parse(extractorPath) as string;
},
async resolveQueriesStartingPacks(queries: string[]): Promise<string[]> {
const codeqlArgs = [
"resolve",
"queries",
"--format=startingpacks",
...getExtraOptionsFromEnv(["resolve", "queries"]),
...queries,
];
const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
try {
return JSON.parse(output) as string[];
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve queries --format=startingpacks: ${e}`,
);
}
},
async mergeResults(
sarifFiles: string[],
outputFile: string,

100
src/config-utils.test.ts
View File

@@ -9,13 +9,16 @@ import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as api from "./api-client";
import { CachingKind } from "./caching-utils";
import { PackDownloadOutput, createStubCodeQL } from "./codeql";
import { createStubCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { Feature } from "./feature-flags";
import * as gitUtils from "./git-utils";
import { KnownLanguage, Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import {
CODEQL_OVERLAY_MINIMUM_VERSION,
OverlayDatabaseMode,
} from "./overlay-database-utils";
import { parseRepositoryNwo } from "./repository";
import {
setupTests,
@@ -140,19 +143,6 @@ test("load empty config", async (t) => {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: { queries: ["query1.ql"] },
python: { queries: ["query2.ql"] },
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const config = await configUtils.initConfig(
@@ -192,19 +182,6 @@ test("loading config saves config", async (t) => {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: { queries: ["query1.ql"] },
python: { queries: ["query2.ql"] },
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
// Sanity check the saved config file does not already exist
@@ -327,21 +304,6 @@ test("load non-empty input", async (t) => {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: {
"/foo/a.ql": {},
"/bar/b.ql": {},
},
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
// Just create a generic config object with non-default values for all fields
@@ -403,25 +365,6 @@ test("load non-empty input", async (t) => {
});
});
/**
* Returns the provided queries, just in the right format for a resolved query
* This way we can test by seeing which returned items are in the final
* configuration.
*/
function queriesToResolvedQueryForm(queries: string[]) {
const dummyResolvedQueries = {};
for (const q of queries) {
dummyResolvedQueries[q] = {};
}
return {
byLanguage: {
javascript: dummyResolvedQueries,
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
}
test("Using config input and file together, config input should be used.", async (t) => {
return await withTmpDir(async (tempDir) => {
process.env["RUNNER_TEMP"] = tempDir;
@@ -446,10 +389,6 @@ test("Using config input and file together, config input should be used.", async
fs.mkdirSync(path.join(tempDir, "foo"));
const resolveQueriesArgs: Array<{
queries: string[];
extraSearchPath: string | undefined;
}> = [];
const codeql = createStubCodeQL({
async betterResolveLanguages() {
return {
@@ -459,16 +398,6 @@ test("Using config input and file together, config input should be used.", async
},
};
},
async resolveQueries(
queries: string[],
extraSearchPath: string | undefined,
) {
resolveQueriesArgs.push({ queries, extraSearchPath });
return queriesToResolvedQueryForm(queries);
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
// Only JS, python packs will be ignored
@@ -499,20 +428,6 @@ test("API client used when reading remote config", async (t) => {
},
};
},
async resolveQueries() {
return {
byLanguage: {
javascript: {
"foo.ql": {},
},
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
const inputFileContents = `
@@ -612,9 +527,6 @@ test("No detected languages", async (t) => {
async resolveLanguages() {
return {};
},
async packDownload(): Promise<PackDownloadOutput> {
return { packs: [] };
},
});
try {
@@ -1269,7 +1181,7 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
repositoryOwner: "github",
buildMode: BuildMode.None,
languages: [KnownLanguage.javascript],
codeqlVersion: "2.21.0",
codeqlVersion: CODEQL_OVERLAY_MINIMUM_VERSION,
gitRoot: "/some/git/root",
codeScanningConfig: {},
};

55
src/init-action.ts
View File

@@ -35,10 +35,11 @@ import { EnvVar } from "./environment";
import { Feature, featureConfig, Features } from "./feature-flags";
import {
checkInstallPython311,
checkPacksForOverlayCompatibility,
cleanupDatabaseClusterDirectory,
initCodeQL,
initConfig,
runInit,
runDatabaseInitCluster,
} from "./init";
import { KnownLanguage } from "./languages";
import { getActionsLogger, Logger } from "./logging";
@@ -59,6 +60,7 @@ import {
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { ToolsFeature } from "./tools-features";
import { getCombinedTracerConfig } from "./tracer-config";
import {
checkDiskUsage,
checkForTimeout,
@@ -743,15 +745,60 @@ async function run() {
}
}
const tracerConfig = await runInit(
const { registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
getOptionalInput("registries"),
config.tempDir,
logger,
);
const databaseInitEnvironment = {
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
};
await runDatabaseInitCluster(
databaseInitEnvironment,
codeql,
config,
sourceRoot,
"Runner.Worker.exe",
getOptionalInput("registries"),
apiDetails,
qlconfigFile,
logger,
);
// To check custom query packs for compatibility with overlay analysis, we
// need to first initialize the database cluster, which downloads the
// user-specified custom query packs. But we also want to check custom query
// pack compatibility first, because database cluster initialization depends
// on the overlay database mode. The solution is to initialize the database
// cluster first, check custom query pack compatibility, and if we need to
// revert to `OverlayDatabaseMode.None`, re-initialize the database cluster
// with the new overlay database mode.
if (
config.augmentationProperties.overlayDatabaseMode !==
OverlayDatabaseMode.None &&
!(await checkPacksForOverlayCompatibility(codeql, config, logger))
) {
logger.info(
"Reverting overlay database mode to None due to incompatible packs.",
);
config.augmentationProperties.overlayDatabaseMode =
OverlayDatabaseMode.None;
cleanupDatabaseClusterDirectory(config, logger, {
disableExistingDirectoryWarning: true,
});
await runDatabaseInitCluster(
databaseInitEnvironment,
codeql,
config,
sourceRoot,
"Runner.Worker.exe",
qlconfigFile,
logger,
);
}
const tracerConfig = await getCombinedTracerConfig(codeql, config);
if (tracerConfig !== undefined) {
for (const [key, value] of Object.entries(tracerConfig.env)) {
core.exportVariable(key, value);

340
src/init.test.ts
View File

@@ -1,9 +1,14 @@
import * as fs from "fs";
import path from "path";
import test from "ava";
import test, { ExecutionContext } from "ava";
import { cleanupDatabaseClusterDirectory } from "./init";
import { createStubCodeQL } from "./codeql";
import {
checkPacksForOverlayCompatibility,
cleanupDatabaseClusterDirectory,
} from "./init";
import { KnownLanguage } from "./languages";
import {
LoggedMessage,
createTestConfig,
@@ -87,6 +92,7 @@ for (const { runnerEnv, ErrorConstructor, message } of [
cleanupDatabaseClusterDirectory(
createTestConfig({ dbLocation }),
getRecordingLogger(messages),
{},
() => {
throw new Error(rmSyncError);
},
@@ -106,3 +112,333 @@ for (const { runnerEnv, ErrorConstructor, message } of [
});
});
}
test("cleanupDatabaseClusterDirectory can disable warning with options", async (t) => {
await withTmpDir(async (tmpDir: string) => {
const dbLocation = path.resolve(tmpDir, "dbs");
fs.mkdirSync(dbLocation, { recursive: true });
const fileToCleanUp = path.resolve(dbLocation, "something-to-cleanup.txt");
fs.writeFileSync(fileToCleanUp, "");
const messages: LoggedMessage[] = [];
cleanupDatabaseClusterDirectory(
createTestConfig({ dbLocation }),
getRecordingLogger(messages),
{ disableExistingDirectoryWarning: true },
);
// Should only have the info message, not the warning
t.is(messages.length, 1);
t.is(messages[0].type, "info");
t.is(
messages[0].message,
`Cleaned up database cluster directory ${dbLocation}.`,
);
t.false(fs.existsSync(fileToCleanUp));
});
});
type PackInfo = {
language: KnownLanguage;
packinfoContents: string | undefined;
sourceOnlyPack?: boolean;
qlpackFileName?: string;
};
const testCheckPacksForOverlayCompatibility = test.macro({
exec: async (
t: ExecutionContext,
_title: string,
{
cliOverlayVersion,
languages,
packs,
expectedResult,
}: {
cliOverlayVersion: number | undefined;
languages: KnownLanguage[];
packs: Record<string, PackInfo>;
expectedResult: boolean;
},
) => {
await withTmpDir(async (tmpDir) => {
const packDirsByLanguage = new Map<KnownLanguage, string[]>();
for (const [packName, packInfo] of Object.entries(packs)) {
const packPath = path.join(tmpDir, packName);
fs.mkdirSync(packPath, { recursive: true });
if (packInfo.packinfoContents) {
fs.writeFileSync(
path.join(packPath, ".packinfo"),
packInfo.packinfoContents,
);
}
const qlpackFileName = packInfo.qlpackFileName || "qlpack.yml";
fs.writeFileSync(
path.join(packPath, qlpackFileName),
packInfo.sourceOnlyPack
? `name: ${packName}\nversion: 1.0.0\n`
: `name: ${packName}\nversion: 1.0.0\nbuildMetadata:\n sha: 123abc\n`,
);
if (!packDirsByLanguage.has(packInfo.language)) {
packDirsByLanguage.set(packInfo.language, []);
}
packDirsByLanguage.get(packInfo.language)!.push(packPath);
}
const codeql = createStubCodeQL({
getVersion: async () => ({
version: "2.22.2",
overlayVersion: cliOverlayVersion,
}),
resolveQueriesStartingPacks: async (suitePaths: string[]) => {
for (const language of packDirsByLanguage.keys()) {
const suiteForLanguage = path.join(
language,
"temp",
"config-queries.qls",
);
if (suitePaths[0].endsWith(suiteForLanguage)) {
return packDirsByLanguage.get(language) || [];
}
}
return [];
},
});
const messages: LoggedMessage[] = [];
const result = await checkPacksForOverlayCompatibility(
codeql,
createTestConfig({ dbLocation: tmpDir, languages }),
getRecordingLogger(messages),
);
t.is(result, expectedResult);
t.deepEqual(
messages.length,
expectedResult ? 0 : 1,
"Expected log messages",
);
});
},
title: (_, title) => `checkPacksForOverlayCompatibility: ${title}`,
});
test(
testCheckPacksForOverlayCompatibility,
"returns false when CLI does not support overlay",
{
cliOverlayVersion: undefined,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: false,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns true when there are no query packs",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {},
expectedResult: true,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns true when query pack has not been compiled",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: undefined,
sourceOnlyPack: true,
},
},
expectedResult: true,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns true when query pack has expected overlay version",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: true,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns true when query packs for all languages to analyze are compatible",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.cpp, KnownLanguage.java],
packs: {
"codeql/cpp-queries": {
language: KnownLanguage.cpp,
packinfoContents: '{"overlayVersion":2}',
},
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: true,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns true when query pack for a language not analyzed is incompatible",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/cpp-queries": {
language: KnownLanguage.cpp,
packinfoContents: undefined,
},
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: true,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns false when query pack for a language to analyze is incompatible",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.cpp, KnownLanguage.java],
packs: {
"codeql/cpp-queries": {
language: KnownLanguage.cpp,
packinfoContents: '{"overlayVersion":1}',
},
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
},
expectedResult: false,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns false when query pack is missing .packinfo",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: KnownLanguage.java,
packinfoContents: undefined,
},
},
expectedResult: false,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns false when query pack has different overlay version",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":1}',
},
},
expectedResult: false,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns false when query pack is missing overlayVersion in .packinfo",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: KnownLanguage.java,
packinfoContents: "{}",
},
},
expectedResult: false,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns false when .packinfo is not valid JSON",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
},
"custom/queries": {
language: KnownLanguage.java,
packinfoContents: "this_is_not_valid_json",
},
},
expectedResult: false,
},
);
test(
testCheckPacksForOverlayCompatibility,
"returns true when query pack uses codeql-pack.yml filename",
{
cliOverlayVersion: 2,
languages: [KnownLanguage.java],
packs: {
"codeql/java-queries": {
language: KnownLanguage.java,
packinfoContents: '{"overlayVersion":2}',
qlpackFileName: "codeql-pack.yml",
},
},
expectedResult: true,
},
);

154
src/init.ts
View File

@@ -3,9 +3,10 @@ import * as path from "path";
import * as toolrunner from "@actions/exec/lib/toolrunner";
import * as io from "@actions/io";
import * as yaml from "js-yaml";
import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
import { GitHubApiDetails } from "./api-client";
import { CodeQL, setupCodeQL } from "./codeql";
import * as configUtils from "./config-utils";
import { CodeQLDefaultVersionInfo } from "./feature-flags";
@@ -14,7 +15,6 @@ import { Logger, withGroupAsync } from "./logging";
import { ToolsSource } from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
import { TracerConfig, getCombinedTracerConfig } from "./tracer-config";
import * as util from "./util";
export async function initCodeQL(
@@ -66,30 +66,18 @@ export async function initConfig(
});
}
export async function runInit(
export async function runDatabaseInitCluster(
databaseInitEnvironment: Record<string, string | undefined>,
codeql: CodeQL,
config: configUtils.Config,
sourceRoot: string,
processName: string | undefined,
registriesInput: string | undefined,
apiDetails: GitHubApiCombinedDetails,
qlconfigFile: string | undefined,
logger: Logger,
): Promise<TracerConfig | undefined> {
): Promise<void> {
fs.mkdirSync(config.dbLocation, { recursive: true });
const { registriesAuthTokens, qlconfigFile } =
await configUtils.generateRegistries(
registriesInput,
config.tempDir,
logger,
);
await configUtils.wrapEnvironment(
{
GITHUB_TOKEN: apiDetails.auth,
CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
},
// Init a database cluster
databaseInitEnvironment,
async () =>
await codeql.databaseInitCluster(
config,
@@ -99,7 +87,124 @@ export async function runInit(
logger,
),
);
return await getCombinedTracerConfig(codeql, config);
}
/**
* Check whether all query packs are compatible with the overlay analysis
* support in the CodeQL CLI. If the check fails, this function will log a
* warning and returns false.
*
* @param codeql A CodeQL instance.
* @param logger A logger.
* @returns `true` if all query packs are compatible with overlay analysis,
* `false` otherwise.
*/
export async function checkPacksForOverlayCompatibility(
codeql: CodeQL,
config: configUtils.Config,
logger: Logger,
): Promise<boolean> {
const codeQlOverlayVersion = (await codeql.getVersion()).overlayVersion;
if (codeQlOverlayVersion === undefined) {
logger.warning("The CodeQL CLI does not support overlay analysis.");
return false;
}
for (const language of config.languages) {
const suitePath = util.getGeneratedSuitePath(config, language);
const packDirs = await codeql.resolveQueriesStartingPacks([suitePath]);
if (
packDirs.some(
(packDir) =>
!checkPackForOverlayCompatibility(
packDir,
codeQlOverlayVersion,
logger,
),
)
) {
return false;
}
}
return true;
}
/** Interface for `qlpack.yml` file contents. */
interface QlPack {
buildMetadata?: string;
}
/**
* Check a single pack for its overlay compatibility. If the check fails, this
* function will log a warning and returns false.
*
* @param packDir Path to the directory containing the pack.
* @param codeQlOverlayVersion The overlay version of the CodeQL CLI.
* @param logger A logger.
* @returns `true` if the pack is compatible with overlay analysis, `false`
* otherwise.
*/
function checkPackForOverlayCompatibility(
packDir: string,
codeQlOverlayVersion: number,
logger: Logger,
): boolean {
try {
let qlpackPath = path.join(packDir, "qlpack.yml");
if (!fs.existsSync(qlpackPath)) {
qlpackPath = path.join(packDir, "codeql-pack.yml");
}
const qlpackContents = yaml.load(
fs.readFileSync(qlpackPath, "utf8"),
) as QlPack;
if (!qlpackContents.buildMetadata) {
// This is a source-only pack, and overlay compatibility checks apply only
// to precompiled packs.
return true;
}
const packInfoPath = path.join(packDir, ".packinfo");
if (!fs.existsSync(packInfoPath)) {
logger.warning(
`The query pack at ${packDir} does not have a .packinfo file, ` +
"so it cannot support overlay analysis. Recompiling the query pack " +
"with the latest CodeQL CLI should solve this problem.",
);
return false;
}
const packInfoFileContents = JSON.parse(
fs.readFileSync(packInfoPath, "utf8"),
);
const packOverlayVersion = packInfoFileContents.overlayVersion;
if (typeof packOverlayVersion !== "number") {
logger.warning(
`The .packinfo file for the query pack at ${packDir} ` +
"does not have the overlayVersion field, which indicates that " +
"the pack is not compatible with overlay analysis.",
);
return false;
}
if (packOverlayVersion !== codeQlOverlayVersion) {
logger.warning(
`The query pack at ${packDir} was compiled with ` +
`overlay version ${packOverlayVersion}, but the CodeQL CLI ` +
`supports overlay version ${codeQlOverlayVersion}. The ` +
"query pack needs to be recompiled to support overlay analysis.",
);
return false;
}
} catch (e) {
logger.warning(
`Error while checking pack at ${packDir} ` +
`for overlay compatibility: ${util.getErrorMessage(e)}`,
);
return false;
}
return true;
}
/**
@@ -129,6 +234,7 @@ export async function checkInstallPython311(
export function cleanupDatabaseClusterDirectory(
config: configUtils.Config,
logger: Logger,
options: { disableExistingDirectoryWarning?: boolean } = {},
// We can't stub the fs module in tests, so we allow the caller to override the rmSync function
// for testing.
rmSync = fs.rmSync,
@@ -138,9 +244,11 @@ export function cleanupDatabaseClusterDirectory(
(fs.statSync(config.dbLocation).isFile() ||
fs.readdirSync(config.dbLocation).length > 0)
) {
logger.warning(
`The database cluster directory ${config.dbLocation} must be empty. Attempting to clean it up.`,
);
if (!options.disableExistingDirectoryWarning) {
logger.warning(
`The database cluster directory ${config.dbLocation} must be empty. Attempting to clean it up.`,
);
}
try {
rmSync(config.dbLocation, {
force: true,

2
src/overlay-database-utils.ts
View File

@@ -16,7 +16,7 @@ export enum OverlayDatabaseMode {
None = "none",
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5";
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3";
/**
* Writes a JSON file containing Git OIDs for all tracked files (represented

5
src/testing-utils.ts
View File

@@ -254,18 +254,21 @@ export function mockLanguagesInRepo(languages: string[]) {
export const makeVersionInfo = (
version: string,
features?: { [name: string]: boolean },
overlayVersion?: number,
): codeql.VersionInfo => ({
version,
features,
overlayVersion,
});
export function mockCodeQLVersion(
version: string,
features?: { [name: string]: boolean },
overlayVersion?: number,
) {
return codeql.createStubCodeQL({
async getVersion() {
return makeVersionInfo(version, features);
return makeVersionInfo(version, features, overlayVersion);
},
});
}

12
src/util.ts
View File

@@ -513,6 +513,18 @@ export function getCodeQLDatabasePath(config: Config, language: Language) {
return path.resolve(config.dbLocation, language);
}
/**
* Get the path where the generated query suite for the given language lives.
*/
export function getGeneratedSuitePath(config: Config, language: Language) {
return path.resolve(
config.dbLocation,
language,
"temp",
"config-queries.qls",
);
}
/**
* Parses user input of a github.com or GHES URL to a canonical form.
* Removes any API prefix or suffix if one is present.