github/codeql-action
0
0
Fork 0
mirror of https://github.com/github/codeql-action.git synced 2026-05-12 00:30:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge 23cdd6bab6 into cbdf0df97b

Browse Source
This commit is contained in:
Andrew Eisenberg
2021-06-09 21:32:22 +00:00
committed by GitHub
parent cbdf0df97b 23cdd6bab6
commit 8780a229c8
53 changed files with 1623 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/post-release-mergeback.yml
+1 -1
View File
@@ -124,4 +124,4 @@ jobs:
--base "$BASE_BRANCH" \
--title "$PR_TITLE" \
--body "$PR_BODY" \
"$DRAFT"
${DRAFT:+"$DRAFT"} # no quotes around $DRAFT. gh will error out if there is an empty ""
.github/workflows/pr-checks.yml
+72 -11
View File
@@ -65,20 +65,81 @@ jobs:
shell: bash
run: ./build.sh
- uses: ./../action/analyze
id: analysis
env:
TEST_MODE: true
- run: |
cd "$RUNNER_TEMP/customDbLocation"
# List all directories as there will be precisely one directory per database
# but there may be other files in this directory such as query suites.
if [ "$(ls -d */ | wc -l)" != 6 ] || \
[[ ! -d cpp ]] || \
[[ ! -d csharp ]] || \
[[ ! -d go ]] || \
[[ ! -d java ]] || \
[[ ! -d javascript ]] || \
[[ ! -d python ]]; then
echo "Did not find expected number of databases. Database dir contains: $(ls)"
CPP_DB=${{ fromJson(steps.analysis.outputs.db-locations).cpp }}
if [[ ! -d $CPP_DB ]] || [[ ! $CPP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for CPP, or created it in the wrong location."
exit 1
fi
CSHARP_DB=${{ fromJson(steps.analysis.outputs.db-locations).csharp }}
if [[ ! -d $CSHARP_DB ]] || [[ ! $CSHARP_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for C Sharp, or created it in the wrong location."
exit 1
fi
GO_DB=${{ fromJson(steps.analysis.outputs.db-locations).go }}
if [[ ! -d $GO_DB ]] || [[ ! $GO_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Go, or created it in the wrong location."
exit 1
fi
JAVA_DB=${{ fromJson(steps.analysis.outputs.db-locations).java }}
if [[ ! -d $JAVA_DB ]] || [[ ! $JAVA_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Java, or created it in the wrong location."
exit 1
fi
JAVASCRIPT_DB=${{ fromJson(steps.analysis.outputs.db-locations).javascript }}
if [[ ! -d $JAVASCRIPT_DB ]] || [[ ! $JAVASCRIPT_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Javascript, or created it in the wrong location."
exit 1
fi
PYTHON_DB=${{ fromJson(steps.analysis.outputs.db-locations).python }}
if [[ ! -d $PYTHON_DB ]] || [[ ! $PYTHON_DB == ${{ runner.temp }}/customDbLocation/* ]]; then
echo "Did not create a database for Python, or created it in the wrong location."
exit 1
fi
# Packaging test that runs against a javascript database
test-packaging-javascript:
needs: [check-js, check-node-modules]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Move codeql-action
shell: bash
run: |
mkdir ../action
mv * .github ../action/
mv ../action/tests/multi-language-repo/{*,.github} .
mv ../action/.github/workflows .github
- uses: ./../action/init
with:
config-file: ".github/codeql/codeql-config-packaging.yml"
languages: javascript
# TODO: this is temporary until we have a release that includes the latest packaging work.
tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210606/codeql-bundle-linux64.tar.gz
- name: Build code
shell: bash
run: ./build.sh
- uses: ./../action/analyze
with:
output: "${{ runner.temp }}/results"
env:
TEST_MODE: true
- name: Assert Results
run: |
cd "$RUNNER_TEMP/results"
# We should have 3 hits from these rules
EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
# use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
echo "Found matching rules '$RULES'"
if [ "$RULES" != "$EXPECTED_RULES" ]; then
echo "Did not match expected rules '$EXPECTED_RULES'."
exit 1
fi
CHANGELOG.md
+5
View File
@@ -2,6 +2,11 @@
## [UNRELEASED]
## 1.0.1 - 07 Jun 2021
- Pass the `--sarif-group-rules-by-pack` argument to CodeQL CLI invocations that generate SARIF. This means the SARIF rule object for each query will now be found underneath its corresponding query pack in `runs[].tool.extensions`. [#546](https://github.com/github/codeql-action/pull/546)
- Output the location of CodeQL databases created in the analyze step. [#543](https://github.com/github/codeql-action/pull/543)
## 1.0.0 - 31 May 2021
- Add this changelog file. [#507](https://github.com/github/codeql-action/pull/507)
analyze/action.yml
+7
View File
@@ -13,6 +13,10 @@ inputs:
description: Upload the SARIF file
required: false
default: "true"
cleanup-level:
description: "Level of cleanup to perform on CodeQL databases at the end of the analyze step. This should either be 'none' to skip cleanup, or be a valid argument for the --mode flag of the CodeQL CLI command 'codeql database cleanup' as documented at https://codeql.github.com/docs/codeql-cli/manual/database-cleanup"
required: false
default: "brutal"
ram:
description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
required: false
@@ -34,6 +38,9 @@ inputs:
default: ${{ github.token }}
matrix:
default: ${{ toJson(matrix) }}
outputs:
db-locations:
description: A map from language to absolute path for each database created by CodeQL.
runs:
using: 'node12'
main: '../lib/analyze-action.js'
lib/analysis-paths.test.js
Generated
+3
View File
@@ -29,6 +29,7 @@ ava_1.default("emptyPaths", async (t) => {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@@ -49,6 +50,7 @@ ava_1.default("nonEmptyPaths", async (t) => {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@@ -70,6 +72,7 @@ ava_1.default("exclude temp dir", async (t) => {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
dbLocation: path.resolve(tempDir, "codeql_databases"),
packs: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
lib/analysis-paths.test.js.map
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;SACrD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;SACrD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO;YACP,YAAY;YACZ,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;SACtD,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,aAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;SACV,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,YAAY,EAAE,MAAM;YACpB,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;SACV,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO;YACP,YAAY;YACZ,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;YACrD,KAAK,EAAE,EAAE;SACV,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;QAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
lib/analyze-action.js
Generated
+8
View File
@@ -50,6 +50,14 @@ async function run() {
};
const outputDir = actionsUtil.getRequiredInput("output");
const queriesStats = await analyze_1.runAnalyze(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger), actionsUtil.getOptionalInput("category"), config, logger);
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
await analyze_1.runCleanup(config, actionsUtil.getOptionalInput("cleanup-level") || "brutal", logger);
}
const dbLocations = {};
for (const language of config.languages) {
dbLocations[language] = util.getCodeQLDatabasePath(config, language);
}
core.setOutput("db-locations", dbLocations);
if (actionsUtil.getRequiredInput("upload") === "true") {
const uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
stats = { ...queriesStats, ...uploadStats };
lib/analyze-action.js.map
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAImB;AACnB,iDAAmD;AACnD,uCAA6C;AAC7C,yDAA2C;AAC3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAqC,SAAS,CAAC;IACxD,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,MAAM,oBAAU,CACnC,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,EACpE,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;QAEF,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACrD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACpD,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;YACF,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;SAC7C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;SAC7B;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;SAC1C;QAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAKmB;AACnB,iDAAmD;AACnD,uCAA6C;AAC7C,yDAA2C;AAC3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAqC,SAAS,CAAC;IACxD,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,MAAM,oBAAU,CACnC,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,EACpE,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;QAEF,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACrD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACpD,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;YACF,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;SAC7C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;SAC7B;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;SAC1C;QAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
lib/analyze.js
Generated
+70 -47
View File
@@ -15,7 +15,6 @@ const codeql_1 = require("./codeql");
const count_loc_1 = require("./count-loc");
const languages_1 = require("./languages");
const sharedEnv = __importStar(require("./shared-environment"));
const upload_lib_1 = require("./upload-lib");
const util = __importStar(require("./util"));
class CodeQLAnalysisError extends Error {
constructor(queriesStatusReport, message) {
@@ -77,6 +76,7 @@ async function finalizeDatabaseCreation(config, threadsFlag, logger) {
}
// Runs queries and creates sarif files in the given folder
async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
var _a, _b;
const statusReport = {};
// count the number of lines in the background
const locPromise = count_loc_1.countLoc(path.resolve(),
@@ -85,96 +85,119 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
// that here.
config.paths, config.pathsIgnore, config.languages, logger);
for (const language of config.languages) {
logger.startGroup(`Analyzing ${language}`);
const queries = config.queries[language];
if (queries === undefined ||
(queries.builtin.length === 0 && queries.custom.length === 0)) {
const packsWithVersion = config.packs[language] || [];
const hasBuiltinQueries = ((_a = queries) === null || _a === void 0 ? void 0 : _a.builtin.length) > 0;
const hasCustomQueries = ((_b = queries) === null || _b === void 0 ? void 0 : _b.custom.length) > 0;
const hasPackWithCustomQueries = packsWithVersion.length > 0;
if (!hasBuiltinQueries && !hasCustomQueries && !hasPackWithCustomQueries) {
throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
}
try {
let analysisSummaryBuiltIn = "";
const customAnalysisSummaries = [];
if (hasPackWithCustomQueries) {
logger.startGroup(`Downloading custom packs for ${language}`);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
const results = await codeql.packDownload(packsWithVersion);
logger.info(`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths = [];
if (queries["builtin"].length > 0) {
const startTimeBuiltIn = new Date().getTime();
const { sarifFile, stdout } = await runQueryGroup(language, "builtin", queries["builtin"], sarifFolder, undefined);
analysisSummaryBuiltIn = stdout;
await injectLinesOfCode(sarifFile, language, locPromise);
querySuitePaths.push(await runQueryGroup(language, "builtin", createQuerySuiteContents(queries["builtin"]), undefined));
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = config.tempDir;
const temporarySarifFiles = [];
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
const { sarifFile, stdout } = await runQueryGroup(language, `custom-${i}`, queries["custom"][i].queries, temporarySarifDir, queries["custom"][i].searchPath);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
querySuitePaths.push(await runQueryGroup(language, `custom-${i}`, createQuerySuiteContents(queries["custom"][i].queries), queries["custom"][i].searchPath));
ranCustom = true;
}
}
if (temporarySarifFiles.length > 0) {
const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
fs.writeFileSync(sarifFile, upload_lib_1.combineSarifFiles(temporarySarifFiles));
await injectLinesOfCode(sarifFile, language, locPromise);
if (packsWithVersion.length > 0) {
querySuitePaths.push(await runQueryGroup(language, "packs", createPackSuiteContents(packsWithVersion), undefined));
ranCustom = true;
}
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
// Print the LoC baseline and the summary results from database analyze for the standard
// query suite and (if appropriate) each custom query suite.
logger.startGroup(`Analysis summary for ${language}`);
printLinesOfCodeSummary(logger, language, await locPromise);
logger.info(analysisSummaryBuiltIn);
for (const [i, customSummary] of customAnalysisSummaries.entries()) {
if (customSummary.trim() === "") {
continue;
}
const description = customAnalysisSummaries.length === 1
? "custom queries"
: `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
logger.info(`Analysis summary for ${description}:`);
logger.info("");
logger.info(customSummary);
logger.info("");
}
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(language, querySuitePaths, sarifFile);
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
printLinesOfCodeSummary(logger, language, await locPromise);
}
catch (e) {
logger.info(e);
logger.info(e.stack);
statusReport.analyze_failure_language = language;
throw new CodeQLAnalysisError(statusReport, `Error running analysis for ${language}: ${e}`);
}
}
return statusReport;
async function runQueryGroup(language, type, queries, destinationFolder, searchPath) {
async function runInterpretResults(language, queries, sarifFile) {
const databasePath = util.getCodeQLDatabasePath(config, language);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
return await codeql.databaseInterpretResults(databasePath, queries, sarifFile, addSnippetsFlag, threadsFlag, automationDetailsId);
}
async function runQueryGroup(language, type, querySuiteContents, searchPath) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries
.map((q) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
const databaseAnalyzeStdout = await codeql.databaseAnalyze(databasePath, sarifFile, searchPath, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId);
logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
return { sarifFile, stdout: databaseAnalyzeStdout };
await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, memoryFlag, threadsFlag);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
}
exports.runQueries = runQueries;
function createQuerySuiteContents(queries) {
return queries.map((q) => `- query: ${q}`).join("\n");
}
function createPackSuiteContents(packsWithVersion) {
return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
}
function packWithVersionToQuerySuiteEntry(pack) {
let text = `- qlpack: ${pack.packName}`;
if (pack.version) {
text += `${"\n"} version: ${pack.version}`;
}
return text;
}
async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
// Delete the tracer config env var to avoid tracing ourselves
delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
fs.mkdirSync(outputDir, { recursive: true });
logger.info("Finalizing database creation");
await finalizeDatabaseCreation(config, threadsFlag, logger);
logger.info("Analyzing database");
const queriesStats = await runQueries(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger);
return { ...queriesStats };
}
exports.runAnalyze = runAnalyze;
async function runCleanup(config, cleanupLevel, logger) {
logger.startGroup("Cleaning up databases");
for (const language of config.languages) {
const codeql = codeql_1.getCodeQL(config.codeQLCmd);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
logger.endGroup();
}
exports.runCleanup = runCleanup;
async function injectLinesOfCode(sarifFile, language, locPromise) {
const lineCounts = await locPromise;
const idPrefix = count_loc_1.getIdPrefix(language);
@@ -199,7 +222,7 @@ async function injectLinesOfCode(sarifFile, language, locPromise) {
}
function printLinesOfCodeSummary(logger, language, lineCounts) {
if (language in lineCounts) {
logger.info(`Counted ${lineCounts[language]} lines of code for ${language} as a baseline.`);
logger.info(`Counted a baseline of ${lineCounts[language]} lines of code for ${language}.`);
}
}
//# sourceMappingURL=analyze.js.map
lib/analyze.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/analyze.test.js
Generated
+87 -9
View File
@@ -13,6 +13,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const ava_1 = __importDefault(require("ava"));
const yaml = __importStar(require("js-yaml"));
const semver_1 = require("semver");
const sinon_1 = __importDefault(require("sinon"));
const analyze_1 = require("./analyze");
const codeql_1 = require("./codeql");
@@ -39,9 +41,27 @@ ava_1.default("status report fields and search path setting", async (t) => {
const memoryFlag = "";
const addSnippetsFlag = "";
const threadsFlag = "";
const packs = {
[languages_1.Language.cpp]: [
{
packName: "a/b",
version: semver_1.clean("1.0.0"),
},
],
[languages_1.Language.java]: [
{
packName: "c/d",
version: semver_1.clean("2.0.0"),
},
],
};
for (const language of Object.values(languages_1.Language)) {
codeql_1.setCodeQL({
databaseAnalyze: async (_, sarifFile, searchPath) => {
packDownload: async () => ({ packs: [] }),
databaseRunQueries: async (_db, searchPath) => {
searchPathsUsed.push(searchPath);
},
databaseInterpretResults: async (_db, _queriesRun, sarifFile) => {
fs.writeFileSync(sarifFile, JSON.stringify({
runs: [
// variant 1 uses ruleId
@@ -71,7 +91,6 @@ ava_1.default("status report fields and search path setting", async (t) => {
{},
],
}));
searchPathsUsed.push(searchPath);
return "";
},
});
@@ -89,6 +108,7 @@ ava_1.default("status report fields and search path setting", async (t) => {
type: util.GitHubVariant.DOTCOM,
},
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs,
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
@@ -98,8 +118,18 @@ ava_1.default("status report fields and search path setting", async (t) => {
custom: [],
};
const builtinStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, config, logging_1.getRunnerLogger(true));
t.deepEqual(Object.keys(builtinStatusReport).length, 1);
t.true(`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport);
const hasPacks = language in packs;
const statusReportKeys = Object.keys(builtinStatusReport).sort();
if (hasPacks) {
t.deepEqual(statusReportKeys.length, 3, statusReportKeys.toString());
t.deepEqual(statusReportKeys[0], `analyze_builtin_queries_${language}_duration_ms`);
t.deepEqual(statusReportKeys[1], `analyze_custom_queries_${language}_duration_ms`);
t.deepEqual(statusReportKeys[2], `interpret_results_${language}_duration_ms`);
}
else {
t.deepEqual(statusReportKeys[0], `analyze_builtin_queries_${language}_duration_ms`);
t.deepEqual(statusReportKeys[1], `interpret_results_${language}_duration_ms`);
}
config.queries[language] = {
builtin: [],
custom: [
@@ -114,17 +144,21 @@ ava_1.default("status report fields and search path setting", async (t) => {
],
};
const customStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, undefined, config, logging_1.getRunnerLogger(true));
t.deepEqual(Object.keys(customStatusReport).length, 1);
t.deepEqual(Object.keys(customStatusReport).length, 2);
t.true(`analyze_custom_queries_${language}_duration_ms` in customStatusReport);
t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
const expectedSearchPathsUsed = hasPacks
? [undefined, undefined, "/1", "/2", undefined]
: [undefined, "/1", "/2"];
t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
t.true(`interpret_results_${language}_duration_ms` in customStatusReport);
}
verifyLineCounts(tmpDir);
verifyQuerySuites(tmpDir);
});
function verifyLineCounts(tmpDir) {
// eslint-disable-next-line github/array-foreach
Object.keys(languages_1.Language).forEach((lang, i) => {
verifyLineCountForFile(lang, path.join(tmpDir, `${lang}-builtin.sarif`), i + 1);
verifyLineCountForFile(lang, path.join(tmpDir, `${lang}-custom.sarif`), i + 1);
verifyLineCountForFile(lang, path.join(tmpDir, `${lang}.sarif`), i + 1);
});
}
function verifyLineCountForFile(lang, filePath, lineCount) {
@@ -146,8 +180,52 @@ ava_1.default("status report fields and search path setting", async (t) => {
baseline: lineCount,
},
]);
// when the rule doesn't exists, it should not be added
// when the rule doesn't exist, it should not be added
t.deepEqual(sarif.runs[2].properties.metricResults, []);
}
function verifyQuerySuites(tmpDir) {
const qlsContent = [
{
query: "foo.ql",
},
];
const qlsContent2 = [
{
query: "bar.ql",
},
];
const qlsPackContentCpp = [
{
qlpack: "a/b",
version: "1.0.0",
},
];
const qlsPackContentJava = [
{
qlpack: "c/d",
version: "2.0.0",
},
];
for (const lang of Object.values(languages_1.Language)) {
t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
const packSuiteName = `${lang}-queries-packs.qls`;
if (lang === languages_1.Language.cpp) {
t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
}
else if (lang === languages_1.Language.java) {
t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
}
else {
t.false(fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName)));
}
}
function readContents(name) {
const x = fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8");
console.log(x);
return yaml.safeLoad(fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8"));
}
}
});
//# sourceMappingURL=analyze.test.js.map
lib/analyze.test.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/api-client.js
Generated
+3 -1
View File
@@ -12,6 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
Object.defineProperty(exports, "__esModule", { value: true });
const path = __importStar(require("path"));
const githubUtils = __importStar(require("@actions/github/lib/utils"));
const retry = __importStar(require("@octokit/plugin-retry"));
const console_log_level_1 = __importDefault(require("console-log-level"));
const actions_util_1 = require("./actions-util");
const util_1 = require("./util");
@@ -24,7 +25,8 @@ var DisallowedAPIVersionReason;
})(DisallowedAPIVersionReason = exports.DisallowedAPIVersionReason || (exports.DisallowedAPIVersionReason = {}));
exports.getApiClient = function (apiDetails, { allowExternal = false } = {}) {
const auth = (allowExternal && apiDetails.externalRepoAuth) || apiDetails.auth;
return new githubUtils.GitHub(githubUtils.getOctokitOptions(auth, {
const retryingOctokit = githubUtils.GitHub.plugin(retry.retry);
return new retryingOctokit(githubUtils.getOctokitOptions(auth, {
baseUrl: getApiUrl(apiDetails.url),
userAgent: `CodeQL-${util_1.getMode()}/${pkg.version}`,
log: console_log_level_1.default({ level: "debug" }),
lib/api-client.js.map
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,uEAAyD;AACzD,0EAAgD;AAEhD,iDAAkD;AAClD,iCAAsD;AAEtD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,GAA1B,kCAA0B,KAA1B,kCAA0B,QAGrC;AAeY,QAAA,YAAY,GAAG,UAC1B,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,OAAO,IAAI,WAAW,CAAC,MAAM,CAC3B,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC;QAClC,SAAS,EAAE,UAAU,cAAO,EAAE,IAAI,GAAG,CAAC,OAAO,EAAE;QAC/C,GAAG,EAAE,2BAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,SAAS,CAAC,SAAiB;IAClC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAE/B,uDAAuD;IACvD,0CAA0C;IAC1C,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY,IAAI,GAAG,CAAC,QAAQ,KAAK,gBAAgB,EAAE;QACtE,OAAO,wBAAwB,CAAC;KACjC;IAED,6BAA6B;IAC7B,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,uFAAuF;AACvF,qFAAqF;AACrF,+CAA+C;AAC/C,SAAgB,mBAAmB;IACjC,MAAM,UAAU,GAAG;QACjB,IAAI,EAAE,+BAAgB,CAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,0BAAmB,CAAC,mBAAmB,CAAC;KAC9C,CAAC;IAEF,OAAO,oBAAY,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAPD,kDAOC"}
{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../src/api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6B;AAE7B,uEAAyD;AACzD,6DAA+C;AAC/C,0EAAgD;AAEhD,iDAAkD;AAClD,iCAAsD;AAEtD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,IAAY,0BAGX;AAHD,WAAY,0BAA0B;IACpC,+FAAc,CAAA;IACd,+FAAc,CAAA;AAChB,CAAC,EAHW,0BAA0B,GAA1B,kCAA0B,KAA1B,kCAA0B,QAGrC;AAeY,QAAA,YAAY,GAAG,UAC1B,UAAoC,EACpC,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,EAAE;IAE9B,MAAM,IAAI,GACR,CAAC,aAAa,IAAI,UAAU,CAAC,gBAAgB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC;IACpE,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/D,OAAO,IAAI,eAAe,CACxB,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE;QAClC,OAAO,EAAE,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC;QAClC,SAAS,EAAE,UAAU,cAAO,EAAE,IAAI,GAAG,CAAC,OAAO,EAAE;QAC/C,GAAG,EAAE,2BAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;KACzC,CAAC,CACH,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,SAAS,CAAC,SAAiB;IAClC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAE/B,uDAAuD;IACvD,0CAA0C;IAC1C,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY,IAAI,GAAG,CAAC,QAAQ,KAAK,gBAAgB,EAAE;QACtE,OAAO,wBAAwB,CAAC;KACjC;IAED,6BAA6B;IAC7B,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,uFAAuF;AACvF,qFAAqF;AACrF,+CAA+C;AAC/C,SAAgB,mBAAmB;IACjC,MAAM,UAAU,GAAG;QACjB,IAAI,EAAE,+BAAgB,CAAC,OAAO,CAAC;QAC/B,GAAG,EAAE,0BAAmB,CAAC,mBAAmB,CAAC;KAC9C,CAAC;IAEF,OAAO,oBAAY,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAPD,kDAOC"}
lib/api-client.test.js
Generated
+4 -1
View File
@@ -19,9 +19,12 @@ const util_1 = require("./util");
// eslint-disable-next-line import/no-commonjs
const pkg = require("../package.json");
testing_utils_1.setupTests(ava_1.default);
let pluginStub;
let githubStub;
ava_1.default.beforeEach(() => {
githubStub = sinon_1.default.stub(githubUtils, "GitHub");
pluginStub = sinon_1.default.stub(githubUtils.GitHub, "plugin");
githubStub = sinon_1.default.stub();
pluginStub.returns(githubStub);
util_1.initializeEnvironment(util_1.Mode.actions, pkg.version);
});
ava_1.default("Get the client API", async (t) => {
lib/api-client.test.js.map
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"api-client.test.js","sourceRoot":"","sources":["../src/api-client.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uEAAyD;AACzD,8CAA6C;AAC7C,kDAA0B;AAE1B,6CAA4C;AAC5C,mDAA6C;AAC7C,iCAAqD;AAErD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,IAAI,UAA2B,CAAC;AAEhC,aAAI,CAAC,UAAU,CAAC,GAAG,EAAE;IACnB,UAAU,GAAG,eAAK,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC/C,4BAAqB,CAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oBAAoB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrC,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,6BAA6B,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC9C,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,yCAAyC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC1D,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oCAAoC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrD,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,qCAAqC;KAC3C,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,wBAAwB;QACjC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,SAAS,MAAM,CACb,CAA4B,EAC5B,UAAe,EACf,aAAkB,EAClB,QAAa;IAEb,yBAAY,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAExC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,iEAAiE;IACjE,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5B,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzC,CAAC"}
{"version":3,"file":"api-client.test.js","sourceRoot":"","sources":["../src/api-client.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uEAAyD;AACzD,8CAA6C;AAC7C,kDAA0B;AAE1B,6CAA4C;AAC5C,mDAA6C;AAC7C,iCAAqD;AAErD,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,IAAI,UAA2B,CAAC;AAChC,IAAI,UAA2B,CAAC;AAEhC,aAAI,CAAC,UAAU,CAAC,GAAG,EAAE;IACnB,UAAU,GAAG,eAAK,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACtD,UAAU,GAAG,eAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC/B,4BAAqB,CAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oBAAoB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrC,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,6BAA6B,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC9C,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,yCAAyC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC1D,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,gBAAgB;KACtB,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,EACvB;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,uBAAuB;QAChC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,aAAI,CAAC,oCAAoC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACrD,MAAM,CACJ,CAAC,EACD;QACE,IAAI,EAAE,KAAK;QACX,GAAG,EAAE,qCAAqC;KAC3C,EACD,SAAS,EACT;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,wBAAwB;QACjC,SAAS,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE;KAC1C,CACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,SAAS,MAAM,CACb,CAA4B,EAC5B,UAAe,EACf,aAAkB,EAClB,QAAa;IAEb,yBAAY,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAExC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,iEAAiE;IACjE,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5B,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzC,CAAC"}
lib/codeql.js
Generated
+96 -45
View File
@@ -285,7 +285,10 @@ function setCodeQL(partialCodeql) {
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
databaseInterpretResults: resolveFunction(partialCodeql, "databaseInterpretResults"),
};
return cachedCodeQL;
}
@@ -310,7 +313,7 @@ function getCodeQLForCmd(cmd) {
return cmd;
},
async printVersion() {
await new toolrunner.ToolRunner(cmd, ["version", "--format=json"]).exec();
await runTool(cmd, ["version", "--format=json"]);
},
async getTracerEnv(databasePath) {
// Write tracer-env.js to a temp location.
@@ -341,7 +344,7 @@ function getCodeQLForCmd(cmd) {
// action/runner has been implemented in `codeql database trace-command`
// _and_ is present in the latest supported CLI release.)
const envFile = path.resolve(databasePath, "working", "env.tmp");
await new toolrunner.ToolRunner(cmd, [
await runTool(cmd, [
"database",
"trace-command",
databasePath,
@@ -349,18 +352,18 @@ function getCodeQLForCmd(cmd) {
process.execPath,
tracerEnvJs,
envFile,
]).exec();
]);
return JSON.parse(fs.readFileSync(envFile, "utf-8"));
},
async databaseInit(databasePath, language, sourceRoot) {
await new toolrunner.ToolRunner(cmd, [
await runTool(cmd, [
"database",
"init",
databasePath,
`--language=${language}`,
`--source-root=${sourceRoot}`,
...getExtraOptionsFromEnv(["database", "init"]),
]).exec();
]);
},
async runAutobuild(language) {
const cmdName = process.platform === "win32" ? "autobuild.cmd" : "autobuild.sh";
@@ -376,7 +379,7 @@ function getCodeQLForCmd(cmd) {
"-Dhttp.keepAlive=false",
"-Dmaven.wagon.http.pool=false",
].join(" ");
await new toolrunner.ToolRunner(autobuildCmd).exec();
await runTool(autobuildCmd);
},
async extractScannedLanguage(databasePath, language) {
// Get extractor location
@@ -415,6 +418,7 @@ function getCodeQLForCmd(cmd) {
await toolrunner_error_catcher_1.toolrunnerErrorCatcher(cmd, [
"database",
"finalize",
"--finalize-dataset",
threadsFlag,
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
@@ -422,14 +426,7 @@ function getCodeQLForCmd(cmd) {
},
async resolveLanguages() {
const codeqlArgs = ["resolve", "languages", "--format=json"];
let output = "";
await new toolrunner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data) => {
output += data.toString();
},
},
}).exec();
const output = await runTool(cmd, codeqlArgs);
try {
return JSON.parse(output);
}
@@ -448,14 +445,7 @@ function getCodeQLForCmd(cmd) {
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
let output = "";
await new toolrunner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data) => {
output += data.toString();
},
},
}).exec();
const output = await runTool(cmd, codeqlArgs);
try {
return JSON.parse(output);
}
@@ -463,43 +453,93 @@ function getCodeQLForCmd(cmd) {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async databaseAnalyze(databasePath, sarifFile, extraSearchPath, querySuite, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId) {
const args = [
async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, memoryFlag, threadsFlag) {
const codeqlArgs = [
"database",
"analyze",
"run-queries",
memoryFlag,
threadsFlag,
databasePath,
"--min-disk-free=1024",
"--format=sarif-latest",
"--sarif-multicause-markdown",
`--output=${sarifFile}`,
addSnippetsFlag,
// Enable progress verbosity so we log each query as it's interpreted. This aids debugging
// when interpretation takes a while for one of the queries being analyzed.
"-v",
...getExtraOptionsFromEnv(["database", "analyze"]),
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (extraSearchPath !== undefined) {
args.push("--additional-packs", extraSearchPath);
codeqlArgs.push("--additional-packs", extraSearchPath);
}
codeqlArgs.push(querySuitePath);
await runTool(cmd, codeqlArgs);
},
async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, automationDetailsId) {
const codeqlArgs = [
"database",
"interpret-results",
threadsFlag,
"--format=sarif-latest",
"--print-metrics-summary",
"--sarif-group-rules-by-pack",
"-v",
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "interpret-results"]),
];
if (automationDetailsId !== undefined) {
args.push("--sarif-category", automationDetailsId);
codeqlArgs.push("--sarif-category", automationDetailsId);
}
args.push(querySuite);
codeqlArgs.push(databasePath, ...querySuitePaths);
// capture stdout, which contains analysis summaries
let output = "";
await new toolrunner.ToolRunner(cmd, args, {
listeners: {
stdout: (data) => {
output += data.toString("utf8");
},
},
}).exec();
return output;
return await runTool(cmd, codeqlArgs);
},
/**
* Download specified packs into the package cache. If the specified
* package and version already exists (e.g., from a previous analysis run),
* then it is not downloaded again (unless the extra option `--force` is
* specified).
*
* If no version is specified, then the latest version is
* downloaded. The check to determine what the latest version is is done
* each time this package is requested.
*/
async packDownload(packs) {
const codeqlArgs = [
"pack",
"download",
"--format=json",
...getExtraOptionsFromEnv(["pack", "download"]),
...packs.map(packWithVersionToString),
];
const output = await runTool(cmd, codeqlArgs);
try {
const parsedOutput = JSON.parse(output);
if (Array.isArray(parsedOutput.packs) &&
// TODO PackDownloadOutput will not include the version if it is not specified
// in the input. The version is always the latest version available.
// It should be added to the output, but this requires a CLI change
parsedOutput.packs.every((p) => p.name /* && p.version */)) {
return parsedOutput;
}
else {
throw new Error("Unexpected output from pack download");
}
}
catch (e) {
throw new Error(`Attempted to download specified packs but got an error:\n${output}\n${e}`);
}
},
async databaseCleanup(databasePath, cleanupLevel) {
const codeqlArgs = [
"database",
"cleanup",
databasePath,
`--mode=${cleanupLevel}`,
];
await runTool(cmd, codeqlArgs);
},
};
}
function packWithVersionToString(pack) {
return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
}
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*/
@@ -546,4 +586,15 @@ function getExtraOptions(options, paths, pathInfo) {
return all.concat(specific);
}
exports.getExtraOptions = getExtraOptions;
async function runTool(cmd, args = []) {
let output = "";
await new toolrunner.ToolRunner(cmd, args, {
listeners: {
stdout: (data) => {
output += data.toString();
},
},
}).exec();
return output;
}
//# sourceMappingURL=codeql.js.map
lib/codeql.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/config-utils.js
Generated
+92 -6
View File
@@ -10,6 +10,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const yaml = __importStar(require("js-yaml"));
const semver = __importStar(require("semver"));
const api = __importStar(require("./api-client"));
const externalQueries = __importStar(require("./external-queries"));
const languages_1 = require("./languages");
@@ -20,6 +21,7 @@ const QUERIES_PROPERTY = "queries";
const QUERIES_USES_PROPERTY = "uses";
const PATHS_IGNORE_PROPERTY = "paths-ignore";
const PATHS_PROPERTY = "paths";
const PACKS_PROPERTY = "packs";
/**
* A list of queries from https://github.com/github/codeql that
* we don't want to run. Disabling them here is a quicker alternative to
@@ -254,6 +256,22 @@ function getPathsInvalid(configFile) {
return getConfigFilePropertyError(configFile, PATHS_PROPERTY, "must be an array of non-empty strings");
}
exports.getPathsInvalid = getPathsInvalid;
function getPacksRequireLanguage(lang, configFile) {
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `has "${lang}", but it is not one of the languages to analyze`);
}
exports.getPacksRequireLanguage = getPacksRequireLanguage;
function getPacksInvalidSplit(configFile) {
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, "must split packages by language");
}
exports.getPacksInvalidSplit = getPacksInvalidSplit;
function getPacksInvalid(configFile) {
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, "must be an array of non-empty strings");
}
exports.getPacksInvalid = getPacksInvalid;
function getPacksStrInvalid(packStr, configFile) {
return getConfigFilePropertyError(configFile, PACKS_PROPERTY, `"${packStr}" is not a valid pack`);
}
exports.getPacksStrInvalid = getPacksStrInvalid;
function getLocalPathOutsideOfRepository(configFile, localPath) {
return getConfigFilePropertyError(configFile, `${QUERIES_PROPERTY}.${QUERIES_USES_PROPERTY}`, `is invalid as the local path "${localPath}" is outside of the repository`);
}
@@ -409,6 +427,7 @@ async function getDefaultConfig(languagesInput, queriesInput, dbLocation, reposi
queries,
pathsIgnore: [],
paths: [],
packs: {},
originalUserInput: {},
tempDir,
toolCacheDir,
@@ -422,6 +441,7 @@ exports.getDefaultConfig = getDefaultConfig;
* Load the config from the given file.
*/
async function loadConfig(languagesInput, queriesInput, configFile, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger) {
var _a;
let parsedYAML;
if (isLocal(configFile)) {
// Treat the config file as relative to the workspace
@@ -470,7 +490,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
}
if (shouldAddConfigFileQueries(queriesInput) &&
QUERIES_PROPERTY in parsedYAML) {
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
if (!Array.isArray(parsedYAML[QUERIES_PROPERTY])) {
throw new Error(getQueriesInvalid(configFile));
}
for (const query of parsedYAML[QUERIES_PROPERTY]) {
@@ -482,7 +502,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
}
}
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
if (!(parsedYAML[PATHS_IGNORE_PROPERTY] instanceof Array)) {
if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
throw new Error(getPathsIgnoreInvalid(configFile));
}
for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]) {
@@ -493,7 +513,7 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
}
}
if (PATHS_PROPERTY in parsedYAML) {
if (!(parsedYAML[PATHS_PROPERTY] instanceof Array)) {
if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
throw new Error(getPathsInvalid(configFile));
}
for (const includePath of parsedYAML[PATHS_PROPERTY]) {
@@ -503,11 +523,13 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
paths.push(validateAndSanitisePath(includePath, PATHS_PROPERTY, configFile, logger));
}
}
const packs = parsePacks((_a = parsedYAML[PACKS_PROPERTY], (_a !== null && _a !== void 0 ? _a : {})), languages, configFile);
return {
languages,
queries,
pathsIgnore,
paths,
packs,
originalUserInput: parsedYAML,
tempDir,
toolCacheDir,
@@ -516,6 +538,68 @@ async function loadConfig(languagesInput, queriesInput, configFile, dbLocation,
dbLocation: dbLocationOrDefault(dbLocation, tempDir),
};
}
/**
* Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
* and `-` allowed as long as not the first or last char.
**/
const PACK_IDENTIFIER_PATTERN = (function () {
const alphaNumeric = "[a-z0-9]";
const alphaNumericDash = "[a-z0-9-]";
const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
return new RegExp(`^${component}/${component}$`);
})();
// Exported for testing
function parsePacks(packsByLanguage, languages, configFile) {
const packs = {};
if (Array.isArray(packsByLanguage)) {
if (languages.length === 1) {
// single language analysis, so language is implicit
packsByLanguage = {
[languages[0]]: packsByLanguage,
};
}
else {
// this is an error since multi-language analysis requires
// packs split by language
throw new Error(getPacksInvalidSplit(configFile));
}
}
for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
if (!Array.isArray(packsArr)) {
throw new Error(getPacksInvalid(configFile));
}
if (!languages.includes(lang)) {
throw new Error(getPacksRequireLanguage(lang, configFile));
}
packs[lang] = [];
for (const packStr of packsArr) {
packs[lang].push(toPackWithVersion(packStr, configFile));
}
}
return packs;
}
exports.parsePacks = parsePacks;
function toPackWithVersion(packStr, configFile) {
if (typeof packStr !== "string") {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
const nameWithVersion = packStr.split("@");
let version;
if (nameWithVersion.length > 2 ||
!PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])) {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
else if (nameWithVersion.length === 2) {
version = semver.clean(nameWithVersion[1]) || undefined;
if (!version) {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
}
return {
packName: nameWithVersion[0],
version,
};
}
function dbLocationOrDefault(dbLocation, tempDir) {
return dbLocation || path.resolve(tempDir, "codeql_databases");
}
@@ -526,6 +610,7 @@ function dbLocationOrDefault(dbLocation, tempDir) {
* a default config. The parsed config is then stored to a known location.
*/
async function initConfig(languagesInput, queriesInput, configFile, dbLocation, repository, tempDir, toolCacheDir, codeQL, checkoutPath, gitHubVersion, apiDetails, logger) {
var _a, _b, _c;
let config;
// If no config file was provided create an empty one
if (!configFile) {
@@ -538,9 +623,10 @@ async function initConfig(languagesInput, queriesInput, configFile, dbLocation,
// The list of queries should not be empty for any language. If it is then
// it is a user configuration error.
for (const language of config.languages) {
if (config.queries[language] === undefined ||
(config.queries[language].builtin.length === 0 &&
config.queries[language].custom.length === 0)) {
const hasBuiltinQueries = ((_a = config.queries[language]) === null || _a === void 0 ? void 0 : _a.builtin.length) > 0;
const hasCustomQueries = ((_b = config.queries[language]) === null || _b === void 0 ? void 0 : _b.custom.length) > 0;
const hasPacks = (((_c = config.packs[language]) === null || _c === void 0 ? void 0 : _c.length) || 0) > 0;
if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
throw new Error(`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run.");
}
lib/config-utils.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/config-utils.test.js
Generated
+147
View File
@@ -14,6 +14,7 @@ const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const github = __importStar(require("@actions/github"));
const ava_1 = __importDefault(require("ava"));
const semver_1 = require("semver");
const sinon_1 = __importDefault(require("sinon"));
const api = __importStar(require("./api-client"));
const codeql_1 = require("./codeql");
@@ -200,6 +201,7 @@ ava_1.default("load non-empty input", async (t) => {
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
};
const languages = "javascript";
const configFilePath = createConfigFile(inputFileContents, tmpDir);
@@ -557,6 +559,101 @@ ava_1.default("Unknown languages", async (t) => {
}
});
});
ava_1.default("Config specifies packages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeQL = codeql_1.setCodeQL({
async resolveQueries() {
return {
byLanguage: {},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
const inputFileContents = `
name: my config
disable-default-queries: true
packs:
- a/b@1.2.3
`;
const configFile = path.join(tmpDir, "codeql-config.yaml");
fs.writeFileSync(configFile, inputFileContents);
const languages = "javascript";
const { packs } = await configUtils.initConfig(languages, undefined, configFile, undefined, { owner: "github", repo: "example " }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
t.deepEqual(packs, {
[languages_1.Language.javascript]: [
{
packName: "a/b",
version: semver_1.clean("1.2.3"),
},
],
});
});
});
ava_1.default("Config specifies packages for multiple languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeQL = codeql_1.setCodeQL({
async resolveQueries() {
return {
byLanguage: {
cpp: { "/foo/a.ql": {} },
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
const inputFileContents = `
name: my config
disable-default-queries: true
queries:
- uses: ./foo
packs:
javascript:
- a/b@1.2.3
python:
- c/d@1.2.3
`;
const configFile = path.join(tmpDir, "codeql-config.yaml");
fs.writeFileSync(configFile, inputFileContents);
fs.mkdirSync(path.join(tmpDir, "foo"));
const languages = "javascript,python,cpp";
const { packs, queries } = await configUtils.initConfig(languages, undefined, configFile, undefined, { owner: "github", repo: "example" }, tmpDir, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, logging_1.getRunnerLogger(true));
t.deepEqual(packs, {
[languages_1.Language.javascript]: [
{
packName: "a/b",
version: semver_1.clean("1.2.3"),
},
],
[languages_1.Language.python]: [
{
packName: "c/d",
version: semver_1.clean("1.2.3"),
},
],
});
t.deepEqual(queries, {
cpp: {
builtin: [],
custom: [
{
queries: ["/foo/a.ql"],
searchPath: tmpDir,
},
],
},
javascript: {
builtin: [],
custom: [],
},
python: {
builtin: [],
custom: [],
},
});
});
});
function doInvalidInputTest(testName, inputFileContents, expectedErrorMessageGenerator) {
ava_1.default(`load invalid input - ${testName}`, async (t) => {
return await util.withTmpDir(async (tmpDir) => {
@@ -644,4 +741,54 @@ ava_1.default("path sanitisation", (t) => {
// Trailing stars are stripped
t.deepEqual(configUtils.validateAndSanitisePath("foo/**", propertyName, configFile, logging_1.getRunnerLogger(true)), "foo/");
});
/**
* Test macro for ensuring the packs block is valid
*/
function parsePacksMacro(t, packsByLanguage, languages, expected) {
t.deepEqual(configUtils.parsePacks(packsByLanguage, languages, "/a/b"), expected);
}
parsePacksMacro.title = (providedTitle) => `Parse Packs: ${providedTitle}`;
/**
* Test macro for testing when the packs block is invalid
*/
function parsePacksErrorMacro(t, packsByLanguage, languages, expected) {
t.throws(() => {
configUtils.parsePacks(packsByLanguage, languages, "/a/b");
}, {
message: expected,
});
}
parsePacksErrorMacro.title = (providedTitle) => `Parse Packs Error: ${providedTitle}`;
function invalidPackNameMacro(t, name) {
parsePacksErrorMacro(t, { [languages_1.Language.cpp]: [name] }, [languages_1.Language.cpp], new RegExp(`The configuration file "/a/b" is invalid: property "packs" "${name}" is not a valid pack`));
}
invalidPackNameMacro.title = (_, arg) => `Invalid pack string: ${arg}`;
ava_1.default("no packs", parsePacksMacro, {}, [], {});
ava_1.default("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [languages_1.Language.cpp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: semver_1.clean("1.2.3") },
],
});
ava_1.default("two packs with language", parsePacksMacro, {
[languages_1.Language.cpp]: ["a/b", "c/d@1.2.3"],
[languages_1.Language.java]: ["d/e", "f/g@1.2.3"],
}, [languages_1.Language.cpp, languages_1.Language.java, languages_1.Language.csharp], {
[languages_1.Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: semver_1.clean("1.2.3") },
],
[languages_1.Language.java]: [
{ packName: "d/e", version: undefined },
{ packName: "f/g", version: semver_1.clean("1.2.3") },
],
});
ava_1.default("no language", parsePacksErrorMacro, ["a/b@1.2.3"], [languages_1.Language.java, languages_1.Language.python], /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/);
ava_1.default("invalid language", parsePacksErrorMacro, { [languages_1.Language.java]: ["c/d"] }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/);
ava_1.default("not an array", parsePacksErrorMacro, { [languages_1.Language.cpp]: "c/d" }, [languages_1.Language.cpp], /The configuration file "\/a\/b" is invalid: property "packs" must be an array of non-empty strings/);
ava_1.default(invalidPackNameMacro, "c"); // all packs require at least a scope and a name
ava_1.default(invalidPackNameMacro, "c-/d");
ava_1.default(invalidPackNameMacro, "-c/d");
ava_1.default(invalidPackNameMacro, "c/d_d");
ava_1.default(invalidPackNameMacro, "c/d@x");
//# sourceMappingURL=config-utils.test.js.map
lib/config-utils.test.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/count-loc.test.js
Generated
+3 -3
View File
@@ -21,13 +21,13 @@ ava_1.default("ensure lines of code works for cpp and js", async (t) => {
const results = await count_loc_1.countLoc(path.join(__dirname, "../tests/multi-language-repo"), [], [], [languages_1.Language.cpp, languages_1.Language.javascript], logging_1.getRunnerLogger(true));
t.deepEqual(results, {
cpp: 6,
javascript: 3,
javascript: 9,
});
});
ava_1.default("ensure lines of code can handle undefined language", async (t) => {
const results = await count_loc_1.countLoc(path.join(__dirname, "../tests/multi-language-repo"), [], [], [languages_1.Language.javascript, languages_1.Language.python, "hucairz"], logging_1.getRunnerLogger(true));
t.deepEqual(results, {
javascript: 3,
javascript: 9,
python: 5,
});
});
@@ -54,7 +54,7 @@ ava_1.default("ensure lines of code can handle empty includes", async (t) => {
ava_1.default("ensure lines of code can handle exclude", async (t) => {
const results = await count_loc_1.countLoc(path.join(__dirname, "../tests/multi-language-repo"), [], ["**/*.py"], [languages_1.Language.javascript, languages_1.Language.python], logging_1.getRunnerLogger(true));
t.deepEqual(results, {
javascript: 3,
javascript: 9,
});
});
//# sourceMappingURL=count-loc.test.js.map
lib/fingerprints.js
Generated
+19 -9
View File
@@ -16,6 +16,7 @@ const tab = "\t".charCodeAt(0);
const space = " ".charCodeAt(0);
const lf = "\n".charCodeAt(0);
const cr = "\r".charCodeAt(0);
const EOF = 65535;
const BLOCK_SIZE = 100;
const MOD = long_1.default.fromInt(37); // L
// Compute the starting point for the hash mod
@@ -36,9 +37,9 @@ function computeFirstMod() {
* the hashes of the lines near the end of the file.
*
* @param callback function that is called with the line number (1-based) and hash for every line
* @param input The file's contents
* @param filepath The path to the file to hash
*/
function hash(callback, input) {
async function hash(callback, filepath) {
// A rolling view in to the input
const window = Array(BLOCK_SIZE).fill(0);
// If the character in the window is the start of a new line
@@ -82,12 +83,11 @@ function hash(callback, input) {
// as we go. Once we reach a point in the window again then we've processed
// BLOCK_SIZE characters and if the last character at this point in the window
// was the start of a line then we should output the hash for that line.
for (let i = 0, len = input.length; i <= len; i++) {
let current = i === len ? 65535 : input.charCodeAt(i);
const processCharacter = function (current) {
// skip tabs, spaces, and line feeds that come directly after a carriage return
if (current === space || current === tab || (prevCR && current === lf)) {
prevCR = false;
continue;
return;
}
// replace CR with LF
if (current === cr) {
@@ -109,7 +109,14 @@ function hash(callback, input) {
lineStart = true;
}
updateHash(current);
};
const readStream = fs.createReadStream(filepath, "utf8");
for await (const data of readStream) {
for (let i = 0; i < data.length; ++i) {
processCharacter(data.charCodeAt(i));
}
}
processCharacter(EOF);
// Flush the remaining lines
for (let i = 0; i < BLOCK_SIZE; i++) {
if (lineNumbers[index] !== -1) {
@@ -206,8 +213,8 @@ function resolveUriToFile(location, artifacts, checkoutPath, logger) {
exports.resolveUriToFile = resolveUriToFile;
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
function addFingerprints(sarifContents, checkoutPath, logger) {
var _a, _b;
async function addFingerprints(sarifContents, checkoutPath, logger) {
var _a, _b, _c, _d, _e;
const sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
// callbacks to accept hashes for that file and update the location
@@ -222,6 +229,10 @@ function addFingerprints(sarifContents, checkoutPath, logger) {
logger.debug(`Unable to compute fingerprint for invalid location: ${JSON.stringify(primaryLocation)}`);
continue;
}
if (((_e = (_d = (_c = primaryLocation) === null || _c === void 0 ? void 0 : _c.physicalLocation) === null || _d === void 0 ? void 0 : _d.region) === null || _e === void 0 ? void 0 : _e.startLine) === undefined) {
// Locations without a line number are unlikely to be source files
continue;
}
const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts, checkoutPath, logger);
if (!filepath) {
continue;
@@ -240,8 +251,7 @@ function addFingerprints(sarifContents, checkoutPath, logger) {
c(lineNumber, hashValue);
}
};
const fileContents = fs.readFileSync(filepath).toString();
hash(teeCallback, fileContents);
await hash(teeCallback, filepath);
}
return JSON.stringify(sarif);
}
lib/fingerprints.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/fingerprints.test.js
Generated
+28 -23
View File
@@ -16,28 +16,33 @@ const ava_1 = __importDefault(require("ava"));
const fingerprints = __importStar(require("./fingerprints"));
const logging_1 = require("./logging");
const testing_utils_1 = require("./testing-utils");
const util = __importStar(require("./util"));
testing_utils_1.setupTests(ava_1.default);
function testHash(t, input, expectedHashes) {
let index = 0;
const callback = function (lineNumber, hash) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
fingerprints.hash(callback, input);
t.is(index, input.split(/\r\n|\r|\n/).length);
async function testHash(t, input, expectedHashes) {
await util.withTmpDir(async (tmpDir) => {
const tmpFile = path.resolve(tmpDir, "testfile");
fs.writeFileSync(tmpFile, input);
let index = 0;
const callback = function (lineNumber, hash) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
await fingerprints.hash(callback, tmpFile);
t.is(index, input.split(/\r\n|\r|\n/).length);
});
}
ava_1.default("hash", (t) => {
ava_1.default("hash", async (t) => {
// Try empty file
testHash(t, "", ["c129715d7a2bc9a3:1"]);
await testHash(t, "", ["c129715d7a2bc9a3:1"]);
// Try various combinations of newline characters
testHash(t, " a\nb\n \t\tc\n d", [
await testHash(t, " a\nb\n \t\tc\n d", [
"271789c17abda88f:1",
"54703d4cd895b18:1",
"180aee12dab6264:1",
"a23a3dc5e078b07b:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
"8b7cf3e952e7aeb2:1",
"b1ae1287ec4718d9:1",
"bff680108adb0fcc:1",
@@ -45,7 +50,7 @@ ava_1.default("hash", (t) => {
"b86d3392aea1be30:1",
"e6ceba753e1a442:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@@ -54,7 +59,7 @@ ava_1.default("hash", (t) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
await testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@@ -63,7 +68,7 @@ ava_1.default("hash", (t) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n", [
await testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@@ -72,7 +77,7 @@ ava_1.default("hash", (t) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
await testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@@ -82,7 +87,7 @@ ava_1.default("hash", (t) => {
"c129715d7a2bc9a3:1",
]);
// Try repeating line that will generate identical hashes
testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
await testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
"a7f2ff13bc495cf2:1",
"a7f2ff13bc495cf2:2",
"a7f2ff13bc495cf2:3",
@@ -95,7 +100,7 @@ ava_1.default("hash", (t) => {
"cc97dc7b1d7d8f7b:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
await testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
@@ -150,7 +155,7 @@ ava_1.default("resolveUriToFile", (t) => {
t.is(testResolveUriToFile(dirpath, undefined, []), undefined);
t.is(testResolveUriToFile(`file://${dirpath}`, undefined, []), undefined);
});
ava_1.default("addFingerprints", (t) => {
ava_1.default("addFingerprints", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
@@ -163,9 +168,9 @@ ava_1.default("addFingerprints", (t) => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
t.deepEqual(await fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
});
ava_1.default("missingRegions", (t) => {
ava_1.default("missingRegions", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
@@ -178,6 +183,6 @@ ava_1.default("missingRegions", (t) => {
expected = JSON.stringify(JSON.parse(expected));
// The URIs in the SARIF files resolve to files in the testdata directory
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
t.deepEqual(await fingerprints.addFingerprints(input, checkoutPath, logging_1.getRunnerLogger(true)), expected);
});
//# sourceMappingURL=fingerprints.test.js.map
lib/fingerprints.test.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/tracer-config.test.js
Generated
+1
View File
@@ -31,6 +31,7 @@ function getTestConfig(tmpDir) {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
};
}
// A very minimal setup
lib/tracer-config.test.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
lib/upload-lib.js
Generated
+1 -1
View File
@@ -243,7 +243,7 @@ async function uploadFiles(sarifFiles, repositoryNwo, commitOid, ref, analysisKe
validateSarifFileSchema(file, logger);
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
sarifPayload = await fingerprints.addFingerprints(sarifPayload, checkoutPath, logger);
sarifPayload = populateRunAutomationDetails(sarifPayload, category, analysisKey, environment);
const zippedSarif = zlib_1.default.gzipSync(sarifPayload).toString("base64");
const checkoutURI = file_url_1.default(checkoutPath);
lib/upload-lib.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
node_modules/.package-lock.json
Generated Vendored
+10
View File
@@ -405,6 +405,12 @@
"@types/node": "*"
}
},
"node_modules/@types/js-yaml": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.1.tgz",
"integrity": "sha512-xdOvNmXmrZqqPy3kuCQ+fz6wA0xU5pji9cd1nDrflWaAWtYLLGk5ykW0H6yg5TVyehHP1pfmuuSaZkhP+kspVA==",
"dev": true
},
"node_modules/@types/json-schema": {
"version": "7.0.6",
"dev": true,
@@ -1253,6 +1259,7 @@
"dependencies": {
"anymatch": "~3.1.1",
"braces": "~3.0.2",
"fsevents": "~2.1.2",
"glob-parent": "~5.1.0",
"is-binary-path": "~2.1.0",
"is-glob": "~4.0.1",
@@ -3238,6 +3245,9 @@
"node_modules/jsonfile": {
"version": "4.0.0",
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.1.6"
},
"optionalDependencies": {
"graceful-fs": "^4.1.6"
}
package-lock.json
Generated
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "1.0.1",
"version": "1.0.2",
"lockfileVersion": 2,
"requires": true,
"packages": {
package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "1.0.1",
"version": "1.0.2",
"private": true,
"description": "CodeQL action",
"scripts": {
runner/package-lock.json
Generated
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "1.0.1",
"version": "1.0.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
runner/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "codeql-runner",
"version": "1.0.1",
"version": "1.0.2",
"private": true,
"description": "CodeQL runner",
"scripts": {
src/analysis-paths.test.ts
+3
View File
@@ -21,6 +21,7 @@ test("emptyPaths", async (t) => {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@@ -42,6 +43,7 @@ test("nonEmptyPaths", async (t) => {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@@ -67,6 +69,7 @@ test("exclude temp dir", async (t) => {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tempDir, "codeql_databases"),
packs: {},
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
src/analyze-action.ts
+15
View File
@@ -8,6 +8,7 @@ import {
runAnalyze,
CodeQLAnalysisError,
QueriesStatusReport,
runCleanup,
} from "./analyze";
import { Config, getConfig } from "./config-utils";
import { getActionsLogger } from "./logging";
@@ -89,6 +90,20 @@ async function run() {
logger
);
if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
await runCleanup(
config,
actionsUtil.getOptionalInput("cleanup-level") || "brutal",
logger
);
}
const dbLocations: { [lang: string]: string } = {};
for (const language of config.languages) {
dbLocations[language] = util.getCodeQLDatabasePath(config, language);
}
core.setOutput("db-locations", dbLocations);
if (actionsUtil.getRequiredInput("upload") === "true") {
const uploadStats = await upload_lib.uploadFromActions(
outputDir,
src/analyze.test.ts
+115 -18
View File
@@ -2,6 +2,8 @@ import * as fs from "fs";
import * as path from "path";
import test from "ava";
import * as yaml from "js-yaml";
import { clean } from "semver";
import sinon from "sinon";
import { runQueries } from "./analyze";
@@ -26,20 +28,41 @@ test("status report fields and search path setting", async (t) => {
return obj;
}, {});
sinon.stub(count, "countLoc").resolves(mockLinesOfCode);
let searchPathsUsed: string[] = [];
let searchPathsUsed: Array<string | undefined> = [];
return await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const memoryFlag = "";
const addSnippetsFlag = "";
const threadsFlag = "";
const packs = {
[Language.cpp]: [
{
packName: "a/b",
version: clean("1.0.0")!,
},
],
[Language.java]: [
{
packName: "c/d",
version: clean("2.0.0")!,
},
],
};
for (const language of Object.values(Language)) {
setCodeQL({
databaseAnalyze: async (
_,
sarifFile: string,
packDownload: async () => ({ packs: [] }),
databaseRunQueries: async (
_db: string,
searchPath: string | undefined
) => {
searchPathsUsed.push(searchPath);
},
databaseInterpretResults: async (
_db: string,
_queriesRun: string[],
sarifFile: string
) => {
fs.writeFileSync(
sarifFile,
@@ -75,7 +98,6 @@ test("status report fields and search path setting", async (t) => {
],
})
);
searchPathsUsed.push(searchPath!);
return "";
},
});
@@ -94,6 +116,7 @@ test("status report fields and search path setting", async (t) => {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs,
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
@@ -112,10 +135,32 @@ test("status report fields and search path setting", async (t) => {
config,
getRunnerLogger(true)
);
t.deepEqual(Object.keys(builtinStatusReport).length, 1);
t.true(
`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport
);
const hasPacks = language in packs;
const statusReportKeys = Object.keys(builtinStatusReport).sort();
if (hasPacks) {
t.deepEqual(statusReportKeys.length, 3, statusReportKeys.toString());
t.deepEqual(
statusReportKeys[0],
`analyze_builtin_queries_${language}_duration_ms`
);
t.deepEqual(
statusReportKeys[1],
`analyze_custom_queries_${language}_duration_ms`
);
t.deepEqual(
statusReportKeys[2],
`interpret_results_${language}_duration_ms`
);
} else {
t.deepEqual(
statusReportKeys[0],
`analyze_builtin_queries_${language}_duration_ms`
);
t.deepEqual(
statusReportKeys[1],
`interpret_results_${language}_duration_ms`
);
}
config.queries[language] = {
builtin: [],
@@ -139,14 +184,19 @@ test("status report fields and search path setting", async (t) => {
config,
getRunnerLogger(true)
);
t.deepEqual(Object.keys(customStatusReport).length, 1);
t.deepEqual(Object.keys(customStatusReport).length, 2);
t.true(
`analyze_custom_queries_${language}_duration_ms` in customStatusReport
);
t.deepEqual(searchPathsUsed, [undefined, "/1", "/2"]);
const expectedSearchPathsUsed = hasPacks
? [undefined, undefined, "/1", "/2", undefined]
: [undefined, "/1", "/2"];
t.deepEqual(searchPathsUsed, expectedSearchPathsUsed);
t.true(`interpret_results_${language}_duration_ms` in customStatusReport);
}
verifyLineCounts(tmpDir);
verifyQuerySuites(tmpDir);
});
function verifyLineCounts(tmpDir: string) {
@@ -154,12 +204,7 @@ test("status report fields and search path setting", async (t) => {
Object.keys(Language).forEach((lang, i) => {
verifyLineCountForFile(
lang as Language,
path.join(tmpDir, `${lang}-builtin.sarif`),
i + 1
);
verifyLineCountForFile(
lang as Language,
path.join(tmpDir, `${lang}-custom.sarif`),
path.join(tmpDir, `${lang}.sarif`),
i + 1
);
});
@@ -188,7 +233,59 @@ test("status report fields and search path setting", async (t) => {
baseline: lineCount,
},
]);
// when the rule doesn't exists, it should not be added
// when the rule doesn't exist, it should not be added
t.deepEqual(sarif.runs[2].properties.metricResults, []);
}
function verifyQuerySuites(tmpDir: string) {
const qlsContent = [
{
query: "foo.ql",
},
];
const qlsContent2 = [
{
query: "bar.ql",
},
];
const qlsPackContentCpp = [
{
qlpack: "a/b",
version: "1.0.0",
},
];
const qlsPackContentJava = [
{
qlpack: "c/d",
version: "2.0.0",
},
];
for (const lang of Object.values(Language)) {
t.deepEqual(readContents(`${lang}-queries-builtin.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-0.qls`), qlsContent);
t.deepEqual(readContents(`${lang}-queries-custom-1.qls`), qlsContent2);
const packSuiteName = `${lang}-queries-packs.qls`;
if (lang === Language.cpp) {
t.deepEqual(readContents(packSuiteName), qlsPackContentCpp);
} else if (lang === Language.java) {
t.deepEqual(readContents(packSuiteName), qlsPackContentJava);
} else {
t.false(
fs.existsSync(path.join(tmpDir, "codeql_databases", packSuiteName))
);
}
}
function readContents(name: string) {
const x = fs.readFileSync(
path.join(tmpDir, "codeql_databases", name),
"utf8"
);
console.log(x);
return yaml.safeLoad(
fs.readFileSync(path.join(tmpDir, "codeql_databases", name), "utf8")
);
}
}
});
src/analyze.ts
+161 -97
View File
@@ -10,7 +10,6 @@ import { countLoc, getIdPrefix } from "./count-loc";
import { isScannedLanguage, Language } from "./languages";
import { Logger } from "./logging";
import * as sharedEnv from "./shared-environment";
import { combineSarifFiles } from "./upload-lib";
import * as util from "./util";
export class CodeQLAnalysisError extends Error {
@@ -25,34 +24,48 @@ export class CodeQLAnalysisError extends Error {
}
export interface QueriesStatusReport {
// Time taken in ms to analyze builtin queries for cpp (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for cpp (or undefined if this language was not analyzed)
analyze_builtin_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for csharp (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for csharp (or undefined if this language was not analyzed)
analyze_builtin_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze builtin queries for go (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for go (or undefined if this language was not analyzed)
analyze_builtin_queries_go_duration_ms?: number;
// Time taken in ms to analyze builtin queries for java (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for java (or undefined if this language was not analyzed)
analyze_builtin_queries_java_duration_ms?: number;
// Time taken in ms to analyze builtin queries for javascript (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for javascript (or undefined if this language was not analyzed)
analyze_builtin_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze builtin queries for python (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for python (or undefined if this language was not analyzed)
analyze_builtin_queries_python_duration_ms?: number;
// Time taken in ms to analyze builtin queries for ruby (or undefined if this language was not analyzed)
// Time taken in ms to run builtin queries for ruby (or undefined if this language was not analyzed)
analyze_builtin_queries_ruby_duration_ms?: number;
// Time taken in ms to analyze custom queries for cpp (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for cpp (or undefined if this language was not analyzed)
analyze_custom_queries_cpp_duration_ms?: number;
// Time taken in ms to analyze custom queries for csharp (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for csharp (or undefined if this language was not analyzed)
analyze_custom_queries_csharp_duration_ms?: number;
// Time taken in ms to analyze custom queries for go (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for go (or undefined if this language was not analyzed)
analyze_custom_queries_go_duration_ms?: number;
// Time taken in ms to analyze custom queries for java (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for java (or undefined if this language was not analyzed)
analyze_custom_queries_java_duration_ms?: number;
// Time taken in ms to analyze custom queries for javascript (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for javascript (or undefined if this language was not analyzed)
analyze_custom_queries_javascript_duration_ms?: number;
// Time taken in ms to analyze custom queries for python (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for python (or undefined if this language was not analyzed)
analyze_custom_queries_python_duration_ms?: number;
// Time taken in ms to analyze custom queries for ruby (or undefined if this language was not analyzed)
// Time taken in ms to run custom queries for ruby (or undefined if this language was not analyzed)
analyze_custom_queries_ruby_duration_ms?: number;
// Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed)
interpret_results_cpp_duration_ms?: number;
// Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed)
interpret_results_csharp_duration_ms?: number;
// Time taken in ms to interpret results for go (or undefined if this language was not analyzed)
interpret_results_go_duration_ms?: number;
// Time taken in ms to interpret results for java (or undefined if this language was not analyzed)
interpret_results_java_duration_ms?: number;
// Time taken in ms to interpret results for javascript (or undefined if this language was not analyzed)
interpret_results_javascript_duration_ms?: number;
// Time taken in ms to interpret results for python (or undefined if this language was not analyzed)
interpret_results_python_duration_ms?: number;
// Time taken in ms to interpret results for ruby (or undefined if this language was not analyzed)
interpret_results_ruby_duration_ms?: number;
// Name of language that errored during analysis (or undefined if no language failed)
analyze_failure_language?: string;
}
@@ -163,86 +176,97 @@ export async function runQueries(
);
for (const language of config.languages) {
logger.startGroup(`Analyzing ${language}`);
const queries = config.queries[language];
if (
queries === undefined ||
(queries.builtin.length === 0 && queries.custom.length === 0)
) {
const packsWithVersion = config.packs[language] || [];
const hasBuiltinQueries = queries?.builtin.length > 0;
const hasCustomQueries = queries?.custom.length > 0;
const hasPackWithCustomQueries = packsWithVersion.length > 0;
if (!hasBuiltinQueries && !hasCustomQueries && !hasPackWithCustomQueries) {
throw new Error(
`Unable to analyse ${language} as no queries were selected for this language`
);
}
try {
let analysisSummaryBuiltIn = "";
const customAnalysisSummaries: string[] = [];
if (hasPackWithCustomQueries) {
logger.startGroup(`Downloading custom packs for ${language}`);
const codeql = getCodeQL(config.codeQLCmd);
const results = await codeql.packDownload(packsWithVersion);
logger.info(
`Downloaded packs: ${results.packs
.map((r) => `${r.name}@${r.version || "latest"}`)
.join(", ")}`
);
logger.endGroup();
}
logger.startGroup(`Running queries for ${language}`);
const querySuitePaths: string[] = [];
if (queries["builtin"].length > 0) {
const startTimeBuiltIn = new Date().getTime();
const { sarifFile, stdout } = await runQueryGroup(
language,
"builtin",
queries["builtin"],
sarifFolder,
undefined
querySuitePaths.push(
await runQueryGroup(
language,
"builtin",
createQuerySuiteContents(queries["builtin"]),
undefined
)
);
analysisSummaryBuiltIn = stdout;
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeBuiltIn;
}
const startTimeCustom = new Date().getTime();
const temporarySarifDir = config.tempDir;
const temporarySarifFiles: string[] = [];
let ranCustom = false;
for (let i = 0; i < queries["custom"].length; ++i) {
if (queries["custom"][i].queries.length > 0) {
const { sarifFile, stdout } = await runQueryGroup(
language,
`custom-${i}`,
queries["custom"][i].queries,
temporarySarifDir,
queries["custom"][i].searchPath
querySuitePaths.push(
await runQueryGroup(
language,
`custom-${i}`,
createQuerySuiteContents(queries["custom"][i].queries),
queries["custom"][i].searchPath
)
);
customAnalysisSummaries.push(stdout);
temporarySarifFiles.push(sarifFile);
ranCustom = true;
}
}
if (temporarySarifFiles.length > 0) {
const sarifFile = path.join(sarifFolder, `${language}-custom.sarif`);
fs.writeFileSync(sarifFile, combineSarifFiles(temporarySarifFiles));
await injectLinesOfCode(sarifFile, language, locPromise);
if (packsWithVersion.length > 0) {
querySuitePaths.push(
await runQueryGroup(
language,
"packs",
createPackSuiteContents(packsWithVersion),
undefined
)
);
ranCustom = true;
}
if (ranCustom) {
statusReport[`analyze_custom_queries_${language}_duration_ms`] =
new Date().getTime() - startTimeCustom;
}
logger.endGroup();
// Print the LoC baseline and the summary results from database analyze for the standard
// query suite and (if appropriate) each custom query suite.
logger.startGroup(`Analysis summary for ${language}`);
printLinesOfCodeSummary(logger, language, await locPromise);
logger.info(analysisSummaryBuiltIn);
for (const [i, customSummary] of customAnalysisSummaries.entries()) {
if (customSummary.trim() === "") {
continue;
}
const description =
customAnalysisSummaries.length === 1
? "custom queries"
: `custom query suite ${i + 1}/${customAnalysisSummaries.length}`;
logger.info(`Analysis summary for ${description}:`);
logger.info("");
logger.info(customSummary);
logger.info("");
}
logger.startGroup(`Interpreting results for ${language}`);
const startTimeInterpretResults = new Date().getTime();
const sarifFile = path.join(sarifFolder, `${language}.sarif`);
const analysisSummary = await runInterpretResults(
language,
querySuitePaths,
sarifFile
);
await injectLinesOfCode(sarifFile, language, locPromise);
statusReport[`interpret_results_${language}_duration_ms`] =
new Date().getTime() - startTimeInterpretResults;
logger.endGroup();
logger.info(analysisSummary);
printLinesOfCodeSummary(logger, language, await locPromise);
} catch (e) {
logger.info(e);
logger.info(e.stack);
statusReport.analyze_failure_language = language;
throw new CodeQLAnalysisError(
statusReport,
@@ -253,42 +277,70 @@ export async function runQueries(
return statusReport;
async function runQueryGroup(
async function runInterpretResults(
language: Language,
type: string,
queries: string[],
destinationFolder: string,
searchPath: string | undefined
): Promise<{ sarifFile: string; stdout: string }> {
sarifFile: string
): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
const querySuiteContents = queries
.map((q: string) => `- query: ${q}`)
.join("\n");
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
const sarifFile = path.join(destinationFolder, `${language}-${type}.sarif`);
const codeql = getCodeQL(config.codeQLCmd);
const databaseAnalyzeStdout = await codeql.databaseAnalyze(
return await codeql.databaseInterpretResults(
databasePath,
queries,
sarifFile,
searchPath,
querySuitePath,
memoryFlag,
addSnippetsFlag,
threadsFlag,
automationDetailsId
);
logger.debug(
`SARIF results for database ${language} created at "${sarifFile}"`
);
return { sarifFile, stdout: databaseAnalyzeStdout };
}
async function runQueryGroup(
language: Language,
type: string,
querySuiteContents: string,
searchPath: string | undefined
): Promise<string> {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = `${databasePath}-queries-${type}.qls`;
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(
`Query suite file for ${language}-${type}...\n${querySuiteContents}`
);
const codeql = getCodeQL(config.codeQLCmd);
await codeql.databaseRunQueries(
databasePath,
searchPath,
querySuitePath,
memoryFlag,
threadsFlag
);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
}
function createQuerySuiteContents(queries: string[]) {
return queries.map((q: string) => `- query: ${q}`).join("\n");
}
function createPackSuiteContents(
packsWithVersion: configUtils.PackWithVersion[]
) {
return packsWithVersion.map(packWithVersionToQuerySuiteEntry).join("\n");
}
function packWithVersionToQuerySuiteEntry(
pack: configUtils.PackWithVersion
): string {
let text = `- qlpack: ${pack.packName}`;
if (pack.version) {
text += `${"\n"} version: ${pack.version}`;
}
return text;
}
export async function runAnalyze(
@@ -305,10 +357,8 @@ export async function runAnalyze(
fs.mkdirSync(outputDir, { recursive: true });
logger.info("Finalizing database creation");
await finalizeDatabaseCreation(config, threadsFlag, logger);
logger.info("Analyzing database");
const queriesStats = await runQueries(
outputDir,
memoryFlag,
@@ -322,6 +372,20 @@ export async function runAnalyze(
return { ...queriesStats };
}
export async function runCleanup(
config: configUtils.Config,
cleanupLevel: string,
logger: Logger
): Promise<void> {
logger.startGroup("Cleaning up databases");
for (const language of config.languages) {
const codeql = getCodeQL(config.codeQLCmd);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseCleanup(databasePath, cleanupLevel);
}
logger.endGroup();
}
async function injectLinesOfCode(
sarifFile: string,
language: Language,
@@ -357,7 +421,7 @@ function printLinesOfCodeSummary(
) {
if (language in lineCounts) {
logger.info(
`Counted ${lineCounts[language]} lines of code for ${language} as a baseline.`
`Counted a baseline of ${lineCounts[language]} lines of code for ${language}.`
);
}
}
src/api-client.test.ts
+4 -1
View File
@@ -11,10 +11,13 @@ const pkg = require("../package.json");
setupTests(test);
let pluginStub: sinon.SinonStub;
let githubStub: sinon.SinonStub;
test.beforeEach(() => {
githubStub = sinon.stub(githubUtils, "GitHub");
pluginStub = sinon.stub(githubUtils.GitHub, "plugin");
githubStub = sinon.stub();
pluginStub.returns(githubStub);
initializeEnvironment(Mode.actions, pkg.version);
});
src/api-client.ts
+3 -1
View File
@@ -1,6 +1,7 @@
import * as path from "path";
import * as githubUtils from "@actions/github/lib/utils";
import * as retry from "@octokit/plugin-retry";
import consoleLogLevel from "console-log-level";
import { getRequiredInput } from "./actions-util";
@@ -33,7 +34,8 @@ export const getApiClient = function (
) {
const auth =
(allowExternal && apiDetails.externalRepoAuth) || apiDetails.auth;
return new githubUtils.GitHub(
const retryingOctokit = githubUtils.GitHub.plugin(retry.retry);
return new retryingOctokit(
githubUtils.getOctokitOptions(auth, {
baseUrl: getApiUrl(apiDetails.url),
userAgent: `CodeQL-${getMode()}/${pkg.version}`,
src/codeql.ts
+152 -55
View File
@@ -13,6 +13,7 @@ import { v4 as uuidV4 } from "uuid";
import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
import * as api from "./api-client";
import { PackWithVersion } from "./config-utils";
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
import { errorMatchers } from "./error-matcher";
import { Language } from "./languages";
@@ -88,15 +89,33 @@ export interface CodeQL {
queries: string[],
extraSearchPath: string | undefined
): Promise<ResolveQueriesOutput>;
/**
* Run 'codeql database analyze'.
* Run 'codeql pack download'.
*/
databaseAnalyze(
packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput>;
/**
* Run 'codeql database cleanup'.
*/
databaseCleanup(databasePath: string, cleanupLevel: string): Promise<void>;
/**
* Run 'codeql database run-queries'.
*/
databaseRunQueries(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
querySuitePath: string,
memoryFlag: string,
threadsFlag: string
): Promise<void>;
/**
* Run 'codeql database interpret-results'.
*/
databaseInterpretResults(
databasePath: string,
querySuitePaths: string[],
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
automationDetailsId: string | undefined
@@ -121,6 +140,17 @@ export interface ResolveQueriesOutput {
};
}
export interface PackDownloadOutput {
packs: PackDownloadItem[];
}
interface PackDownloadItem {
name: string;
version: string;
packDir: string;
installResult: string;
}
/**
* Stores the CodeQL object, and is populated by `setupCodeQL` or `getCodeQL`.
* Can be overridden in tests using `setCodeQL`.
@@ -480,7 +510,13 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
databaseAnalyze: resolveFunction(partialCodeql, "databaseAnalyze"),
packDownload: resolveFunction(partialCodeql, "packDownload"),
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
databaseInterpretResults: resolveFunction(
partialCodeql,
"databaseInterpretResults"
),
};
return cachedCodeQL;
}
@@ -505,7 +541,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
return cmd;
},
async printVersion() {
await new toolrunner.ToolRunner(cmd, ["version", "--format=json"]).exec();
await runTool(cmd, ["version", "--format=json"]);
},
async getTracerEnv(databasePath: string) {
// Write tracer-env.js to a temp location.
@@ -546,7 +582,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
// _and_ is present in the latest supported CLI release.)
const envFile = path.resolve(databasePath, "working", "env.tmp");
await new toolrunner.ToolRunner(cmd, [
await runTool(cmd, [
"database",
"trace-command",
databasePath,
@@ -554,7 +590,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
process.execPath,
tracerEnvJs,
envFile,
]).exec();
]);
return JSON.parse(fs.readFileSync(envFile, "utf-8"));
},
async databaseInit(
@@ -562,14 +598,14 @@ function getCodeQLForCmd(cmd: string): CodeQL {
language: Language,
sourceRoot: string
) {
await new toolrunner.ToolRunner(cmd, [
await runTool(cmd, [
"database",
"init",
databasePath,
`--language=${language}`,
`--source-root=${sourceRoot}`,
...getExtraOptionsFromEnv(["database", "init"]),
]).exec();
]);
},
async runAutobuild(language: Language) {
const cmdName =
@@ -593,7 +629,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
"-Dmaven.wagon.http.pool=false",
].join(" ");
await new toolrunner.ToolRunner(autobuildCmd).exec();
await runTool(autobuildCmd);
},
async extractScannedLanguage(databasePath: string, language: Language) {
// Get extractor location
@@ -648,6 +684,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
[
"database",
"finalize",
"--finalize-dataset",
threadsFlag,
...getExtraOptionsFromEnv(["database", "finalize"]),
databasePath,
@@ -657,14 +694,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
},
async resolveLanguages() {
const codeqlArgs = ["resolve", "languages", "--format=json"];
let output = "";
await new toolrunner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
},
},
}).exec();
const output = await runTool(cmd, codeqlArgs);
try {
return JSON.parse(output);
@@ -688,14 +718,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
let output = "";
await new toolrunner.ToolRunner(cmd, codeqlArgs, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
},
},
}).exec();
const output = await runTool(cmd, codeqlArgs);
try {
return JSON.parse(output);
@@ -703,53 +726,115 @@ function getCodeQLForCmd(cmd: string): CodeQL {
throw new Error(`Unexpected output from codeql resolve queries: ${e}`);
}
},
async databaseAnalyze(
async databaseRunQueries(
databasePath: string,
sarifFile: string,
extraSearchPath: string | undefined,
querySuite: string,
querySuitePath: string,
memoryFlag: string,
addSnippetsFlag: string,
threadsFlag: string,
automationDetailsId: string | undefined
): Promise<string> {
const args = [
threadsFlag: string
): Promise<void> {
const codeqlArgs = [
"database",
"analyze",
"run-queries",
memoryFlag,
threadsFlag,
databasePath,
"--min-disk-free=1024", // Try to leave at least 1GB free
"--format=sarif-latest",
"--sarif-multicause-markdown",
`--output=${sarifFile}`,
addSnippetsFlag,
// Enable progress verbosity so we log each query as it's interpreted. This aids debugging
// when interpretation takes a while for one of the queries being analyzed.
"-v",
...getExtraOptionsFromEnv(["database", "analyze"]),
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (extraSearchPath !== undefined) {
args.push("--additional-packs", extraSearchPath);
codeqlArgs.push("--additional-packs", extraSearchPath);
}
codeqlArgs.push(querySuitePath);
await runTool(cmd, codeqlArgs);
},
async databaseInterpretResults(
databasePath: string,
querySuitePaths: string[],
sarifFile: string,
addSnippetsFlag: string,
threadsFlag: string,
automationDetailsId: string | undefined
): Promise<string> {
const codeqlArgs = [
"database",
"interpret-results",
threadsFlag,
"--format=sarif-latest",
"--print-metrics-summary",
"--sarif-group-rules-by-pack",
"-v",
`--output=${sarifFile}`,
addSnippetsFlag,
...getExtraOptionsFromEnv(["database", "interpret-results"]),
];
if (automationDetailsId !== undefined) {
args.push("--sarif-category", automationDetailsId);
codeqlArgs.push("--sarif-category", automationDetailsId);
}
args.push(querySuite);
codeqlArgs.push(databasePath, ...querySuitePaths);
// capture stdout, which contains analysis summaries
let output = "";
await new toolrunner.ToolRunner(cmd, args, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString("utf8");
},
},
}).exec();
return output;
return await runTool(cmd, codeqlArgs);
},
/**
* Download specified packs into the package cache. If the specified
* package and version already exists (e.g., from a previous analysis run),
* then it is not downloaded again (unless the extra option `--force` is
* specified).
*
* If no version is specified, then the latest version is
* downloaded. The check to determine what the latest version is is done
* each time this package is requested.
*/
async packDownload(packs: PackWithVersion[]): Promise<PackDownloadOutput> {
const codeqlArgs = [
"pack",
"download",
"--format=json",
...getExtraOptionsFromEnv(["pack", "download"]),
...packs.map(packWithVersionToString),
];
const output = await runTool(cmd, codeqlArgs);
try {
const parsedOutput: PackDownloadOutput = JSON.parse(output);
if (
Array.isArray(parsedOutput.packs) &&
// TODO PackDownloadOutput will not include the version if it is not specified
// in the input. The version is always the latest version available.
// It should be added to the output, but this requires a CLI change
parsedOutput.packs.every((p) => p.name /* && p.version */)
) {
return parsedOutput;
} else {
throw new Error("Unexpected output from pack download");
}
} catch (e) {
throw new Error(
`Attempted to download specified packs but got an error:\n${output}\n${e}`
);
}
},
async databaseCleanup(
databasePath: string,
cleanupLevel: string
): Promise<void> {
const codeqlArgs = [
"database",
"cleanup",
databasePath,
`--mode=${cleanupLevel}`,
];
await runTool(cmd, codeqlArgs);
},
};
}
function packWithVersionToString(pack: PackWithVersion): string {
return pack.version ? `${pack.packName}@${pack.version}` : pack.packName;
}
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*/
@@ -809,3 +894,15 @@ export function getExtraOptions(
);
return all.concat(specific);
}
async function runTool(cmd: string, args: string[] = []) {
let output = "";
await new toolrunner.ToolRunner(cmd, args, {
listeners: {
stdout: (data: Buffer) => {
output += data.toString();
},
},
}).exec();
return output;
}
src/config-utils.test.ts
+240 -1
View File
@@ -2,7 +2,8 @@ import * as fs from "fs";
import * as path from "path";
import * as github from "@actions/github";
import test from "ava";
import test, { ExecutionContext } from "ava";
import { clean } from "semver";
import sinon from "sinon";
import * as api from "./api-client";
@@ -318,6 +319,7 @@ test("load non-empty input", async (t) => {
codeQLCmd: codeQL.getPath(),
gitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {} as configUtils.Packs,
};
const languages = "javascript";
@@ -983,6 +985,137 @@ test("Unknown languages", async (t) => {
});
});
test("Config specifies packages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async resolveQueries() {
return {
byLanguage: {},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
const inputFileContents = `
name: my config
disable-default-queries: true
packs:
- a/b@1.2.3
`;
const configFile = path.join(tmpDir, "codeql-config.yaml");
fs.writeFileSync(configFile, inputFileContents);
const languages = "javascript";
const { packs } = await configUtils.initConfig(
languages,
undefined,
configFile,
undefined,
{ owner: "github", repo: "example " },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true)
);
t.deepEqual(packs as unknown, {
[Language.javascript]: [
{
packName: "a/b",
version: clean("1.2.3"),
},
],
});
});
});
test("Config specifies packages for multiple languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeQL = setCodeQL({
async resolveQueries() {
return {
byLanguage: {
cpp: { "/foo/a.ql": {} },
},
noDeclaredLanguage: {},
multipleDeclaredLanguages: {},
};
},
});
const inputFileContents = `
name: my config
disable-default-queries: true
queries:
- uses: ./foo
packs:
javascript:
- a/b@1.2.3
python:
- c/d@1.2.3
`;
const configFile = path.join(tmpDir, "codeql-config.yaml");
fs.writeFileSync(configFile, inputFileContents);
fs.mkdirSync(path.join(tmpDir, "foo"));
const languages = "javascript,python,cpp";
const { packs, queries } = await configUtils.initConfig(
languages,
undefined,
configFile,
undefined,
{ owner: "github", repo: "example" },
tmpDir,
tmpDir,
codeQL,
tmpDir,
gitHubVersion,
sampleApiDetails,
getRunnerLogger(true)
);
t.deepEqual(packs as unknown, {
[Language.javascript]: [
{
packName: "a/b",
version: clean("1.2.3"),
},
],
[Language.python]: [
{
packName: "c/d",
version: clean("1.2.3"),
},
],
});
t.deepEqual(queries, {
cpp: {
builtin: [],
custom: [
{
queries: ["/foo/a.ql"],
searchPath: tmpDir,
},
],
},
javascript: {
builtin: [],
custom: [],
},
python: {
builtin: [],
custom: [],
},
});
});
});
function doInvalidInputTest(
testName: string,
inputFileContents: string,
@@ -1177,3 +1310,109 @@ test("path sanitisation", (t) => {
"foo/"
);
});
/**
* Test macro for ensuring the packs block is valid
*/
function parsePacksMacro(
t: ExecutionContext<unknown>,
packsByLanguage: string[] | Record<string, string[]>,
languages: Language[],
expected
) {
t.deepEqual(
configUtils.parsePacks(packsByLanguage, languages, "/a/b"),
expected
);
}
parsePacksMacro.title = (providedTitle: string) =>
`Parse Packs: ${providedTitle}`;
/**
* Test macro for testing when the packs block is invalid
*/
function parsePacksErrorMacro(
t: ExecutionContext<unknown>,
packsByLanguage,
languages: Language[],
expected: RegExp
) {
t.throws(
() => {
configUtils.parsePacks(packsByLanguage, languages, "/a/b");
},
{
message: expected,
}
);
}
parsePacksErrorMacro.title = (providedTitle: string) =>
`Parse Packs Error: ${providedTitle}`;
function invalidPackNameMacro(t: ExecutionContext<unknown>, name: string) {
parsePacksErrorMacro(
t,
{ [Language.cpp]: [name] },
[Language.cpp],
new RegExp(
`The configuration file "/a/b" is invalid: property "packs" "${name}" is not a valid pack`
)
);
}
invalidPackNameMacro.title = (_: string, arg: string) =>
`Invalid pack string: ${arg}`;
test("no packs", parsePacksMacro, {}, [], {});
test("two packs", parsePacksMacro, ["a/b", "c/d@1.2.3"], [Language.cpp], {
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: clean("1.2.3") },
],
});
test(
"two packs with language",
parsePacksMacro,
{
[Language.cpp]: ["a/b", "c/d@1.2.3"],
[Language.java]: ["d/e", "f/g@1.2.3"],
},
[Language.cpp, Language.java, Language.csharp],
{
[Language.cpp]: [
{ packName: "a/b", version: undefined },
{ packName: "c/d", version: clean("1.2.3") },
],
[Language.java]: [
{ packName: "d/e", version: undefined },
{ packName: "f/g", version: clean("1.2.3") },
],
}
);
test(
"no language",
parsePacksErrorMacro,
["a/b@1.2.3"],
[Language.java, Language.python],
/The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/
);
test(
"invalid language",
parsePacksErrorMacro,
{ [Language.java]: ["c/d"] },
[Language.cpp],
/The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/
);
test(
"not an array",
parsePacksErrorMacro,
{ [Language.cpp]: "c/d" },
[Language.cpp],
/The configuration file "\/a\/b" is invalid: property "packs" must be an array of non-empty strings/
);
test(invalidPackNameMacro, "c"); // all packs require at least a scope and a name
test(invalidPackNameMacro, "c-/d");
test(invalidPackNameMacro, "-c/d");
test(invalidPackNameMacro, "c/d_d");
test(invalidPackNameMacro, "c/d@x");
src/config-utils.ts
+145 -8
View File
@@ -2,6 +2,7 @@ import * as fs from "fs";
import * as path from "path";
import * as yaml from "js-yaml";
import * as semver from "semver";
import * as api from "./api-client";
import { CodeQL, ResolveQueriesOutput } from "./codeql";
@@ -18,6 +19,7 @@ const QUERIES_PROPERTY = "queries";
const QUERIES_USES_PROPERTY = "uses";
const PATHS_IGNORE_PROPERTY = "paths-ignore";
const PATHS_PROPERTY = "paths";
const PACKS_PROPERTY = "packs";
/**
* Format of the config file supplied by the user.
@@ -31,6 +33,11 @@ export interface UserConfig {
}>;
"paths-ignore"?: string[];
paths?: string[];
// If this is a multi-language analysis, then the packages must be split by
// language. If this is a single language analysis, then no split by
// language is necessary.
packs?: Record<string, string[]> | string[];
}
/**
@@ -114,6 +121,19 @@ export interface Config {
* The location where CodeQL databases should be stored.
*/
dbLocation: string;
/**
* List of packages, separated by language to download before any analysis.
*/
packs: Packs;
}
export type Packs = Partial<Record<Language, PackWithVersion[]>>;
export interface PackWithVersion {
/** qualified name of a package reference */
packName: string;
/** version of the package, or undefined, which means latest version */
version?: string;
}
/**
@@ -536,6 +556,44 @@ export function getPathsInvalid(configFile: string): string {
);
}
export function getPacksRequireLanguage(
lang: string,
configFile: string
): string {
return getConfigFilePropertyError(
configFile,
PACKS_PROPERTY,
`has "${lang}", but it is not one of the languages to analyze`
);
}
export function getPacksInvalidSplit(configFile: string): string {
return getConfigFilePropertyError(
configFile,
PACKS_PROPERTY,
"must split packages by language"
);
}
export function getPacksInvalid(configFile: string): string {
return getConfigFilePropertyError(
configFile,
PACKS_PROPERTY,
"must be an array of non-empty strings"
);
}
export function getPacksStrInvalid(
packStr: string,
configFile: string
): string {
return getConfigFilePropertyError(
configFile,
PACKS_PROPERTY,
`"${packStr}" is not a valid pack`
);
}
export function getLocalPathOutsideOfRepository(
configFile: string | undefined,
localPath: string
@@ -787,6 +845,7 @@ export async function getDefaultConfig(
queries,
pathsIgnore: [],
paths: [],
packs: {} as Record<Language, PackWithVersion[]>,
originalUserInput: {},
tempDir,
toolCacheDir,
@@ -883,7 +942,7 @@ async function loadConfig(
shouldAddConfigFileQueries(queriesInput) &&
QUERIES_PROPERTY in parsedYAML
) {
if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
if (!Array.isArray(parsedYAML[QUERIES_PROPERTY])) {
throw new Error(getQueriesInvalid(configFile));
}
for (const query of parsedYAML[QUERIES_PROPERTY]!) {
@@ -908,7 +967,7 @@ async function loadConfig(
}
if (PATHS_IGNORE_PROPERTY in parsedYAML) {
if (!(parsedYAML[PATHS_IGNORE_PROPERTY] instanceof Array)) {
if (!Array.isArray(parsedYAML[PATHS_IGNORE_PROPERTY])) {
throw new Error(getPathsIgnoreInvalid(configFile));
}
for (const ignorePath of parsedYAML[PATHS_IGNORE_PROPERTY]!) {
@@ -927,7 +986,7 @@ async function loadConfig(
}
if (PATHS_PROPERTY in parsedYAML) {
if (!(parsedYAML[PATHS_PROPERTY] instanceof Array)) {
if (!Array.isArray(parsedYAML[PATHS_PROPERTY])) {
throw new Error(getPathsInvalid(configFile));
}
for (const includePath of parsedYAML[PATHS_PROPERTY]!) {
@@ -940,11 +999,18 @@ async function loadConfig(
}
}
const packs = parsePacks(
parsedYAML[PACKS_PROPERTY] ?? {},
languages,
configFile
);
return {
languages,
queries,
pathsIgnore,
paths,
packs,
originalUserInput: parsedYAML,
tempDir,
toolCacheDir,
@@ -954,6 +1020,78 @@ async function loadConfig(
};
}
/**
* Pack names must be in the form of `scope/name`, with only alpha-numeric characters,
* and `-` allowed as long as not the first or last char.
**/
const PACK_IDENTIFIER_PATTERN = (function () {
const alphaNumeric = "[a-z0-9]";
const alphaNumericDash = "[a-z0-9-]";
const component = `${alphaNumeric}(${alphaNumericDash}*${alphaNumeric})?`;
return new RegExp(`^${component}/${component}$`);
})();
// Exported for testing
export function parsePacks(
packsByLanguage: string[] | Record<string, string[]>,
languages: Language[],
configFile: string
): Packs {
const packs = {};
if (Array.isArray(packsByLanguage)) {
if (languages.length === 1) {
// single language analysis, so language is implicit
packsByLanguage = {
[languages[0]]: packsByLanguage,
};
} else {
// this is an error since multi-language analysis requires
// packs split by language
throw new Error(getPacksInvalidSplit(configFile));
}
}
for (const [lang, packsArr] of Object.entries(packsByLanguage)) {
if (!Array.isArray(packsArr)) {
throw new Error(getPacksInvalid(configFile));
}
if (!languages.includes(lang as Language)) {
throw new Error(getPacksRequireLanguage(lang, configFile));
}
packs[lang] = [];
for (const packStr of packsArr) {
packs[lang].push(toPackWithVersion(packStr, configFile));
}
}
return packs;
}
function toPackWithVersion(packStr, configFile: string): PackWithVersion {
if (typeof packStr !== "string") {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
const nameWithVersion = packStr.split("@");
let version: string | undefined;
if (
nameWithVersion.length > 2 ||
!PACK_IDENTIFIER_PATTERN.test(nameWithVersion[0])
) {
throw new Error(getPacksStrInvalid(packStr, configFile));
} else if (nameWithVersion.length === 2) {
version = semver.clean(nameWithVersion[1]) || undefined;
if (!version) {
throw new Error(getPacksStrInvalid(packStr, configFile));
}
}
return {
packName: nameWithVersion[0],
version,
};
}
function dbLocationOrDefault(
dbLocation: string | undefined,
tempDir: string
@@ -1019,11 +1157,10 @@ export async function initConfig(
// The list of queries should not be empty for any language. If it is then
// it is a user configuration error.
for (const language of config.languages) {
if (
config.queries[language] === undefined ||
(config.queries[language].builtin.length === 0 &&
config.queries[language].custom.length === 0)
) {
const hasBuiltinQueries = config.queries[language]?.builtin.length > 0;
const hasCustomQueries = config.queries[language]?.custom.length > 0;
const hasPacks = (config.packs[language]?.length || 0) > 0;
if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
throw new Error(
`Did not detect any queries to run for ${language}. ` +
"Please make sure that the default queries are enabled, or you are specifying queries to run."
src/count-loc.test.ts
+3 -3
View File
@@ -20,7 +20,7 @@ test("ensure lines of code works for cpp and js", async (t) => {
t.deepEqual(results, {
cpp: 6,
javascript: 3,
javascript: 9,
});
});
@@ -34,7 +34,7 @@ test("ensure lines of code can handle undefined language", async (t) => {
);
t.deepEqual(results, {
javascript: 3,
javascript: 9,
python: 5,
});
});
@@ -93,6 +93,6 @@ test("ensure lines of code can handle exclude", async (t) => {
);
t.deepEqual(results, {
javascript: 3,
javascript: 9,
});
});
src/fingerprints.test.ts
+65 -40
View File
@@ -7,32 +7,41 @@ import test from "ava";
import * as fingerprints from "./fingerprints";
import { getRunnerLogger } from "./logging";
import { setupTests } from "./testing-utils";
import * as util from "./util";
setupTests(test);
function testHash(t: ava.Assertions, input: string, expectedHashes: string[]) {
let index = 0;
const callback = function (lineNumber: number, hash: string) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
fingerprints.hash(callback, input);
t.is(index, input.split(/\r\n|\r|\n/).length);
async function testHash(
t: ava.Assertions,
input: string,
expectedHashes: string[]
) {
await util.withTmpDir(async (tmpDir) => {
const tmpFile = path.resolve(tmpDir, "testfile");
fs.writeFileSync(tmpFile, input);
let index = 0;
const callback = function (lineNumber: number, hash: string) {
t.is(lineNumber, index + 1);
t.is(hash, expectedHashes[index]);
index++;
};
await fingerprints.hash(callback, tmpFile);
t.is(index, input.split(/\r\n|\r|\n/).length);
});
}
test("hash", (t: ava.Assertions) => {
test("hash", async (t: ava.Assertions) => {
// Try empty file
testHash(t, "", ["c129715d7a2bc9a3:1"]);
await testHash(t, "", ["c129715d7a2bc9a3:1"]);
// Try various combinations of newline characters
testHash(t, " a\nb\n \t\tc\n d", [
await testHash(t, " a\nb\n \t\tc\n d", [
"271789c17abda88f:1",
"54703d4cd895b18:1",
"180aee12dab6264:1",
"a23a3dc5e078b07b:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End", [
"8b7cf3e952e7aeb2:1",
"b1ae1287ec4718d9:1",
"bff680108adb0fcc:1",
@@ -40,7 +49,7 @@ test("hash", (t: ava.Assertions) => {
"b86d3392aea1be30:1",
"e6ceba753e1a442:1",
]);
testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
await testHash(t, " hello; \t\nworld!!!\n\n\n \t\tGreetings\n End\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@@ -49,7 +58,7 @@ test("hash", (t: ava.Assertions) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
await testHash(t, " hello; \t\nworld!!!\r\r\r \t\tGreetings\r End\r", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@@ -58,16 +67,20 @@ test("hash", (t: ava.Assertions) => {
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]);
testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
await testHash(
t,
" hello; \t\r\nworld!!!\r\n\r\n\r\n \t\tGreetings\r\n End\r\n",
[
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
"e20e36e16fcb0cc8:1",
"b3edc88f2938467e:1",
"c8e28b0b4002a3a0:1",
"c129715d7a2bc9a3:1",
]
);
await testHash(t, " hello; \t\nworld!!!\r\n\n\r \t\tGreetings\r End\r\n", [
"e9496ae3ebfced30:1",
"fb7c023a8b9ccb3f:1",
"ce8ba1a563dcdaca:1",
@@ -78,7 +91,7 @@ test("hash", (t: ava.Assertions) => {
]);
// Try repeating line that will generate identical hashes
testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
await testHash(t, "Lorem ipsum dolor sit amet.\n".repeat(10), [
"a7f2ff13bc495cf2:1",
"a7f2ff13bc495cf2:2",
"a7f2ff13bc495cf2:3",
@@ -92,16 +105,20 @@ test("hash", (t: ava.Assertions) => {
"c129715d7a2bc9a3:1",
]);
testHash(t, "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n", [
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
"5c482a0f8b35ea28:1",
"54517377da7028d2:1",
"2c644846cb18d53e:1",
"f1b89f20de0d133:1",
"c129715d7a2bc9a3:1",
]);
await testHash(
t,
"x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n",
[
"e54938cc54b302f1:1",
"bb609acbe9138d60:1",
"1131fd5871777f34:1",
"5c482a0f8b35ea28:1",
"54517377da7028d2:1",
"2c644846cb18d53e:1",
"f1b89f20de0d133:1",
"c129715d7a2bc9a3:1",
]
);
});
function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
@@ -170,7 +187,7 @@ test("resolveUriToFile", (t) => {
t.is(testResolveUriToFile(`file://${dirpath}`, undefined, []), undefined);
});
test("addFingerprints", (t) => {
test("addFingerprints", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting.input.sarif`)
@@ -187,12 +204,16 @@ test("addFingerprints", (t) => {
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(
fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
await fingerprints.addFingerprints(
input,
checkoutPath,
getRunnerLogger(true)
),
expected
);
});
test("missingRegions", (t) => {
test("missingRegions", async (t) => {
// Run an end-to-end test on a test file
let input = fs
.readFileSync(`${__dirname}/../src/testdata/fingerprinting2.input.sarif`)
@@ -209,7 +230,11 @@ test("missingRegions", (t) => {
const checkoutPath = path.normalize(`${__dirname}/../src/testdata`);
t.deepEqual(
fingerprints.addFingerprints(input, checkoutPath, getRunnerLogger(true)),
await fingerprints.addFingerprints(
input,
checkoutPath,
getRunnerLogger(true)
),
expected
);
});
src/fingerprints.ts
+21 -9
View File
@@ -8,6 +8,7 @@ const tab = "\t".charCodeAt(0);
const space = " ".charCodeAt(0);
const lf = "\n".charCodeAt(0);
const cr = "\r".charCodeAt(0);
const EOF = 65535;
const BLOCK_SIZE = 100;
const MOD = Long.fromInt(37); // L
@@ -34,9 +35,9 @@ type hashCallback = (lineNumber: number, hash: string) => void;
* the hashes of the lines near the end of the file.
*
* @param callback function that is called with the line number (1-based) and hash for every line
* @param input The file's contents
* @param filepath The path to the file to hash
*/
export function hash(callback: hashCallback, input: string) {
export async function hash(callback: hashCallback, filepath: string) {
// A rolling view in to the input
const window = Array(BLOCK_SIZE).fill(0);
@@ -87,12 +88,11 @@ export function hash(callback: hashCallback, input: string) {
// as we go. Once we reach a point in the window again then we've processed
// BLOCK_SIZE characters and if the last character at this point in the window
// was the start of a line then we should output the hash for that line.
for (let i = 0, len = input.length; i <= len; i++) {
let current = i === len ? 65535 : input.charCodeAt(i);
const processCharacter = function (current: number) {
// skip tabs, spaces, and line feeds that come directly after a carriage return
if (current === space || current === tab || (prevCR && current === lf)) {
prevCR = false;
continue;
return;
}
// replace CR with LF
if (current === cr) {
@@ -113,7 +113,15 @@ export function hash(callback: hashCallback, input: string) {
lineStart = true;
}
updateHash(current);
};
const readStream = fs.createReadStream(filepath, "utf8");
for await (const data of readStream) {
for (let i = 0; i < data.length; ++i) {
processCharacter(data.charCodeAt(i));
}
}
processCharacter(EOF);
// Flush the remaining lines
for (let i = 0; i < BLOCK_SIZE; i++) {
@@ -237,11 +245,11 @@ export function resolveUriToFile(
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
export function addFingerprints(
export async function addFingerprints(
sarifContents: string,
checkoutPath: string,
logger: Logger
): string {
): Promise<string> {
const sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
@@ -263,6 +271,11 @@ export function addFingerprints(
continue;
}
if (primaryLocation?.physicalLocation?.region?.startLine === undefined) {
// Locations without a line number are unlikely to be source files
continue;
}
const filepath = resolveUriToFile(
primaryLocation.physicalLocation.artifactLocation,
artifacts,
@@ -289,8 +302,7 @@ export function addFingerprints(
c(lineNumber, hashValue);
}
};
const fileContents = fs.readFileSync(filepath).toString();
hash(teeCallback, fileContents);
await hash(teeCallback, filepath);
}
return JSON.stringify(sarif);
src/testdata/fingerprinting2.expected.sarif
Vendored
+1 -3
View File
@@ -30,9 +30,7 @@
"message": {
"text": "This header file should contain a header guard to prevent multiple inclusion."
},
"partialFingerprints": {
"primaryLocationLineHash": "599c824c91d0f75e:1"
},
"partialFingerprints": {},
"ruleId": "cpp/missing-header-guard",
"ruleIndex": 0
}
src/tracer-config.test.ts
+1
View File
@@ -28,6 +28,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
codeQLCmd: "",
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
packs: {} as configUtils.Packs,
};
}
src/upload-lib.ts
+1 -1
View File
@@ -356,7 +356,7 @@ async function uploadFiles(
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(
sarifPayload = await fingerprints.addFingerprints(
sarifPayload,
checkoutPath,
logger
tests/multi-language-repo/.github/codeql/codeql-config-packaging.yml
Vendored
+10
View File
@@ -0,0 +1,10 @@
name: Pack testing in the CodeQL Action
disable-default-queries: true
packs:
javascript:
- dsp-testing/codeql-pack1@0.0.3
- dsp-testing/codeql-pack2 # latest
paths-ignore:
- tests
- lib
tests/multi-language-repo/main.js
+9
View File
@@ -1,3 +1,12 @@
if (true) {
console.log("Hello, World!");
console.log("Good-bye, World!");
}
if (true) {
console.log("Hello, World!");
}
if (true) {
// empty
}