mirror of
https://github.com/github/codeql-action.git
synced 2026-05-09 15:20:28 +00:00
Compare commits
216 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f56d0ec51b | |||
| 33edb83959 | |||
| 33276bfbdb | |||
| 16adc4e672 | |||
| 2a607fea25 | |||
| e51b6a9a52 | |||
| 160d27baf0 | |||
| 28737ec792 | |||
| e5f9d3b55e | |||
| dc00a6f08f | |||
| ab56c02e0c | |||
| 25bde03dfb | |||
| c4dca28336 | |||
| 1aad2787ec | |||
| b6cf67a711 | |||
| f59338d600 | |||
| 2a07b6e3c7 | |||
| fba33f686a | |||
| 48094d2b6e | |||
| cb4e075f11 | |||
| 1847416575 | |||
| 11dd746d70 | |||
| a754a57c21 | |||
| 466da5ec2d | |||
| 0a9b98b511 | |||
| bce7dc4616 | |||
| b13ab62bc0 | |||
| 4ea06e96f5 | |||
| c9223eb0a0 | |||
| f0767c48a1 | |||
| 4e71011f44 | |||
| 710e294578 | |||
| b948539dd4 | |||
| c54531587d | |||
| 559d85d1fa | |||
| 8e010557a9 | |||
| 37d6d1ca27 | |||
| 68b53dc641 | |||
| 89a39a4e59 | |||
| e5d84c885c | |||
| 0c202097b5 | |||
| 314172e5a1 | |||
| cdda72d36b | |||
| cfda84cc55 | |||
| 39ba80c475 | |||
| 00150dad95 | |||
| d97dce6561 | |||
| 50fdbb9ec8 | |||
| f7905e8415 | |||
| 4191f52110 | |||
| 79a913656c | |||
| 167b47e60c | |||
| 5e7a52feb2 | |||
| 76cf404c99 | |||
| 7407d38386 | |||
| 015d8c7cbc | |||
| 09bd46dda5 | |||
| b927a69f96 | |||
| 61f7dd3d0d | |||
| 64300e453b | |||
| 906dd890a5 | |||
| 898ae16413 | |||
| fa56ea8dc0 | |||
| 657f337cd1 | |||
| 05d4e25296 | |||
| 5c583bbb19 | |||
| 554b93127b | |||
| 3dd1275368 | |||
| d24014a749 | |||
| cc0dce044b | |||
| ef58c00dfe | |||
| 7b7a951e08 | |||
| 0c47ae1c18 | |||
| 6c405c2562 | |||
| 827bba691f | |||
| 96961e0ee3 | |||
| ebad062f08 | |||
| e275d63e1d | |||
| 69c2819972 | |||
| d28d9967fe | |||
| d1bdc0ea05 | |||
| b1b1e44da9 | |||
| 46473e05b7 | |||
| 32ab108bfd | |||
| 971592501c | |||
| 2abec3f0c3 | |||
| 6d55dfff02 | |||
| 5c96b6e3db | |||
| 44a4bea367 | |||
| 11c6c18818 | |||
| 99fcc7b2a1 | |||
| c1d6ee5477 | |||
| ef9cfd91a8 | |||
| 4250b466b2 | |||
| a3d7d36aa6 | |||
| 33e2dff082 | |||
| bff89dcba4 | |||
| d6ea6709b9 | |||
| f315d82bd7 | |||
| ebce69a4b7 | |||
| ab2580041c | |||
| d1689c9307 | |||
| 147d1495e4 | |||
| 3e37216660 | |||
| ad5a6c0147 | |||
| aee29a19d7 | |||
| ac74c2835a | |||
| f8c75d3f32 | |||
| e315c6fd3b | |||
| e6a312a771 | |||
| 73f5a29960 | |||
| 8b734d3bc2 | |||
| e21e4ca93f | |||
| 595ce2dc3e | |||
| a61e3cb9f2 | |||
| d5f0374a1f | |||
| 466a4f00eb | |||
| 817d568ca0 | |||
| 34d43db4c6 | |||
| db834c9e1d | |||
| 7af50a43c1 | |||
| 60dee3dbd3 | |||
| 0874cf9f8b | |||
| bc76ceafaf | |||
| 377300bcda | |||
| ee8360df59 | |||
| 9dcfdf2c9c | |||
| 2c9bc45d46 | |||
| 368f322a09 | |||
| 5283c3ba5a | |||
| ea1a400e13 | |||
| 248d7971c2 | |||
| 64940fad4a | |||
| ef618feace | |||
| 6bddc7956d | |||
| 01fcdceb89 | |||
| 9e907b5e64 | |||
| 1814c9fbfd | |||
| 4bf6fa4e2d | |||
| 9658e23e5b | |||
| e1933c66bd | |||
| edf36092cf | |||
| 15a3d32df0 | |||
| 9835994414 | |||
| 0ce6420f8e | |||
| be75dd92ea | |||
| 05bca54402 | |||
| 2d6b98c7cf | |||
| 876cecb383 | |||
| 43b46a19be | |||
| 8ad4b6ec58 | |||
| 4edc7d2e82 | |||
| 2adcb6464e | |||
| da67096c6f | |||
| c48cd247df | |||
| 0cfcceb4b8 | |||
| cbb92e7ff6 | |||
| db9346285d | |||
| 2de76b6faa | |||
| 6a17f4e258 | |||
| 8cc4d2539b | |||
| 406bbfcef1 | |||
| 5132eb53f2 | |||
| 5b3261bcbf | |||
| 9267d8d51e | |||
| bc1164e014 | |||
| 7801eda177 | |||
| b1d963ed8f | |||
| d636fb3f63 | |||
| d155ebf27f | |||
| e8f0116911 | |||
| 713a293090 | |||
| ff33514494 | |||
| efb92e2714 | |||
| d73644591f | |||
| 41d2cc39b6 | |||
| be578c7735 | |||
| fa6e24cf12 | |||
| 2b5b614c85 | |||
| 555ee17b0b | |||
| e114998dda | |||
| bd36637537 | |||
| 4d0bec12bf | |||
| 0387f55b70 | |||
| 27b3b6586d | |||
| c4b0f60beb | |||
| 51357000d2 | |||
| 4d44b570d2 | |||
| 700fc11b44 | |||
| 9f2f6d0d2e | |||
| 01ee641f14 | |||
| c7eff3f0b1 | |||
| c4717c9c74 | |||
| b030333651 | |||
| 70eae154c6 | |||
| 93302bc63a | |||
| 310177a1fb | |||
| b13d724d35 | |||
| 4b8e16f54f | |||
| 481be99883 | |||
| 9b3a0d2c26 | |||
| d2901f5537 | |||
| 46c411a7f4 | |||
| 5a82333186 | |||
| 45cbd0c69e | |||
| cb528be87e | |||
| 7aee932974 | |||
| b5f028a984 | |||
| 9702c27ab9 | |||
| c36c94846f | |||
| 3d0331896c | |||
| 77591e2c4a | |||
| 7a44a9db3f | |||
| e2ac371513 | |||
| 4f6ea84c21 | |||
| 73dbc8364d |
@@ -23,13 +23,13 @@ For internal use only. Please select the risk level of this change:
|
||||
Workflow types:
|
||||
|
||||
- **Advanced setup** - Impacts users who have custom CodeQL workflows.
|
||||
- **Managed** - Impacts users with `dynamic` workflows (Default Setup, CCR, ...).
|
||||
- **Managed** - Impacts users with `dynamic` workflows (Default Setup, Code Quality, ...).
|
||||
|
||||
Products:
|
||||
|
||||
- **Code Scanning** - The changes impact analyses when `analysis-kinds: code-scanning`.
|
||||
- **Code Quality** - The changes impact analyses when `analysis-kinds: code-quality`.
|
||||
- **CCR** - The changes impact analyses for Copilot Code Reviews.
|
||||
- **Other first-party** - The changes impact other first-party analyses.
|
||||
- **Third-party analyses** - The changes affect the `upload-sarif` action.
|
||||
|
||||
Environments:
|
||||
@@ -54,6 +54,7 @@ Environments:
|
||||
|
||||
- **Feature flags** - All new or changed code paths can be fully disabled with corresponding feature flags.
|
||||
- **Rollback** - Change can only be disabled by rolling back the release or releasing a new version with a fix.
|
||||
- **Development/testing only** - This change cannot cause any failures in production.
|
||||
- **Other** - Please provide details.
|
||||
|
||||
#### How will you know if something goes wrong after this change is released?
|
||||
|
||||
@@ -71,8 +71,9 @@ def open_pr(
|
||||
body.append('')
|
||||
body.append('Contains the following pull requests:')
|
||||
for pr in pull_requests:
|
||||
merger = get_merger_of_pr(repo, pr)
|
||||
body.append(f'- #{pr.number} (@{merger})')
|
||||
# Use PR author if they are GitHub staff, otherwise use the merger
|
||||
display_user = get_pr_author_if_staff(pr) or get_merger_of_pr(repo, pr)
|
||||
body.append(f'- #{pr.number} (@{display_user})')
|
||||
|
||||
# List all commits not part of a PR
|
||||
if len(commits_without_pull_requests) > 0:
|
||||
@@ -168,6 +169,14 @@ def get_pr_for_commit(commit):
|
||||
def get_merger_of_pr(repo, pr):
|
||||
return repo.get_commit(pr.merge_commit_sha).author.login
|
||||
|
||||
# Get the PR author if they are GitHub staff, otherwise None.
|
||||
def get_pr_author_if_staff(pr):
|
||||
if pr.user is None:
|
||||
return None
|
||||
if getattr(pr.user, 'site_admin', False):
|
||||
return pr.user.login
|
||||
return None
|
||||
|
||||
def get_current_version():
|
||||
with open('package.json', 'r') as f:
|
||||
return json.load(f)['version']
|
||||
@@ -181,9 +190,9 @@ def replace_version_package_json(prev_version, new_version):
|
||||
print(line.replace(prev_version, new_version), end='')
|
||||
else:
|
||||
prev_line_is_codeql = False
|
||||
print(line, end='')
|
||||
print(line, end='')
|
||||
if '\"name\": \"codeql\",' in line:
|
||||
prev_line_is_codeql = True
|
||||
prev_line_is_codeql = True
|
||||
|
||||
def get_today_string():
|
||||
today = datetime.datetime.today()
|
||||
|
||||
+18
-17
@@ -3,7 +3,7 @@
|
||||
# pr-checks/sync.sh
|
||||
# to regenerate this file.
|
||||
|
||||
name: PR Check - Quality queries input
|
||||
name: PR Check - Analysis kinds
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GO111MODULE: auto
|
||||
@@ -29,9 +29,9 @@ defaults:
|
||||
shell: bash
|
||||
concurrency:
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || false }}
|
||||
group: quality-queries-${{github.ref}}
|
||||
group: analysis-kinds-${{github.ref}}
|
||||
jobs:
|
||||
quality-queries:
|
||||
analysis-kinds:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -45,6 +45,9 @@ jobs:
|
||||
- os: ubuntu-latest
|
||||
version: linked
|
||||
analysis-kinds: code-scanning,code-quality
|
||||
- os: ubuntu-latest
|
||||
version: linked
|
||||
analysis-kinds: risk-assessment
|
||||
- os: ubuntu-latest
|
||||
version: nightly-latest
|
||||
analysis-kinds: code-scanning
|
||||
@@ -54,7 +57,10 @@ jobs:
|
||||
- os: ubuntu-latest
|
||||
version: nightly-latest
|
||||
analysis-kinds: code-scanning,code-quality
|
||||
name: Quality queries input
|
||||
- os: ubuntu-latest
|
||||
version: nightly-latest
|
||||
analysis-kinds: risk-assessment
|
||||
name: Analysis kinds
|
||||
if: github.triggering_actor != 'dependabot[bot]'
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -81,30 +87,24 @@ jobs:
|
||||
output: ${{ runner.temp }}/results
|
||||
upload-database: false
|
||||
post-processed-sarif-path: ${{ runner.temp }}/post-processed
|
||||
- name: Upload security SARIF
|
||||
if: contains(matrix.analysis-kinds, 'code-scanning')
|
||||
|
||||
- name: Upload SARIF files
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: |
|
||||
quality-queries-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}.sarif.json
|
||||
path: ${{ runner.temp }}/results/javascript.sarif
|
||||
retention-days: 7
|
||||
- name: Upload quality SARIF
|
||||
if: contains(matrix.analysis-kinds, 'code-quality')
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: |
|
||||
quality-queries-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}.quality.sarif.json
|
||||
path: ${{ runner.temp }}/results/javascript.quality.sarif
|
||||
analysis-kinds-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}
|
||||
path: ${{ runner.temp }}/results/*.sarif
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload post-processed SARIF
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: |
|
||||
post-processed-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}.sarif.json
|
||||
post-processed-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}
|
||||
path: ${{ runner.temp }}/post-processed
|
||||
retention-days: 7
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Check quality query does not appear in security SARIF
|
||||
if: contains(matrix.analysis-kinds, 'code-scanning')
|
||||
uses: actions/github-script@v8
|
||||
@@ -122,6 +122,7 @@ jobs:
|
||||
with:
|
||||
script: ${{ env.CHECK_SCRIPT }}
|
||||
env:
|
||||
CODEQL_ACTION_RISK_ASSESSMENT_ID: 1
|
||||
CHECK_SCRIPT: |
|
||||
const fs = require('fs');
|
||||
|
||||
+12
-30
@@ -3,7 +3,7 @@
|
||||
# pr-checks/sync.sh
|
||||
# to regenerate this file.
|
||||
|
||||
name: PR Check - CCR
|
||||
name: 'PR Check - Bundle: From nightly'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GO111MODULE: auto
|
||||
@@ -29,32 +29,16 @@ defaults:
|
||||
shell: bash
|
||||
concurrency:
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' || false }}
|
||||
group: ccr-${{github.ref}}
|
||||
group: bundle-from-nightly-${{github.ref}}
|
||||
jobs:
|
||||
ccr:
|
||||
bundle-from-nightly:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
version: stable-v2.17.6
|
||||
- os: ubuntu-latest
|
||||
version: stable-v2.18.4
|
||||
- os: ubuntu-latest
|
||||
version: stable-v2.19.4
|
||||
- os: ubuntu-latest
|
||||
version: stable-v2.20.7
|
||||
- os: ubuntu-latest
|
||||
version: stable-v2.21.4
|
||||
- os: ubuntu-latest
|
||||
version: stable-v2.22.4
|
||||
- os: ubuntu-latest
|
||||
version: default
|
||||
- os: ubuntu-latest
|
||||
version: linked
|
||||
- os: ubuntu-latest
|
||||
version: nightly-latest
|
||||
name: CCR
|
||||
name: 'Bundle: From nightly'
|
||||
if: github.triggering_actor != 'dependabot[bot]'
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -71,17 +55,15 @@ jobs:
|
||||
version: ${{ matrix.version }}
|
||||
use-all-platform-bundle: 'false'
|
||||
setup-kotlin: 'true'
|
||||
- uses: ./../action/init
|
||||
id: init
|
||||
- id: init
|
||||
uses: ./../action/init
|
||||
env:
|
||||
CODEQL_ACTION_FORCE_NIGHTLY: true
|
||||
with:
|
||||
languages: javascript
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
|
||||
- uses: ./../action/analyze
|
||||
id: analysis
|
||||
with:
|
||||
upload-database: false
|
||||
|
||||
languages: javascript
|
||||
- name: Fail if the CodeQL version is not a nightly
|
||||
if: "!contains(steps.init.outputs.codeql-version, '+')"
|
||||
run: exit 1
|
||||
env:
|
||||
CODEQL_ACTION_ANALYSIS_KEY: dynamic/copilot-pull-request-reviewer/codeql-action-test
|
||||
CODEQL_ACTION_TEST_MODE: true
|
||||
+1
-1
@@ -56,7 +56,7 @@ jobs:
|
||||
use-all-platform-bundle: 'false'
|
||||
setup-kotlin: 'true'
|
||||
- name: Set up Ruby
|
||||
uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0
|
||||
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
||||
with:
|
||||
ruby-version: 2.6
|
||||
- name: Install Code Scanning integration
|
||||
|
||||
@@ -17,6 +17,7 @@ jobs:
|
||||
sizeup:
|
||||
name: Label PR with size
|
||||
runs-on: ubuntu-slim
|
||||
if: github.event.pull_request.merged != true
|
||||
|
||||
steps:
|
||||
- name: Run sizeup
|
||||
|
||||
@@ -111,7 +111,7 @@ jobs:
|
||||
# Otherwise, just commit the changes.
|
||||
if git rev-parse --verify MERGE_HEAD >/dev/null 2>&1; then
|
||||
echo "In progress merge detected, finishing it up."
|
||||
git merge --continue
|
||||
git merge --continue --no-edit
|
||||
else
|
||||
echo "No in-progress merge detected, committing changes."
|
||||
git commit -m "Rebuild"
|
||||
|
||||
+17
-1
@@ -4,7 +4,23 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
|
||||
|
||||
## [UNRELEASED]
|
||||
|
||||
- The experimental `quality-queries` input that was deprecated in CodeQL Action 3.30.2 has now been removed.
|
||||
No user facing changes.
|
||||
|
||||
## 4.32.4 - 20 Feb 2026
|
||||
|
||||
- Update default CodeQL bundle version to [2.24.2](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.24.2). [#3493](https://github.com/github/codeql-action/pull/3493)
|
||||
- Added an experimental change which improves how certificates are generated for the authentication proxy that is used by the CodeQL Action in Default Setup when [private package registries are configured](https://docs.github.com/en/code-security/how-tos/secure-at-scale/configure-organization-security/manage-usage-and-access/giving-org-access-private-registries). This is expected to generate more widely compatible certificates and should have no impact on analyses which are working correctly already. We expect to roll this change out to everyone in February. [#3473](https://github.com/github/codeql-action/pull/3473)
|
||||
- When the CodeQL Action is run [with debugging enabled in Default Setup](https://docs.github.com/en/code-security/how-tos/scan-code-for-vulnerabilities/troubleshooting/troubleshooting-analysis-errors/logs-not-detailed-enough#creating-codeql-debugging-artifacts-for-codeql-default-setup) and [private package registries are configured](https://docs.github.com/en/code-security/how-tos/secure-at-scale/configure-organization-security/manage-usage-and-access/giving-org-access-private-registries), the "Setup proxy for registries" step will output additional diagnostic information that can be used for troubleshooting. [#3486](https://github.com/github/codeql-action/pull/3486)
|
||||
- Added a setting which allows the CodeQL Action to enable network debugging for Java programs. This will help GitHub staff support customers with troubleshooting issues in GitHub-managed CodeQL workflows, such as Default Setup. This setting can only be enabled by GitHub staff. [#3485](https://github.com/github/codeql-action/pull/3485)
|
||||
- Added a setting which enables GitHub-managed workflows, such as Default Setup, to use a [nightly CodeQL CLI release](https://github.com/dsp-testing/codeql-cli-nightlies) instead of the latest, stable release that is used by default. This will help GitHub staff support customers whose analyses for a given repository or organization require early access to a change in an upcoming CodeQL CLI release. This setting can only be enabled by GitHub staff. [#3484](https://github.com/github/codeql-action/pull/3484)
|
||||
|
||||
## 4.32.3 - 13 Feb 2026
|
||||
|
||||
- Added experimental support for testing connections to [private package registries](https://docs.github.com/en/code-security/how-tos/secure-at-scale/configure-organization-security/manage-usage-and-access/giving-org-access-private-registries). This feature is not currently enabled for any analysis. In the future, it may be enabled by default for Default Setup. [#3466](https://github.com/github/codeql-action/pull/3466)
|
||||
|
||||
## 4.32.2 - 05 Feb 2026
|
||||
|
||||
- Update default CodeQL bundle version to [2.24.1](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.24.1). [#3460](https://github.com/github/codeql-action/pull/3460)
|
||||
|
||||
## 4.32.1 - 02 Feb 2026
|
||||
|
||||
|
||||
@@ -80,6 +80,12 @@ We typically release new minor versions of the CodeQL Action and Bundle when a n
|
||||
|
||||
See the full list of GHES release and deprecation dates at [GitHub Enterprise Server releases](https://docs.github.com/en/enterprise-server/admin/all-releases#releases-of-github-enterprise-server).
|
||||
|
||||
## Keeping the CodeQL Action up to date in advanced setups
|
||||
|
||||
If you are using an [advanced setup](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/configuring-advanced-setup-for-code-scanning), we recommend referencing the CodeQL Action using a major version tag (e.g. `v4`) in your workflow file. This ensures your workflow automatically picks up the latest release within that major version, including bug fixes, new features, and updated CodeQL CLI versions.
|
||||
|
||||
If you pin to a specific commit SHA or patch version tag, ensure you keep it updated (e.g. via [Dependabot](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot)). Some CodeQL Action features are enabled by server-side flags that may be removed over time, which can cause old versions to lose functionality.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
Read about [troubleshooting code scanning](https://docs.github.com/en/code-security/code-scanning/troubleshooting-code-scanning).
|
||||
|
||||
+35
-37
@@ -1,27 +1,14 @@
|
||||
// Automatically generated by running npx @eslint/migrate-config .eslintrc.json
|
||||
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { fixupConfigRules, fixupPluginRules } from "@eslint/compat";
|
||||
import { FlatCompat } from "@eslint/eslintrc";
|
||||
import { fixupPluginRules } from "@eslint/compat";
|
||||
import js from "@eslint/js";
|
||||
import typescriptEslint from "@typescript-eslint/eslint-plugin";
|
||||
import tsParser from "@typescript-eslint/parser";
|
||||
import filenames from "eslint-plugin-filenames";
|
||||
import github from "eslint-plugin-github";
|
||||
import _import from "eslint-plugin-import";
|
||||
import { importX, createNodeResolver } from "eslint-plugin-import-x";
|
||||
import { createTypeScriptImportResolver } from "eslint-import-resolver-typescript";
|
||||
import noAsyncForeach from "eslint-plugin-no-async-foreach";
|
||||
import jsdoc from "eslint-plugin-jsdoc";
|
||||
import tseslint from "typescript-eslint";
|
||||
import globals from "globals";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const compat = new FlatCompat({
|
||||
baseDirectory: __dirname,
|
||||
recommendedConfig: js.configs.recommended,
|
||||
allConfig: js.configs.all,
|
||||
});
|
||||
const githubFlatConfigs = github.getFlatConfigs();
|
||||
|
||||
export default [
|
||||
{
|
||||
@@ -36,29 +23,29 @@ export default [
|
||||
".github/**/*",
|
||||
],
|
||||
},
|
||||
...fixupConfigRules(
|
||||
compat.extends(
|
||||
"eslint:recommended",
|
||||
"plugin:@typescript-eslint/recommended",
|
||||
"plugin:@typescript-eslint/recommended-requiring-type-checking",
|
||||
"plugin:github/recommended",
|
||||
"plugin:github/typescript",
|
||||
"plugin:import/typescript",
|
||||
),
|
||||
),
|
||||
// eslint recommended config
|
||||
js.configs.recommended,
|
||||
// Type-checked rules from typescript-eslint
|
||||
...tseslint.configs.recommendedTypeChecked,
|
||||
...tseslint.configs.strict,
|
||||
// eslint-plugin-github recommended config
|
||||
githubFlatConfigs.recommended,
|
||||
// eslint-plugin-github typescript config
|
||||
...githubFlatConfigs.typescript,
|
||||
// import-x TypeScript settings
|
||||
// This is needed for import-x rules to properly parse TypeScript files.
|
||||
{
|
||||
settings: importX.flatConfigs.typescript.settings,
|
||||
},
|
||||
{
|
||||
plugins: {
|
||||
"@typescript-eslint": fixupPluginRules(typescriptEslint),
|
||||
filenames: fixupPluginRules(filenames),
|
||||
github: fixupPluginRules(github),
|
||||
import: fixupPluginRules(_import),
|
||||
"no-async-foreach": noAsyncForeach,
|
||||
"import-x": importX,
|
||||
"no-async-foreach": fixupPluginRules(noAsyncForeach),
|
||||
"jsdoc": jsdoc,
|
||||
},
|
||||
|
||||
languageOptions: {
|
||||
parser: tsParser,
|
||||
ecmaVersion: 5,
|
||||
ecmaVersion: "latest",
|
||||
sourceType: "module",
|
||||
|
||||
globals: {
|
||||
@@ -79,10 +66,16 @@ export default [
|
||||
typescript: {},
|
||||
},
|
||||
"import/ignore": ["sinon", "uuid", "@octokit/plugin-retry", "del", "get-folder-size"],
|
||||
"import-x/resolver-next": [
|
||||
createTypeScriptImportResolver(),
|
||||
createNodeResolver({
|
||||
extensions: [".ts", ".js", ".json"],
|
||||
}),
|
||||
],
|
||||
},
|
||||
|
||||
rules: {
|
||||
"filenames/match-regex": ["error", "^[a-z0-9-]+(\\.test)?$"],
|
||||
"github/filenames-match-regex": ["error", "^[a-z0-9-]+(\\.test)?$"],
|
||||
"i18n-text/no-en": "off",
|
||||
|
||||
"import/extensions": [
|
||||
@@ -94,7 +87,10 @@ export default [
|
||||
|
||||
"import/no-amd": "error",
|
||||
"import/no-commonjs": "error",
|
||||
"import/no-cycle": "error",
|
||||
// import/no-cycle does not seem to work with ESLint 9.
|
||||
// Use import-x/no-cycle from eslint-plugin-import-x instead.
|
||||
"import/no-cycle": "off",
|
||||
"import-x/no-cycle": "error",
|
||||
"import/no-dynamic-require": "error",
|
||||
|
||||
"import/no-extraneous-dependencies": [
|
||||
@@ -132,6 +128,8 @@ export default [
|
||||
"no-async-foreach/no-async-foreach": "error",
|
||||
"no-sequences": "error",
|
||||
"no-shadow": "off",
|
||||
// This is overly restrictive with unsetting `EnvVar`s
|
||||
"@typescript-eslint/no-dynamic-delete": "off",
|
||||
"@typescript-eslint/no-shadow": "error",
|
||||
"@typescript-eslint/prefer-optional-chain": "error",
|
||||
"one-var": ["error", "never"],
|
||||
|
||||
@@ -99,6 +99,9 @@ inputs:
|
||||
queries:
|
||||
description: Comma-separated list of additional queries to run. By default, this overrides the same setting in a configuration file; prefix with "+" to use both sets of queries.
|
||||
required: false
|
||||
quality-queries:
|
||||
description: '[Internal] DEPRECATED. Comma-separated list of code quality queries to run.'
|
||||
required: false
|
||||
packs:
|
||||
description: >-
|
||||
Comma-separated list of packs to run. Reference a pack in the format `scope/name[@version]`. If `version` is not
|
||||
|
||||
Generated
+26691
-8380
File diff suppressed because one or more lines are too long
Generated
+10074
-8404
File diff suppressed because one or more lines are too long
Generated
+9962
-8363
File diff suppressed because one or more lines are too long
+4
-4
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"bundleVersion": "codeql-bundle-v2.24.0",
|
||||
"cliVersion": "2.24.0",
|
||||
"priorBundleVersion": "codeql-bundle-v2.23.9",
|
||||
"priorCliVersion": "2.23.9"
|
||||
"bundleVersion": "codeql-bundle-v2.24.2",
|
||||
"cliVersion": "2.24.2",
|
||||
"priorBundleVersion": "codeql-bundle-v2.24.1",
|
||||
"priorCliVersion": "2.24.1"
|
||||
}
|
||||
|
||||
Generated
+27773
-9211
File diff suppressed because one or more lines are too long
Generated
+10536
-8748
File diff suppressed because one or more lines are too long
Generated
+9874
-8322
File diff suppressed because one or more lines are too long
Generated
+10263
-8579
File diff suppressed because one or more lines are too long
Generated
+26685
-8378
File diff suppressed because one or more lines are too long
Generated
+32072
-29841
File diff suppressed because one or more lines are too long
Generated
+10259
-8583
File diff suppressed because one or more lines are too long
Generated
+26697
-8390
File diff suppressed because one or more lines are too long
Generated
+10332
-8620
File diff suppressed because one or more lines are too long
Generated
+2114
-1399
File diff suppressed because it is too large
Load Diff
+14
-16
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "codeql",
|
||||
"version": "4.32.2",
|
||||
"version": "4.32.5",
|
||||
"private": true,
|
||||
"description": "CodeQL action",
|
||||
"scripts": {
|
||||
@@ -29,7 +29,7 @@
|
||||
"@actions/cache": "^5.0.5",
|
||||
"@actions/core": "^2.0.3",
|
||||
"@actions/exec": "^2.0.0",
|
||||
"@actions/github": "^8.0.0",
|
||||
"@actions/github": "^8.0.1",
|
||||
"@actions/glob": "^0.5.0",
|
||||
"@actions/http-client": "^3.0.0",
|
||||
"@actions/io": "^2.0.0",
|
||||
@@ -40,18 +40,17 @@
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"follow-redirects": "^1.15.11",
|
||||
"get-folder-size": "^5.0.0",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"js-yaml": "^4.1.1",
|
||||
"jsonschema": "1.4.1",
|
||||
"long": "^5.3.2",
|
||||
"node-forge": "^1.3.3",
|
||||
"semver": "^7.7.3",
|
||||
"semver": "^7.7.4",
|
||||
"uuid": "^13.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ava/typescript": "6.0.0",
|
||||
"@eslint/compat": "^2.0.1",
|
||||
"@eslint/eslintrc": "^3.3.3",
|
||||
"@eslint/js": "^9.39.2",
|
||||
"@eslint/compat": "^2.0.2",
|
||||
"@microsoft/eslint-formatter-sarif": "^3.1.0",
|
||||
"@octokit/types": "^16.0.0",
|
||||
"@types/archiver": "^7.0.0",
|
||||
@@ -61,21 +60,20 @@
|
||||
"@types/node-forge": "^1.3.14",
|
||||
"@types/semver": "^7.7.1",
|
||||
"@types/sinon": "^21.0.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.53.1",
|
||||
"@typescript-eslint/parser": "^8.48.0",
|
||||
"ava": "^6.4.1",
|
||||
"esbuild": "^0.27.2",
|
||||
"eslint": "^8.57.1",
|
||||
"esbuild": "^0.27.3",
|
||||
"eslint": "^9.39.2",
|
||||
"eslint-import-resolver-typescript": "^3.8.7",
|
||||
"eslint-plugin-filenames": "^1.3.2",
|
||||
"eslint-plugin-github": "^5.1.8",
|
||||
"eslint-plugin-import": "2.29.1",
|
||||
"eslint-plugin-jsdoc": "^62.3.0",
|
||||
"eslint-plugin-github": "^6.0.0",
|
||||
"eslint-plugin-import-x": "^4.16.1",
|
||||
"eslint-plugin-jsdoc": "^62.5.0",
|
||||
"eslint-plugin-no-async-foreach": "^0.1.1",
|
||||
"glob": "^11.1.0",
|
||||
"nock": "^14.0.10",
|
||||
"globals": "^16.5.0",
|
||||
"nock": "^14.0.11",
|
||||
"sinon": "^21.0.1",
|
||||
"typescript": "^5.9.3"
|
||||
"typescript": "^5.9.3",
|
||||
"typescript-eslint": "^8.56.0"
|
||||
},
|
||||
"overrides": {
|
||||
"@actions/tool-cache": {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
name: "Quality queries input"
|
||||
description: "Tests that queries specified in the quality-queries input are used."
|
||||
name: "Analysis kinds"
|
||||
description: "Tests basic functionality for different `analysis-kinds` inputs."
|
||||
versions: ["linked", "nightly-latest"]
|
||||
analysisKinds: ["code-scanning", "code-quality", "code-scanning,code-quality"]
|
||||
analysisKinds: ["code-scanning", "code-quality", "code-scanning,code-quality", "risk-assessment"]
|
||||
env:
|
||||
CODEQL_ACTION_RISK_ASSESSMENT_ID: 1
|
||||
CHECK_SCRIPT: |
|
||||
const fs = require('fs');
|
||||
|
||||
@@ -37,30 +38,24 @@ steps:
|
||||
output: "${{ runner.temp }}/results"
|
||||
upload-database: false
|
||||
post-processed-sarif-path: "${{ runner.temp }}/post-processed"
|
||||
- name: Upload security SARIF
|
||||
if: contains(matrix.analysis-kinds, 'code-scanning')
|
||||
|
||||
- name: Upload SARIF files
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: |
|
||||
quality-queries-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}.sarif.json
|
||||
path: "${{ runner.temp }}/results/javascript.sarif"
|
||||
retention-days: 7
|
||||
- name: Upload quality SARIF
|
||||
if: contains(matrix.analysis-kinds, 'code-quality')
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: |
|
||||
quality-queries-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}.quality.sarif.json
|
||||
path: "${{ runner.temp }}/results/javascript.quality.sarif"
|
||||
analysis-kinds-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}
|
||||
path: "${{ runner.temp }}/results/*.sarif"
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload post-processed SARIF
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: |
|
||||
post-processed-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}.sarif.json
|
||||
post-processed-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.analysis-kinds }}
|
||||
path: "${{ runner.temp }}/post-processed"
|
||||
retention-days: 7
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Check quality query does not appear in security SARIF
|
||||
if: contains(matrix.analysis-kinds, 'code-scanning')
|
||||
uses: actions/github-script@v8
|
||||
@@ -0,0 +1,15 @@
|
||||
name: "Bundle: From nightly"
|
||||
description: "The nightly CodeQL bundle should be used when forced"
|
||||
versions:
|
||||
- linked # overruled by the FF set below
|
||||
steps:
|
||||
- id: init
|
||||
uses: ./../action/init
|
||||
env:
|
||||
CODEQL_ACTION_FORCE_NIGHTLY: true
|
||||
with:
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
languages: javascript
|
||||
- name: Fail if the CodeQL version is not a nightly
|
||||
if: "!contains(steps.init.outputs.codeql-version, '+')"
|
||||
run: exit 1
|
||||
@@ -1,16 +0,0 @@
|
||||
name: "CCR"
|
||||
description: "A standard analysis in CCR mode"
|
||||
env:
|
||||
CODEQL_ACTION_ANALYSIS_KEY: "dynamic/copilot-pull-request-reviewer/codeql-action-test"
|
||||
steps:
|
||||
- uses: ./../action/init
|
||||
id: init
|
||||
with:
|
||||
languages: javascript
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
|
||||
- uses: ./../action/analyze
|
||||
id: analysis
|
||||
with:
|
||||
upload-database: false
|
||||
|
||||
@@ -4,7 +4,7 @@ description: "Tests using RuboCop to analyze a multi-language repository and the
|
||||
versions: ["default"]
|
||||
steps:
|
||||
- name: Set up Ruby
|
||||
uses: ruby/setup-ruby@90be1154f987f4dc0fe0dd0feedac9e473aa4ba8 # v1.286.0
|
||||
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
||||
with:
|
||||
ruby-version: 2.6
|
||||
- name: Install Code Scanning integration
|
||||
|
||||
@@ -5,7 +5,6 @@ import {
|
||||
fixCodeQualityCategory,
|
||||
getPullRequestBranches,
|
||||
isAnalyzingPullRequest,
|
||||
isCCR,
|
||||
isDefaultSetup,
|
||||
isDynamicWorkflow,
|
||||
} from "./actions-util";
|
||||
@@ -257,16 +256,8 @@ test("isDynamicWorkflow() returns true if event name is `dynamic`", (t) => {
|
||||
t.false(isDynamicWorkflow());
|
||||
});
|
||||
|
||||
test("isCCR() returns true when expected", (t) => {
|
||||
process.env.GITHUB_EVENT_NAME = "dynamic";
|
||||
process.env[EnvVar.ANALYSIS_KEY] = "dynamic/copilot-pull-request-reviewer";
|
||||
t.assert(isCCR());
|
||||
t.false(isDefaultSetup());
|
||||
});
|
||||
|
||||
test("isDefaultSetup() returns true when expected", (t) => {
|
||||
process.env.GITHUB_EVENT_NAME = "dynamic";
|
||||
process.env[EnvVar.ANALYSIS_KEY] = "dynamic/github-code-scanning";
|
||||
t.assert(isDefaultSetup());
|
||||
t.false(isCCR());
|
||||
});
|
||||
|
||||
+1
-10
@@ -8,7 +8,6 @@ import * as io from "@actions/io";
|
||||
import { JSONSchemaForNPMPackageJsonFiles } from "@schemastore/package";
|
||||
|
||||
import type { Config } from "./config-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Logger } from "./logging";
|
||||
import {
|
||||
doesDirectoryExist,
|
||||
@@ -255,15 +254,7 @@ export function isDynamicWorkflow(): boolean {
|
||||
|
||||
/** Determines whether we are running in default setup. */
|
||||
export function isDefaultSetup(): boolean {
|
||||
return isDynamicWorkflow() && !isCCR();
|
||||
}
|
||||
|
||||
/* The analysis key prefix used for CCR. */
|
||||
const CCR_KEY_PREFIX = "dynamic/copilot-pull-request-reviewer";
|
||||
|
||||
/** Determines whether we are running in CCR. */
|
||||
export function isCCR(): boolean {
|
||||
return process.env[EnvVar.ANALYSIS_KEY]?.startsWith(CCR_KEY_PREFIX) || false;
|
||||
return isDynamicWorkflow();
|
||||
}
|
||||
|
||||
export function prettyPrintInvocation(cmd: string, args: string[]): string {
|
||||
|
||||
+116
-5
@@ -1,15 +1,23 @@
|
||||
import path from "path";
|
||||
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import {
|
||||
AnalysisKind,
|
||||
CodeScanning,
|
||||
compatibilityMatrix,
|
||||
RiskAssessment,
|
||||
getAnalysisConfig,
|
||||
getAnalysisKinds,
|
||||
parseAnalysisKinds,
|
||||
supportedAnalysisKinds,
|
||||
} from "./analyses";
|
||||
import { EnvVar } from "./environment";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { setupTests } from "./testing-utils";
|
||||
import { AssessmentPayload } from "./upload-lib/types";
|
||||
import { ConfigurationError } from "./util";
|
||||
|
||||
setupTests(test);
|
||||
@@ -52,15 +60,14 @@ test("getAnalysisKinds - returns expected analysis kinds for `analysis-kinds` in
|
||||
t.assert(result.includes(AnalysisKind.CodeQuality));
|
||||
});
|
||||
|
||||
test("getAnalysisKinds - throws ConfigurationError when deprecated `quality-queries` input is used", async (t) => {
|
||||
test("getAnalysisKinds - includes `code-quality` when deprecated `quality-queries` input is used", async (t) => {
|
||||
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
requiredInputStub.withArgs("analysis-kinds").returns("code-scanning");
|
||||
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
|
||||
optionalInputStub.withArgs("quality-queries").returns("code-quality");
|
||||
|
||||
await t.throwsAsync(getAnalysisKinds(getRunnerLogger(true), true), {
|
||||
instanceOf: ConfigurationError,
|
||||
});
|
||||
const result = await getAnalysisKinds(getRunnerLogger(true), true);
|
||||
t.assert(result.includes(AnalysisKind.CodeScanning));
|
||||
t.assert(result.includes(AnalysisKind.CodeQuality));
|
||||
});
|
||||
|
||||
test("getAnalysisKinds - throws if `analysis-kinds` input is invalid", async (t) => {
|
||||
@@ -68,3 +75,107 @@ test("getAnalysisKinds - throws if `analysis-kinds` input is invalid", async (t)
|
||||
requiredInputStub.withArgs("analysis-kinds").returns("no-such-thing");
|
||||
await t.throwsAsync(getAnalysisKinds(getRunnerLogger(true), true));
|
||||
});
|
||||
|
||||
// Test the compatibility matrix by looping through all analysis kinds.
|
||||
const analysisKinds = Object.values(AnalysisKind);
|
||||
for (let i = 0; i < analysisKinds.length; i++) {
|
||||
const analysisKind = analysisKinds[i];
|
||||
|
||||
for (let j = i + 1; j < analysisKinds.length; j++) {
|
||||
const otherAnalysis = analysisKinds[j];
|
||||
|
||||
if (analysisKind === otherAnalysis) continue;
|
||||
if (compatibilityMatrix[analysisKind].has(otherAnalysis)) {
|
||||
test(`getAnalysisKinds - allows ${analysisKind} with ${otherAnalysis}`, async (t) => {
|
||||
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
requiredInputStub
|
||||
.withArgs("analysis-kinds")
|
||||
.returns([analysisKind, otherAnalysis].join(","));
|
||||
const result = await getAnalysisKinds(getRunnerLogger(true), true);
|
||||
t.is(result.length, 2);
|
||||
});
|
||||
} else {
|
||||
test(`getAnalysisKinds - throws if ${analysisKind} is enabled with ${otherAnalysis}`, async (t) => {
|
||||
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
requiredInputStub
|
||||
.withArgs("analysis-kinds")
|
||||
.returns([analysisKind, otherAnalysis].join(","));
|
||||
await t.throwsAsync(getAnalysisKinds(getRunnerLogger(true), true), {
|
||||
instanceOf: ConfigurationError,
|
||||
message: `${analysisKind} and ${otherAnalysis} cannot be enabled at the same time`,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test("Code Scanning configuration does not accept other SARIF extensions", (t) => {
|
||||
for (const analysisKind of supportedAnalysisKinds) {
|
||||
if (analysisKind === AnalysisKind.CodeScanning) continue;
|
||||
|
||||
const analysis = getAnalysisConfig(analysisKind);
|
||||
const sarifPath = path.join("path", "to", `file${analysis.sarifExtension}`);
|
||||
|
||||
// The Code Scanning configuration's `sarifPredicate` should not accept a path which
|
||||
// ends in a different configuration's `sarifExtension`.
|
||||
t.false(CodeScanning.sarifPredicate(sarifPath));
|
||||
}
|
||||
});
|
||||
|
||||
test("Risk Assessment configuration transforms SARIF upload payload", (t) => {
|
||||
process.env[EnvVar.RISK_ASSESSMENT_ID] = "1";
|
||||
const payload = RiskAssessment.transformPayload({
|
||||
commit_oid: "abc",
|
||||
sarif: "sarif",
|
||||
ref: "ref",
|
||||
workflow_run_attempt: 1,
|
||||
workflow_run_id: 1,
|
||||
checkout_uri: "uri",
|
||||
tool_names: [],
|
||||
}) as AssessmentPayload;
|
||||
|
||||
const expected: AssessmentPayload = { sarif: "sarif", assessment_id: 1 };
|
||||
t.deepEqual(expected, payload);
|
||||
});
|
||||
|
||||
test("Risk Assessment configuration throws for negative assessment IDs", (t) => {
|
||||
process.env[EnvVar.RISK_ASSESSMENT_ID] = "-1";
|
||||
t.throws(
|
||||
() =>
|
||||
RiskAssessment.transformPayload({
|
||||
commit_oid: "abc",
|
||||
sarif: "sarif",
|
||||
ref: "ref",
|
||||
workflow_run_attempt: 1,
|
||||
workflow_run_id: 1,
|
||||
checkout_uri: "uri",
|
||||
tool_names: [],
|
||||
}),
|
||||
{
|
||||
instanceOf: Error,
|
||||
message: (msg) =>
|
||||
msg.startsWith(`${EnvVar.RISK_ASSESSMENT_ID} must not be negative: `),
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("Risk Assessment configuration throws for invalid IDs", (t) => {
|
||||
process.env[EnvVar.RISK_ASSESSMENT_ID] = "foo";
|
||||
t.throws(
|
||||
() =>
|
||||
RiskAssessment.transformPayload({
|
||||
commit_oid: "abc",
|
||||
sarif: "sarif",
|
||||
ref: "ref",
|
||||
workflow_run_attempt: 1,
|
||||
workflow_run_id: 1,
|
||||
checkout_uri: "uri",
|
||||
tool_names: [],
|
||||
}),
|
||||
{
|
||||
instanceOf: Error,
|
||||
message: (msg) =>
|
||||
msg.startsWith(`${EnvVar.RISK_ASSESSMENT_ID} must not be NaN: `),
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
+95
-9
@@ -3,14 +3,30 @@ import {
|
||||
getOptionalInput,
|
||||
getRequiredInput,
|
||||
} from "./actions-util";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Logger } from "./logging";
|
||||
import { ConfigurationError } from "./util";
|
||||
import {
|
||||
AssessmentPayload,
|
||||
BasePayload,
|
||||
UploadPayload,
|
||||
} from "./upload-lib/types";
|
||||
import { ConfigurationError, getRequiredEnvParam } from "./util";
|
||||
|
||||
export enum AnalysisKind {
|
||||
CodeScanning = "code-scanning",
|
||||
CodeQuality = "code-quality",
|
||||
RiskAssessment = "risk-assessment",
|
||||
}
|
||||
|
||||
export type CompatibilityMatrix = Record<AnalysisKind, Set<AnalysisKind>>;
|
||||
|
||||
/** A mapping from analysis kinds to other analysis kinds which can be enabled concurrently. */
|
||||
export const compatibilityMatrix: CompatibilityMatrix = {
|
||||
[AnalysisKind.CodeScanning]: new Set([AnalysisKind.CodeQuality]),
|
||||
[AnalysisKind.CodeQuality]: new Set([AnalysisKind.CodeScanning]),
|
||||
[AnalysisKind.RiskAssessment]: new Set(),
|
||||
};
|
||||
|
||||
// Exported for testing. A set of all known analysis kinds.
|
||||
export const supportedAnalysisKinds = new Set(Object.values(AnalysisKind));
|
||||
|
||||
@@ -50,35 +66,62 @@ let cachedAnalysisKinds: AnalysisKind[] | undefined;
|
||||
|
||||
/**
|
||||
* Initialises the analysis kinds for the analysis based on the `analysis-kinds` input.
|
||||
* This function will also use the deprecated `quality-queries` input as an indicator to enable `code-quality`.
|
||||
* If the `analysis-kinds` input cannot be parsed, a `ConfigurationError` is thrown.
|
||||
*
|
||||
* @param _logger The logger to use.
|
||||
* @param logger The logger to use.
|
||||
* @param skipCache For testing, whether to ignore the cached values (default: false).
|
||||
*
|
||||
* @returns The array of enabled analysis kinds.
|
||||
* @throws A `ConfigurationError` if the `analysis-kinds` input cannot be parsed.
|
||||
*/
|
||||
export async function getAnalysisKinds(
|
||||
_logger: Logger,
|
||||
logger: Logger,
|
||||
skipCache: boolean = false,
|
||||
): Promise<AnalysisKind[]> {
|
||||
if (!skipCache && cachedAnalysisKinds !== undefined) {
|
||||
return cachedAnalysisKinds;
|
||||
}
|
||||
|
||||
cachedAnalysisKinds = await parseAnalysisKinds(
|
||||
const analysisKinds = await parseAnalysisKinds(
|
||||
getRequiredInput("analysis-kinds"),
|
||||
);
|
||||
|
||||
// Throw if there is an argument for `quality-queries`.
|
||||
// Warn that `quality-queries` is deprecated if there is an argument for it.
|
||||
const qualityQueriesInput = getOptionalInput("quality-queries");
|
||||
|
||||
if (qualityQueriesInput !== undefined) {
|
||||
throw new ConfigurationError(
|
||||
"The `quality-queries` input is no longer supported. Use `analysis-kinds` instead.",
|
||||
logger.warning(
|
||||
"The `quality-queries` input is deprecated and will be removed in a future version of the CodeQL Action. " +
|
||||
"Use the `analysis-kinds` input to configure different analysis kinds instead.",
|
||||
);
|
||||
}
|
||||
|
||||
// For backwards compatibility, add Code Quality to the enabled analysis kinds
|
||||
// if an input to `quality-queries` was specified. We should remove this once
|
||||
// `quality-queries` is no longer used.
|
||||
if (
|
||||
!analysisKinds.includes(AnalysisKind.CodeQuality) &&
|
||||
qualityQueriesInput !== undefined
|
||||
) {
|
||||
analysisKinds.push(AnalysisKind.CodeQuality);
|
||||
}
|
||||
|
||||
// Check that all enabled analysis kinds are compatible with each other.
|
||||
for (const analysisKind of analysisKinds) {
|
||||
for (const otherAnalysisKind of analysisKinds) {
|
||||
if (analysisKind === otherAnalysisKind) continue;
|
||||
|
||||
if (!compatibilityMatrix[analysisKind].has(otherAnalysisKind)) {
|
||||
throw new ConfigurationError(
|
||||
`${analysisKind} and ${otherAnalysisKind} cannot be enabled at the same time`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cache the analysis kinds and return them.
|
||||
cachedAnalysisKinds = analysisKinds;
|
||||
return cachedAnalysisKinds;
|
||||
}
|
||||
|
||||
@@ -89,6 +132,7 @@ export const codeQualityQueries: string[] = ["code-quality"];
|
||||
enum SARIF_UPLOAD_ENDPOINT {
|
||||
CODE_SCANNING = "PUT /repos/:owner/:repo/code-scanning/analysis",
|
||||
CODE_QUALITY = "PUT /repos/:owner/:repo/code-quality/analysis",
|
||||
RISK_ASSESSMENT = "PUT /repos/:owner/:repo/code-scanning/risk-assessment",
|
||||
}
|
||||
|
||||
// Represents configurations for different analysis kinds.
|
||||
@@ -108,6 +152,8 @@ export interface AnalysisConfig {
|
||||
fixCategory: (logger: Logger, category?: string) => string | undefined;
|
||||
/** A prefix for environment variables used to track the uniqueness of SARIF uploads. */
|
||||
sentinelPrefix: string;
|
||||
/** Transforms the upload payload in an analysis-specific way. */
|
||||
transformPayload: (payload: UploadPayload) => BasePayload;
|
||||
}
|
||||
|
||||
// Represents the Code Scanning analysis configuration.
|
||||
@@ -118,9 +164,11 @@ export const CodeScanning: AnalysisConfig = {
|
||||
sarifExtension: ".sarif",
|
||||
sarifPredicate: (name) =>
|
||||
name.endsWith(CodeScanning.sarifExtension) &&
|
||||
!CodeQuality.sarifPredicate(name),
|
||||
!CodeQuality.sarifPredicate(name) &&
|
||||
!RiskAssessment.sarifPredicate(name),
|
||||
fixCategory: (_, category) => category,
|
||||
sentinelPrefix: "CODEQL_UPLOAD_SARIF_",
|
||||
transformPayload: (payload) => payload,
|
||||
};
|
||||
|
||||
// Represents the Code Quality analysis configuration.
|
||||
@@ -132,6 +180,38 @@ export const CodeQuality: AnalysisConfig = {
|
||||
sarifPredicate: (name) => name.endsWith(CodeQuality.sarifExtension),
|
||||
fixCategory: fixCodeQualityCategory,
|
||||
sentinelPrefix: "CODEQL_UPLOAD_QUALITY_SARIF_",
|
||||
transformPayload: (payload) => payload,
|
||||
};
|
||||
|
||||
/**
|
||||
* Retrieves the CSRA assessment id from an environment variable and adds it to the payload.
|
||||
* @param payload The base payload.
|
||||
*/
|
||||
function addAssessmentId(payload: UploadPayload): AssessmentPayload {
|
||||
const rawAssessmentId = getRequiredEnvParam(EnvVar.RISK_ASSESSMENT_ID);
|
||||
const assessmentId = parseInt(rawAssessmentId, 10);
|
||||
if (Number.isNaN(assessmentId)) {
|
||||
throw new Error(
|
||||
`${EnvVar.RISK_ASSESSMENT_ID} must not be NaN: ${rawAssessmentId}`,
|
||||
);
|
||||
}
|
||||
if (assessmentId < 0) {
|
||||
throw new Error(
|
||||
`${EnvVar.RISK_ASSESSMENT_ID} must not be negative: ${rawAssessmentId}`,
|
||||
);
|
||||
}
|
||||
return { sarif: payload.sarif, assessment_id: assessmentId };
|
||||
}
|
||||
|
||||
export const RiskAssessment: AnalysisConfig = {
|
||||
kind: AnalysisKind.RiskAssessment,
|
||||
name: "code scanning risk assessment",
|
||||
target: SARIF_UPLOAD_ENDPOINT.RISK_ASSESSMENT,
|
||||
sarifExtension: ".csra.sarif",
|
||||
sarifPredicate: (name) => name.endsWith(RiskAssessment.sarifExtension),
|
||||
fixCategory: (_, category) => category,
|
||||
sentinelPrefix: "CODEQL_UPLOAD_CSRA_SARIF_",
|
||||
transformPayload: addAssessmentId,
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -148,6 +228,8 @@ export function getAnalysisConfig(kind: AnalysisKind): AnalysisConfig {
|
||||
return CodeScanning;
|
||||
case AnalysisKind.CodeQuality:
|
||||
return CodeQuality;
|
||||
case AnalysisKind.RiskAssessment:
|
||||
return RiskAssessment;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,4 +237,8 @@ export function getAnalysisConfig(kind: AnalysisKind): AnalysisConfig {
|
||||
// we want to scan a folder containing SARIF files in an order that finds the more
|
||||
// specific extensions first. This constant defines an array in the order of analyis
|
||||
// configurations with more specific extensions to less specific extensions.
|
||||
export const SarifScanOrder = [CodeQuality, CodeScanning];
|
||||
export const SarifScanOrder: AnalysisConfig[] = [
|
||||
RiskAssessment,
|
||||
CodeQuality,
|
||||
CodeScanning,
|
||||
];
|
||||
|
||||
@@ -30,10 +30,10 @@ import {
|
||||
} from "./dependency-caching";
|
||||
import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Features } from "./feature-flags";
|
||||
import { initFeatures } from "./feature-flags";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { getActionsLogger, Logger } from "./logging";
|
||||
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
|
||||
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import * as statusReport from "./status-report";
|
||||
import {
|
||||
@@ -293,7 +293,7 @@ async function run(startedAt: Date) {
|
||||
|
||||
util.checkActionVersion(actionsUtil.getActionVersion(), gitHubVersion);
|
||||
|
||||
const features = new Features(
|
||||
const features = initFeatures(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
actionsUtil.getTemporaryDirectory(),
|
||||
|
||||
+2
-1
@@ -4,7 +4,7 @@ import * as path from "path";
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import { CodeQuality, CodeScanning } from "./analyses";
|
||||
import { CodeQuality, CodeScanning, RiskAssessment } from "./analyses";
|
||||
import {
|
||||
runQueries,
|
||||
defaultSuites,
|
||||
@@ -155,5 +155,6 @@ test("addSarifExtension", (t) => {
|
||||
addSarifExtension(CodeQuality, language),
|
||||
`${language}.quality.sarif`,
|
||||
);
|
||||
t.is(addSarifExtension(RiskAssessment, language), `${language}.csra.sarif`);
|
||||
}
|
||||
});
|
||||
|
||||
+4
-7
@@ -24,7 +24,7 @@ import { EnvVar } from "./environment";
|
||||
import { FeatureEnablement, Feature } from "./feature-flags";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { Logger, withGroupAsync } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay-database-utils";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import { DatabaseCreationTimings, EventReport } from "./status-report";
|
||||
import { endTracingForCluster } from "./tracer-config";
|
||||
import * as util from "./util";
|
||||
@@ -549,12 +549,9 @@ export async function runQueries(
|
||||
): Promise<{ summary: string; sarifFile: string }> {
|
||||
logger.info(`Interpreting ${analysis.name} results for ${language}`);
|
||||
|
||||
// If this is a Code Quality analysis, correct the category to one
|
||||
// accepted by the Code Quality backend.
|
||||
let category = automationDetailsId;
|
||||
if (analysis.kind === analyses.AnalysisKind.CodeQuality) {
|
||||
category = analysis.fixCategory(logger, automationDetailsId);
|
||||
}
|
||||
// Apply the analysis configuration's `fixCategory` function to adjust the category if needed.
|
||||
// This is a no-op for Code Scanning.
|
||||
const category = analysis.fixCategory(logger, automationDetailsId);
|
||||
|
||||
const sarifFile = path.join(
|
||||
sarifFolder,
|
||||
|
||||
@@ -36,6 +36,9 @@ test("getApiClient", async (t) => {
|
||||
baseUrl: "http://api.github.localhost",
|
||||
log: sinon.match.any,
|
||||
userAgent: `CodeQL-Action/${actionsUtil.getActionVersion()}`,
|
||||
retry: {
|
||||
doNotRetry: [400, 410, 422, 451],
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -51,6 +51,12 @@ function createApiClientWithDetails(
|
||||
warn: core.warning,
|
||||
error: core.error,
|
||||
},
|
||||
retry: {
|
||||
// The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
|
||||
// with authentication, so we remove 401, 403, and 404 from the default list to ensure that
|
||||
// these errors are retried.
|
||||
doNotRetry: [400, 410, 422, 451],
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@ import { CodeQL, getCodeQL } from "./codeql";
|
||||
import * as configUtils from "./config-utils";
|
||||
import { DocUrl } from "./doc-url";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature, featureConfig, Features } from "./feature-flags";
|
||||
import { Feature, featureConfig, initFeatures } from "./feature-flags";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
@@ -117,7 +117,7 @@ export async function setupCppAutobuild(codeql: CodeQL, logger: Logger) {
|
||||
const featureName = "C++ automatic installation of dependencies";
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
const repositoryNwo = getRepositoryNwo();
|
||||
const features = new Features(
|
||||
const features = initFeatures(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
getTemporaryDirectory(),
|
||||
|
||||
+10
-2
@@ -28,7 +28,7 @@ import {
|
||||
OverlayDatabaseMode,
|
||||
writeBaseDatabaseOidsFile,
|
||||
writeOverlayChangesFile,
|
||||
} from "./overlay-database-utils";
|
||||
} from "./overlay";
|
||||
import * as setupCodeql from "./setup-codeql";
|
||||
import { ZstdAvailability } from "./tar";
|
||||
import { ToolsDownloadStatusReport } from "./tools-download";
|
||||
@@ -160,6 +160,7 @@ export interface CodeQL {
|
||||
databasePath: string,
|
||||
outputFilePath: string,
|
||||
dbName: string,
|
||||
includeDiagnostics: boolean,
|
||||
alsoIncludeRelativePaths: string[],
|
||||
): Promise<void>;
|
||||
/**
|
||||
@@ -912,15 +913,22 @@ async function getCodeQLForCmd(
|
||||
databasePath: string,
|
||||
outputFilePath: string,
|
||||
databaseName: string,
|
||||
includeDiagnostics: boolean,
|
||||
alsoIncludeRelativePaths: string[],
|
||||
): Promise<void> {
|
||||
const includeDiagnosticsArgs = includeDiagnostics
|
||||
? ["--include-diagnostics"]
|
||||
: [];
|
||||
const args = [
|
||||
"database",
|
||||
"bundle",
|
||||
databasePath,
|
||||
`--output=${outputFilePath}`,
|
||||
`--name=${databaseName}`,
|
||||
...getExtraOptionsFromEnv(["database", "bundle"]),
|
||||
...includeDiagnosticsArgs,
|
||||
...getExtraOptionsFromEnv(["database", "bundle"], {
|
||||
ignoringOptions: includeDiagnosticsArgs,
|
||||
}),
|
||||
];
|
||||
if (
|
||||
await this.supportsFeature(ToolsFeature.BundleSupportsIncludeOption)
|
||||
|
||||
+138
-6
@@ -7,7 +7,7 @@ import * as yaml from "js-yaml";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { AnalysisKind } from "./analyses";
|
||||
import { AnalysisKind, supportedAnalysisKinds } from "./analyses";
|
||||
import * as api from "./api-client";
|
||||
import { CachingKind } from "./caching-utils";
|
||||
import { createStubCodeQL } from "./codeql";
|
||||
@@ -18,10 +18,8 @@ import * as gitUtils from "./git-utils";
|
||||
import { GitVersionInfo } from "./git-utils";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import {
|
||||
CODEQL_OVERLAY_MINIMUM_VERSION,
|
||||
OverlayDatabaseMode,
|
||||
} from "./overlay-database-utils";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION, OverlayDatabaseMode } from "./overlay";
|
||||
import * as overlayStatus from "./overlay/status";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import {
|
||||
setupTests,
|
||||
@@ -984,6 +982,7 @@ interface OverlayDatabaseModeTestSetup {
|
||||
codeScanningConfig: configUtils.UserConfig;
|
||||
diskUsage: DiskUsage | undefined;
|
||||
memoryFlagValue: number;
|
||||
shouldSkipOverlayAnalysisDueToCachedStatus: boolean;
|
||||
}
|
||||
|
||||
const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
|
||||
@@ -1005,6 +1004,7 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
memoryFlagValue: 6920,
|
||||
shouldSkipOverlayAnalysisDueToCachedStatus: false,
|
||||
};
|
||||
|
||||
const getOverlayDatabaseModeMacro = test.macro({
|
||||
@@ -1015,6 +1015,7 @@ const getOverlayDatabaseModeMacro = test.macro({
|
||||
expected: {
|
||||
overlayDatabaseMode: OverlayDatabaseMode;
|
||||
useOverlayDatabaseCaching: boolean;
|
||||
skippedDueToCachedStatus?: boolean;
|
||||
},
|
||||
) => {
|
||||
return await withTmpDir(async (tempDir) => {
|
||||
@@ -1039,6 +1040,10 @@ const getOverlayDatabaseModeMacro = test.macro({
|
||||
|
||||
sinon.stub(util, "checkDiskUsage").resolves(setup.diskUsage);
|
||||
|
||||
sinon
|
||||
.stub(overlayStatus, "shouldSkipOverlayAnalysis")
|
||||
.resolves(setup.shouldSkipOverlayAnalysisDueToCachedStatus);
|
||||
|
||||
// Mock feature flags
|
||||
const features = createFeatures(setup.features);
|
||||
|
||||
@@ -1081,7 +1086,10 @@ const getOverlayDatabaseModeMacro = test.macro({
|
||||
logger,
|
||||
);
|
||||
|
||||
t.deepEqual(result, expected);
|
||||
t.deepEqual(result, {
|
||||
skippedDueToCachedStatus: false,
|
||||
...expected,
|
||||
});
|
||||
} finally {
|
||||
// Restore the original environment
|
||||
process.env = originalEnv;
|
||||
@@ -1261,6 +1269,71 @@ test(
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
getOverlayDatabaseModeMacro,
|
||||
"No overlay-base database on default branch if runner disk space is below v2 limit and v2 resource checks enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
Feature.OverlayAnalysisResourceChecksV2,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: 5_000_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
useOverlayDatabaseCaching: false,
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
getOverlayDatabaseModeMacro,
|
||||
"Overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
Feature.OverlayAnalysisResourceChecksV2,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: 15_000_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
useOverlayDatabaseCaching: true,
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
getOverlayDatabaseModeMacro,
|
||||
"No overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks not enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: 15_000_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
useOverlayDatabaseCaching: false,
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
getOverlayDatabaseModeMacro,
|
||||
"No overlay-base database on default branch if memory flag is too low",
|
||||
@@ -1298,6 +1371,46 @@ test(
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
getOverlayDatabaseModeMacro,
|
||||
"No overlay-base database on default branch when cached status indicates previous failure",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisJavascript,
|
||||
Feature.OverlayAnalysisStatusCheck,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
shouldSkipOverlayAnalysisDueToCachedStatus: true,
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
useOverlayDatabaseCaching: false,
|
||||
skippedDueToCachedStatus: true,
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
getOverlayDatabaseModeMacro,
|
||||
"No overlay analysis on PR when cached status indicates previous failure",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisJavascript,
|
||||
Feature.OverlayAnalysisStatusCheck,
|
||||
],
|
||||
isPullRequest: true,
|
||||
shouldSkipOverlayAnalysisDueToCachedStatus: true,
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
useOverlayDatabaseCaching: false,
|
||||
skippedDueToCachedStatus: true,
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
getOverlayDatabaseModeMacro,
|
||||
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
|
||||
@@ -1829,3 +1942,22 @@ test("hasActionsWorkflows doesn't throw if workflows folder doesn't exist", asyn
|
||||
t.notThrows(() => configUtils.hasActionsWorkflows(tmpDir));
|
||||
});
|
||||
});
|
||||
|
||||
test("getPrimaryAnalysisConfig - single analysis kind", (t) => {
|
||||
// If only one analysis kind is configured, we expect to get the matching configuration.
|
||||
for (const analysisKind of supportedAnalysisKinds) {
|
||||
const singleKind = createTestConfig({ analysisKinds: [analysisKind] });
|
||||
t.is(configUtils.getPrimaryAnalysisConfig(singleKind).kind, analysisKind);
|
||||
}
|
||||
});
|
||||
|
||||
test("getPrimaryAnalysisConfig - Code Scanning + Code Quality", (t) => {
|
||||
// For CS+CQ, we expect to get the Code Scanning configuration.
|
||||
const codeScanningAndCodeQuality = createTestConfig({
|
||||
analysisKinds: [AnalysisKind.CodeScanning, AnalysisKind.CodeQuality],
|
||||
});
|
||||
t.is(
|
||||
configUtils.getPrimaryAnalysisConfig(codeScanningAndCodeQuality).kind,
|
||||
AnalysisKind.CodeScanning,
|
||||
);
|
||||
});
|
||||
|
||||
+121
-37
@@ -7,14 +7,13 @@ import * as yaml from "js-yaml";
|
||||
import {
|
||||
getActionVersion,
|
||||
isAnalyzingPullRequest,
|
||||
isCCR,
|
||||
isDynamicWorkflow,
|
||||
} from "./actions-util";
|
||||
import {
|
||||
AnalysisConfig,
|
||||
AnalysisKind,
|
||||
CodeQuality,
|
||||
codeQualityQueries,
|
||||
CodeScanning,
|
||||
getAnalysisConfig,
|
||||
} from "./analyses";
|
||||
import * as api from "./api-client";
|
||||
import { CachingKind, getCachingKind } from "./caching-utils";
|
||||
@@ -28,9 +27,11 @@ import {
|
||||
} from "./config/db-config";
|
||||
import {
|
||||
addNoLanguageDiagnostic,
|
||||
makeDiagnostic,
|
||||
makeTelemetryDiagnostic,
|
||||
} from "./diagnostics";
|
||||
import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils";
|
||||
import { DocUrl } from "./doc-url";
|
||||
import { EnvVar } from "./environment";
|
||||
import * as errorMessages from "./error-messages";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
@@ -45,10 +46,8 @@ import {
|
||||
} from "./git-utils";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import {
|
||||
CODEQL_OVERLAY_MINIMUM_VERSION,
|
||||
OverlayDatabaseMode,
|
||||
} from "./overlay-database-utils";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION, OverlayDatabaseMode } from "./overlay";
|
||||
import { shouldSkipOverlayAnalysis } from "./overlay/status";
|
||||
import { RepositoryNwo } from "./repository";
|
||||
import { ToolsFeature } from "./tools-features";
|
||||
import { downloadTrapCaches } from "./trap-caching";
|
||||
@@ -64,6 +63,7 @@ import {
|
||||
getErrorMessage,
|
||||
isInTestMode,
|
||||
joinAtMost,
|
||||
DiskUsage,
|
||||
} from "./util";
|
||||
|
||||
export * from "./config/db-config";
|
||||
@@ -79,6 +79,15 @@ const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
|
||||
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
|
||||
|
||||
/**
|
||||
* The v2 minimum available disk space (in MB) required to perform overlay
|
||||
* analysis. This is a lower threshold than the v1 limit, allowing overlay
|
||||
* analysis to run on runners with less available disk space.
|
||||
*/
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14000;
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES =
|
||||
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1_000_000;
|
||||
|
||||
/**
|
||||
* The minimum memory (in MB) that must be available for CodeQL to perform overlay
|
||||
* analysis. If CodeQL will be given less memory than this threshold, then the
|
||||
@@ -676,21 +685,26 @@ async function isOverlayAnalysisFeatureEnabled(
|
||||
* and the maximum memory CodeQL will be allowed to use.
|
||||
*/
|
||||
async function runnerSupportsOverlayAnalysis(
|
||||
diskUsage: DiskUsage | undefined,
|
||||
ramInput: string | undefined,
|
||||
logger: Logger,
|
||||
useV2ResourceChecks: boolean,
|
||||
): Promise<boolean> {
|
||||
const diskUsage = await checkDiskUsage(logger);
|
||||
const minimumDiskSpaceBytes = useV2ResourceChecks
|
||||
? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES
|
||||
: OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES;
|
||||
if (
|
||||
diskUsage === undefined ||
|
||||
diskUsage.numAvailableBytes < OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES
|
||||
diskUsage.numAvailableBytes < minimumDiskSpaceBytes
|
||||
) {
|
||||
const diskSpaceMb =
|
||||
diskUsage === undefined
|
||||
? 0
|
||||
: Math.round(diskUsage.numAvailableBytes / 1_000_000);
|
||||
const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000);
|
||||
logger.info(
|
||||
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
|
||||
`due to insufficient disk space (${diskSpaceMb} MB).`,
|
||||
`due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
@@ -699,7 +713,7 @@ async function runnerSupportsOverlayAnalysis(
|
||||
if (memoryFlagValue < OVERLAY_MINIMUM_MEMORY_MB) {
|
||||
logger.info(
|
||||
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
|
||||
`due to insufficient memory for CodeQL analysis (${memoryFlagValue} MB).`,
|
||||
`due to insufficient memory for CodeQL analysis (${memoryFlagValue} MB, needed ${OVERLAY_MINIMUM_MEMORY_MB} MB).`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
@@ -741,9 +755,11 @@ export async function getOverlayDatabaseMode(
|
||||
): Promise<{
|
||||
overlayDatabaseMode: OverlayDatabaseMode;
|
||||
useOverlayDatabaseCaching: boolean;
|
||||
skippedDueToCachedStatus: boolean;
|
||||
}> {
|
||||
let overlayDatabaseMode = OverlayDatabaseMode.None;
|
||||
let useOverlayDatabaseCaching = false;
|
||||
let skippedDueToCachedStatus = false;
|
||||
|
||||
const modeEnv = process.env.CODEQL_OVERLAY_DATABASE_MODE;
|
||||
// Any unrecognized CODEQL_OVERLAY_DATABASE_MODE value will be ignored and
|
||||
@@ -770,11 +786,43 @@ export async function getOverlayDatabaseMode(
|
||||
Feature.OverlayAnalysisSkipResourceChecks,
|
||||
codeql,
|
||||
));
|
||||
const useV2ResourceChecks = await features.getValue(
|
||||
Feature.OverlayAnalysisResourceChecksV2,
|
||||
);
|
||||
const checkOverlayStatus = await features.getValue(
|
||||
Feature.OverlayAnalysisStatusCheck,
|
||||
);
|
||||
const diskUsage =
|
||||
performResourceChecks || checkOverlayStatus
|
||||
? await checkDiskUsage(logger)
|
||||
: undefined;
|
||||
if (
|
||||
performResourceChecks &&
|
||||
!(await runnerSupportsOverlayAnalysis(ramInput, logger))
|
||||
!(await runnerSupportsOverlayAnalysis(
|
||||
diskUsage,
|
||||
ramInput,
|
||||
logger,
|
||||
useV2ResourceChecks,
|
||||
))
|
||||
) {
|
||||
overlayDatabaseMode = OverlayDatabaseMode.None;
|
||||
} else if (checkOverlayStatus && diskUsage === undefined) {
|
||||
logger.warning(
|
||||
`Unable to determine disk usage, therefore setting overlay database mode to ${OverlayDatabaseMode.None}.`,
|
||||
);
|
||||
overlayDatabaseMode = OverlayDatabaseMode.None;
|
||||
} else if (
|
||||
checkOverlayStatus &&
|
||||
diskUsage &&
|
||||
(await shouldSkipOverlayAnalysis(codeql, languages, diskUsage, logger))
|
||||
) {
|
||||
logger.info(
|
||||
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
|
||||
"because overlay analysis previously failed with this combination of languages, " +
|
||||
"disk space, and CodeQL version.",
|
||||
);
|
||||
overlayDatabaseMode = OverlayDatabaseMode.None;
|
||||
skippedDueToCachedStatus = true;
|
||||
} else if (isAnalyzingPullRequest()) {
|
||||
overlayDatabaseMode = OverlayDatabaseMode.Overlay;
|
||||
useOverlayDatabaseCaching = true;
|
||||
@@ -795,6 +843,7 @@ export async function getOverlayDatabaseMode(
|
||||
const nonOverlayAnalysis = {
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
useOverlayDatabaseCaching: false,
|
||||
skippedDueToCachedStatus,
|
||||
};
|
||||
|
||||
if (overlayDatabaseMode === OverlayDatabaseMode.None) {
|
||||
@@ -859,6 +908,7 @@ export async function getOverlayDatabaseMode(
|
||||
return {
|
||||
overlayDatabaseMode,
|
||||
useOverlayDatabaseCaching,
|
||||
skippedDueToCachedStatus,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -965,10 +1015,13 @@ export async function initConfig(
|
||||
}
|
||||
}
|
||||
|
||||
// If we are in CCR or the corresponding FF is enabled, try to determine
|
||||
// If we are in a dynamic workflow or the corresponding FF is enabled, try to determine
|
||||
// which files in the repository are marked as generated and add them to
|
||||
// the `paths-ignore` configuration.
|
||||
if ((await features.getValue(Feature.IgnoreGeneratedFiles)) && isCCR()) {
|
||||
if (
|
||||
(await features.getValue(Feature.IgnoreGeneratedFiles)) &&
|
||||
isDynamicWorkflow()
|
||||
) {
|
||||
try {
|
||||
const generatedFilesCheckStartedAt = performance.now();
|
||||
const generatedFiles = await getGeneratedFiles(inputs.sourceRoot);
|
||||
@@ -1002,18 +1055,21 @@ export async function initConfig(
|
||||
// and queries, which in turn depends on the user config and the augmentation
|
||||
// properties. So we need to calculate the overlay database mode after the
|
||||
// rest of the config has been populated.
|
||||
const { overlayDatabaseMode, useOverlayDatabaseCaching } =
|
||||
await getOverlayDatabaseMode(
|
||||
inputs.codeql,
|
||||
inputs.features,
|
||||
config.languages,
|
||||
inputs.sourceRoot,
|
||||
config.buildMode,
|
||||
inputs.ramInput,
|
||||
config.computedConfig,
|
||||
gitVersion,
|
||||
logger,
|
||||
);
|
||||
const {
|
||||
overlayDatabaseMode,
|
||||
useOverlayDatabaseCaching,
|
||||
skippedDueToCachedStatus: overlaySkippedDueToCachedStatus,
|
||||
} = await getOverlayDatabaseMode(
|
||||
inputs.codeql,
|
||||
inputs.features,
|
||||
config.languages,
|
||||
inputs.sourceRoot,
|
||||
config.buildMode,
|
||||
inputs.ramInput,
|
||||
config.computedConfig,
|
||||
gitVersion,
|
||||
logger,
|
||||
);
|
||||
logger.info(
|
||||
`Using overlay database mode: ${overlayDatabaseMode} ` +
|
||||
`${useOverlayDatabaseCaching ? "with" : "without"} caching.`,
|
||||
@@ -1021,6 +1077,35 @@ export async function initConfig(
|
||||
config.overlayDatabaseMode = overlayDatabaseMode;
|
||||
config.useOverlayDatabaseCaching = useOverlayDatabaseCaching;
|
||||
|
||||
if (overlaySkippedDueToCachedStatus) {
|
||||
addNoLanguageDiagnostic(
|
||||
config,
|
||||
makeDiagnostic(
|
||||
"codeql-action/overlay-skipped-due-to-cached-status",
|
||||
"Skipped improved incremental analysis because it failed previously with similar hardware resources",
|
||||
{
|
||||
attributes: {
|
||||
languages: config.languages,
|
||||
},
|
||||
markdownMessage:
|
||||
`Improved incremental analysis was skipped because it previously failed for this repository ` +
|
||||
`with CodeQL version ${(await inputs.codeql.getVersion()).version} on a runner with similar hardware resources. ` +
|
||||
"Improved incremental analysis may require a significant amount of disk space for some repositories. " +
|
||||
"If you want to enable improved incremental analysis, increase the disk space available " +
|
||||
"to the runner. If that doesn't help, contact GitHub Support for further assistance.\n\n" +
|
||||
"Improved incremental analysis will be automatically retried when the next version of CodeQL is released. " +
|
||||
`You can also manually trigger a retry by [removing](${DocUrl.DELETE_ACTIONS_CACHE_ENTRIES}) \`codeql-overlay-status-*\` entries from the Actions cache.`,
|
||||
severity: "note",
|
||||
visibility: {
|
||||
cliSummaryTable: true,
|
||||
statusPage: true,
|
||||
telemetry: true,
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
overlayDatabaseMode === OverlayDatabaseMode.Overlay ||
|
||||
(await shouldPerformDiffInformedAnalysis(
|
||||
@@ -1389,28 +1474,27 @@ export function isCodeQualityEnabled(config: Config): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the primary analysis kind that the Action is initialised with. This is
|
||||
* always `AnalysisKind.CodeScanning` unless `AnalysisKind.CodeScanning` is not enabled.
|
||||
* Returns the primary analysis kind that the Action is initialised with. If there is only
|
||||
* one analysis kind, then that is returned.
|
||||
*
|
||||
* @returns Returns `AnalysisKind.CodeScanning` if `AnalysisKind.CodeScanning` is enabled;
|
||||
* otherwise `AnalysisKind.CodeQuality`.
|
||||
* The special case is Code Scanning + Code Quality, which can be enabled at the same time.
|
||||
* In that case, this function returns Code Scanning.
|
||||
*/
|
||||
function getPrimaryAnalysisKind(config: Config): AnalysisKind {
|
||||
if (config.analysisKinds.length === 1) {
|
||||
return config.analysisKinds[0];
|
||||
}
|
||||
|
||||
return isCodeScanningEnabled(config)
|
||||
? AnalysisKind.CodeScanning
|
||||
: AnalysisKind.CodeQuality;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the primary analysis configuration that the Action is initialised with. This is
|
||||
* always `CodeScanning` unless `CodeScanning` is not enabled.
|
||||
*
|
||||
* @returns Returns `CodeScanning` if `AnalysisKind.CodeScanning` is enabled; otherwise `CodeQuality`.
|
||||
* Returns the primary analysis configuration that the Action is initialised with.
|
||||
*/
|
||||
export function getPrimaryAnalysisConfig(config: Config): AnalysisConfig {
|
||||
return getPrimaryAnalysisKind(config) === AnalysisKind.CodeScanning
|
||||
? CodeScanning
|
||||
: CodeQuality;
|
||||
return getAnalysisConfig(getPrimaryAnalysisKind(config));
|
||||
}
|
||||
|
||||
/** Logs the Git version as a telemetry diagnostic. */
|
||||
|
||||
@@ -8,7 +8,7 @@ import { Config } from "./config-utils";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import * as gitUtils from "./git-utils";
|
||||
import { Logger, withGroupAsync } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay-database-utils";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import { RepositoryNwo } from "./repository";
|
||||
import * as util from "./util";
|
||||
import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
|
||||
@@ -101,7 +101,9 @@ export async function cleanupAndUploadDatabases(
|
||||
// Although we are uploading arbitrary file contents to the API, it's worth
|
||||
// noting that it's the API's job to validate that the contents is acceptable.
|
||||
// This API method is available to anyone with write access to the repo.
|
||||
const bundledDb = await bundleDb(config, language, codeql, language);
|
||||
const bundledDb = await bundleDb(config, language, codeql, language, {
|
||||
includeDiagnostics: false,
|
||||
});
|
||||
bundledDbSize = fs.statSync(bundledDb).size;
|
||||
const bundledDbReadStream = fs.createReadStream(bundledDb);
|
||||
const commitOid = await gitUtils.getCommitOid(
|
||||
|
||||
@@ -429,6 +429,7 @@ async function createDatabaseBundleCli(
|
||||
language,
|
||||
codeql,
|
||||
`${config.debugDatabaseName}-${language}`,
|
||||
{ includeDiagnostics: true },
|
||||
);
|
||||
return databaseBundlePath;
|
||||
}
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"bundleVersion": "codeql-bundle-v2.24.0",
|
||||
"cliVersion": "2.24.0",
|
||||
"priorBundleVersion": "codeql-bundle-v2.23.9",
|
||||
"priorCliVersion": "2.23.9"
|
||||
"bundleVersion": "codeql-bundle-v2.24.2",
|
||||
"cliVersion": "2.24.2",
|
||||
"priorBundleVersion": "codeql-bundle-v2.24.1",
|
||||
"priorCliVersion": "2.24.1"
|
||||
}
|
||||
|
||||
+27
-12
@@ -66,6 +66,12 @@ interface UnwrittenDiagnostic {
|
||||
/** A list of diagnostics which have not yet been written to disk. */
|
||||
let unwrittenDiagnostics: UnwrittenDiagnostic[] = [];
|
||||
|
||||
/**
|
||||
* A list of diagnostics which have not yet been written to disk,
|
||||
* and where the language does not matter.
|
||||
*/
|
||||
let unwrittenDefaultLanguageDiagnostics: DiagnosticMessage[] = [];
|
||||
|
||||
/**
|
||||
* Constructs a new diagnostic message with the specified id and name, as well as optional additional data.
|
||||
*
|
||||
@@ -119,16 +125,20 @@ export function addDiagnostic(
|
||||
|
||||
/** Adds a diagnostic that is not specific to any language. */
|
||||
export function addNoLanguageDiagnostic(
|
||||
config: Config,
|
||||
config: Config | undefined,
|
||||
diagnostic: DiagnosticMessage,
|
||||
) {
|
||||
addDiagnostic(
|
||||
config,
|
||||
// Arbitrarily choose the first language. We could also choose all languages, but that
|
||||
// increases the risk of misinterpreting the data.
|
||||
config.languages[0],
|
||||
diagnostic,
|
||||
);
|
||||
if (config !== undefined) {
|
||||
addDiagnostic(
|
||||
config,
|
||||
// Arbitrarily choose the first language. We could also choose all languages, but that
|
||||
// increases the risk of misinterpreting the data.
|
||||
config.languages[0],
|
||||
diagnostic,
|
||||
);
|
||||
} else {
|
||||
unwrittenDefaultLanguageDiagnostics.push(diagnostic);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -188,16 +198,21 @@ export function logUnwrittenDiagnostics() {
|
||||
/** Writes all unwritten diagnostics to disk. */
|
||||
export function flushDiagnostics(config: Config) {
|
||||
const logger = getActionsLogger();
|
||||
logger.debug(
|
||||
`Writing ${unwrittenDiagnostics.length} diagnostic(s) to database.`,
|
||||
);
|
||||
|
||||
const diagnosticsCount =
|
||||
unwrittenDiagnostics.length + unwrittenDefaultLanguageDiagnostics.length;
|
||||
logger.debug(`Writing ${diagnosticsCount} diagnostic(s) to database.`);
|
||||
|
||||
for (const unwritten of unwrittenDiagnostics) {
|
||||
writeDiagnostic(config, unwritten.language, unwritten.diagnostic);
|
||||
}
|
||||
for (const unwritten of unwrittenDefaultLanguageDiagnostics) {
|
||||
addNoLanguageDiagnostic(config, unwritten);
|
||||
}
|
||||
|
||||
// Reset the unwritten diagnostics array.
|
||||
// Reset the unwritten diagnostics arrays.
|
||||
unwrittenDiagnostics = [];
|
||||
unwrittenDefaultLanguageDiagnostics = [];
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
shouldPerformDiffInformedAnalysis,
|
||||
exportedForTesting,
|
||||
} from "./diff-informed-analysis-utils";
|
||||
import { Feature, Features } from "./feature-flags";
|
||||
import { Feature, initFeatures } from "./feature-flags";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import {
|
||||
@@ -63,7 +63,7 @@ const testShouldPerformDiffInformedAnalysis = test.macro({
|
||||
delete process.env.CODEQL_ACTION_DIFF_INFORMED_QUERIES;
|
||||
}
|
||||
|
||||
const features = new Features(
|
||||
const features = initFeatures(
|
||||
testCase.gitHubVersion,
|
||||
parseRepositoryNwo("github/example"),
|
||||
tmpDir,
|
||||
|
||||
+3
-2
@@ -5,10 +5,11 @@
|
||||
export enum DocUrl {
|
||||
ASSIGNING_PERMISSIONS_TO_JOBS = "https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs",
|
||||
AUTOMATIC_BUILD_FAILED = "https://docs.github.com/en/code-security/code-scanning/troubleshooting-code-scanning/automatic-build-failed",
|
||||
CODEQL_BUILD_MODES = "https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#codeql-build-modes",
|
||||
DEFINE_ENV_VARIABLES = "https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow",
|
||||
DELETE_ACTIONS_CACHE_ENTRIES = "https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-caches#deleting-cache-entries",
|
||||
SCANNING_ON_PUSH = "https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#scanning-on-push",
|
||||
SPECIFY_BUILD_STEPS_MANUALLY = "https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#about-specifying-build-steps-manually",
|
||||
TRACK_CODE_SCANNING_ALERTS_ACROSS_RUNS = "https://docs.github.com/en/enterprise-cloud@latest/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning#providing-data-to-track-code-scanning-alerts-across-runs",
|
||||
CODEQL_BUILD_MODES = "https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#codeql-build-modes",
|
||||
SYSTEM_REQUIREMENTS = "https://codeql.github.com/docs/codeql-overview/system-requirements/",
|
||||
TRACK_CODE_SCANNING_ALERTS_ACROSS_RUNS = "https://docs.github.com/en/code-security/reference/code-scanning/sarif-support-for-code-scanning#data-for-preventing-duplicated-alerts",
|
||||
}
|
||||
|
||||
@@ -141,4 +141,7 @@ export enum EnvVar {
|
||||
* `getAnalysisKey`, but can also be set manually for testing and non-standard applications.
|
||||
*/
|
||||
ANALYSIS_KEY = "CODEQL_ACTION_ANALYSIS_KEY",
|
||||
|
||||
/** Used by Code Scanning Risk Assessment to communicate the assessment ID to the CodeQL Action. */
|
||||
RISK_ASSESSMENT_ID = "CODEQL_ACTION_RISK_ASSESSMENT_ID",
|
||||
}
|
||||
|
||||
+19
-79
@@ -1,32 +1,31 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import test, { ExecutionContext } from "ava";
|
||||
import test from "ava";
|
||||
|
||||
import * as defaults from "./defaults.json";
|
||||
import {
|
||||
Feature,
|
||||
featureConfig,
|
||||
FeatureEnablement,
|
||||
Features,
|
||||
FEATURE_FLAGS_FILE_NAME,
|
||||
FeatureConfig,
|
||||
FeatureWithoutCLI,
|
||||
} from "./feature-flags";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import {
|
||||
setUpFeatureFlagTests,
|
||||
getFeatureIncludingCodeQlIfRequired,
|
||||
assertAllFeaturesUndefinedInApi,
|
||||
assertAllFeaturesHaveDefaultValues,
|
||||
} from "./feature-flags/testing-util";
|
||||
import {
|
||||
checkExpectedLogMessages,
|
||||
getRecordingLogger,
|
||||
initializeFeatures,
|
||||
LoggedMessage,
|
||||
mockCodeQLVersion,
|
||||
mockFeatureFlagApiEndpoint,
|
||||
setupActionsVars,
|
||||
setupTests,
|
||||
stubFeatureFlagApiEndpoint,
|
||||
} from "./testing-utils";
|
||||
import { ToolsFeature } from "./tools-features";
|
||||
import * as util from "./util";
|
||||
import { GitHubVariant, initializeEnvironment, withTmpDir } from "./util";
|
||||
|
||||
setupTests(test);
|
||||
@@ -35,9 +34,7 @@ test.beforeEach(() => {
|
||||
initializeEnvironment("1.2.3");
|
||||
});
|
||||
|
||||
const testRepositoryNwo = parseRepositoryNwo("github/example");
|
||||
|
||||
test(`All features are disabled if running against GHES`, async (t) => {
|
||||
test(`All features use default values if running against GHES`, async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const loggedMessages = [];
|
||||
const features = setUpFeatureFlagTests(
|
||||
@@ -46,21 +43,10 @@ test(`All features are disabled if running against GHES`, async (t) => {
|
||||
{ type: GitHubVariant.GHES, version: "3.0.0" },
|
||||
);
|
||||
|
||||
for (const feature of Object.values(Feature)) {
|
||||
t.deepEqual(
|
||||
await getFeatureIncludingCodeQlIfRequired(features, feature),
|
||||
featureConfig[feature].defaultValue,
|
||||
);
|
||||
}
|
||||
|
||||
t.assert(
|
||||
loggedMessages.find(
|
||||
(v: LoggedMessage) =>
|
||||
v.type === "debug" &&
|
||||
v.message ===
|
||||
"Not running against github.com. Disabling all toggleable features.",
|
||||
) !== undefined,
|
||||
);
|
||||
await assertAllFeaturesHaveDefaultValues(t, features);
|
||||
checkExpectedLogMessages(t, loggedMessages, [
|
||||
"Not running against github.com. Using default values for all features.",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -542,55 +528,9 @@ test("non-legacy feature flags should not start with codeql_action_", async (t)
|
||||
}
|
||||
});
|
||||
|
||||
function assertAllFeaturesUndefinedInApi(
|
||||
t: ExecutionContext<unknown>,
|
||||
loggedMessages: LoggedMessage[],
|
||||
) {
|
||||
for (const feature of Object.keys(featureConfig)) {
|
||||
t.assert(
|
||||
loggedMessages.find(
|
||||
(v) =>
|
||||
v.type === "debug" &&
|
||||
(v.message as string).includes(feature) &&
|
||||
(v.message as string).includes("undefined in API response"),
|
||||
) !== undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function setUpFeatureFlagTests(
|
||||
tmpDir: string,
|
||||
logger = getRunnerLogger(true),
|
||||
gitHubVersion = { type: GitHubVariant.DOTCOM } as util.GitHubVersion,
|
||||
): FeatureEnablement {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
return new Features(gitHubVersion, testRepositoryNwo, tmpDir, logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an argument to pass to `getValue` that if required includes a CodeQL object meeting the
|
||||
* minimum version or tool feature requirements specified by the feature.
|
||||
*/
|
||||
function getFeatureIncludingCodeQlIfRequired(
|
||||
features: FeatureEnablement,
|
||||
feature: Feature,
|
||||
) {
|
||||
const config = featureConfig[
|
||||
feature
|
||||
] satisfies FeatureConfig as FeatureConfig;
|
||||
if (
|
||||
config.minimumVersion === undefined &&
|
||||
config.toolsFeature === undefined
|
||||
) {
|
||||
return features.getValue(feature as FeatureWithoutCLI);
|
||||
}
|
||||
|
||||
return features.getValue(
|
||||
feature,
|
||||
mockCodeQLVersion(
|
||||
"9.9.9",
|
||||
Object.fromEntries(Object.values(ToolsFeature).map((v) => [v, true])),
|
||||
),
|
||||
);
|
||||
}
|
||||
test("initFeatures returns a `Features` instance by default", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const features = setUpFeatureFlagTests(tmpDir);
|
||||
t.is("Features", features.constructor.name);
|
||||
});
|
||||
});
|
||||
|
||||
+158
-57
@@ -7,7 +7,7 @@ import { getApiClient } from "./api-client";
|
||||
import type { CodeQL } from "./codeql";
|
||||
import * as defaults from "./defaults.json";
|
||||
import { Logger } from "./logging";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION } from "./overlay-database-utils";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION } from "./overlay";
|
||||
import { RepositoryNwo } from "./repository";
|
||||
import { ToolsFeature } from "./tools-features";
|
||||
import * as util from "./util";
|
||||
@@ -45,7 +45,10 @@ export enum Feature {
|
||||
DisableJavaBuildlessEnabled = "disable_java_buildless_enabled",
|
||||
DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
|
||||
ExportDiagnosticsEnabled = "export_diagnostics_enabled",
|
||||
ForceNightly = "force_nightly",
|
||||
IgnoreGeneratedFiles = "ignore_generated_files",
|
||||
ImprovedProxyCertificates = "improved_proxy_certificates",
|
||||
JavaNetworkDebugging = "java_network_debugging",
|
||||
OverlayAnalysis = "overlay_analysis",
|
||||
OverlayAnalysisActions = "overlay_analysis_actions",
|
||||
OverlayAnalysisCodeScanningActions = "overlay_analysis_code_scanning_actions",
|
||||
@@ -60,10 +63,13 @@ export enum Feature {
|
||||
OverlayAnalysisCodeScanningSwift = "overlay_analysis_code_scanning_swift",
|
||||
OverlayAnalysisCpp = "overlay_analysis_cpp",
|
||||
OverlayAnalysisCsharp = "overlay_analysis_csharp",
|
||||
OverlayAnalysisStatusCheck = "overlay_analysis_status_check",
|
||||
OverlayAnalysisStatusSave = "overlay_analysis_status_save",
|
||||
OverlayAnalysisGo = "overlay_analysis_go",
|
||||
OverlayAnalysisJava = "overlay_analysis_java",
|
||||
OverlayAnalysisJavascript = "overlay_analysis_javascript",
|
||||
OverlayAnalysisPython = "overlay_analysis_python",
|
||||
OverlayAnalysisResourceChecksV2 = "overlay_analysis_resource_checks_v2",
|
||||
OverlayAnalysisRuby = "overlay_analysis_ruby",
|
||||
OverlayAnalysisRust = "overlay_analysis_rust",
|
||||
OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks",
|
||||
@@ -73,7 +79,7 @@ export enum Feature {
|
||||
/** Note that this currently only disables baseline file coverage information. */
|
||||
SkipFileCoverageOnPrs = "skip_file_coverage_on_prs",
|
||||
UploadOverlayDbToApi = "upload_overlay_db_to_api",
|
||||
UseRepositoryProperties = "use_repository_properties",
|
||||
UseRepositoryProperties = "use_repository_properties_v2",
|
||||
ValidateDbConfig = "validate_db_config",
|
||||
}
|
||||
|
||||
@@ -161,11 +167,26 @@ export const featureConfig = {
|
||||
legacyApi: true,
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.ForceNightly]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_FORCE_NIGHTLY",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.IgnoreGeneratedFiles]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_IGNORE_GENERATED_FILES",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.ImprovedProxyCertificates]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_IMPROVED_PROXY_CERTIFICATES",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.JavaNetworkDebugging]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_JAVA_NETWORK_DEBUGGING",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysis]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS",
|
||||
@@ -236,6 +257,16 @@ export const featureConfig = {
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_CSHARP",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisStatusCheck]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisStatusSave]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_SAVE",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisGo]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_GO",
|
||||
@@ -256,6 +287,11 @@ export const featureConfig = {
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_PYTHON",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisResourceChecksV2]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisRuby]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RUBY",
|
||||
@@ -348,51 +384,60 @@ type GitHubFeatureFlagsApiResponse = Partial<Record<Feature, boolean>>;
|
||||
export const FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json";
|
||||
|
||||
/**
|
||||
* Determines the enablement status of a number of features.
|
||||
* If feature enablement is not able to be determined locally, a request to the
|
||||
* GitHub API is made to determine the enablement status.
|
||||
* Determines the enablement status of a number of features locally without
|
||||
* consulting the GitHub API.
|
||||
*/
|
||||
export class Features implements FeatureEnablement {
|
||||
private gitHubFeatureFlags: GitHubFeatureFlags;
|
||||
|
||||
constructor(
|
||||
gitHubVersion: util.GitHubVersion,
|
||||
repositoryNwo: RepositoryNwo,
|
||||
tempDir: string,
|
||||
private readonly logger: Logger,
|
||||
) {
|
||||
this.gitHubFeatureFlags = new GitHubFeatureFlags(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
path.join(tempDir, FEATURE_FLAGS_FILE_NAME),
|
||||
logger,
|
||||
);
|
||||
}
|
||||
class OfflineFeatures implements FeatureEnablement {
|
||||
constructor(protected readonly logger: Logger) {}
|
||||
|
||||
async getDefaultCliVersion(
|
||||
variant: util.GitHubVariant,
|
||||
_variant: util.GitHubVariant,
|
||||
): Promise<CodeQLDefaultVersionInfo> {
|
||||
return await this.gitHubFeatureFlags.getDefaultCliVersion(variant);
|
||||
return {
|
||||
cliVersion: defaults.cliVersion,
|
||||
tagName: defaults.bundleVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the `FeatureConfig` for `feature`.
|
||||
*/
|
||||
getFeatureConfig(feature: Feature): FeatureConfig {
|
||||
// Narrow the type to FeatureConfig to avoid type errors. To avoid unsafe use of `as`, we
|
||||
// check that the required properties exist using `satisfies`.
|
||||
return featureConfig[feature] satisfies FeatureConfig as FeatureConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether `feature` is enabled without consulting the GitHub API.
|
||||
*
|
||||
* @param feature The feature to check.
|
||||
* @param codeql An optional CodeQL object. If provided, and a `minimumVersion` is specified for the
|
||||
* feature, the version of the CodeQL CLI will be checked against the minimum version.
|
||||
* If the version is less than the minimum version, the feature will be considered
|
||||
* disabled. If not provided, and a `minimumVersion` is specified for the feature, the
|
||||
* disabled. If not provided, and a `minimumVersion` is specified for the feature, then
|
||||
* this function will throw.
|
||||
* @returns true if the feature is enabled, false otherwise.
|
||||
*
|
||||
* @throws if a `minimumVersion` is specified for the feature, and `codeql` is not provided.
|
||||
*/
|
||||
async getValue(feature: Feature, codeql?: CodeQL): Promise<boolean> {
|
||||
// Narrow the type to FeatureConfig to avoid type errors. To avoid unsafe use of `as`, we
|
||||
// check that the required properties exist using `satisfies`.
|
||||
const config = featureConfig[
|
||||
feature
|
||||
] satisfies FeatureConfig as FeatureConfig;
|
||||
const offlineValue = await this.getOfflineValue(feature, codeql);
|
||||
if (offlineValue !== undefined) {
|
||||
return offlineValue;
|
||||
}
|
||||
|
||||
return this.getDefaultValue(feature);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether `feature` is enabled using the CLI and environment variables.
|
||||
*/
|
||||
protected async getOfflineValue(
|
||||
feature: Feature,
|
||||
codeql?: CodeQL,
|
||||
): Promise<boolean | undefined> {
|
||||
const config = this.getFeatureConfig(feature);
|
||||
|
||||
if (!codeql && config.minimumVersion) {
|
||||
throw new Error(
|
||||
@@ -458,6 +503,68 @@ export class Features implements FeatureEnablement {
|
||||
return true;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Gets the default value of `feature`. */
|
||||
protected async getDefaultValue(feature: Feature): Promise<boolean> {
|
||||
const config = this.getFeatureConfig(feature);
|
||||
const defaultValue = config.defaultValue;
|
||||
this.logger.debug(
|
||||
`Feature ${feature} is ${
|
||||
defaultValue ? "enabled" : "disabled"
|
||||
} due to its default value.`,
|
||||
);
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines the enablement status of a number of features.
|
||||
* If feature enablement is not able to be determined locally, a request to the
|
||||
* GitHub API is made to determine the enablement status.
|
||||
*/
|
||||
class Features extends OfflineFeatures {
|
||||
private gitHubFeatureFlags: GitHubFeatureFlags;
|
||||
|
||||
constructor(repositoryNwo: RepositoryNwo, tempDir: string, logger: Logger) {
|
||||
super(logger);
|
||||
|
||||
this.gitHubFeatureFlags = new GitHubFeatureFlags(
|
||||
repositoryNwo,
|
||||
path.join(tempDir, FEATURE_FLAGS_FILE_NAME),
|
||||
logger,
|
||||
);
|
||||
}
|
||||
|
||||
async getDefaultCliVersion(
|
||||
variant: util.GitHubVariant,
|
||||
): Promise<CodeQLDefaultVersionInfo> {
|
||||
if (supportsFeatureFlags(variant)) {
|
||||
return await this.gitHubFeatureFlags.getDefaultCliVersionFromFlags();
|
||||
}
|
||||
return super.getDefaultCliVersion(variant);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param feature The feature to check.
|
||||
* @param codeql An optional CodeQL object. If provided, and a `minimumVersion` is specified for the
|
||||
* feature, the version of the CodeQL CLI will be checked against the minimum version.
|
||||
* If the version is less than the minimum version, the feature will be considered
|
||||
* disabled. If not provided, and a `minimumVersion` is specified for the feature, then
|
||||
* this function will throw.
|
||||
* @returns true if the feature is enabled, false otherwise.
|
||||
*
|
||||
* @throws if a `minimumVersion` is specified for the feature, and `codeql` is not provided.
|
||||
*/
|
||||
async getValue(feature: Feature, codeql?: CodeQL): Promise<boolean> {
|
||||
// Check whether the feature is enabled locally.
|
||||
const offlineValue = await this.getOfflineValue(feature, codeql);
|
||||
if (offlineValue !== undefined) {
|
||||
return offlineValue;
|
||||
}
|
||||
|
||||
// Ask the GitHub API if the feature is enabled.
|
||||
const apiValue = await this.gitHubFeatureFlags.getValue(feature);
|
||||
if (apiValue !== undefined) {
|
||||
@@ -469,13 +576,8 @@ export class Features implements FeatureEnablement {
|
||||
return apiValue;
|
||||
}
|
||||
|
||||
const defaultValue = config.defaultValue;
|
||||
this.logger.debug(
|
||||
`Feature ${feature} is ${
|
||||
defaultValue ? "enabled" : "disabled"
|
||||
} due to its default value.`,
|
||||
);
|
||||
return defaultValue;
|
||||
// Return the default value.
|
||||
return this.getDefaultValue(feature);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -487,7 +589,6 @@ class GitHubFeatureFlags {
|
||||
private hasAccessedRemoteFeatureFlags: boolean;
|
||||
|
||||
constructor(
|
||||
private readonly gitHubVersion: util.GitHubVersion,
|
||||
private readonly repositoryNwo: RepositoryNwo,
|
||||
private readonly featureFlagsFile: string,
|
||||
private readonly logger: Logger,
|
||||
@@ -518,18 +619,6 @@ class GitHubFeatureFlags {
|
||||
return version;
|
||||
}
|
||||
|
||||
async getDefaultCliVersion(
|
||||
variant: util.GitHubVariant,
|
||||
): Promise<CodeQLDefaultVersionInfo> {
|
||||
if (supportsFeatureFlags(variant)) {
|
||||
return await this.getDefaultCliVersionFromFlags();
|
||||
}
|
||||
return {
|
||||
cliVersion: defaults.cliVersion,
|
||||
tagName: defaults.bundleVersion,
|
||||
};
|
||||
}
|
||||
|
||||
async getDefaultCliVersionFromFlags(): Promise<CodeQLDefaultVersionInfo> {
|
||||
const response = await this.getAllFeatures();
|
||||
|
||||
@@ -655,14 +744,6 @@ class GitHubFeatureFlags {
|
||||
}
|
||||
|
||||
private async loadApiResponse(): Promise<GitHubFeatureFlagsApiResponse> {
|
||||
// Do nothing when not running against github.com
|
||||
if (!supportsFeatureFlags(this.gitHubVersion.type)) {
|
||||
this.logger.debug(
|
||||
"Not running against github.com. Disabling all toggleable features.",
|
||||
);
|
||||
this.hasAccessedRemoteFeatureFlags = false;
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
const featuresToRequest = Object.entries(featureConfig)
|
||||
.filter(
|
||||
@@ -732,3 +813,23 @@ function supportsFeatureFlags(githubVariant: util.GitHubVariant): boolean {
|
||||
githubVariant === util.GitHubVariant.GHEC_DR
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialises an instance of a `FeatureEnablement` implementation. The implementation used
|
||||
* is determined by the environment we are running in.
|
||||
*/
|
||||
export function initFeatures(
|
||||
gitHubVersion: util.GitHubVersion,
|
||||
repositoryNwo: RepositoryNwo,
|
||||
tempDir: string,
|
||||
logger: Logger,
|
||||
): FeatureEnablement {
|
||||
if (!supportsFeatureFlags(gitHubVersion.type)) {
|
||||
logger.debug(
|
||||
"Not running against github.com. Using default values for all features.",
|
||||
);
|
||||
return new OfflineFeatures(logger);
|
||||
} else {
|
||||
return new Features(repositoryNwo, tempDir, logger);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as apiClient from "../api-client";
|
||||
import {
|
||||
checkExpectedLogMessages,
|
||||
getRecordingLogger,
|
||||
LoggedMessage,
|
||||
setupTests,
|
||||
} from "../testing-utils";
|
||||
import { GitHubVariant, initializeEnvironment, withTmpDir } from "../util";
|
||||
|
||||
import {
|
||||
assertAllFeaturesHaveDefaultValues,
|
||||
setUpFeatureFlagTests,
|
||||
} from "./testing-util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
test.beforeEach(() => {
|
||||
initializeEnvironment("1.2.3");
|
||||
});
|
||||
|
||||
test("OfflineFeatures makes no API requests", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const loggedMessages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
const features = setUpFeatureFlagTests(tmpDir, logger, {
|
||||
type: GitHubVariant.GHES,
|
||||
version: "3.0.0",
|
||||
});
|
||||
t.is("OfflineFeatures", features.constructor.name);
|
||||
|
||||
sinon
|
||||
.stub(apiClient, "getApiClient")
|
||||
.throws(new Error("Should not have called getApiClient"));
|
||||
|
||||
await assertAllFeaturesHaveDefaultValues(t, features);
|
||||
checkExpectedLogMessages(t, loggedMessages, [
|
||||
"Not running against github.com. Using default values for all features.",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import { type ExecutionContext } from "ava";
|
||||
|
||||
import {
|
||||
Feature,
|
||||
featureConfig,
|
||||
FeatureConfig,
|
||||
FeatureEnablement,
|
||||
FeatureWithoutCLI,
|
||||
initFeatures,
|
||||
} from "../feature-flags";
|
||||
import { getRunnerLogger } from "../logging";
|
||||
import { parseRepositoryNwo } from "../repository";
|
||||
import {
|
||||
LoggedMessage,
|
||||
mockCodeQLVersion,
|
||||
setupActionsVars,
|
||||
} from "../testing-utils";
|
||||
import { ToolsFeature } from "../tools-features";
|
||||
import { GitHubVariant } from "../util";
|
||||
import * as util from "../util";
|
||||
|
||||
const testRepositoryNwo = parseRepositoryNwo("github/example");
|
||||
|
||||
export async function assertAllFeaturesHaveDefaultValues(
|
||||
t: ExecutionContext<unknown>,
|
||||
features: FeatureEnablement,
|
||||
) {
|
||||
for (const feature of Object.values(Feature)) {
|
||||
t.deepEqual(
|
||||
await getFeatureIncludingCodeQlIfRequired(features, feature),
|
||||
featureConfig[feature].defaultValue,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function assertAllFeaturesUndefinedInApi(
|
||||
t: ExecutionContext<unknown>,
|
||||
loggedMessages: LoggedMessage[],
|
||||
) {
|
||||
for (const feature of Object.keys(featureConfig)) {
|
||||
t.assert(
|
||||
loggedMessages.find(
|
||||
(v) =>
|
||||
v.type === "debug" &&
|
||||
(v.message as string).includes(feature) &&
|
||||
(v.message as string).includes("undefined in API response"),
|
||||
) !== undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function setUpFeatureFlagTests(
|
||||
tmpDir: string,
|
||||
logger = getRunnerLogger(true),
|
||||
gitHubVersion = { type: GitHubVariant.DOTCOM } as util.GitHubVersion,
|
||||
): FeatureEnablement {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
return initFeatures(gitHubVersion, testRepositoryNwo, tmpDir, logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an argument to pass to `getValue` that if required includes a CodeQL object meeting the
|
||||
* minimum version or tool feature requirements specified by the feature.
|
||||
*/
|
||||
export function getFeatureIncludingCodeQlIfRequired(
|
||||
features: FeatureEnablement,
|
||||
feature: Feature,
|
||||
) {
|
||||
const config = featureConfig[
|
||||
feature
|
||||
] satisfies FeatureConfig as FeatureConfig;
|
||||
if (
|
||||
config.minimumVersion === undefined &&
|
||||
config.toolsFeature === undefined
|
||||
) {
|
||||
return features.getValue(feature as FeatureWithoutCLI);
|
||||
}
|
||||
|
||||
return features.getValue(
|
||||
feature,
|
||||
mockCodeQLVersion(
|
||||
"9.9.9",
|
||||
Object.fromEntries(Object.values(ToolsFeature).map((v) => [v, true])),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -5,9 +5,12 @@ import * as actionsUtil from "./actions-util";
|
||||
import { AnalysisKind } from "./analyses";
|
||||
import * as codeql from "./codeql";
|
||||
import * as configUtils from "./config-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature } from "./feature-flags";
|
||||
import * as initActionPostHelper from "./init-action-post-helper";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import * as overlayStatus from "./overlay/status";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import {
|
||||
createFeatures,
|
||||
@@ -19,9 +22,11 @@ import * as uploadLib from "./upload-lib";
|
||||
import * as util from "./util";
|
||||
import * as workflow from "./workflow";
|
||||
|
||||
const NUM_BYTES_PER_GIB = 1024 * 1024 * 1024;
|
||||
|
||||
setupTests(test);
|
||||
|
||||
test("post: init action with debug mode off", async (t) => {
|
||||
test("init-post action with debug mode off", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
@@ -55,7 +60,7 @@ test("post: init action with debug mode off", async (t) => {
|
||||
});
|
||||
});
|
||||
|
||||
test("post: init action with debug mode on", async (t) => {
|
||||
test("init-post action with debug mode on", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
@@ -308,6 +313,179 @@ test("not uploading failed SARIF when `code-scanning` is not an enabled analysis
|
||||
);
|
||||
});
|
||||
|
||||
test("saves overlay status when overlay-base analysis did not complete successfully", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
// Ensure analyze did not complete successfully.
|
||||
delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY];
|
||||
|
||||
const diskUsage: util.DiskUsage = {
|
||||
numAvailableBytes: 100 * NUM_BYTES_PER_GIB,
|
||||
numTotalBytes: 200 * NUM_BYTES_PER_GIB,
|
||||
};
|
||||
sinon.stub(util, "checkDiskUsage").resolves(diskUsage);
|
||||
|
||||
const saveOverlayStatusStub = sinon
|
||||
.stub(overlayStatus, "saveOverlayStatus")
|
||||
.resolves(true);
|
||||
|
||||
const stubCodeQL = codeql.createStubCodeQL({});
|
||||
|
||||
await initActionPostHelper.run(
|
||||
sinon.spy(),
|
||||
sinon.spy(),
|
||||
stubCodeQL,
|
||||
createTestConfig({
|
||||
debugMode: false,
|
||||
languages: ["javascript"],
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([Feature.OverlayAnalysisStatusSave]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.calledOnce,
|
||||
"saveOverlayStatus should be called exactly once",
|
||||
);
|
||||
t.deepEqual(
|
||||
saveOverlayStatusStub.firstCall.args[0],
|
||||
stubCodeQL,
|
||||
"first arg should be the CodeQL instance",
|
||||
);
|
||||
t.deepEqual(
|
||||
saveOverlayStatusStub.firstCall.args[1],
|
||||
["javascript"],
|
||||
"second arg should be the languages",
|
||||
);
|
||||
t.deepEqual(
|
||||
saveOverlayStatusStub.firstCall.args[2],
|
||||
diskUsage,
|
||||
"third arg should be the disk usage",
|
||||
);
|
||||
t.deepEqual(
|
||||
saveOverlayStatusStub.firstCall.args[3],
|
||||
{
|
||||
attemptedToBuildOverlayBaseDatabase: true,
|
||||
builtOverlayBaseDatabase: false,
|
||||
},
|
||||
"fourth arg should be the overlay status recording an unsuccessful build attempt",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("does not save overlay status when OverlayAnalysisStatusSave feature flag is disabled", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
// Ensure analyze did not complete successfully.
|
||||
delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY];
|
||||
|
||||
sinon.stub(util, "checkDiskUsage").resolves({
|
||||
numAvailableBytes: 100 * NUM_BYTES_PER_GIB,
|
||||
numTotalBytes: 200 * NUM_BYTES_PER_GIB,
|
||||
});
|
||||
|
||||
const saveOverlayStatusStub = sinon
|
||||
.stub(overlayStatus, "saveOverlayStatus")
|
||||
.resolves(true);
|
||||
|
||||
await initActionPostHelper.run(
|
||||
sinon.spy(),
|
||||
sinon.spy(),
|
||||
codeql.createStubCodeQL({}),
|
||||
createTestConfig({
|
||||
debugMode: false,
|
||||
languages: ["javascript"],
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.notCalled,
|
||||
"saveOverlayStatus should not be called when OverlayAnalysisStatusSave feature flag is disabled",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("does not save overlay status when build successful", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
// Mark analyze as having completed successfully.
|
||||
process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] = "true";
|
||||
|
||||
sinon.stub(util, "checkDiskUsage").resolves({
|
||||
numAvailableBytes: 100 * NUM_BYTES_PER_GIB,
|
||||
numTotalBytes: 200 * NUM_BYTES_PER_GIB,
|
||||
});
|
||||
|
||||
const saveOverlayStatusStub = sinon
|
||||
.stub(overlayStatus, "saveOverlayStatus")
|
||||
.resolves(true);
|
||||
|
||||
await initActionPostHelper.run(
|
||||
sinon.spy(),
|
||||
sinon.spy(),
|
||||
codeql.createStubCodeQL({}),
|
||||
createTestConfig({
|
||||
debugMode: false,
|
||||
languages: ["javascript"],
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([Feature.OverlayAnalysisStatusSave]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.notCalled,
|
||||
"saveOverlayStatus should not be called when build completed successfully",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("does not save overlay status when overlay not enabled", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_REPOSITORY"] = "github/codeql-action-fake-repository";
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
delete process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY];
|
||||
|
||||
sinon.stub(util, "checkDiskUsage").resolves({
|
||||
numAvailableBytes: 100 * NUM_BYTES_PER_GIB,
|
||||
numTotalBytes: 200 * NUM_BYTES_PER_GIB,
|
||||
});
|
||||
|
||||
const saveOverlayStatusStub = sinon
|
||||
.stub(overlayStatus, "saveOverlayStatus")
|
||||
.resolves(true);
|
||||
|
||||
await initActionPostHelper.run(
|
||||
sinon.spy(),
|
||||
sinon.spy(),
|
||||
codeql.createStubCodeQL({}),
|
||||
createTestConfig({
|
||||
debugMode: false,
|
||||
languages: ["javascript"],
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
}),
|
||||
parseRepositoryNwo("github/codeql-action"),
|
||||
createFeatures([]),
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.true(
|
||||
saveOverlayStatusStub.notCalled,
|
||||
"saveOverlayStatus should not be called when overlay is not enabled",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
function createTestWorkflow(
|
||||
steps: workflow.WorkflowJobStep[],
|
||||
): workflow.Workflow {
|
||||
|
||||
@@ -11,10 +11,13 @@ import * as dependencyCaching from "./dependency-caching";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { Logger } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import { OverlayStatus, saveOverlayStatus } from "./overlay/status";
|
||||
import { RepositoryNwo, getRepositoryNwo } from "./repository";
|
||||
import { JobStatus } from "./status-report";
|
||||
import * as uploadLib from "./upload-lib";
|
||||
import {
|
||||
checkDiskUsage,
|
||||
delay,
|
||||
getErrorMessage,
|
||||
getRequiredEnvParam,
|
||||
@@ -169,6 +172,8 @@ export async function run(
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
) {
|
||||
await recordOverlayStatus(codeql, config, features, logger);
|
||||
|
||||
const uploadFailedSarifResult = await tryUploadSarifIfRunFailed(
|
||||
config,
|
||||
repositoryNwo,
|
||||
@@ -246,6 +251,68 @@ export async function run(
|
||||
return uploadFailedSarifResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* If overlay base database creation was attempted but the analysis did not complete
|
||||
* successfully, save the failure status to the Actions cache so that subsequent runs
|
||||
* can skip overlay analysis until something changes (e.g. a new CodeQL version).
|
||||
*/
|
||||
async function recordOverlayStatus(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
) {
|
||||
if (
|
||||
config.overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase ||
|
||||
process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] === "true" ||
|
||||
!(await features.getValue(Feature.OverlayAnalysisStatusSave))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const overlayStatus: OverlayStatus = {
|
||||
attemptedToBuildOverlayBaseDatabase: true,
|
||||
builtOverlayBaseDatabase: false,
|
||||
};
|
||||
|
||||
const diskUsage = await checkDiskUsage(logger);
|
||||
if (diskUsage === undefined) {
|
||||
logger.warning(
|
||||
"Unable to save overlay status to the Actions cache because the available disk space could not be determined.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const saved = await saveOverlayStatus(
|
||||
codeql,
|
||||
config.languages,
|
||||
diskUsage,
|
||||
overlayStatus,
|
||||
logger,
|
||||
);
|
||||
|
||||
const blurb =
|
||||
"This job attempted to run with improved incremental analysis but it did not complete successfully. " +
|
||||
"This may have been due to disk space constraints: using improved incremental analysis can " +
|
||||
"require a significant amount of disk space for some repositories.";
|
||||
|
||||
if (saved) {
|
||||
logger.error(
|
||||
`${blurb} ` +
|
||||
"This failure has been recorded in the Actions cache, so the next CodeQL analysis will run " +
|
||||
"without improved incremental analysis. If you want to enable improved incremental analysis, " +
|
||||
"increase the disk space available to the runner. " +
|
||||
"If that doesn't help, contact GitHub Support for further assistance.",
|
||||
);
|
||||
} else {
|
||||
logger.error(
|
||||
`${blurb} ` +
|
||||
"The attempt to save this failure status to the Actions cache failed. The Action will attempt to " +
|
||||
"run with improved incremental analysis again.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function removeUploadedSarif(
|
||||
uploadFailedSarifResult: UploadFailedSarifResult,
|
||||
logger: Logger,
|
||||
|
||||
@@ -21,7 +21,7 @@ import {
|
||||
getDependencyCacheUsage,
|
||||
} from "./dependency-caching";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Features } from "./feature-flags";
|
||||
import { initFeatures } from "./feature-flags";
|
||||
import * as gitUtils from "./git-utils";
|
||||
import * as initActionPostHelper from "./init-action-post-helper";
|
||||
import { getActionsLogger } from "./logging";
|
||||
@@ -62,7 +62,7 @@ async function run(startedAt: Date) {
|
||||
checkGitHubVersionInRange(gitHubVersion, logger);
|
||||
|
||||
const repositoryNwo = getRepositoryNwo();
|
||||
const features = new Features(
|
||||
const features = initFeatures(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
getTemporaryDirectory(),
|
||||
|
||||
+25
-9
@@ -38,7 +38,7 @@ import {
|
||||
makeTelemetryDiagnostic,
|
||||
} from "./diagnostics";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature, FeatureEnablement, Features } from "./feature-flags";
|
||||
import { Feature, FeatureEnablement, initFeatures } from "./feature-flags";
|
||||
import {
|
||||
loadPropertiesFromApi,
|
||||
RepositoryProperties,
|
||||
@@ -52,13 +52,13 @@ import {
|
||||
initConfig,
|
||||
runDatabaseInitCluster,
|
||||
} from "./init";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { JavaEnvVars, KnownLanguage } from "./languages";
|
||||
import { getActionsLogger, Logger } from "./logging";
|
||||
import {
|
||||
downloadOverlayBaseDatabaseFromCache,
|
||||
OverlayBaseDatabaseDownloadStats,
|
||||
OverlayDatabaseMode,
|
||||
} from "./overlay-database-utils";
|
||||
} from "./overlay";
|
||||
import { getRepositoryNwo, RepositoryNwo } from "./repository";
|
||||
import { ToolsSource } from "./setup-codeql";
|
||||
import {
|
||||
@@ -95,6 +95,9 @@ import {
|
||||
BuildMode,
|
||||
GitHubVersion,
|
||||
Result,
|
||||
getOptionalEnvVar,
|
||||
Success,
|
||||
Failure,
|
||||
} from "./util";
|
||||
import { checkWorkflow } from "./workflow";
|
||||
|
||||
@@ -210,7 +213,7 @@ async function run(startedAt: Date) {
|
||||
let config: configUtils.Config | undefined;
|
||||
let configFile: string | undefined;
|
||||
let codeql: CodeQL;
|
||||
let features: Features;
|
||||
let features: FeatureEnablement;
|
||||
let sourceRoot: string;
|
||||
let toolsDownloadStatusReport: ToolsDownloadStatusReport | undefined;
|
||||
let toolsFeatureFlagsValid: boolean | undefined;
|
||||
@@ -237,7 +240,7 @@ async function run(startedAt: Date) {
|
||||
|
||||
const repositoryNwo = getRepositoryNwo();
|
||||
|
||||
features = new Features(
|
||||
features = initFeatures(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
getTemporaryDirectory(),
|
||||
@@ -753,6 +756,19 @@ async function run(startedAt: Date) {
|
||||
}
|
||||
}
|
||||
|
||||
// Enable Java network debugging if the FF is enabled.
|
||||
if (await features.getValue(Feature.JavaNetworkDebugging)) {
|
||||
// Get the existing value of `JAVA_TOOL_OPTIONS`, if any.
|
||||
const existingJavaToolOptions =
|
||||
getOptionalEnvVar(JavaEnvVars.JAVA_TOOL_OPTIONS) || "";
|
||||
|
||||
// Add the network debugging options.
|
||||
core.exportVariable(
|
||||
JavaEnvVars.JAVA_TOOL_OPTIONS,
|
||||
`${existingJavaToolOptions} -Djavax.net.debug=all`,
|
||||
);
|
||||
}
|
||||
|
||||
// Write diagnostics to the database that we previously stored in memory because the database
|
||||
// did not exist until now.
|
||||
flushDiagnostics(config);
|
||||
@@ -820,25 +836,25 @@ async function loadRepositoryProperties(
|
||||
"Skipping loading repository properties because the repository is owned by a user and " +
|
||||
"therefore cannot have repository properties.",
|
||||
);
|
||||
return Result.success({});
|
||||
return new Success({});
|
||||
}
|
||||
|
||||
if (!(await features.getValue(Feature.UseRepositoryProperties))) {
|
||||
logger.debug(
|
||||
"Skipping loading repository properties because the UseRepositoryProperties feature flag is disabled.",
|
||||
);
|
||||
return Result.success({});
|
||||
return new Success({});
|
||||
}
|
||||
|
||||
try {
|
||||
return Result.success(
|
||||
return new Success(
|
||||
await loadPropertiesFromApi(gitHubVersion, logger, repositoryNwo),
|
||||
);
|
||||
} catch (error) {
|
||||
logger.warning(
|
||||
`Failed to load repository properties: ${getErrorMessage(error)}`,
|
||||
);
|
||||
return Result.failure(error);
|
||||
return new Failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,3 +19,11 @@ export enum KnownLanguage {
|
||||
rust = "rust",
|
||||
swift = "swift",
|
||||
}
|
||||
|
||||
/** Java-specific environment variable names that we may care about. */
|
||||
export enum JavaEnvVars {
|
||||
JAVA_HOME = "JAVA_HOME",
|
||||
JAVA_TOOL_OPTIONS = "JAVA_TOOL_OPTIONS",
|
||||
JDK_JAVA_OPTIONS = "JDK_JAVA_OPTIONS",
|
||||
_JAVA_OPTIONS = "_JAVA_OPTIONS",
|
||||
}
|
||||
|
||||
@@ -5,12 +5,20 @@ import * as actionsCache from "@actions/cache";
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import * as apiClient from "./api-client";
|
||||
import { ResolveDatabaseOutput } from "./codeql";
|
||||
import * as gitUtils from "./git-utils";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import * as actionsUtil from "../actions-util";
|
||||
import * as apiClient from "../api-client";
|
||||
import { ResolveDatabaseOutput } from "../codeql";
|
||||
import * as gitUtils from "../git-utils";
|
||||
import { KnownLanguage } from "../languages";
|
||||
import { getRunnerLogger } from "../logging";
|
||||
import {
|
||||
createTestConfig,
|
||||
mockCodeQLVersion,
|
||||
setupTests,
|
||||
} from "../testing-utils";
|
||||
import * as utils from "../util";
|
||||
import { withTmpDir } from "../util";
|
||||
|
||||
import {
|
||||
downloadOverlayBaseDatabaseFromCache,
|
||||
getCacheRestoreKeyPrefix,
|
||||
@@ -18,14 +26,7 @@ import {
|
||||
OverlayDatabaseMode,
|
||||
writeBaseDatabaseOidsFile,
|
||||
writeOverlayChangesFile,
|
||||
} from "./overlay-database-utils";
|
||||
import {
|
||||
createTestConfig,
|
||||
mockCodeQLVersion,
|
||||
setupTests,
|
||||
} from "./testing-utils";
|
||||
import * as utils from "./util";
|
||||
import { withTmpDir } from "./util";
|
||||
} from ".";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
@@ -8,13 +8,13 @@ import {
|
||||
getTemporaryDirectory,
|
||||
getWorkflowRunAttempt,
|
||||
getWorkflowRunID,
|
||||
} from "./actions-util";
|
||||
import { getAutomationID } from "./api-client";
|
||||
import { createCacheKeyHash } from "./caching-utils";
|
||||
import { type CodeQL } from "./codeql";
|
||||
import { type Config } from "./config-utils";
|
||||
import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
|
||||
import { Logger, withGroupAsync } from "./logging";
|
||||
} from "../actions-util";
|
||||
import { getAutomationID } from "../api-client";
|
||||
import { createCacheKeyHash } from "../caching-utils";
|
||||
import { type CodeQL } from "../codeql";
|
||||
import { type Config } from "../config-utils";
|
||||
import { getCommitOid, getFileOidsUnderPath } from "../git-utils";
|
||||
import { Logger, withGroupAsync } from "../logging";
|
||||
import {
|
||||
CleanupLevel,
|
||||
getBaseDatabaseOidsFilePath,
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
isInTestMode,
|
||||
tryGetFolderBytes,
|
||||
waitForResultWithTimeLimit,
|
||||
} from "./util";
|
||||
} from "../util";
|
||||
|
||||
export enum OverlayDatabaseMode {
|
||||
Overlay = "overlay",
|
||||
@@ -0,0 +1,172 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as actionsCache from "@actions/cache";
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import {
|
||||
getRecordingLogger,
|
||||
LoggedMessage,
|
||||
mockCodeQLVersion,
|
||||
setupTests,
|
||||
} from "../testing-utils";
|
||||
import { DiskUsage, withTmpDir } from "../util";
|
||||
|
||||
import { getCacheKey, shouldSkipOverlayAnalysis } from "./status";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
function makeDiskUsage(totalGiB: number): DiskUsage {
|
||||
return {
|
||||
numTotalBytes: totalGiB * 1024 * 1024 * 1024,
|
||||
numAvailableBytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
test("getCacheKey incorporates language, CodeQL version, and disk space", async (t) => {
|
||||
const codeql = mockCodeQLVersion("2.20.0");
|
||||
t.is(
|
||||
await getCacheKey(codeql, ["javascript"], makeDiskUsage(50)),
|
||||
"codeql-overlay-status-javascript-2.20.0-runner-50GB",
|
||||
);
|
||||
t.is(
|
||||
await getCacheKey(codeql, ["python"], makeDiskUsage(50)),
|
||||
"codeql-overlay-status-python-2.20.0-runner-50GB",
|
||||
);
|
||||
t.is(
|
||||
await getCacheKey(
|
||||
mockCodeQLVersion("2.21.0"),
|
||||
["javascript"],
|
||||
makeDiskUsage(50),
|
||||
),
|
||||
"codeql-overlay-status-javascript-2.21.0-runner-50GB",
|
||||
);
|
||||
t.is(
|
||||
await getCacheKey(codeql, ["javascript"], makeDiskUsage(100)),
|
||||
"codeql-overlay-status-javascript-2.20.0-runner-100GB",
|
||||
);
|
||||
});
|
||||
|
||||
test("getCacheKey sorts and joins multiple languages", async (t) => {
|
||||
const codeql = mockCodeQLVersion("2.20.0");
|
||||
t.is(
|
||||
await getCacheKey(codeql, ["python", "javascript"], makeDiskUsage(50)),
|
||||
"codeql-overlay-status-javascript+python-2.20.0-runner-50GB",
|
||||
);
|
||||
t.is(
|
||||
await getCacheKey(codeql, ["javascript", "python"], makeDiskUsage(50)),
|
||||
"codeql-overlay-status-javascript+python-2.20.0-runner-50GB",
|
||||
);
|
||||
});
|
||||
|
||||
test("getCacheKey rounds disk space down to nearest 10 GiB", async (t) => {
|
||||
const codeql = mockCodeQLVersion("2.20.0");
|
||||
t.is(
|
||||
await getCacheKey(codeql, ["javascript"], makeDiskUsage(14)),
|
||||
"codeql-overlay-status-javascript-2.20.0-runner-10GB",
|
||||
);
|
||||
t.is(
|
||||
await getCacheKey(codeql, ["javascript"], makeDiskUsage(19)),
|
||||
"codeql-overlay-status-javascript-2.20.0-runner-10GB",
|
||||
);
|
||||
});
|
||||
|
||||
test("shouldSkipOverlayAnalysis returns false when no cached status exists", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
const codeql = mockCodeQLVersion("2.20.0");
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
sinon.stub(actionsCache, "restoreCache").resolves(undefined);
|
||||
|
||||
const result = await shouldSkipOverlayAnalysis(
|
||||
codeql,
|
||||
["javascript"],
|
||||
makeDiskUsage(50),
|
||||
logger,
|
||||
);
|
||||
|
||||
t.false(result);
|
||||
t.true(
|
||||
messages.some(
|
||||
(m) =>
|
||||
m.type === "debug" &&
|
||||
typeof m.message === "string" &&
|
||||
m.message.includes("No overlay status found in Actions cache."),
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("shouldSkipOverlayAnalysis returns true when cached status indicates failed build", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
const codeql = mockCodeQLVersion("2.20.0");
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
const status = {
|
||||
attemptedToBuildOverlayBaseDatabase: true,
|
||||
builtOverlayBaseDatabase: false,
|
||||
};
|
||||
|
||||
// Stub restoreCache to write the status file and return a key
|
||||
sinon.stub(actionsCache, "restoreCache").callsFake(async (paths) => {
|
||||
const statusFile = paths[0];
|
||||
await fs.promises.mkdir(path.dirname(statusFile), { recursive: true });
|
||||
await fs.promises.writeFile(statusFile, JSON.stringify(status));
|
||||
return "found-key";
|
||||
});
|
||||
|
||||
const result = await shouldSkipOverlayAnalysis(
|
||||
codeql,
|
||||
["javascript"],
|
||||
makeDiskUsage(50),
|
||||
logger,
|
||||
);
|
||||
|
||||
t.true(result);
|
||||
});
|
||||
});
|
||||
|
||||
test("shouldSkipOverlayAnalysis returns false when cached status indicates successful build", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
process.env["RUNNER_TEMP"] = tmpDir;
|
||||
const codeql = mockCodeQLVersion("2.20.0");
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
const status = {
|
||||
attemptedToBuildOverlayBaseDatabase: true,
|
||||
builtOverlayBaseDatabase: true,
|
||||
};
|
||||
|
||||
sinon.stub(actionsCache, "restoreCache").callsFake(async (paths) => {
|
||||
const statusFile = paths[0];
|
||||
await fs.promises.mkdir(path.dirname(statusFile), { recursive: true });
|
||||
await fs.promises.writeFile(statusFile, JSON.stringify(status));
|
||||
return "found-key";
|
||||
});
|
||||
|
||||
const result = await shouldSkipOverlayAnalysis(
|
||||
codeql,
|
||||
["javascript"],
|
||||
makeDiskUsage(50),
|
||||
logger,
|
||||
);
|
||||
|
||||
t.false(result);
|
||||
t.true(
|
||||
messages.some(
|
||||
(m) =>
|
||||
m.type === "debug" &&
|
||||
typeof m.message === "string" &&
|
||||
m.message.includes(
|
||||
"Cached overlay status does not indicate a previous unsuccessful attempt",
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
/*
|
||||
* We perform enablement checks for overlay analysis to avoid using it on runners that are too small
|
||||
* to support it. However these checks cannot avoid every potential issue without being overly
|
||||
* conservative. Therefore, if our enablement checks enable overlay analysis for a runner that is
|
||||
* too small, we want to remember that, so that we will not try to use overlay analysis until
|
||||
* something changes (e.g. a larger runner is provisioned, or a new CodeQL version is released).
|
||||
*
|
||||
* We use the Actions cache as a lightweight way of providing this functionality.
|
||||
*/
|
||||
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as actionsCache from "@actions/cache";
|
||||
|
||||
import { getTemporaryDirectory } from "../actions-util";
|
||||
import { type CodeQL } from "../codeql";
|
||||
import { Logger } from "../logging";
|
||||
import {
|
||||
DiskUsage,
|
||||
getErrorMessage,
|
||||
waitForResultWithTimeLimit,
|
||||
} from "../util";
|
||||
|
||||
/** The maximum time to wait for a cache operation to complete. */
|
||||
const MAX_CACHE_OPERATION_MS = 30_000;
|
||||
|
||||
/** File name for the serialized overlay status. */
|
||||
const STATUS_FILE_NAME = "overlay-status.json";
|
||||
|
||||
/** Path to the local overlay status file. */
|
||||
function getStatusFilePath(languages: string[]): string {
|
||||
return path.join(
|
||||
getTemporaryDirectory(),
|
||||
"overlay-status",
|
||||
[...languages].sort().join("+"),
|
||||
STATUS_FILE_NAME,
|
||||
);
|
||||
}
|
||||
|
||||
/** Status of an overlay analysis for a group of languages. */
|
||||
export interface OverlayStatus {
|
||||
/** Whether the job attempted to build an overlay base database. */
|
||||
attemptedToBuildOverlayBaseDatabase: boolean;
|
||||
/** Whether the job successfully built an overlay base database. */
|
||||
builtOverlayBaseDatabase: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether overlay analysis should be skipped, based on the cached status for the given languages and disk usage.
|
||||
*/
|
||||
export async function shouldSkipOverlayAnalysis(
|
||||
codeql: CodeQL,
|
||||
languages: string[],
|
||||
diskUsage: DiskUsage,
|
||||
logger: Logger,
|
||||
): Promise<boolean> {
|
||||
const status = await getOverlayStatus(codeql, languages, diskUsage, logger);
|
||||
if (status === undefined) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
status.attemptedToBuildOverlayBaseDatabase &&
|
||||
!status.builtOverlayBaseDatabase
|
||||
) {
|
||||
logger.debug(
|
||||
"Cached overlay status indicates that building an overlay base database was unsuccessful.",
|
||||
);
|
||||
return true;
|
||||
}
|
||||
logger.debug(
|
||||
"Cached overlay status does not indicate a previous unsuccessful attempt to build an overlay base database.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve overlay status from the Actions cache, if available.
|
||||
*
|
||||
* @returns `undefined` if no status was found in the cache (e.g. first run with
|
||||
* this cache key) or if the cache operation fails.
|
||||
*/
|
||||
export async function getOverlayStatus(
|
||||
codeql: CodeQL,
|
||||
languages: string[],
|
||||
diskUsage: DiskUsage,
|
||||
logger: Logger,
|
||||
): Promise<OverlayStatus | undefined> {
|
||||
const cacheKey = await getCacheKey(codeql, languages, diskUsage);
|
||||
const statusFile = getStatusFilePath(languages);
|
||||
|
||||
try {
|
||||
await fs.promises.mkdir(path.dirname(statusFile), { recursive: true });
|
||||
const foundKey = await waitForResultWithTimeLimit(
|
||||
MAX_CACHE_OPERATION_MS,
|
||||
actionsCache.restoreCache([statusFile], cacheKey),
|
||||
() => {
|
||||
logger.warning("Timed out restoring overlay status from cache.");
|
||||
},
|
||||
);
|
||||
if (foundKey === undefined) {
|
||||
logger.debug("No overlay status found in Actions cache.");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!fs.existsSync(statusFile)) {
|
||||
logger.debug(
|
||||
"Overlay status cache entry found but status file is missing.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const contents = await fs.promises.readFile(statusFile, "utf-8");
|
||||
const parsed: unknown = JSON.parse(contents);
|
||||
if (
|
||||
typeof parsed !== "object" ||
|
||||
parsed === null ||
|
||||
typeof parsed["attemptedToBuildOverlayBaseDatabase"] !== "boolean" ||
|
||||
typeof parsed["builtOverlayBaseDatabase"] !== "boolean"
|
||||
) {
|
||||
logger.debug(
|
||||
"Ignoring overlay status cache entry with unexpected format.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
return parsed as OverlayStatus;
|
||||
} catch (error) {
|
||||
logger.warning(
|
||||
`Failed to restore overlay status from cache: ${getErrorMessage(error)}`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save overlay status to the Actions cache.
|
||||
*
|
||||
* @returns `true` if the status was saved successfully, `false` otherwise.
|
||||
*/
|
||||
export async function saveOverlayStatus(
|
||||
codeql: CodeQL,
|
||||
languages: string[],
|
||||
diskUsage: DiskUsage,
|
||||
status: OverlayStatus,
|
||||
logger: Logger,
|
||||
): Promise<boolean> {
|
||||
const cacheKey = await getCacheKey(codeql, languages, diskUsage);
|
||||
const statusFile = getStatusFilePath(languages);
|
||||
|
||||
try {
|
||||
await fs.promises.mkdir(path.dirname(statusFile), { recursive: true });
|
||||
await fs.promises.writeFile(statusFile, JSON.stringify(status));
|
||||
const cacheId = await waitForResultWithTimeLimit(
|
||||
MAX_CACHE_OPERATION_MS,
|
||||
actionsCache.saveCache([statusFile], cacheKey),
|
||||
() => {
|
||||
logger.warning("Timed out saving overlay status to cache.");
|
||||
},
|
||||
);
|
||||
if (cacheId === undefined) {
|
||||
return false;
|
||||
}
|
||||
logger.debug(`Saved overlay status to Actions cache with key ${cacheKey}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.warning(
|
||||
`Failed to save overlay status to cache: ${getErrorMessage(error)}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getCacheKey(
|
||||
codeql: CodeQL,
|
||||
languages: string[],
|
||||
diskUsage: DiskUsage,
|
||||
): Promise<string> {
|
||||
// Total disk space, rounded to the nearest 10 GB. This is included in the cache key so that if a
|
||||
// customer upgrades their runner, we will try again to use overlay analysis, even if the CodeQL
|
||||
// version has not changed. We round to the nearest 10 GB to work around small differences in disk
|
||||
// space.
|
||||
//
|
||||
// Limitation: this can still flip from "too small" to "large enough" and back again if the disk
|
||||
// space fluctuates above and below a multiple of 10 GB.
|
||||
const diskSpaceToNearest10Gb = `${10 * Math.floor(diskUsage.numTotalBytes / (10 * 1024 * 1024 * 1024))}GB`;
|
||||
|
||||
// Include the CodeQL version in the cache key so we will try again to use overlay analysis when
|
||||
// new queries and libraries that may be more efficient are released.
|
||||
return `codeql-overlay-status-${[...languages].sort().join("+")}-${(await codeql.getVersion()).version}-runner-${diskSpaceToNearest10Gb}`;
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import { CodeQL } from "./codeql";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Features } from "./feature-flags";
|
||||
import { initFeatures } from "./feature-flags";
|
||||
import { initCodeQL } from "./init";
|
||||
import { getActionsLogger, Logger } from "./logging";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
@@ -114,7 +114,7 @@ async function run(startedAt: Date): Promise<void> {
|
||||
|
||||
const repositoryNwo = getRepositoryNwo();
|
||||
|
||||
const features = new Features(
|
||||
const features = initFeatures(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
getTemporaryDirectory(),
|
||||
|
||||
+127
-6
@@ -1,18 +1,22 @@
|
||||
import * as path from "path";
|
||||
|
||||
import * as github from "@actions/github";
|
||||
import * as toolcache from "@actions/tool-cache";
|
||||
import test, { ExecutionContext } from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import * as api from "./api-client";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import * as setupCodeql from "./setup-codeql";
|
||||
import * as tar from "./tar";
|
||||
import {
|
||||
LINKED_CLI_VERSION,
|
||||
LoggedMessage,
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
checkExpectedLogMessages,
|
||||
createFeatures,
|
||||
getRecordingLogger,
|
||||
initializeFeatures,
|
||||
@@ -268,13 +272,127 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
|
||||
});
|
||||
});
|
||||
|
||||
test("getCodeQLSource correctly returns nightly CLI version when tools == nightly", async (t) => {
|
||||
const loggedMessages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
const features = createFeatures([]);
|
||||
|
||||
const expectedDate = "30260213";
|
||||
const expectedTag = `codeql-bundle-${expectedDate}`;
|
||||
|
||||
// Ensure that we consistently select "zstd" for the test.
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
"nightly",
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
false,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
|
||||
// Check that the `CodeQLToolsSource` object matches our expectations.
|
||||
const expectedVersion = `0.0.0-${expectedDate}`;
|
||||
const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`;
|
||||
t.deepEqual(source, {
|
||||
bundleVersion: expectedDate,
|
||||
cliVersion: undefined,
|
||||
codeqlURL: expectedURL,
|
||||
compressionMethod: "zstd",
|
||||
sourceType: "download",
|
||||
toolsVersion: expectedVersion,
|
||||
} satisfies setupCodeql.CodeQLToolsSource);
|
||||
|
||||
// Afterwards, ensure that we see the expected messages in the log.
|
||||
checkExpectedLogMessages(t, loggedMessages, [
|
||||
"Using the latest CodeQL CLI nightly, as requested by 'tools: nightly'.",
|
||||
`Bundle version ${expectedDate} is not in SemVer format. Will treat it as pre-release ${expectedVersion}.`,
|
||||
`Attempting to obtain CodeQL tools. CLI version: unknown, bundle tag name: ${expectedTag}`,
|
||||
`Using CodeQL CLI sourced from ${expectedURL}`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("getCodeQLSource correctly returns nightly CLI version when forced by FF", async (t) => {
|
||||
const loggedMessages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
const features = createFeatures([Feature.ForceNightly]);
|
||||
|
||||
const expectedDate = "30260213";
|
||||
const expectedTag = `codeql-bundle-${expectedDate}`;
|
||||
|
||||
// Ensure that we consistently select "zstd" for the test.
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
sinon.stub(tar, "isZstdAvailable").resolves({
|
||||
available: true,
|
||||
foundZstdBinary: true,
|
||||
});
|
||||
|
||||
const client = github.getOctokit("123");
|
||||
const listReleases = sinon.stub(client.rest.repos, "listReleases");
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
listReleases.resolves({
|
||||
data: [{ tag_name: expectedTag }],
|
||||
} as any);
|
||||
sinon.stub(api, "getApiClient").value(() => client);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
process.env["GITHUB_EVENT_NAME"] = "dynamic";
|
||||
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
undefined,
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
SAMPLE_DOTCOM_API_DETAILS,
|
||||
GitHubVariant.DOTCOM,
|
||||
false,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
|
||||
// Check that the `CodeQLToolsSource` object matches our expectations.
|
||||
const expectedVersion = `0.0.0-${expectedDate}`;
|
||||
const expectedURL = `https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/${expectedTag}/${setupCodeql.getCodeQLBundleName("zstd")}`;
|
||||
t.deepEqual(source, {
|
||||
bundleVersion: expectedDate,
|
||||
cliVersion: undefined,
|
||||
codeqlURL: expectedURL,
|
||||
compressionMethod: "zstd",
|
||||
sourceType: "download",
|
||||
toolsVersion: expectedVersion,
|
||||
} satisfies setupCodeql.CodeQLToolsSource);
|
||||
|
||||
// Afterwards, ensure that we see the expected messages in the log.
|
||||
checkExpectedLogMessages(t, loggedMessages, [
|
||||
`Using the latest CodeQL CLI nightly, as forced by the ${Feature.ForceNightly} feature flag.`,
|
||||
`Bundle version ${expectedDate} is not in SemVer format. Will treat it as pre-release ${expectedVersion}.`,
|
||||
`Attempting to obtain CodeQL tools. CLI version: unknown, bundle tag name: ${expectedTag}`,
|
||||
`Using CodeQL CLI sourced from ${expectedURL}`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("getCodeQLSource correctly returns latest version from toolcache when tools == toolcache", async (t) => {
|
||||
const loggedMessages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
const features = createFeatures([Feature.AllowToolcacheInput]);
|
||||
|
||||
process.env["GITHUB_EVENT_NAME"] = "dynamic";
|
||||
|
||||
const latestToolcacheVersion = "3.2.1";
|
||||
const latestVersionPath = "/path/to/latest";
|
||||
const testVersions = ["2.3.1", latestToolcacheVersion, "1.2.3"];
|
||||
@@ -288,6 +406,8 @@ test("getCodeQLSource correctly returns latest version from toolcache when tools
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
process.env["GITHUB_EVENT_NAME"] = "dynamic";
|
||||
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
"toolcache",
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
@@ -343,16 +463,17 @@ const toolcacheInputFallbackMacro = test.macro({
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
const features = createFeatures(featureList);
|
||||
|
||||
for (const [k, v] of Object.entries(environment)) {
|
||||
process.env[k] = v;
|
||||
}
|
||||
|
||||
const findAllVersionsStub = sinon
|
||||
.stub(toolcache, "findAllVersions")
|
||||
.returns(testVersions);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
for (const [k, v] of Object.entries(environment)) {
|
||||
process.env[k] = v;
|
||||
}
|
||||
|
||||
const source = await setupCodeql.getCodeQLSource(
|
||||
"toolcache",
|
||||
SAMPLE_DEFAULT_CLI_VERSION,
|
||||
|
||||
+62
-8
@@ -10,6 +10,7 @@ import { v4 as uuidV4 } from "uuid";
|
||||
import { isDynamicWorkflow, isRunningLocalAction } from "./actions-util";
|
||||
import * as api from "./api-client";
|
||||
import * as defaults from "./defaults.json";
|
||||
import { addNoLanguageDiagnostic, makeDiagnostic } from "./diagnostics";
|
||||
import {
|
||||
CODEQL_VERSION_ZSTD_BUNDLE,
|
||||
CodeQLDefaultVersionInfo,
|
||||
@@ -55,7 +56,9 @@ function getCodeQLBundleExtension(
|
||||
}
|
||||
}
|
||||
|
||||
function getCodeQLBundleName(compressionMethod: tar.CompressionMethod): string {
|
||||
export function getCodeQLBundleName(
|
||||
compressionMethod: tar.CompressionMethod,
|
||||
): string {
|
||||
const extension = getCodeQLBundleExtension(compressionMethod);
|
||||
|
||||
let platform: string;
|
||||
@@ -196,7 +199,7 @@ export function convertToSemVer(version: string, logger: Logger): string {
|
||||
return s;
|
||||
}
|
||||
|
||||
type CodeQLToolsSource =
|
||||
export type CodeQLToolsSource =
|
||||
| {
|
||||
codeqlTarPath: string;
|
||||
compressionMethod: tar.CompressionMethod;
|
||||
@@ -261,6 +264,20 @@ async function findOverridingToolsInCache(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines where the CodeQL CLI we want to use comes from. This can be from a local file,
|
||||
* the Actions toolcache, or a download.
|
||||
*
|
||||
* @param toolsInput The argument provided for the `tools` input, if any.
|
||||
* @param defaultCliVersion The default CLI version that's linked to the CodeQL Action.
|
||||
* @param apiDetails Information about the GitHub API.
|
||||
* @param variant The GitHub variant we are running on.
|
||||
* @param tarSupportsZstd Whether zstd is supported by `tar`.
|
||||
* @param features Information about enabled features.
|
||||
* @param logger The logger to use.
|
||||
*
|
||||
* @returns Information about where the CodeQL CLI we want to use comes from.
|
||||
*/
|
||||
export async function getCodeQLSource(
|
||||
toolsInput: string | undefined,
|
||||
defaultCliVersion: CodeQLDefaultVersionInfo,
|
||||
@@ -270,6 +287,9 @@ export async function getCodeQLSource(
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<CodeQLToolsSource> {
|
||||
// If there is an explicit `tools` input, it's not one of the reserved values, and it doesn't appear
|
||||
// to point to a URL, then we assume it is a local path and use the CLI from there.
|
||||
// TODO: This appears to misclassify filenames that happen to start with `http` as URLs.
|
||||
if (
|
||||
toolsInput &&
|
||||
!isReservedToolsValue(toolsInput) &&
|
||||
@@ -302,13 +322,47 @@ export async function getCodeQLSource(
|
||||
*/
|
||||
let url: string | undefined;
|
||||
|
||||
if (
|
||||
// We allow forcing the nightly CLI via the FF for `dynamic` events (or in test mode) where the
|
||||
// `tools` input cannot be adjusted to explicitly request it.
|
||||
const canForceNightlyWithFF = isDynamicWorkflow() || util.isInTestMode();
|
||||
const forceNightlyValueFF = await features.getValue(Feature.ForceNightly);
|
||||
const forceNightly = forceNightlyValueFF && canForceNightlyWithFF;
|
||||
|
||||
// For advanced workflows, a value from `CODEQL_NIGHTLY_TOOLS_INPUTS` can be specified explicitly
|
||||
// for the `tools` input in the workflow file.
|
||||
const nightlyRequestedByToolsInput =
|
||||
toolsInput !== undefined &&
|
||||
CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput)
|
||||
) {
|
||||
logger.info(
|
||||
`Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
|
||||
);
|
||||
CODEQL_NIGHTLY_TOOLS_INPUTS.includes(toolsInput);
|
||||
|
||||
if (forceNightly || nightlyRequestedByToolsInput) {
|
||||
if (forceNightly) {
|
||||
logger.info(
|
||||
`Using the latest CodeQL CLI nightly, as forced by the ${Feature.ForceNightly} feature flag.`,
|
||||
);
|
||||
addNoLanguageDiagnostic(
|
||||
undefined,
|
||||
makeDiagnostic(
|
||||
"codeql-action/forced-nightly-cli",
|
||||
"A nightly release of CodeQL was used",
|
||||
{
|
||||
markdownMessage:
|
||||
"GitHub configured this analysis to use a nightly release of CodeQL to allow you to preview changes from an upcoming release.\n\n" +
|
||||
"Nightly releases do not undergo the same validation as regular releases and may lead to analysis instability.\n\n" +
|
||||
"If use of a nightly CodeQL release for this analysis is unexpected, please contact GitHub support.",
|
||||
visibility: {
|
||||
cliSummaryTable: true,
|
||||
statusPage: true,
|
||||
telemetry: true,
|
||||
},
|
||||
severity: "note",
|
||||
},
|
||||
),
|
||||
);
|
||||
} else {
|
||||
logger.info(
|
||||
`Using the latest CodeQL CLI nightly, as requested by 'tools: ${toolsInput}'.`,
|
||||
);
|
||||
}
|
||||
toolsInput = await getNightlyToolsUrl(logger);
|
||||
}
|
||||
|
||||
|
||||
+52
-74
@@ -2,96 +2,37 @@ import { ChildProcess, spawn } from "child_process";
|
||||
import * as path from "path";
|
||||
|
||||
import * as core from "@actions/core";
|
||||
import { pki } from "node-forge";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import { Feature, FeatureEnablement, initFeatures } from "./feature-flags";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { getActionsLogger, Logger } from "./logging";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import {
|
||||
Credential,
|
||||
credentialToStr,
|
||||
getCredentials,
|
||||
getProxyBinaryPath,
|
||||
getSafeErrorMessage,
|
||||
parseLanguage,
|
||||
ProxyInfo,
|
||||
sendFailedStatusReport,
|
||||
sendSuccessStatusReport,
|
||||
Registry,
|
||||
ProxyConfig,
|
||||
} from "./start-proxy";
|
||||
import { generateCertificateAuthority } from "./start-proxy/ca";
|
||||
import { checkProxyEnvironment } from "./start-proxy/environment";
|
||||
import { checkConnections } from "./start-proxy/reachability";
|
||||
import { ActionName, sendUnhandledErrorStatusReport } from "./status-report";
|
||||
import * as util from "./util";
|
||||
|
||||
const KEY_SIZE = 2048;
|
||||
const KEY_EXPIRY_YEARS = 2;
|
||||
|
||||
type CertificateAuthority = {
|
||||
cert: string;
|
||||
key: string;
|
||||
};
|
||||
|
||||
type BasicAuthCredentials = {
|
||||
username: string;
|
||||
password: string;
|
||||
};
|
||||
|
||||
type ProxyConfig = {
|
||||
all_credentials: Credential[];
|
||||
ca: CertificateAuthority;
|
||||
proxy_auth?: BasicAuthCredentials;
|
||||
};
|
||||
|
||||
const CERT_SUBJECT = [
|
||||
{
|
||||
name: "commonName",
|
||||
value: "Dependabot Internal CA",
|
||||
},
|
||||
{
|
||||
name: "organizationName",
|
||||
value: "GitHub inc.",
|
||||
},
|
||||
{
|
||||
shortName: "OU",
|
||||
value: "Dependabot",
|
||||
},
|
||||
{
|
||||
name: "countryName",
|
||||
value: "US",
|
||||
},
|
||||
{
|
||||
shortName: "ST",
|
||||
value: "California",
|
||||
},
|
||||
{
|
||||
name: "localityName",
|
||||
value: "San Francisco",
|
||||
},
|
||||
];
|
||||
|
||||
function generateCertificateAuthority(): CertificateAuthority {
|
||||
const keys = pki.rsa.generateKeyPair(KEY_SIZE);
|
||||
const cert = pki.createCertificate();
|
||||
cert.publicKey = keys.publicKey;
|
||||
cert.serialNumber = "01";
|
||||
cert.validity.notBefore = new Date();
|
||||
cert.validity.notAfter = new Date();
|
||||
cert.validity.notAfter.setFullYear(
|
||||
cert.validity.notBefore.getFullYear() + KEY_EXPIRY_YEARS,
|
||||
);
|
||||
|
||||
cert.setSubject(CERT_SUBJECT);
|
||||
cert.setIssuer(CERT_SUBJECT);
|
||||
cert.setExtensions([{ name: "basicConstraints", cA: true }]);
|
||||
cert.sign(keys.privateKey);
|
||||
|
||||
const pem = pki.certificateToPem(cert);
|
||||
const key = pki.privateKeyToPem(keys.privateKey);
|
||||
return { cert: pem, key };
|
||||
}
|
||||
|
||||
async function run(startedAt: Date) {
|
||||
// To capture errors appropriately, keep as much code within the try-catch as
|
||||
// possible, and only use safe functions outside.
|
||||
|
||||
const logger = getActionsLogger();
|
||||
let features: FeatureEnablement | undefined;
|
||||
let language: KnownLanguage | undefined;
|
||||
|
||||
try {
|
||||
@@ -103,9 +44,21 @@ async function run(startedAt: Date) {
|
||||
const proxyLogFilePath = path.resolve(tempDir, "proxy.log");
|
||||
core.saveState("proxy-log-file", proxyLogFilePath);
|
||||
|
||||
// Get the configuration options
|
||||
// Initialise FFs.
|
||||
const repositoryNwo = getRepositoryNwo();
|
||||
const gitHubVersion = await getGitHubVersion();
|
||||
features = initFeatures(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
actionsUtil.getTemporaryDirectory(),
|
||||
logger,
|
||||
);
|
||||
|
||||
// Get the language input.
|
||||
const languageInput = actionsUtil.getOptionalInput("language");
|
||||
language = languageInput ? parseLanguage(languageInput) : undefined;
|
||||
|
||||
// Get the registry configurations from one of the inputs.
|
||||
const credentials = getCredentials(
|
||||
logger,
|
||||
actionsUtil.getOptionalInput("registry_secrets"),
|
||||
@@ -124,7 +77,22 @@ async function run(startedAt: Date) {
|
||||
.join("\n")}`,
|
||||
);
|
||||
|
||||
const ca = generateCertificateAuthority();
|
||||
// Check the environment for any configurations which may affect the proxy.
|
||||
// This is a best effort process to give us insights into potential factors
|
||||
// which may affect the operation of our proxy.
|
||||
if (core.isDebug() || util.isInTestMode()) {
|
||||
try {
|
||||
await checkProxyEnvironment(logger, language);
|
||||
} catch (err) {
|
||||
logger.debug(
|
||||
`Unable to inspect runner environment: ${util.getErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const ca = generateCertificateAuthority(
|
||||
await features.getValue(Feature.ImprovedProxyCertificates),
|
||||
);
|
||||
|
||||
const proxyConfig: ProxyConfig = {
|
||||
all_credentials: credentials,
|
||||
@@ -133,7 +101,15 @@ async function run(startedAt: Date) {
|
||||
|
||||
// Start the Proxy
|
||||
const proxyBin = await getProxyBinaryPath(logger);
|
||||
await startProxy(proxyBin, proxyConfig, proxyLogFilePath, logger);
|
||||
const proxyInfo = await startProxy(
|
||||
proxyBin,
|
||||
proxyConfig,
|
||||
proxyLogFilePath,
|
||||
logger,
|
||||
);
|
||||
|
||||
// Check that the private registries are reachable.
|
||||
await checkConnections(logger, proxyInfo);
|
||||
|
||||
// Report success if we have reached this point.
|
||||
await sendSuccessStatusReport(
|
||||
@@ -171,7 +147,7 @@ async function startProxy(
|
||||
config: ProxyConfig,
|
||||
logFilePath: string,
|
||||
logger: Logger,
|
||||
) {
|
||||
): Promise<ProxyInfo> {
|
||||
const host = "127.0.0.1";
|
||||
let port = 49152;
|
||||
let subprocess: ChildProcess | undefined = undefined;
|
||||
@@ -214,13 +190,15 @@ async function startProxy(
|
||||
core.setOutput("proxy_port", port.toString());
|
||||
core.setOutput("proxy_ca_certificate", config.ca.cert);
|
||||
|
||||
const registry_urls = config.all_credentials
|
||||
const registry_urls: Registry[] = config.all_credentials
|
||||
.filter((credential) => credential.url !== undefined)
|
||||
.map((credential) => ({
|
||||
type: credential.type,
|
||||
url: credential.url,
|
||||
}));
|
||||
core.setOutput("proxy_urls", JSON.stringify(registry_urls));
|
||||
|
||||
return { host, port, cert: config.ca.cert, registries: registry_urls };
|
||||
}
|
||||
|
||||
void runWrapper();
|
||||
|
||||
+25
-2
@@ -175,6 +175,27 @@ test("getCredentials throws error when credential is not an object", async (t) =
|
||||
}
|
||||
});
|
||||
|
||||
test("getCredentials throws error when credential is missing type", async (t) => {
|
||||
const testCredentials = [[{ token: "abc", url: "https://localhost" }]].map(
|
||||
toEncodedJSON,
|
||||
);
|
||||
|
||||
for (const testCredential of testCredentials) {
|
||||
t.throws(
|
||||
() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
testCredential,
|
||||
undefined,
|
||||
),
|
||||
{
|
||||
message: "Invalid credentials - must have a type",
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("getCredentials throws error when credential missing host and url", async (t) => {
|
||||
const testCredentials = [
|
||||
[{ type: "npm_registry", token: "abc" }],
|
||||
@@ -396,13 +417,14 @@ test("credentialToStr - hides passwords", (t) => {
|
||||
const credential = {
|
||||
type: "maven_credential",
|
||||
password: secret,
|
||||
url: "https://localhost",
|
||||
};
|
||||
|
||||
const str = startProxyExports.credentialToStr(credential);
|
||||
|
||||
t.false(str.includes(secret));
|
||||
t.is(
|
||||
"Type: maven_credential; Host: undefined; Url: undefined Username: undefined; Password: true; Token: false",
|
||||
"Type: maven_credential; Host: undefined; Url: https://localhost Username: undefined; Password: true; Token: false",
|
||||
str,
|
||||
);
|
||||
});
|
||||
@@ -412,13 +434,14 @@ test("credentialToStr - hides tokens", (t) => {
|
||||
const credential = {
|
||||
type: "maven_credential",
|
||||
token: secret,
|
||||
url: "https://localhost",
|
||||
};
|
||||
|
||||
const str = startProxyExports.credentialToStr(credential);
|
||||
|
||||
t.false(str.includes(secret));
|
||||
t.is(
|
||||
"Type: maven_credential; Host: undefined; Url: undefined Username: undefined; Password: false; Token: true",
|
||||
"Type: maven_credential; Host: undefined; Url: https://localhost Username: undefined; Password: false; Token: true",
|
||||
str,
|
||||
);
|
||||
});
|
||||
|
||||
+42
-19
@@ -13,6 +13,12 @@ import { Config } from "./config-utils";
|
||||
import * as defaults from "./defaults.json";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import {
|
||||
Address,
|
||||
RawCredential,
|
||||
Registry,
|
||||
Credential,
|
||||
} from "./start-proxy/types";
|
||||
import {
|
||||
ActionName,
|
||||
createStatusReportBase,
|
||||
@@ -23,6 +29,8 @@ import {
|
||||
import * as util from "./util";
|
||||
import { ConfigurationError, getErrorMessage, isDefined } from "./util";
|
||||
|
||||
export * from "./start-proxy/types";
|
||||
|
||||
/**
|
||||
* Enumerates specific error types for which we have corresponding error messages that
|
||||
* are safe to include in status reports.
|
||||
@@ -161,15 +169,6 @@ export const UPDATEJOB_PROXY_VERSION = "v2.0.20250624110901";
|
||||
const UPDATEJOB_PROXY_URL_PREFIX =
|
||||
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.22.0/";
|
||||
|
||||
export type Credential = {
|
||||
type: string;
|
||||
host?: string;
|
||||
url?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
token?: string;
|
||||
};
|
||||
|
||||
/*
|
||||
* Language aliases supported by the start-proxy Action.
|
||||
*
|
||||
@@ -229,6 +228,31 @@ const LANGUAGE_TO_REGISTRY_TYPE: Partial<Record<KnownLanguage, string[]>> = {
|
||||
go: ["goproxy_server", "git_source"],
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Extracts an `Address` value from the given `Registry` value by determining whether it has
|
||||
* a `url` value, or no `url` value but a `host` value.
|
||||
*
|
||||
* @throws A `ConfigurationError` if the `Registry` value contains neither a `url` or `host` field.
|
||||
*/
|
||||
function getRegistryAddress(registry: Partial<Registry>): Address {
|
||||
if (isDefined(registry.url)) {
|
||||
return {
|
||||
url: registry.url,
|
||||
host: registry.host,
|
||||
};
|
||||
} else if (isDefined(registry.host)) {
|
||||
return {
|
||||
url: undefined,
|
||||
host: registry.host,
|
||||
};
|
||||
} else {
|
||||
// The proxy needs one of these to work. If both are defined, the url has the precedence.
|
||||
throw new ConfigurationError(
|
||||
"Invalid credentials - must specify host or url",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// getCredentials returns registry credentials from action inputs.
|
||||
// It prefers `registries_credentials` over `registry_secrets`.
|
||||
// If neither is set, it returns an empty array.
|
||||
@@ -255,9 +279,9 @@ export function getCredentials(
|
||||
}
|
||||
|
||||
// Parse and validate the credentials
|
||||
let parsed: Credential[];
|
||||
let parsed: RawCredential[];
|
||||
try {
|
||||
parsed = JSON.parse(credentialsStr) as Credential[];
|
||||
parsed = JSON.parse(credentialsStr) as RawCredential[];
|
||||
} catch {
|
||||
// Don't log the error since it might contain sensitive information.
|
||||
logger.error("Failed to parse the credentials data.");
|
||||
@@ -277,6 +301,11 @@ export function getCredentials(
|
||||
throw new ConfigurationError("Invalid credentials - must be an object");
|
||||
}
|
||||
|
||||
// The configuration must have a type.
|
||||
if (!isDefined(e.type)) {
|
||||
throw new ConfigurationError("Invalid credentials - must have a type");
|
||||
}
|
||||
|
||||
// Mask credentials to reduce chance of accidental leakage in logs.
|
||||
if (isDefined(e.password)) {
|
||||
core.setSecret(e.password);
|
||||
@@ -285,12 +314,7 @@ export function getCredentials(
|
||||
core.setSecret(e.token);
|
||||
}
|
||||
|
||||
if (!isDefined(e.url) && !isDefined(e.host)) {
|
||||
// The proxy needs one of these to work. If both are defined, the url has the precedence.
|
||||
throw new ConfigurationError(
|
||||
"Invalid credentials - must specify host or url",
|
||||
);
|
||||
}
|
||||
const address = getRegistryAddress(e);
|
||||
|
||||
// Filter credentials based on language if specified. `type` is the registry type.
|
||||
// E.g., "maven_feed" for Java/Kotlin, "nuget_repository" for C#.
|
||||
@@ -333,11 +357,10 @@ export function getCredentials(
|
||||
|
||||
out.push({
|
||||
type: e.type,
|
||||
host: e.host,
|
||||
url: e.url,
|
||||
username: e.username,
|
||||
password: e.password,
|
||||
token: e.token,
|
||||
...address,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import test, { ExecutionContext } from "ava";
|
||||
import { pki } from "node-forge";
|
||||
|
||||
import { setupTests } from "../testing-utils";
|
||||
|
||||
import * as ca from "./ca";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
const toMap = <T>(array: T[], func: (e: T) => string) =>
|
||||
new Map<string, T>(array.map((val) => [func(val), val]));
|
||||
|
||||
function checkCertAttributes(
|
||||
t: ExecutionContext<unknown>,
|
||||
cert: pki.Certificate,
|
||||
) {
|
||||
const subjectMap = toMap(
|
||||
cert.subject.attributes,
|
||||
(attr) => attr.name as string,
|
||||
);
|
||||
const issuerMap = toMap(
|
||||
cert.issuer.attributes,
|
||||
(attr) => attr.name as string,
|
||||
);
|
||||
|
||||
t.is(subjectMap.get("commonName")?.value, "Dependabot Internal CA");
|
||||
t.is(issuerMap.get("commonName")?.value, "Dependabot Internal CA");
|
||||
|
||||
for (const attrName of subjectMap.keys()) {
|
||||
t.deepEqual(subjectMap.get(attrName), issuerMap.get(attrName));
|
||||
}
|
||||
}
|
||||
|
||||
test("generateCertificateAuthority - generates certificates", (t) => {
|
||||
const result = ca.generateCertificateAuthority(false);
|
||||
const cert = pki.certificateFromPem(result.cert);
|
||||
const key = pki.privateKeyFromPem(result.key);
|
||||
|
||||
t.truthy(cert);
|
||||
t.truthy(key);
|
||||
|
||||
checkCertAttributes(t, cert);
|
||||
|
||||
// Check the validity.
|
||||
t.true(
|
||||
cert.validity.notBefore <= new Date(),
|
||||
"notBefore date is in the future",
|
||||
);
|
||||
t.true(cert.validity.notAfter > new Date(), "notAfter date is in the past");
|
||||
|
||||
// Check that the extensions are set as we'd expect.
|
||||
const exts = cert.extensions as ca.Extension[];
|
||||
t.is(exts.length, 1);
|
||||
t.is(exts[0].name, "basicConstraints");
|
||||
t.is(exts[0].cA, true);
|
||||
|
||||
t.truthy(cert.siginfo);
|
||||
});
|
||||
|
||||
test("generateCertificateAuthority - generates certificates with FF", (t) => {
|
||||
const result = ca.generateCertificateAuthority(true);
|
||||
const cert = pki.certificateFromPem(result.cert);
|
||||
const key = pki.privateKeyFromPem(result.key);
|
||||
|
||||
t.truthy(cert);
|
||||
t.truthy(key);
|
||||
|
||||
checkCertAttributes(t, cert);
|
||||
|
||||
// Check the validity.
|
||||
t.true(
|
||||
cert.validity.notBefore <= new Date(),
|
||||
"notBefore date is in the future",
|
||||
);
|
||||
t.true(cert.validity.notAfter > new Date(), "notAfter date is in the past");
|
||||
|
||||
// Check that the extensions are set as we'd expect.
|
||||
const exts = toMap(cert.extensions as ca.Extension[], (ext) => ext.name);
|
||||
t.is(exts.size, 4);
|
||||
t.true(exts.get("basicConstraints")?.cA);
|
||||
t.truthy(exts.get("subjectKeyIdentifier"));
|
||||
t.truthy(exts.get("authorityKeyIdentifier"));
|
||||
|
||||
const keyUsage = exts.get("keyUsage");
|
||||
if (t.truthy(keyUsage)) {
|
||||
t.true(keyUsage.critical);
|
||||
t.true(keyUsage.keyCertSign);
|
||||
t.true(keyUsage.cRLSign);
|
||||
t.true(keyUsage.digitalSignature);
|
||||
}
|
||||
|
||||
t.truthy(cert.siginfo);
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import { md, pki } from "node-forge";
|
||||
|
||||
import { CertificateAuthority } from "./types";
|
||||
|
||||
const KEY_SIZE = 2048;
|
||||
const KEY_EXPIRY_YEARS = 2;
|
||||
|
||||
const CERT_SUBJECT = [
|
||||
{
|
||||
name: "commonName",
|
||||
value: "Dependabot Internal CA",
|
||||
},
|
||||
{
|
||||
name: "organizationName",
|
||||
value: "GitHub inc.",
|
||||
},
|
||||
{
|
||||
shortName: "OU",
|
||||
value: "Dependabot",
|
||||
},
|
||||
{
|
||||
name: "countryName",
|
||||
value: "US",
|
||||
},
|
||||
{
|
||||
shortName: "ST",
|
||||
value: "California",
|
||||
},
|
||||
{
|
||||
name: "localityName",
|
||||
value: "San Francisco",
|
||||
},
|
||||
];
|
||||
|
||||
export type Extension = {
|
||||
name: string;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
const extraExtensions: Extension[] = [
|
||||
{
|
||||
name: "keyUsage",
|
||||
critical: true,
|
||||
keyCertSign: true,
|
||||
cRLSign: true,
|
||||
digitalSignature: true,
|
||||
},
|
||||
{ name: "subjectKeyIdentifier" },
|
||||
{ name: "authorityKeyIdentifier", keyIdentifier: true },
|
||||
];
|
||||
|
||||
/**
|
||||
* Generates a CA certificate for the proxy.
|
||||
*
|
||||
* @param newCertGenFF Whether to use the updated certificate generation.
|
||||
* @returns The private and public keys.
|
||||
*/
|
||||
export function generateCertificateAuthority(
|
||||
newCertGenFF: boolean,
|
||||
): CertificateAuthority {
|
||||
const keys = pki.rsa.generateKeyPair(KEY_SIZE);
|
||||
const cert = pki.createCertificate();
|
||||
cert.publicKey = keys.publicKey;
|
||||
cert.serialNumber = "01";
|
||||
cert.validity.notBefore = new Date();
|
||||
cert.validity.notAfter = new Date();
|
||||
cert.validity.notAfter.setFullYear(
|
||||
cert.validity.notBefore.getFullYear() + KEY_EXPIRY_YEARS,
|
||||
);
|
||||
|
||||
cert.setSubject(CERT_SUBJECT);
|
||||
cert.setIssuer(CERT_SUBJECT);
|
||||
|
||||
const extensions: Extension[] = [{ name: "basicConstraints", cA: true }];
|
||||
|
||||
// Add the extra CA extensions if the FF is enabled.
|
||||
if (newCertGenFF) {
|
||||
extensions.push(...extraExtensions);
|
||||
}
|
||||
|
||||
cert.setExtensions(extensions);
|
||||
|
||||
// Specifically use SHA256 when the FF is enabled.
|
||||
if (newCertGenFF) {
|
||||
cert.sign(keys.privateKey, md.sha256.create());
|
||||
} else {
|
||||
cert.sign(keys.privateKey);
|
||||
}
|
||||
|
||||
const pem = pki.certificateToPem(cert);
|
||||
const key = pki.privateKeyToPem(keys.privateKey);
|
||||
return { cert: pem, key };
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
import * as fs from "fs";
|
||||
import * as os from "os";
|
||||
import path from "path";
|
||||
|
||||
import * as toolrunner from "@actions/exec/lib/toolrunner";
|
||||
import * as io from "@actions/io";
|
||||
import test, { ExecutionContext } from "ava";
|
||||
import sinon from "sinon";
|
||||
|
||||
import { JavaEnvVars, KnownLanguage } from "../languages";
|
||||
import {
|
||||
checkExpectedLogMessages,
|
||||
getRecordingLogger,
|
||||
LoggedMessage,
|
||||
setupTests,
|
||||
} from "../testing-utils";
|
||||
import { withTmpDir } from "../util";
|
||||
|
||||
import {
|
||||
checkJavaEnvVars,
|
||||
checkJdkSettings,
|
||||
checkProxyEnvironment,
|
||||
checkProxyEnvVars,
|
||||
discoverActionsJdks,
|
||||
JAVA_PROXY_ENV_VARS,
|
||||
ProxyEnvVars,
|
||||
} from "./environment";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
function stubToolrunner() {
|
||||
sinon.stub(io, "which").throws(new Error("Java not installed"));
|
||||
sinon.stub(toolrunner, "ToolRunner").returns({
|
||||
exec: async () => {
|
||||
return 0;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function assertEnvVarLogMessages(
|
||||
t: ExecutionContext<any>,
|
||||
envVars: string[],
|
||||
messages: LoggedMessage[],
|
||||
expectSet: boolean | string,
|
||||
) {
|
||||
const template = (envVar: string) => {
|
||||
if (typeof expectSet === "string") {
|
||||
return `Environment variable '${envVar}' is set to '${expectSet}'`;
|
||||
}
|
||||
return expectSet
|
||||
? `Environment variable '${envVar}' is set to '${envVar}'`
|
||||
: `Environment variable '${envVar}' is not set`;
|
||||
};
|
||||
|
||||
const expected: string[] = [];
|
||||
|
||||
for (const envVar of envVars) {
|
||||
expected.push(template(envVar));
|
||||
}
|
||||
|
||||
checkExpectedLogMessages(t, messages, expected);
|
||||
}
|
||||
|
||||
test("checkJavaEnvironment - none set", (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
checkJavaEnvVars(logger);
|
||||
assertEnvVarLogMessages(t, JAVA_PROXY_ENV_VARS, messages, false);
|
||||
});
|
||||
|
||||
test("checkJavaEnvironment - logs values when variables are set", (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
for (const envVar of Object.values(JavaEnvVars)) {
|
||||
process.env[envVar] = envVar;
|
||||
}
|
||||
|
||||
checkJavaEnvVars(logger);
|
||||
assertEnvVarLogMessages(t, JAVA_PROXY_ENV_VARS, messages, true);
|
||||
});
|
||||
|
||||
test("discoverActionsJdks - discovers JDK paths", (t) => {
|
||||
// Clear GHA variables that may interfere with this test in CI.
|
||||
for (const envVar of Object.keys(process.env)) {
|
||||
if (envVar.startsWith("JAVA_HOME_")) {
|
||||
delete process.env[envVar];
|
||||
}
|
||||
}
|
||||
|
||||
const jdk8 = "/usr/lib/jvm/temurin-8-jdk-amd64";
|
||||
const jdk17 = "/usr/lib/jvm/temurin-17-jdk-amd64";
|
||||
const jdk21 = "/usr/lib/jvm/temurin-21-jdk-amd64";
|
||||
|
||||
process.env[JavaEnvVars.JAVA_HOME] = jdk17;
|
||||
process.env["JAVA_HOME_8_X64"] = jdk8;
|
||||
process.env["JAVA_HOME_17_X64"] = jdk17;
|
||||
process.env["JAVA_HOME_21_X64"] = jdk21;
|
||||
|
||||
const results = discoverActionsJdks();
|
||||
t.is(results.size, 3);
|
||||
t.true(results.has(jdk8));
|
||||
t.true(results.has(jdk17));
|
||||
t.true(results.has(jdk21));
|
||||
});
|
||||
|
||||
test("checkJdkSettings - does not throw for an empty directory", async (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
t.notThrows(() => checkJdkSettings(logger, tmpDir));
|
||||
});
|
||||
});
|
||||
|
||||
test("checkJdkSettings - finds files and logs relevant properties", async (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const dir = path.join(tmpDir, "conf");
|
||||
fs.mkdirSync(dir);
|
||||
|
||||
const file = path.join(dir, "net.properties");
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
[
|
||||
"irrelevant.property=foo",
|
||||
"http.proxyHost=proxy.example.com",
|
||||
"http.unrelated=bar",
|
||||
].join(os.EOL),
|
||||
{},
|
||||
);
|
||||
checkJdkSettings(logger, tmpDir);
|
||||
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
`Found '${file}'.`,
|
||||
`Found 'http.proxyHost=proxy.example.com' in '${file}'`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("checkProxyEnvVars - none set", (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
checkProxyEnvVars(logger);
|
||||
assertEnvVarLogMessages(t, Object.values(ProxyEnvVars), messages, false);
|
||||
});
|
||||
|
||||
test("checkProxyEnvVars - logs values when variables are set", (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
for (const envVar of Object.values(ProxyEnvVars)) {
|
||||
process.env[envVar] = envVar;
|
||||
}
|
||||
|
||||
checkProxyEnvVars(logger);
|
||||
assertEnvVarLogMessages(t, Object.values(ProxyEnvVars), messages, true);
|
||||
});
|
||||
|
||||
test("checkProxyEnvVars - credentials are removed from URLs", (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
for (const envVar of Object.values(ProxyEnvVars)) {
|
||||
process.env[envVar] = "https://secret:password@proxy.local";
|
||||
}
|
||||
|
||||
checkProxyEnvVars(logger);
|
||||
assertEnvVarLogMessages(
|
||||
t,
|
||||
Object.values(ProxyEnvVars),
|
||||
messages,
|
||||
"https://proxy.local/",
|
||||
);
|
||||
});
|
||||
|
||||
test("checkProxyEnvironment - includes base checks for all known languages", async (t) => {
|
||||
stubToolrunner();
|
||||
|
||||
for (const language of Object.values(KnownLanguage)) {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
await checkProxyEnvironment(logger, language);
|
||||
assertEnvVarLogMessages(t, Object.keys(ProxyEnvVars), messages, false);
|
||||
}
|
||||
});
|
||||
|
||||
test("checkProxyEnvironment - includes Java checks for Java", async (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
stubToolrunner();
|
||||
|
||||
await checkProxyEnvironment(logger, KnownLanguage.java);
|
||||
assertEnvVarLogMessages(t, Object.keys(ProxyEnvVars), messages, false);
|
||||
assertEnvVarLogMessages(t, JAVA_PROXY_ENV_VARS, messages, false);
|
||||
});
|
||||
|
||||
test("checkProxyEnvironment - includes language-specific checks if the language is undefined", async (t) => {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
stubToolrunner();
|
||||
|
||||
await checkProxyEnvironment(logger, undefined);
|
||||
assertEnvVarLogMessages(t, Object.keys(ProxyEnvVars), messages, false);
|
||||
assertEnvVarLogMessages(t, JAVA_PROXY_ENV_VARS, messages, false);
|
||||
});
|
||||
@@ -0,0 +1,209 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as toolrunner from "@actions/exec/lib/toolrunner";
|
||||
import * as io from "@actions/io";
|
||||
|
||||
import { JavaEnvVars, KnownLanguage, Language } from "../languages";
|
||||
import { Logger } from "../logging";
|
||||
import { getErrorMessage, isDefined } from "../util";
|
||||
|
||||
/**
|
||||
* Checks whether an environment variable named `name` is set and logs its value if set.
|
||||
*
|
||||
* @param logger The logger to use.
|
||||
* @param name The name of the environment variable.
|
||||
* @returns True if set or false otherwise.
|
||||
*/
|
||||
function checkEnvVar(logger: Logger, name: string): boolean {
|
||||
const value = process.env[name];
|
||||
if (isDefined(value)) {
|
||||
const url = URL.parse(value);
|
||||
if (isDefined(url)) {
|
||||
url.username = "";
|
||||
url.password = "";
|
||||
logger.info(`Environment variable '${name}' is set to '${url}'.`);
|
||||
} else {
|
||||
logger.info(`Environment variable '${name}' is set to '${value}'.`);
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
logger.debug(`Environment variable '${name}' is not set.`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// The JRE properties that may affect the proxy.
|
||||
const javaProperties = [
|
||||
"http.proxyHost",
|
||||
"http.proxyPort",
|
||||
"https.proxyHost",
|
||||
"https.proxyPort",
|
||||
"http.nonProxyHosts",
|
||||
"java.net.useSystemProxies",
|
||||
"javax.net.ssl.trustStore",
|
||||
"javax.net.ssl.trustStoreType",
|
||||
"javax.net.ssl.trustStoreProvider",
|
||||
"jdk.tls.client.protocols",
|
||||
"jdk.tls.disabledAlgorithms",
|
||||
"jdk.security.allowNonCaAnchor",
|
||||
"https.protocols",
|
||||
"com.sun.net.ssl.enableAIAcaIssuers",
|
||||
"com.sun.net.ssl.checkRevocation",
|
||||
"com.sun.security.enableCRLDP",
|
||||
"ocsp.enable",
|
||||
];
|
||||
|
||||
/** Java-specific environment variables which may contain information about proxy settings. */
|
||||
export const JAVA_PROXY_ENV_VARS: JavaEnvVars[] = [
|
||||
JavaEnvVars.JAVA_TOOL_OPTIONS,
|
||||
JavaEnvVars.JDK_JAVA_OPTIONS,
|
||||
JavaEnvVars._JAVA_OPTIONS,
|
||||
];
|
||||
|
||||
/**
|
||||
* Checks whether any Java-specific environment variables which may contain proxy
|
||||
* configurations are set and logs their values if so.
|
||||
*/
|
||||
export function checkJavaEnvVars(logger: Logger) {
|
||||
for (const envVar of JAVA_PROXY_ENV_VARS) {
|
||||
checkEnvVar(logger, envVar);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discovers paths to JDK directories based on JAVA_HOME and GHA-specific environment variables.
|
||||
* @returns A set of JDK paths.
|
||||
*/
|
||||
export function discoverActionsJdks(): Set<string> {
|
||||
const paths: Set<string> = new Set();
|
||||
|
||||
// Check whether JAVA_HOME is set.
|
||||
const javaHome = process.env[JavaEnvVars.JAVA_HOME];
|
||||
if (isDefined(javaHome)) {
|
||||
paths.add(javaHome);
|
||||
}
|
||||
|
||||
for (const [envVar, value] of Object.entries(process.env)) {
|
||||
if (isDefined(value) && envVar.match(/^JAVA_HOME_\d+_/)) {
|
||||
paths.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries to inspect JDK configuration files for the specified JDK path which may contain proxy settings.
|
||||
*
|
||||
* @param logger The logger to use.
|
||||
* @param jdkHome The JDK home directory.
|
||||
*/
|
||||
export function checkJdkSettings(logger: Logger, jdkHome: string) {
|
||||
const filesToCheck = [
|
||||
// JDK 9+
|
||||
path.join("conf", "net.properties"),
|
||||
// JDK 8 and below
|
||||
path.join("lib", "net.properties"),
|
||||
];
|
||||
|
||||
for (const fileToCheck of filesToCheck) {
|
||||
const file = path.join(jdkHome, fileToCheck);
|
||||
|
||||
try {
|
||||
if (fs.existsSync(file)) {
|
||||
logger.debug(`Found '${file}'.`);
|
||||
|
||||
const lines = String(fs.readFileSync(file)).split("\n");
|
||||
for (const line of lines) {
|
||||
for (const property of javaProperties) {
|
||||
if (line.startsWith(`${property}=`)) {
|
||||
logger.info(`Found '${line.trimEnd()}' in '${file}'.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.debug(`'${file}' does not exist.`);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.debug(`Failed to read '${file}': ${getErrorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Invokes `java` to get it to show us the active configuration. */
|
||||
async function showJavaSettings(logger: Logger): Promise<void> {
|
||||
try {
|
||||
const java = await io.which("java", true);
|
||||
|
||||
let output = "";
|
||||
await new toolrunner.ToolRunner(
|
||||
java,
|
||||
["-XshowSettings:all", "-XshowSettings:security:all", "-version"],
|
||||
{
|
||||
silent: true,
|
||||
listeners: {
|
||||
stdout: (data) => {
|
||||
output += String(data);
|
||||
},
|
||||
stderr: (data) => {
|
||||
output += String(data);
|
||||
},
|
||||
},
|
||||
},
|
||||
).exec();
|
||||
|
||||
logger.startGroup("Java settings");
|
||||
logger.info(output);
|
||||
logger.endGroup();
|
||||
} catch (err) {
|
||||
logger.debug(`Failed to query java settings: ${getErrorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Enumerates environment variable names which may contain information about proxy settings. */
|
||||
export enum ProxyEnvVars {
|
||||
HTTP_PROXY = "HTTP_PROXY",
|
||||
HTTPS_PROXY = "HTTPS_PROXY",
|
||||
ALL_PROXY = "ALL_PROXY",
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether any proxy-related environment variables are set and logs their values if so.
|
||||
*/
|
||||
export function checkProxyEnvVars(logger: Logger) {
|
||||
// Both upper-case and lower-case variants of these environment variables are used.
|
||||
for (const envVar of Object.values(ProxyEnvVars)) {
|
||||
checkEnvVar(logger, envVar);
|
||||
checkEnvVar(logger, envVar.toLowerCase());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspects environment variables and other configurations on the runner to determine whether
|
||||
* any settings that may affect the operation of the proxy are present. All relevant information
|
||||
* is written to the log.
|
||||
*
|
||||
* @param logger The logger to use.
|
||||
* @param language The enabled language, if known.
|
||||
*/
|
||||
export async function checkProxyEnvironment(
|
||||
logger: Logger,
|
||||
language: Language | undefined,
|
||||
): Promise<void> {
|
||||
// Determine whether there is an existing proxy configured.
|
||||
checkProxyEnvVars(logger);
|
||||
|
||||
// Check language-specific configurations. If we don't know the language,
|
||||
// then we perform all checks.
|
||||
if (language === undefined || language === KnownLanguage.java) {
|
||||
checkJavaEnvVars(logger);
|
||||
|
||||
await showJavaSettings(logger);
|
||||
|
||||
const jdks = discoverActionsJdks();
|
||||
for (const jdk of jdks) {
|
||||
checkJdkSettings(logger, jdk);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import {
|
||||
checkExpectedLogMessages,
|
||||
setupTests,
|
||||
withRecordingLoggerAsync,
|
||||
} from "./../testing-utils";
|
||||
import {
|
||||
checkConnections,
|
||||
ReachabilityBackend,
|
||||
ReachabilityError,
|
||||
} from "./reachability";
|
||||
import { ProxyInfo, Registry } from "./types";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
class MockReachabilityBackend implements ReachabilityBackend {
|
||||
public async checkConnection(_url: URL): Promise<number> {
|
||||
return 200;
|
||||
}
|
||||
}
|
||||
|
||||
const mavenRegistry: Registry = {
|
||||
type: "maven_registry",
|
||||
url: "https://repo.maven.apache.org/maven2/",
|
||||
};
|
||||
|
||||
const nugetFeed: Registry = {
|
||||
type: "nuget_feed",
|
||||
url: "https://api.nuget.org/v3/index.json",
|
||||
};
|
||||
|
||||
const proxyInfo: ProxyInfo = {
|
||||
host: "127.0.0.1",
|
||||
port: 1080,
|
||||
cert: "",
|
||||
registries: [mavenRegistry, nugetFeed],
|
||||
};
|
||||
|
||||
test("checkConnections - basic functionality", async (t) => {
|
||||
const backend = new MockReachabilityBackend();
|
||||
const messages = await withRecordingLoggerAsync(async (logger) => {
|
||||
const reachable = await checkConnections(logger, proxyInfo, backend);
|
||||
t.is(reachable.size, proxyInfo.registries.length);
|
||||
t.true(reachable.has(mavenRegistry));
|
||||
t.true(reachable.has(nugetFeed));
|
||||
});
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
`Testing connection to ${mavenRegistry.url}`,
|
||||
`Successfully tested connection to ${mavenRegistry.url}`,
|
||||
`Testing connection to ${nugetFeed.url}`,
|
||||
`Successfully tested connection to ${nugetFeed.url}`,
|
||||
`Finished testing connections`,
|
||||
]);
|
||||
});
|
||||
|
||||
test("checkConnections - excludes failed status codes", async (t) => {
|
||||
const backend = new MockReachabilityBackend();
|
||||
sinon
|
||||
.stub(backend, "checkConnection")
|
||||
.onSecondCall()
|
||||
.throws(new ReachabilityError(400));
|
||||
const messages = await withRecordingLoggerAsync(async (logger) => {
|
||||
const reachable = await checkConnections(logger, proxyInfo, backend);
|
||||
t.is(reachable.size, 1);
|
||||
t.true(reachable.has(mavenRegistry));
|
||||
});
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
`Testing connection to ${mavenRegistry.url}`,
|
||||
`Successfully tested connection to ${mavenRegistry.url}`,
|
||||
`Testing connection to ${nugetFeed.url}`,
|
||||
`Connection test to ${nugetFeed.url} failed. (400)`,
|
||||
`Finished testing connections`,
|
||||
]);
|
||||
});
|
||||
|
||||
test("checkConnections - handles other exceptions", async (t) => {
|
||||
const backend = new MockReachabilityBackend();
|
||||
sinon
|
||||
.stub(backend, "checkConnection")
|
||||
.onSecondCall()
|
||||
.throws(new Error("Some generic error"));
|
||||
const messages = await withRecordingLoggerAsync(async (logger) => {
|
||||
const reachable = await checkConnections(logger, proxyInfo, backend);
|
||||
t.is(reachable.size, 1);
|
||||
t.true(reachable.has(mavenRegistry));
|
||||
});
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
`Testing connection to ${mavenRegistry.url}`,
|
||||
`Successfully tested connection to ${mavenRegistry.url}`,
|
||||
`Testing connection to ${nugetFeed.url}`,
|
||||
`Connection test to ${nugetFeed.url} failed: Some generic error`,
|
||||
`Finished testing connections`,
|
||||
]);
|
||||
});
|
||||
|
||||
test("checkConnections - handles invalid URLs", async (t) => {
|
||||
const backend = new MockReachabilityBackend();
|
||||
const messages = await withRecordingLoggerAsync(async (logger) => {
|
||||
const reachable = await checkConnections(
|
||||
logger,
|
||||
{
|
||||
...proxyInfo,
|
||||
registries: [
|
||||
{
|
||||
type: "nuget_feed",
|
||||
url: "localhost",
|
||||
},
|
||||
],
|
||||
},
|
||||
backend,
|
||||
);
|
||||
t.is(reachable.size, 0);
|
||||
});
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
`Skipping check for localhost since it is not a valid URL.`,
|
||||
`Finished testing connections`,
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
import * as https from "https";
|
||||
|
||||
import { HttpsProxyAgent } from "https-proxy-agent";
|
||||
|
||||
import { Logger } from "../logging";
|
||||
import { getErrorMessage } from "../util";
|
||||
|
||||
import { getAddressString, ProxyInfo, Registry } from "./types";
|
||||
|
||||
export class ReachabilityError extends Error {
|
||||
constructor(public readonly statusCode?: number | undefined) {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstracts over the backend for the reachability checks,
|
||||
* to allow actual networking to be replaced with stubs.
|
||||
*/
|
||||
export interface ReachabilityBackend {
|
||||
/**
|
||||
* Performs a test HTTP request to the specified `url`. Resolves to the status code,
|
||||
* if a successful status code was obtained. Otherwise throws
|
||||
*
|
||||
* @param url The URL of the registry to try and reach.
|
||||
* @returns The successful status code (in the `<400` range).
|
||||
*/
|
||||
checkConnection: (url: URL) => Promise<number>;
|
||||
}
|
||||
|
||||
class NetworkReachabilityBackend implements ReachabilityBackend {
|
||||
private agent: https.Agent;
|
||||
|
||||
constructor(private readonly proxy: ProxyInfo) {
|
||||
this.agent = new HttpsProxyAgent(`http://${proxy.host}:${proxy.port}`);
|
||||
}
|
||||
|
||||
public async checkConnection(url: URL): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request(
|
||||
url,
|
||||
{
|
||||
agent: this.agent,
|
||||
method: "HEAD",
|
||||
ca: this.proxy.cert,
|
||||
timeout: 5 * 1000, // 5 seconds
|
||||
},
|
||||
(res) => {
|
||||
res.destroy();
|
||||
|
||||
if (res.statusCode !== undefined && res.statusCode < 400) {
|
||||
resolve(res.statusCode);
|
||||
} else {
|
||||
reject(new ReachabilityError(res.statusCode));
|
||||
}
|
||||
},
|
||||
);
|
||||
req.on("error", (e) => {
|
||||
reject(e);
|
||||
});
|
||||
req.on("timeout", () => {
|
||||
req.destroy();
|
||||
reject(new Error("Connection timeout."));
|
||||
});
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines which configured registries can be reached by performing test requests to them.
|
||||
*
|
||||
* @param logger The logger to use.
|
||||
* @param proxy Information about the proxy, including the configured registries.
|
||||
* @param backend Optionally for testing, a `ReachabilityBackend` to use.
|
||||
* @returns The set of registries which passed the checks.
|
||||
*/
|
||||
export async function checkConnections(
|
||||
logger: Logger,
|
||||
proxy: ProxyInfo,
|
||||
backend?: ReachabilityBackend,
|
||||
): Promise<Set<Registry>> {
|
||||
const result: Set<Registry> = new Set();
|
||||
|
||||
// Don't do anything if there are no registries.
|
||||
if (proxy.registries.length === 0) return result;
|
||||
|
||||
try {
|
||||
// Initialise a networking backend if no backend was provided.
|
||||
if (backend === undefined) {
|
||||
backend = new NetworkReachabilityBackend(proxy);
|
||||
}
|
||||
|
||||
for (const registry of proxy.registries) {
|
||||
const address = getAddressString(registry);
|
||||
const url = URL.parse(address);
|
||||
|
||||
if (url === null) {
|
||||
logger.info(
|
||||
`Skipping check for ${address} since it is not a valid URL.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
logger.debug(`Testing connection to ${url}...`);
|
||||
const statusCode = await backend.checkConnection(url);
|
||||
|
||||
logger.info(`Successfully tested connection to ${url} (${statusCode})`);
|
||||
result.add(registry);
|
||||
} catch (e) {
|
||||
if (e instanceof ReachabilityError && e.statusCode !== undefined) {
|
||||
logger.error(`Connection test to ${url} failed. (${e.statusCode})`);
|
||||
} else {
|
||||
logger.error(
|
||||
`Connection test to ${url} failed: ${getErrorMessage(e)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(`Finished testing connections to private registries.`);
|
||||
} catch (e) {
|
||||
logger.error(
|
||||
`Failed to test connections to private registries: ${getErrorMessage(e)}`,
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* After parsing configurations from JSON, we don't know whether all the keys we expect are
|
||||
* present or not. This type is used to represent such values, which we expect to be
|
||||
* `Credential` values, but haven't validated yet.
|
||||
*/
|
||||
export type RawCredential = Partial<Credential>;
|
||||
|
||||
/**
|
||||
* A package registry configuration includes identifying information as well as
|
||||
* authentication credentials.
|
||||
*/
|
||||
export type Credential = {
|
||||
/** The username needed to authenticate to the package registry, if any. */
|
||||
username?: string;
|
||||
/** The password needed to authenticate to the package registry, if any. */
|
||||
password?: string;
|
||||
/** The token needed to authenticate to the package registry, if any. */
|
||||
token?: string;
|
||||
} & Registry;
|
||||
|
||||
/** A package registry is identified by its type and address. */
|
||||
export type Registry = {
|
||||
/** The type of the package registry. */
|
||||
type: string;
|
||||
} & Address;
|
||||
|
||||
// If a registry has an `url`, then that takes precedence over the `host` which may or may
|
||||
// not be defined.
|
||||
interface HasUrl {
|
||||
url: string;
|
||||
host?: string;
|
||||
}
|
||||
|
||||
// If a registry does not have an `url`, then it must have a `host`.
|
||||
interface WithoutUrl {
|
||||
url: undefined;
|
||||
host: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A valid `Registry` value must either have a `url` or a `host` value. If it has a `url` value,
|
||||
* then that takes precedence over the `host` value. If there is no `url` value, then it must
|
||||
* have a `host` value.
|
||||
*/
|
||||
export type Address = HasUrl | WithoutUrl;
|
||||
|
||||
/** Gets the address as a string. This will either be the `url` if present, or the `host` if not. */
|
||||
export function getAddressString(address: Address): string {
|
||||
if (address.url === undefined) {
|
||||
return address.host;
|
||||
} else {
|
||||
return address.url;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ProxyInfo {
|
||||
host: string;
|
||||
port: number;
|
||||
cert: string;
|
||||
registries: Registry[];
|
||||
}
|
||||
|
||||
export type CertificateAuthority = {
|
||||
cert: string;
|
||||
key: string;
|
||||
};
|
||||
|
||||
export type BasicAuthCredentials = {
|
||||
username: string;
|
||||
password: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Represents configurations for the authentication proxy.
|
||||
*/
|
||||
export type ProxyConfig = {
|
||||
/** The validated configurations for the proxy. */
|
||||
all_credentials: Credential[];
|
||||
ca: CertificateAuthority;
|
||||
proxy_auth?: BasicAuthCredentials;
|
||||
};
|
||||
@@ -18,7 +18,7 @@ import { DocUrl } from "./doc-url";
|
||||
import { EnvVar } from "./environment";
|
||||
import { getRef } from "./git-utils";
|
||||
import { Logger } from "./logging";
|
||||
import { OverlayBaseDatabaseDownloadStats } from "./overlay-database-utils";
|
||||
import { OverlayBaseDatabaseDownloadStats } from "./overlay";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import { ToolsSource } from "./setup-codeql";
|
||||
import {
|
||||
|
||||
+128
-43
@@ -21,7 +21,7 @@ import {
|
||||
FeatureEnablement,
|
||||
} from "./feature-flags";
|
||||
import { Logger } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay-database-utils";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import {
|
||||
DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
DEFAULT_DEBUG_DATABASE_NAME,
|
||||
@@ -145,67 +145,152 @@ export function setupActionsVars(tempDir: string, toolsDir: string) {
|
||||
process.env["RUNNER_TEMP"] = tempDir;
|
||||
process.env["RUNNER_TOOL_CACHE"] = toolsDir;
|
||||
process.env["GITHUB_WORKSPACE"] = tempDir;
|
||||
process.env["GITHUB_EVENT_NAME"] = "push";
|
||||
}
|
||||
|
||||
type LogLevel = "debug" | "info" | "warning" | "error";
|
||||
|
||||
export interface LoggedMessage {
|
||||
type: "debug" | "info" | "warning" | "error";
|
||||
type: LogLevel;
|
||||
message: string | Error;
|
||||
}
|
||||
|
||||
export class RecordingLogger implements Logger {
|
||||
messages: LoggedMessage[] = [];
|
||||
readonly groups: string[] = [];
|
||||
readonly unfinishedGroups: Set<string> = new Set();
|
||||
private currentGroup: string | undefined = undefined;
|
||||
|
||||
constructor(private readonly logToConsole: boolean = true) {}
|
||||
|
||||
private addMessage(level: LogLevel, message: string | Error): void {
|
||||
this.messages.push({ type: level, message });
|
||||
|
||||
if (this.logToConsole) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug(message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether the logged messages contain `messageOrRegExp`.
|
||||
*
|
||||
* If `messageOrRegExp` is a string, this function returns true as long as
|
||||
* `messageOrRegExp` appears as part of one of the `messages`.
|
||||
*
|
||||
* If `messageOrRegExp` is a regular expression, this function returns true as long as
|
||||
* one of the `messages` matches `messageOrRegExp`.
|
||||
*/
|
||||
hasMessage(messageOrRegExp: string | RegExp): boolean {
|
||||
return hasLoggedMessage(this.messages, messageOrRegExp);
|
||||
}
|
||||
|
||||
isDebug() {
|
||||
return true;
|
||||
}
|
||||
|
||||
debug(message: string) {
|
||||
this.addMessage("debug", message);
|
||||
}
|
||||
|
||||
info(message: string) {
|
||||
this.addMessage("info", message);
|
||||
}
|
||||
|
||||
warning(message: string | Error) {
|
||||
this.addMessage("warning", message);
|
||||
}
|
||||
|
||||
error(message: string | Error) {
|
||||
this.addMessage("error", message);
|
||||
}
|
||||
|
||||
startGroup(name: string) {
|
||||
this.groups.push(name);
|
||||
this.currentGroup = name;
|
||||
this.unfinishedGroups.add(name);
|
||||
}
|
||||
|
||||
endGroup() {
|
||||
if (this.currentGroup !== undefined) {
|
||||
this.unfinishedGroups.delete(this.currentGroup);
|
||||
}
|
||||
this.currentGroup = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function getRecordingLogger(
|
||||
messages: LoggedMessage[],
|
||||
{ logToConsole }: { logToConsole?: boolean } = { logToConsole: true },
|
||||
): Logger {
|
||||
return {
|
||||
debug: (message: string) => {
|
||||
messages.push({ type: "debug", message });
|
||||
if (logToConsole) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug(message);
|
||||
}
|
||||
},
|
||||
info: (message: string) => {
|
||||
messages.push({ type: "info", message });
|
||||
if (logToConsole) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.info(message);
|
||||
}
|
||||
},
|
||||
warning: (message: string | Error) => {
|
||||
messages.push({ type: "warning", message });
|
||||
if (logToConsole) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(message);
|
||||
}
|
||||
},
|
||||
error: (message: string | Error) => {
|
||||
messages.push({ type: "error", message });
|
||||
if (logToConsole) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(message);
|
||||
}
|
||||
},
|
||||
isDebug: () => true,
|
||||
startGroup: () => undefined,
|
||||
endGroup: () => undefined,
|
||||
};
|
||||
const logger = new RecordingLogger(logToConsole);
|
||||
logger.messages = messages;
|
||||
return logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether `messages` contains `messageOrRegExp`.
|
||||
*
|
||||
* If `messageOrRegExp` is a string, this function returns true as long as
|
||||
* `messageOrRegExp` appears as part of one of the `messages`.
|
||||
*
|
||||
* If `messageOrRegExp` is a regular expression, this function returns true as long as
|
||||
* one of the `messages` matches `messageOrRegExp`.
|
||||
*/
|
||||
function hasLoggedMessage(
|
||||
messages: LoggedMessage[],
|
||||
messageOrRegExp: string | RegExp,
|
||||
): boolean {
|
||||
const check = (val: string) =>
|
||||
typeof messageOrRegExp === "string"
|
||||
? val.includes(messageOrRegExp)
|
||||
: messageOrRegExp.test(val);
|
||||
|
||||
return messages.some(
|
||||
(msg) => typeof msg.message === "string" && check(msg.message),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks that `messages` contains all of `expectedMessages`.
|
||||
*/
|
||||
export function checkExpectedLogMessages(
|
||||
t: ExecutionContext<any>,
|
||||
messages: LoggedMessage[],
|
||||
expectedMessages: string[],
|
||||
) {
|
||||
const missingMessages: string[] = [];
|
||||
|
||||
for (const expectedMessage of expectedMessages) {
|
||||
t.assert(
|
||||
messages.some(
|
||||
(msg) =>
|
||||
typeof msg.message === "string" &&
|
||||
msg.message.includes(expectedMessage),
|
||||
),
|
||||
`Expected '${expectedMessage}' in the logger output, but didn't find it in:\n ${messages.map((m) => ` - '${m.message}'`).join("\n")}`,
|
||||
);
|
||||
if (!hasLoggedMessage(messages, expectedMessage)) {
|
||||
missingMessages.push(expectedMessage);
|
||||
}
|
||||
}
|
||||
|
||||
if (missingMessages.length > 0) {
|
||||
const listify = (lines: string[]) =>
|
||||
lines.map((m) => ` - '${m}'`).join("\n");
|
||||
|
||||
t.fail(
|
||||
`Expected\n\n${listify(missingMessages)}\n\nin the logger output, but didn't find it in:\n\n${messages.map((m) => ` - '${m.message}'`).join("\n")}`,
|
||||
);
|
||||
} else {
|
||||
t.pass();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that `message` should not have been logged to `logger`.
|
||||
*/
|
||||
export function assertNotLogged(
|
||||
t: ExecutionContext<any>,
|
||||
logger: RecordingLogger,
|
||||
message: string | RegExp,
|
||||
) {
|
||||
t.false(
|
||||
logger.hasMessage(message),
|
||||
`'${message}' should not have been logged, but was.`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+70
-25
@@ -12,6 +12,7 @@ import * as api from "./api-client";
|
||||
import { getRunnerLogger, Logger } from "./logging";
|
||||
import { setupTests } from "./testing-utils";
|
||||
import * as uploadLib from "./upload-lib";
|
||||
import { UploadPayload } from "./upload-lib/types";
|
||||
import { GitHubVariant, initializeEnvironment, withTmpDir } from "./util";
|
||||
|
||||
setupTests(test);
|
||||
@@ -128,11 +129,21 @@ test("finding SARIF files", async (t) => {
|
||||
"file",
|
||||
);
|
||||
|
||||
// add some `.quality.sarif` files that should be ignored, unless we look for them specifically
|
||||
fs.writeFileSync(path.join(tmpDir, "a.quality.sarif"), "");
|
||||
fs.writeFileSync(path.join(tmpDir, "dir1", "b.quality.sarif"), "");
|
||||
// add some non-Code Scanning files that should be ignored, unless we look for them specifically
|
||||
for (const analysisKind of analyses.supportedAnalysisKinds) {
|
||||
if (analysisKind === AnalysisKind.CodeScanning) continue;
|
||||
|
||||
const expectedSarifFiles = [
|
||||
const analysis = analyses.getAnalysisConfig(analysisKind);
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, `a${analysis.sarifExtension}`), "");
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, "dir1", `b${analysis.sarifExtension}`),
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
const expectedSarifFiles: Partial<Record<AnalysisKind, string[]>> = {};
|
||||
expectedSarifFiles[AnalysisKind.CodeScanning] = [
|
||||
path.join(tmpDir, "a.sarif"),
|
||||
path.join(tmpDir, "b.sarif"),
|
||||
path.join(tmpDir, "dir1", "d.sarif"),
|
||||
@@ -143,18 +154,24 @@ test("finding SARIF files", async (t) => {
|
||||
CodeScanning.sarifPredicate,
|
||||
);
|
||||
|
||||
t.deepEqual(sarifFiles, expectedSarifFiles);
|
||||
t.deepEqual(sarifFiles, expectedSarifFiles[AnalysisKind.CodeScanning]);
|
||||
|
||||
const expectedQualitySarifFiles = [
|
||||
path.join(tmpDir, "a.quality.sarif"),
|
||||
path.join(tmpDir, "dir1", "b.quality.sarif"),
|
||||
];
|
||||
const qualitySarifFiles = uploadLib.findSarifFilesInDir(
|
||||
tmpDir,
|
||||
CodeQuality.sarifPredicate,
|
||||
);
|
||||
for (const analysisKind of analyses.supportedAnalysisKinds) {
|
||||
if (analysisKind === AnalysisKind.CodeScanning) continue;
|
||||
|
||||
t.deepEqual(qualitySarifFiles, expectedQualitySarifFiles);
|
||||
const analysis = analyses.getAnalysisConfig(analysisKind);
|
||||
|
||||
expectedSarifFiles[analysisKind] = [
|
||||
path.join(tmpDir, `a${analysis.sarifExtension}`),
|
||||
path.join(tmpDir, "dir1", `b${analysis.sarifExtension}`),
|
||||
];
|
||||
const foundSarifFiles = uploadLib.findSarifFilesInDir(
|
||||
tmpDir,
|
||||
analysis.sarifPredicate,
|
||||
);
|
||||
|
||||
t.deepEqual(foundSarifFiles, expectedSarifFiles[analysisKind]);
|
||||
}
|
||||
|
||||
const groupedSarifFiles = await uploadLib.getGroupedSarifFilePaths(
|
||||
getRunnerLogger(true),
|
||||
@@ -162,16 +179,31 @@ test("finding SARIF files", async (t) => {
|
||||
);
|
||||
|
||||
t.not(groupedSarifFiles, undefined);
|
||||
t.not(groupedSarifFiles[AnalysisKind.CodeScanning], undefined);
|
||||
t.not(groupedSarifFiles[AnalysisKind.CodeQuality], undefined);
|
||||
t.deepEqual(
|
||||
groupedSarifFiles[AnalysisKind.CodeScanning],
|
||||
expectedSarifFiles,
|
||||
);
|
||||
t.deepEqual(
|
||||
groupedSarifFiles[AnalysisKind.CodeQuality],
|
||||
expectedQualitySarifFiles,
|
||||
for (const analysisKind of analyses.supportedAnalysisKinds) {
|
||||
t.not(groupedSarifFiles[analysisKind], undefined);
|
||||
t.deepEqual(
|
||||
groupedSarifFiles[analysisKind],
|
||||
expectedSarifFiles[analysisKind],
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("getGroupedSarifFilePaths - Risk Assessment files", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const sarifPath = path.join(tmpDir, "a.csra.sarif");
|
||||
fs.writeFileSync(sarifPath, "");
|
||||
|
||||
const groupedSarifFiles = await uploadLib.getGroupedSarifFilePaths(
|
||||
getRunnerLogger(true),
|
||||
sarifPath,
|
||||
);
|
||||
|
||||
t.not(groupedSarifFiles, undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.CodeScanning], undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.CodeQuality], undefined);
|
||||
t.not(groupedSarifFiles[AnalysisKind.RiskAssessment], undefined);
|
||||
t.deepEqual(groupedSarifFiles[AnalysisKind.RiskAssessment], [sarifPath]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -188,6 +220,7 @@ test("getGroupedSarifFilePaths - Code Quality file", async (t) => {
|
||||
t.not(groupedSarifFiles, undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.CodeScanning], undefined);
|
||||
t.not(groupedSarifFiles[AnalysisKind.CodeQuality], undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.RiskAssessment], undefined);
|
||||
t.deepEqual(groupedSarifFiles[AnalysisKind.CodeQuality], [sarifPath]);
|
||||
});
|
||||
});
|
||||
@@ -205,6 +238,7 @@ test("getGroupedSarifFilePaths - Code Scanning file", async (t) => {
|
||||
t.not(groupedSarifFiles, undefined);
|
||||
t.not(groupedSarifFiles[AnalysisKind.CodeScanning], undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.CodeQuality], undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.RiskAssessment], undefined);
|
||||
t.deepEqual(groupedSarifFiles[AnalysisKind.CodeScanning], [sarifPath]);
|
||||
});
|
||||
});
|
||||
@@ -222,6 +256,7 @@ test("getGroupedSarifFilePaths - Other file", async (t) => {
|
||||
t.not(groupedSarifFiles, undefined);
|
||||
t.not(groupedSarifFiles[AnalysisKind.CodeScanning], undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.CodeQuality], undefined);
|
||||
t.is(groupedSarifFiles[AnalysisKind.RiskAssessment], undefined);
|
||||
t.deepEqual(groupedSarifFiles[AnalysisKind.CodeScanning], [sarifPath]);
|
||||
});
|
||||
});
|
||||
@@ -875,7 +910,15 @@ function createMockSarif(id?: string, tool?: string) {
|
||||
|
||||
function uploadPayloadFixtures(analysis: analyses.AnalysisConfig) {
|
||||
const mockData = {
|
||||
payload: { sarif: "base64data", commit_sha: "abc123" },
|
||||
payload: {
|
||||
commit_oid: "abc123",
|
||||
ref: "ref",
|
||||
sarif: "base64data",
|
||||
workflow_run_id: 1,
|
||||
workflow_run_attempt: 1,
|
||||
checkout_uri: "uri",
|
||||
tool_names: ["codeql"],
|
||||
} satisfies UploadPayload,
|
||||
owner: "test-owner",
|
||||
repo: "test-repo",
|
||||
response: {
|
||||
@@ -907,7 +950,9 @@ function uploadPayloadFixtures(analysis: analyses.AnalysisConfig) {
|
||||
};
|
||||
}
|
||||
|
||||
for (const analysis of [CodeScanning, CodeQuality]) {
|
||||
for (const analysisKind of analyses.supportedAnalysisKinds) {
|
||||
const analysis = analyses.getAnalysisConfig(analysisKind);
|
||||
|
||||
test(`uploadPayload on ${analysis.name} uploads successfully`, async (t) => {
|
||||
const { upload, requestStub, mockData } = uploadPayloadFixtures(analysis);
|
||||
requestStub
|
||||
|
||||
+19
-21
@@ -21,6 +21,7 @@ import * as gitUtils from "./git-utils";
|
||||
import { initCodeQL } from "./init";
|
||||
import { Logger } from "./logging";
|
||||
import { getRepositoryNwo, RepositoryNwo } from "./repository";
|
||||
import { BasePayload, UploadPayload } from "./upload-lib/types";
|
||||
import * as util from "./util";
|
||||
import {
|
||||
ConfigurationError,
|
||||
@@ -326,7 +327,7 @@ function getAutomationID(
|
||||
* This is exported for testing purposes only.
|
||||
*/
|
||||
export async function uploadPayload(
|
||||
payload: any,
|
||||
payload: BasePayload,
|
||||
repositoryNwo: RepositoryNwo,
|
||||
logger: Logger,
|
||||
analysis: analyses.AnalysisConfig,
|
||||
@@ -618,8 +619,8 @@ export function buildPayload(
|
||||
environment: string | undefined,
|
||||
toolNames: string[],
|
||||
mergeBaseCommitOid: string | undefined,
|
||||
) {
|
||||
const payloadObj = {
|
||||
): UploadPayload {
|
||||
const payloadObj: UploadPayload = {
|
||||
commit_oid: commitOid,
|
||||
ref,
|
||||
analysis_key: analysisKey,
|
||||
@@ -847,18 +848,20 @@ export async function uploadPostProcessedFiles(
|
||||
const zippedSarif = zlib.gzipSync(sarifPayload).toString("base64");
|
||||
const checkoutURI = url.pathToFileURL(checkoutPath).href;
|
||||
|
||||
const payload = buildPayload(
|
||||
await gitUtils.getCommitOid(checkoutPath),
|
||||
await gitUtils.getRef(),
|
||||
postProcessingResults.analysisKey,
|
||||
util.getRequiredEnvParam("GITHUB_WORKFLOW"),
|
||||
zippedSarif,
|
||||
actionsUtil.getWorkflowRunID(),
|
||||
actionsUtil.getWorkflowRunAttempt(),
|
||||
checkoutURI,
|
||||
postProcessingResults.environment,
|
||||
toolNames,
|
||||
await gitUtils.determineBaseBranchHeadCommitOid(),
|
||||
const payload = uploadTarget.transformPayload(
|
||||
buildPayload(
|
||||
await gitUtils.getCommitOid(checkoutPath),
|
||||
await gitUtils.getRef(),
|
||||
postProcessingResults.analysisKey,
|
||||
util.getRequiredEnvParam("GITHUB_WORKFLOW"),
|
||||
zippedSarif,
|
||||
actionsUtil.getWorkflowRunID(),
|
||||
actionsUtil.getWorkflowRunAttempt(),
|
||||
checkoutURI,
|
||||
postProcessingResults.environment,
|
||||
toolNames,
|
||||
await gitUtils.determineBaseBranchHeadCommitOid(),
|
||||
),
|
||||
);
|
||||
|
||||
// Log some useful debug info about the info
|
||||
@@ -939,7 +942,6 @@ export async function waitForProcessing(
|
||||
const client = api.getApiClient();
|
||||
|
||||
const statusCheckingStarted = Date.now();
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
if (
|
||||
Date.now() >
|
||||
@@ -1128,11 +1130,7 @@ function sanitize(str?: string) {
|
||||
/**
|
||||
* An error that occurred due to an invalid SARIF upload request.
|
||||
*/
|
||||
export class InvalidSarifUploadError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
export class InvalidSarifUploadError extends Error {}
|
||||
|
||||
function filterAlertsByDiffRange(logger: Logger, sarif: SarifFile): SarifFile {
|
||||
const diffRanges = readDiffRangesJsonFile(logger);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Represents the minimum, common payload for SARIF upload endpoints that we support.
|
||||
*/
|
||||
export interface BasePayload {
|
||||
/** The gzipped contents of a SARIF file. */
|
||||
sarif: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the payload expected for Code Scanning and Code Quality SARIF uploads.
|
||||
*/
|
||||
export interface UploadPayload extends BasePayload {
|
||||
/** The SHA of the commit that was analysed. */
|
||||
commit_oid: string;
|
||||
/** The ref that was analysed. */
|
||||
ref: string;
|
||||
/** The analysis key that identifies the analysis. */
|
||||
analysis_key?: string;
|
||||
/** The name of the analysis. */
|
||||
analysis_name?: string;
|
||||
/** The ID of the workflow run that performed the analysis. */
|
||||
workflow_run_id: number;
|
||||
/** The attempt number. */
|
||||
workflow_run_attempt: number;
|
||||
/** The URI where the repository was checked out. */
|
||||
checkout_uri: string;
|
||||
/** The matrix value. */
|
||||
environment?: string;
|
||||
/** A string representation of when the analysis was started. */
|
||||
started_at?: string;
|
||||
/** The names of the tools that performed the analysis. */
|
||||
tool_names: string[];
|
||||
/** For a pull request, the ref of the base the PR is targeting. */
|
||||
base_ref?: string;
|
||||
/** For a pull request, the commit SHA of the merge base. */
|
||||
base_sha?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the payload expected for Code Scanning Risk Assessment SARIF uploads.
|
||||
*/
|
||||
export interface AssessmentPayload extends BasePayload {
|
||||
/** The ID of the assessment for which the SARIF is for. */
|
||||
assessment_id: number;
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import * as actionsUtil from "./actions-util";
|
||||
import { getActionVersion, getTemporaryDirectory } from "./actions-util";
|
||||
import * as analyses from "./analyses";
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import { Features } from "./feature-flags";
|
||||
import { initFeatures } from "./feature-flags";
|
||||
import { Logger, getActionsLogger } from "./logging";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import {
|
||||
@@ -70,7 +70,7 @@ async function run(startedAt: Date) {
|
||||
actionsUtil.persistInputs();
|
||||
|
||||
const repositoryNwo = getRepositoryNwo();
|
||||
const features = new Features(
|
||||
const features = initFeatures(
|
||||
gitHubVersion,
|
||||
repositoryNwo,
|
||||
getTemporaryDirectory(),
|
||||
|
||||
+8
-8
@@ -564,27 +564,27 @@ test("joinAtMost - truncates list if array is > than limit", (t) => {
|
||||
t.false(result.includes("test6"));
|
||||
});
|
||||
|
||||
test("Result.success creates a success result", (t) => {
|
||||
const result = util.Result.success("test value");
|
||||
test("Success creates a success result", (t) => {
|
||||
const result = new util.Success("test value");
|
||||
t.true(result.isSuccess());
|
||||
t.false(result.isFailure());
|
||||
t.is(result.value, "test value");
|
||||
});
|
||||
|
||||
test("Result.failure creates a failure result", (t) => {
|
||||
test("Failure creates a failure result", (t) => {
|
||||
const error = new Error("test error");
|
||||
const result = util.Result.failure(error);
|
||||
const result = new util.Failure(error);
|
||||
t.false(result.isSuccess());
|
||||
t.true(result.isFailure());
|
||||
t.is(result.value, error);
|
||||
});
|
||||
|
||||
test("Result.orElse returns the value for a success result", (t) => {
|
||||
const result = util.Result.success("success value");
|
||||
test("Success.orElse returns the value for a success result", (t) => {
|
||||
const result = new util.Success("success value");
|
||||
t.is(result.orElse("default value"), "success value");
|
||||
});
|
||||
|
||||
test("Result.orElse returns the default value for a failure result", (t) => {
|
||||
const result = util.Result.failure(new Error("test error"));
|
||||
test("Failure.orElse returns the default value for a failure result", (t) => {
|
||||
const result = new util.Failure(new Error("test error"));
|
||||
t.is(result.orElse("default value"), "default value");
|
||||
});
|
||||
|
||||
+45
-38
@@ -690,11 +690,7 @@ export class HTTPError extends Error {
|
||||
* An Error class that indicates an error that occurred due to
|
||||
* a misconfiguration of the action or the CodeQL CLI.
|
||||
*/
|
||||
export class ConfigurationError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
export class ConfigurationError extends Error {}
|
||||
|
||||
export function asHTTPError(arg: any): HTTPError | undefined {
|
||||
if (
|
||||
@@ -744,6 +740,7 @@ export async function bundleDb(
|
||||
language: Language,
|
||||
codeql: CodeQL,
|
||||
dbName: string,
|
||||
{ includeDiagnostics }: { includeDiagnostics: boolean },
|
||||
) {
|
||||
const databasePath = getCodeQLDatabasePath(config, language);
|
||||
const databaseBundlePath = path.resolve(config.dbLocation, `${dbName}.zip`);
|
||||
@@ -774,6 +771,7 @@ export async function bundleDb(
|
||||
databasePath,
|
||||
databaseBundlePath,
|
||||
dbName,
|
||||
includeDiagnostics,
|
||||
additionalFiles,
|
||||
);
|
||||
return databaseBundlePath;
|
||||
@@ -1293,42 +1291,51 @@ export function joinAtMost(
|
||||
return array.join(separator);
|
||||
}
|
||||
|
||||
/** A success result. */
|
||||
type Success<T> = Result<T, never>;
|
||||
/** A failure result. */
|
||||
type Failure<E> = Result<never, E>;
|
||||
|
||||
/**
|
||||
* A simple result type representing either a success or a failure.
|
||||
*/
|
||||
export class Result<T, E> {
|
||||
private constructor(
|
||||
private readonly _ok: boolean,
|
||||
public readonly value: T | E,
|
||||
) {}
|
||||
|
||||
/** Creates a success result. */
|
||||
static success<T>(value: T): Success<T> {
|
||||
return new Result(true, value) as Success<T>;
|
||||
}
|
||||
|
||||
/** Creates a failure result. */
|
||||
static failure<E>(value: E): Failure<E> {
|
||||
return new Result(false, value) as Failure<E>;
|
||||
}
|
||||
|
||||
/** An interface representing something that is either a success or a failure. */
|
||||
interface ResultLike<T, E> {
|
||||
/** The value of the result, which can be either a success value or a failure value. */
|
||||
value: T | E;
|
||||
/** Whether this result represents a success. */
|
||||
isSuccess(): this is Success<T> {
|
||||
return this._ok;
|
||||
}
|
||||
|
||||
isSuccess(): this is Success<T>;
|
||||
/** Whether this result represents a failure. */
|
||||
isFailure(): this is Failure<E> {
|
||||
return !this._ok;
|
||||
isFailure(): this is Failure<E>;
|
||||
/** Get the value if this is a success, or return the default value if this is a failure. */
|
||||
orElse<U>(defaultValue: U): T | U;
|
||||
}
|
||||
|
||||
/** A simple result type representing either a success or a failure. */
|
||||
export type Result<T, E> = Success<T> | Failure<E>;
|
||||
|
||||
/** A result representing a success. */
|
||||
export class Success<T> implements ResultLike<T, never> {
|
||||
constructor(public readonly value: T) {}
|
||||
|
||||
isSuccess(): this is Success<T> {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Get the value if this is a success, or return the default value if this is a failure. */
|
||||
orElse<U>(defaultValue: U): T | U {
|
||||
return this.isSuccess() ? this.value : defaultValue;
|
||||
isFailure(): this is Failure<never> {
|
||||
return false;
|
||||
}
|
||||
|
||||
orElse<U>(_defaultValue: U): T {
|
||||
return this.value;
|
||||
}
|
||||
}
|
||||
|
||||
/** A result representing a failure. */
|
||||
export class Failure<E> implements ResultLike<never, E> {
|
||||
constructor(public readonly value: E) {}
|
||||
|
||||
isSuccess(): this is Success<never> {
|
||||
return false;
|
||||
}
|
||||
|
||||
isFailure(): this is Failure<E> {
|
||||
return true;
|
||||
}
|
||||
|
||||
orElse<U>(defaultValue: U): U {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -19,8 +19,8 @@
|
||||
"alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
|
||||
|
||||
/* Additional Checks */
|
||||
"noUnusedLocals": true, /* Report errors on unused locals. */
|
||||
"noUnusedParameters": true, /* Report errors on unused parameters. */
|
||||
"noUnusedLocals": false, /* Report errors on unused locals. */
|
||||
"noUnusedParameters": false, /* Report errors on unused parameters. */
|
||||
"noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
|
||||
"noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
|
||||
|
||||
|
||||
Reference in New Issue
Block a user