mirror of
https://github.com/github/codeql-action.git
synced 2026-05-03 12:20:09 +00:00
Compare commits
257 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a723e99345 | |||
| fbba1e03be | |||
| 933238e8d5 | |||
| e46ed2cbd0 | |||
| b73d1d1634 | |||
| 24e0bb00a9 | |||
| ec298daba7 | |||
| 8c6e48dbe0 | |||
| 719098349e | |||
| 2bb209555a | |||
| 7851e55dc3 | |||
| 262a15f6cf | |||
| a6109b1c07 | |||
| 022ff3c73f | |||
| 0a4d574ac4 | |||
| d1edf2e4de | |||
| facd53f789 | |||
| b77983290b | |||
| fcf29e3d86 | |||
| 1fed3e9ba8 | |||
| 549683cee5 | |||
| 7a6ed56219 | |||
| 91fbc51606 | |||
| 35715ef8fe | |||
| bac7fdaf42 | |||
| 1517969c90 | |||
| f073360456 | |||
| 5145c112e7 | |||
| 7108503ac6 | |||
| 4fe9b1e243 | |||
| 56733fb5ae | |||
| 0a636086c9 | |||
| 97be3af35a | |||
| de303a9db5 | |||
| 7a818e6977 | |||
| 30e0f4391d | |||
| 7c5585e5cf | |||
| 245f6828c4 | |||
| c109008fac | |||
| e73c940c9b | |||
| cdb655d6d4 | |||
| 6153577cab | |||
| 0ed734b61b | |||
| efdcb31f11 | |||
| 4d2c7c6e10 | |||
| 70b2658d23 | |||
| 530fcb3bbf | |||
| 2acf81942b | |||
| d2a54a4507 | |||
| bc4097bbe1 | |||
| c8e26e209a | |||
| 0752451507 | |||
| 243c274daf | |||
| 19b3a84f58 | |||
| 858a6149c1 | |||
| c60c75576d | |||
| 59aede2113 | |||
| 6c35f8607b | |||
| c486cacf49 | |||
| 365478cc5b | |||
| f0e6490756 | |||
| 860353f245 | |||
| 4fb8483ef0 | |||
| c2574efbee | |||
| 4cbe7bef85 | |||
| f6a5638305 | |||
| 1dcdb940d5 | |||
| 0b7b740d4c | |||
| 0ac85966ba | |||
| 5019ed041c | |||
| d64d81d41f | |||
| 6777c894e9 | |||
| 79f9c0517c | |||
| 3b3a77544b | |||
| 9f95de42d6 | |||
| e2d518d895 | |||
| 9df9e9176e | |||
| 6847a42aa8 | |||
| f820c80d4d | |||
| ca7d6d3b79 | |||
| 8d9c36a0ce | |||
| 95e58e9a2c | |||
| 6f31bfe060 | |||
| d2e135a73a | |||
| 60abb65df0 | |||
| 5a0a562209 | |||
| f8b62132ab | |||
| 65216971a1 | |||
| 3c45af2dd2 | |||
| f1c339364c | |||
| 1024fc496c | |||
| 9dd4cfed96 | |||
| c1403f094c | |||
| 90d7616015 | |||
| 1aef4ed505 | |||
| cb52ba6486 | |||
| 7c9e131894 | |||
| 130ab2d721 | |||
| 8cf2dc52f9 | |||
| 8339b9254e | |||
| 97bcdd8c1e | |||
| e6c21da23c | |||
| bad0a744dd | |||
| ee09113642 | |||
| b669eab7e3 | |||
| 4e8c9ce33c | |||
| 1cf0431149 | |||
| a26cb68cc7 | |||
| 60991e61ac | |||
| 7197c2b792 | |||
| 597e12aa85 | |||
| d277a56348 | |||
| 111a537cd9 | |||
| 51d833290e | |||
| 5a17511bf0 | |||
| 43d8420a42 | |||
| 76a687e1d8 | |||
| 751f3e2f7c | |||
| 808513f048 | |||
| e452857e57 | |||
| b623f5fd57 | |||
| 35a38985d3 | |||
| 14ed573199 | |||
| 43d8864b35 | |||
| f8aff3ad8b | |||
| e6c83948f5 | |||
| 347f0c676d | |||
| 6eed62b035 | |||
| de1752b85d | |||
| 1065967b50 | |||
| e25c0a535a | |||
| 5f323cad05 | |||
| 212e28374b | |||
| 36075a4980 | |||
| 34950e1b11 | |||
| 57ec7e1000 | |||
| 311573e58e | |||
| 1f4c852aeb | |||
| 2e3aaaefca | |||
| e2203c62cf | |||
| 7b0c5b1669 | |||
| faf45e07f9 | |||
| 8b5e60477c | |||
| 99b8dd4d57 | |||
| c618c9bddb | |||
| 9fd9b64766 | |||
| 0c7c298b2a | |||
| a507a542a4 | |||
| be0a156326 | |||
| f98bf5e347 | |||
| 3db32b5d27 | |||
| 4e0952a3c0 | |||
| 0592832ed8 | |||
| 88a7e5118e | |||
| 6643a7d207 | |||
| 47f1709a3c | |||
| b1981a5480 | |||
| a899987af2 | |||
| 4ed3c0efe6 | |||
| 191d7c6f13 | |||
| aa69c483cd | |||
| fe775da508 | |||
| 353802f9f2 | |||
| cc7db4a1f9 | |||
| 6010f9d8e2 | |||
| c10b8064de | |||
| c5ffd06837 | |||
| d6d1743b8e | |||
| 999119ba45 | |||
| 65d2efa733 | |||
| 2437b20ab3 | |||
| f13c600724 | |||
| 7dcea06663 | |||
| ea5f71947c | |||
| 45ceeea896 | |||
| 24448c9843 | |||
| 7c51060631 | |||
| b8bb9f28b8 | |||
| e9cf68bb33 | |||
| 36791d8d66 | |||
| 22eba96a28 | |||
| 0078ad667e | |||
| fa7a15b909 | |||
| 8c29faa7ab | |||
| f94817b9f0 | |||
| dd060970a5 | |||
| 5cc552f43e | |||
| 6b1a9f2131 | |||
| 9d3ec5727a | |||
| 3ff82aacd0 | |||
| 4bdd4e7526 | |||
| 23a0098b57 | |||
| ea7b090925 | |||
| a663d0174a | |||
| b659882aae | |||
| d5bb39fa0b | |||
| 521c3536d3 | |||
| 972365e142 | |||
| 8a0b4f2746 | |||
| a5418e172c | |||
| fae4c28b51 | |||
| 661a8fbbe3 | |||
| e7c7b68c5f | |||
| fa568ebc69 | |||
| 0da3139813 | |||
| 0abe92ed20 | |||
| 07f235e5f2 | |||
| 9fd40ff508 | |||
| 75ed461aaa | |||
| cfc18781e0 | |||
| 9fe42f69b7 | |||
| c5a984e1aa | |||
| 0543156694 | |||
| 4cec5d2830 | |||
| 74dd691a45 | |||
| a5244bf7dd | |||
| 1bc611ed0c | |||
| d2008eee7c | |||
| 9481177f3d | |||
| 9813849e61 | |||
| 4867f5927a | |||
| 49af37b7ab | |||
| b72f4fec40 | |||
| 0d87a75829 | |||
| 3db9a05c73 | |||
| aa2773169b | |||
| 054745baee | |||
| 3d564d9359 | |||
| 137e0dec2b | |||
| d128e5daa8 | |||
| eedab83377 | |||
| 8c023a6b07 | |||
| 28f56f2bed | |||
| d48d054533 | |||
| 72c0b0efb7 | |||
| 05b1a5d28f | |||
| 8dc2e5d9d2 | |||
| 8fd6c0e573 | |||
| 3869755554 | |||
| 20e68ac12b | |||
| 095e0fe505 | |||
| 47b94fe61c | |||
| 51a1d6917f | |||
| 510cf736e3 | |||
| 89f0c86efa | |||
| c3f90ba975 | |||
| 64507ed148 | |||
| 1a45a9b9d0 | |||
| f88d49ee5d | |||
| 28f515d9ad | |||
| de06821112 | |||
| ddafddb826 | |||
| 740f177889 | |||
| 0393130759 | |||
| f86097dfdb | |||
| 6e67ef61f2 | |||
| 193dd19c2d |
@@ -22,7 +22,8 @@ runs:
|
||||
MAJOR_VERSION: ${{ inputs.major_version }}
|
||||
LATEST_TAG: ${{ inputs.latest_tag }}
|
||||
run: |
|
||||
python ${{ github.action_path }}/release-branches.py \
|
||||
npm ci
|
||||
npx tsx ./pr-checks/release-branches.ts \
|
||||
--major-version "$MAJOR_VERSION" \
|
||||
--latest-tag "$LATEST_TAG"
|
||||
shell: bash
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import configparser
|
||||
|
||||
# Name of the remote
|
||||
ORIGIN = 'origin'
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
grandparent_dir = os.path.dirname(os.path.dirname(script_dir))
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
with open(os.path.join(grandparent_dir, 'releases.ini')) as stream:
|
||||
config.read_string('[default]\n' + stream.read())
|
||||
|
||||
OLDEST_SUPPORTED_MAJOR_VERSION = int(config['default']['OLDEST_SUPPORTED_MAJOR_VERSION'])
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--major-version", required=True, type=str, help="The major version of the release")
|
||||
parser.add_argument("--latest-tag", required=True, type=str, help="The most recent tag published to the repository")
|
||||
args = parser.parse_args()
|
||||
|
||||
major_version = args.major_version
|
||||
latest_tag = args.latest_tag
|
||||
|
||||
print("major_version: " + major_version)
|
||||
print("latest_tag: " + latest_tag)
|
||||
|
||||
# If this is a primary release, we backport to all supported branches,
|
||||
# so we check whether the major_version taken from the package.json
|
||||
# is greater than or equal to the latest tag pulled from the repo.
|
||||
# For example...
|
||||
# 'v1' >= 'v2' is False # we're operating from an older release branch and should not backport
|
||||
# 'v2' >= 'v2' is True # the normal case where we're updating the current version
|
||||
# 'v3' >= 'v2' is True # in this case we are making the first release of a new major version
|
||||
consider_backports = ( major_version >= latest_tag.split(".")[0] )
|
||||
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
|
||||
f.write(f"backport_source_branch=releases/{major_version}\n")
|
||||
|
||||
backport_target_branches = []
|
||||
|
||||
if consider_backports:
|
||||
for i in range(int(major_version.strip("v"))-1, 0, -1):
|
||||
branch_name = f"releases/v{i}"
|
||||
if i >= OLDEST_SUPPORTED_MAJOR_VERSION:
|
||||
backport_target_branches.append(branch_name)
|
||||
|
||||
f.write("backport_target_branches="+json.dumps(backport_target_branches)+"\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -15,6 +15,12 @@ runs:
|
||||
run: echo "$GITHUB_CONTEXT"
|
||||
shell: bash
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
name: "CodeQL config"
|
||||
queries:
|
||||
queries:
|
||||
- name: Run custom queries
|
||||
uses: ./queries
|
||||
# Run all extra query suites, both because we want to
|
||||
@@ -13,3 +13,5 @@ queries:
|
||||
paths-ignore:
|
||||
- lib
|
||||
- tests
|
||||
- "**/*.test.ts"
|
||||
- "**/testing-util.ts"
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: npm
|
||||
directory: "/"
|
||||
directories:
|
||||
- "/"
|
||||
- "/pr-checks"
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
OLDEST_SUPPORTED_MAJOR_VERSION=3
|
||||
Generated
+2
-2
@@ -60,12 +60,12 @@ jobs:
|
||||
setup-kotlin: 'true'
|
||||
- uses: ./../action/init
|
||||
with:
|
||||
languages: C#,java-kotlin,swift,typescript
|
||||
languages: C#,java-kotlin,typescript
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
|
||||
- name: 'Check languages'
|
||||
run: |
|
||||
expected_languages="csharp,java,swift,javascript"
|
||||
expected_languages="csharp,java,javascript"
|
||||
actual_languages=$(jq -r '.languages | join(",")' "$RUNNER_TEMP"/config)
|
||||
|
||||
if [ "$expected_languages" != "$actual_languages" ]; then
|
||||
|
||||
+1
-1
@@ -59,7 +59,7 @@ jobs:
|
||||
use-all-platform-bundle: 'false'
|
||||
setup-kotlin: 'true'
|
||||
- name: Set up Ruby
|
||||
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
||||
uses: ruby/setup-ruby@0cb964fd540e0a24c900370abf38a33466142735 # v1.305.0
|
||||
with:
|
||||
ruby-version: 2.6
|
||||
- name: Install Code Scanning integration
|
||||
|
||||
Generated
+18
-1
@@ -71,7 +71,17 @@ jobs:
|
||||
id: proxy
|
||||
uses: ./../action/start-proxy
|
||||
with:
|
||||
registry_secrets: '[{ "type": "nuget_feed", "url": "https://api.nuget.org/v3/index.json" }]'
|
||||
registry_secrets: |
|
||||
[
|
||||
{
|
||||
"type": "maven_repository",
|
||||
"url": "https://repo.maven.apache.org/maven2/"
|
||||
},
|
||||
{
|
||||
"type": "maven_repository",
|
||||
"url": "https://repo1.maven.org/maven2"
|
||||
}
|
||||
]
|
||||
|
||||
- name: Print proxy outputs
|
||||
run: |
|
||||
@@ -82,5 +92,12 @@ jobs:
|
||||
- name: Fail if proxy outputs are not set
|
||||
if: (!steps.proxy.outputs.proxy_host) || (!steps.proxy.outputs.proxy_port) || (!steps.proxy.outputs.proxy_ca_certificate) || (!steps.proxy.outputs.proxy_urls)
|
||||
run: exit 1
|
||||
|
||||
- name: Fail if proxy_urls does not contain all registries
|
||||
if: |
|
||||
join(fromJSON(steps.proxy.outputs.proxy_urls)[*].type, ',') != 'maven_repository,maven_repository'
|
||||
|| !contains(steps.proxy.outputs.proxy_urls, 'https://repo.maven.apache.org/maven2/')
|
||||
|| !contains(steps.proxy.outputs.proxy_urls, 'https://repo1.maven.org/maven2')
|
||||
run: exit 1
|
||||
env:
|
||||
CODEQL_ACTION_TEST_MODE: true
|
||||
|
||||
@@ -66,6 +66,7 @@ jobs:
|
||||
uses: ./../action/.github/actions/verify-debug-artifact-scan-completed
|
||||
- uses: ./../action/init
|
||||
with:
|
||||
languages: cpp,csharp,go,java,javascript,python
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
debug: true
|
||||
debug-artifact-name: my-debug-artifacts
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# Workflow runs on main, on a release branch, and that were triggered as part of a merge group have
|
||||
# already passed CI before being merged. Therefore if they fail, we should make sure that there
|
||||
# wasn't a transient failure by rerunning the failed jobs once before investigating further.
|
||||
name: Deflake
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
types: [completed]
|
||||
# Exclude workflows that have significant side effects, like publishing releases. It's OK to
|
||||
# retry CodeQL analysis.
|
||||
workflows:
|
||||
- Check Expected Release Files
|
||||
- Code-Scanning config CLI tests
|
||||
- CodeQL action
|
||||
- Manual Check - go
|
||||
- "PR Check - All-platform bundle"
|
||||
- "PR Check - Analysis kinds"
|
||||
- "PR Check - Analyze: 'ref' and 'sha' from inputs"
|
||||
- "PR Check - autobuild-action"
|
||||
- "PR Check - Autobuild direct tracing (custom working directory)"
|
||||
- "PR Check - Autobuild working directory"
|
||||
- "PR Check - Build mode autobuild"
|
||||
- "PR Check - Build mode manual"
|
||||
- "PR Check - Build mode none"
|
||||
- "PR Check - Build mode rollback"
|
||||
- "PR Check - Bundle: Caching checks"
|
||||
- "PR Check - Bundle: From nightly"
|
||||
- "PR Check - Bundle: From toolcache"
|
||||
- "PR Check - Bundle: Zstandard checks"
|
||||
- "PR Check - C/C\\+\\+: autoinstalling dependencies (Linux)"
|
||||
- "PR Check - C/C\\+\\+: autoinstalling dependencies is skipped (macOS)"
|
||||
- "PR Check - C/C\\+\\+: disabling autoinstalling dependencies (Linux)"
|
||||
- "PR Check - Clean up database cluster directory"
|
||||
- "PR Check - CodeQL Bundle All"
|
||||
- "PR Check - Config export"
|
||||
- "PR Check - Config input"
|
||||
- "PR Check - Custom source root"
|
||||
- "PR Check - Debug artifact upload"
|
||||
- "PR Check - Debug artifacts after failure"
|
||||
- "PR Check - Diagnostic export"
|
||||
- "PR Check - Export file baseline information"
|
||||
- "PR Check - Extractor ram and threads options test"
|
||||
- "PR Check - Go: Custom queries"
|
||||
- "PR Check - Go: diagnostic when Go is changed after init step"
|
||||
- "PR Check - Go: diagnostic when `file` is not installed"
|
||||
- "PR Check - Go: tracing with autobuilder step"
|
||||
- "PR Check - Go: tracing with custom build steps"
|
||||
- "PR Check - Go: tracing with legacy workflow"
|
||||
- "PR Check - Go: workaround for indirect tracing"
|
||||
- "PR Check - Job run UUID added to SARIF"
|
||||
- "PR Check - Language aliases"
|
||||
- "PR Check - Local CodeQL bundle"
|
||||
- "PR Check - Multi-language repository"
|
||||
- "PR Check - Overlay database init fallback"
|
||||
- "PR Check - Packaging: Action input"
|
||||
- "PR Check - Packaging: Config and input"
|
||||
- "PR Check - Packaging: Config and input passed to the CLI"
|
||||
- "PR Check - Packaging: Config file"
|
||||
- "PR Check - Packaging: Download using registries"
|
||||
- "PR Check - Proxy test"
|
||||
- "PR Check - Remote config file"
|
||||
- "PR Check - Resolve environment"
|
||||
- "PR Check - RuboCop multi-language"
|
||||
- "PR Check - Ruby analysis"
|
||||
- "PR Check - Rust analysis"
|
||||
- "PR Check - Split workflow"
|
||||
- "PR Check - Start proxy"
|
||||
- "PR Check - Submit SARIF after failure"
|
||||
- "PR Check - Swift analysis using a custom build command"
|
||||
- "PR Check - Swift analysis using autobuild"
|
||||
- "PR Check - Test different uses of `upload-sarif`"
|
||||
- "PR Check - Test unsetting environment variables"
|
||||
- "PR Check - Upload-sarif: ref and sha from inputs"
|
||||
- "PR Check - Use a custom `checkout_path`"
|
||||
- PR Checks
|
||||
- Query filters tests
|
||||
- Test that the workaround for python 3.12 on windows works
|
||||
|
||||
jobs:
|
||||
rerun-on-failure:
|
||||
name: Rerun failed jobs
|
||||
if: >-
|
||||
github.event.workflow_run.conclusion == 'failure' &&
|
||||
github.event.workflow_run.run_attempt == 1 &&
|
||||
(
|
||||
github.event.workflow_run.head_branch == 'main' ||
|
||||
startsWith(github.event.workflow_run.head_branch, 'releases/') ||
|
||||
github.event.workflow_run.event == 'merge_group'
|
||||
)
|
||||
runs-on: ubuntu-slim
|
||||
permissions:
|
||||
actions: write
|
||||
steps:
|
||||
- name: Rerun failed jobs in ${{ github.event.workflow_run.name }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RUN_NAME: ${{ github.event.workflow_run.name }}
|
||||
RUN_URL: ${{ github.event.workflow_run.html_url }}
|
||||
run: |
|
||||
echo "Rerunning failed jobs for workflow run ${RUN_ID}"
|
||||
gh run rerun "${RUN_ID}" --failed
|
||||
echo "### Reran failed jobs :recycle:" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Workflow: [${RUN_NAME}](${RUN_URL})" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -24,7 +24,7 @@ defaults:
|
||||
|
||||
jobs:
|
||||
merge-back:
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: ubuntu-latest
|
||||
environment: Automation
|
||||
if: github.repository == 'github/codeql-action'
|
||||
env:
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Generate token
|
||||
uses: actions/create-github-app-token@v3.0.0
|
||||
uses: actions/create-github-app-token@v3.1.1
|
||||
id: app-token
|
||||
with:
|
||||
app-id: ${{ vars.AUTOMATION_APP_ID }}
|
||||
|
||||
@@ -29,7 +29,7 @@ defaults:
|
||||
jobs:
|
||||
prepare:
|
||||
name: "Prepare release"
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'github/codeql-action'
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -136,7 +136,7 @@ jobs:
|
||||
|
||||
- name: Generate token
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
uses: actions/create-github-app-token@v3.0.0
|
||||
uses: actions/create-github-app-token@v3.1.1
|
||||
id: app-token
|
||||
with:
|
||||
app-id: ${{ vars.AUTOMATION_APP_ID }}
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Update the required checks based on the current branch.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
REPO_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
GRANDPARENT_DIR="$(dirname "$REPO_DIR")"
|
||||
source "$GRANDPARENT_DIR/releases.ini"
|
||||
|
||||
if ! gh auth status 2>/dev/null; then
|
||||
gh auth status
|
||||
echo "Failed: Not authorized. This script requires admin access to github/codeql-action through the gh CLI."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$#" -eq 1 ]; then
|
||||
# If we were passed an argument, use that as the SHA
|
||||
GITHUB_SHA="$1"
|
||||
elif [ "$#" -gt 1 ]; then
|
||||
echo "Usage: $0 [SHA]"
|
||||
echo "Update the required checks based on the SHA, or main."
|
||||
exit 1
|
||||
elif [ -z "$GITHUB_SHA" ]; then
|
||||
# If we don't have a SHA, use main
|
||||
GITHUB_SHA="$(git rev-parse main)"
|
||||
fi
|
||||
|
||||
echo "Getting checks for $GITHUB_SHA"
|
||||
|
||||
# Ignore any checks with "https://", CodeQL, LGTM, Update, and ESLint checks.
|
||||
CHECKS="$(gh api repos/github/codeql-action/commits/"${GITHUB_SHA}"/check-runs --paginate | jq --slurp --compact-output --raw-output '[.[].check_runs.[] | select(.conclusion != "skipped") | .name | select(contains("https://") or . == "CodeQL" or . == "Dependabot" or . == "check-expected-release-files" or contains("Update") or contains("ESLint") or contains("update") or contains("test-setup-python-scripts") or . == "Agent" or . == "Cleanup artifacts" or . == "Prepare" or . == "Upload results" or . == "Label PR with size" | not)] | unique | sort')"
|
||||
|
||||
echo "$CHECKS" | jq
|
||||
|
||||
# Fail if there are no checks
|
||||
if [ -z "$CHECKS" ] || [ "$(echo "$CHECKS" | jq '. | length')" -eq 0 ]; then
|
||||
echo "No checks found for $GITHUB_SHA"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "{\"contexts\": ${CHECKS}}" > checks.json
|
||||
|
||||
echo "Updating main"
|
||||
gh api --silent -X "PATCH" "repos/github/codeql-action/branches/main/protection/required_status_checks" --input checks.json
|
||||
|
||||
# list all branchs on origin remote matching releases/v*
|
||||
BRANCHES="$(git ls-remote --heads origin 'releases/v*' | sed 's?.*refs/heads/??' | sort -V)"
|
||||
|
||||
for BRANCH in $BRANCHES; do
|
||||
|
||||
# strip exact 'releases/v' prefix from $BRANCH using count of characters
|
||||
VERSION="${BRANCH:10}"
|
||||
|
||||
if [ "$VERSION" -lt "$OLDEST_SUPPORTED_MAJOR_VERSION" ]; then
|
||||
echo "Skipping $BRANCH"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Updating $BRANCH"
|
||||
gh api --silent -X "PATCH" "repos/github/codeql-action/branches/$BRANCH/protection/required_status_checks" --input checks.json
|
||||
done
|
||||
|
||||
rm checks.json
|
||||
@@ -20,7 +20,7 @@ defaults:
|
||||
jobs:
|
||||
update-bundle:
|
||||
if: github.event.release.prerelease && startsWith(github.event.release.tag_name, 'codeql-bundle-')
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # needed to push commits
|
||||
pull-requests: write # needed to create pull requests
|
||||
@@ -57,6 +57,17 @@ jobs:
|
||||
- name: Update bundle
|
||||
uses: ./.github/actions/update-bundle
|
||||
|
||||
- name: Set up CodeQL CLI from new bundle
|
||||
id: setup-codeql
|
||||
uses: ./setup-codeql
|
||||
with:
|
||||
tools: https://github.com/github/codeql-action/releases/download/${{ github.event.release.tag_name }}/codeql-bundle-linux64.tar.gz
|
||||
|
||||
- name: Update built-in languages
|
||||
run: npx tsx pr-checks/update-builtin-languages.ts "$CODEQL_PATH"
|
||||
env:
|
||||
CODEQL_PATH: ${{ steps.setup-codeql.outputs.codeql-path }}
|
||||
|
||||
- name: Bump Action minor version if new CodeQL minor version series
|
||||
id: bump-action-version
|
||||
run: |
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
update:
|
||||
timeout-minutes: 45
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
needs: [prepare]
|
||||
env:
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
|
||||
backport:
|
||||
timeout-minutes: 45
|
||||
runs-on: ubuntu-slim
|
||||
runs-on: ubuntu-latest
|
||||
environment: Automation
|
||||
needs: [prepare]
|
||||
if: ${{ (github.event_name == 'push') && needs.prepare.outputs.backport_target_branches != '[]' }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
pull-requests: write # needed to create pull request
|
||||
steps:
|
||||
- name: Generate token
|
||||
uses: actions/create-github-app-token@v3.0.0
|
||||
uses: actions/create-github-app-token@v3.1.1
|
||||
id: app-token
|
||||
with:
|
||||
app-id: ${{ vars.AUTOMATION_APP_ID }}
|
||||
|
||||
@@ -11,3 +11,5 @@ build/
|
||||
eslint.sarif
|
||||
# for local incremental compilation
|
||||
tsconfig.tsbuildinfo
|
||||
# esbuild metadata file
|
||||
meta.json
|
||||
|
||||
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
// Place your codeql-action workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and
|
||||
// description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope
|
||||
// is left empty or omitted, the snippet gets applied to all languages. The prefix is what is
|
||||
// used to trigger the snippet and the body will be expanded and inserted. Possible variables are:
|
||||
// $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders.
|
||||
// Placeholders with the same ids are connected.
|
||||
// Example:
|
||||
// "Print to console": {
|
||||
// "scope": "javascript,typescript",
|
||||
// "prefix": "log",
|
||||
// "body": [
|
||||
// "console.log('$1');",
|
||||
// "$2"
|
||||
// ],
|
||||
// "description": "Log output to console"
|
||||
// }
|
||||
"Test Macro": {
|
||||
"scope": "javascript, typescript",
|
||||
"prefix": "testMacro",
|
||||
"body": [
|
||||
"const ${1:nameMacro} = test.macro({",
|
||||
" exec: async (t: ExecutionContext<unknown>) => {},",
|
||||
"",
|
||||
" title: (providedTitle = \"\") => `${2:common title} - \\${providedTitle}`,",
|
||||
"});",
|
||||
],
|
||||
"description": "An Ava test macro",
|
||||
},
|
||||
}
|
||||
@@ -2,6 +2,39 @@
|
||||
|
||||
See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
|
||||
|
||||
## [UNRELEASED]
|
||||
|
||||
No user facing changes.
|
||||
|
||||
## 4.35.3 - 01 May 2026
|
||||
|
||||
- _Upcoming breaking change_: Add a deprecation warning for customers using CodeQL version 2.19.3 and earlier. These versions of CodeQL were discontinued on 9 April 2026 alongside GitHub Enterprise Server 3.15, and will be unsupported by the next minor release of the CodeQL Action. [#3837](https://github.com/github/codeql-action/pull/3837)
|
||||
- Configurations for private registries that use Cloudsmith or GCP OIDC are now accepted. [#3850](https://github.com/github/codeql-action/pull/3850)
|
||||
- Best-effort connection tests for private registries now use `GET` requests instead of `HEAD` for better compatibility with various registry implementations. For NuGet feeds, the test is now always performed against the service index. [#3853](https://github.com/github/codeql-action/pull/3853)
|
||||
- Fixed a bug where two diagnostics produced within the same millisecond could overwrite each other on disk, causing one of them to be lost. [#3852](https://github.com/github/codeql-action/pull/3852)
|
||||
- Update default CodeQL bundle version to [2.25.3](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.25.3). [#3865](https://github.com/github/codeql-action/pull/3865)
|
||||
|
||||
## 4.35.2 - 15 Apr 2026
|
||||
|
||||
- The undocumented TRAP cache cleanup feature that could be enabled using the `CODEQL_ACTION_CLEANUP_TRAP_CACHES` environment variable is deprecated and will be removed in May 2026. If you are affected by this, we recommend disabling TRAP caching by passing the `trap-caching: false` input to the `init` Action. [#3795](https://github.com/github/codeql-action/pull/3795)
|
||||
- The Git version 2.36.0 requirement for improved incremental analysis now only applies to repositories that contain submodules. [#3789](https://github.com/github/codeql-action/pull/3789)
|
||||
- Python analysis on GHES no longer extracts the standard library, relying instead on models of the standard library. This should result in significantly faster extraction and analysis times, while the effect on alerts should be minimal. [#3794](https://github.com/github/codeql-action/pull/3794)
|
||||
- Fixed a bug in the validation of OIDC configurations for private registries that was added in CodeQL Action 4.33.0 / 3.33.0. [#3807](https://github.com/github/codeql-action/pull/3807)
|
||||
- Update default CodeQL bundle version to [2.25.2](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.25.2). [#3823](https://github.com/github/codeql-action/pull/3823)
|
||||
|
||||
## 4.35.1 - 27 Mar 2026
|
||||
|
||||
- Fix incorrect minimum required Git version for [improved incremental analysis](https://github.com/github/roadmap/issues/1158): it should have been 2.36.0, not 2.11.0. [#3781](https://github.com/github/codeql-action/pull/3781)
|
||||
|
||||
## 4.35.0 - 27 Mar 2026
|
||||
|
||||
- Reduced the minimum Git version required for [improved incremental analysis](https://github.com/github/roadmap/issues/1158) from 2.38.0 to 2.11.0. [#3767](https://github.com/github/codeql-action/pull/3767)
|
||||
- Update default CodeQL bundle version to [2.25.1](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.25.1). [#3773](https://github.com/github/codeql-action/pull/3773)
|
||||
|
||||
## 4.34.1 - 20 Mar 2026
|
||||
|
||||
- Downgrade default CodeQL bundle version to [2.24.3](https://github.com/github/codeql-action/releases/tag/codeql-bundle-v2.24.3) due to issues with a small percentage of Actions and JavaScript analyses. [#3762](https://github.com/github/codeql-action/pull/3762)
|
||||
|
||||
## 4.34.0 - 20 Mar 2026
|
||||
|
||||
- Added an experimental change which disables TRAP caching when [improved incremental analysis](https://github.com/github/roadmap/issues/1158) is enabled, since improved incremental analysis supersedes TRAP caching. This will improve performance and reduce Actions cache usage. We expect to roll this change out to everyone in March. [#3569](https://github.com/github/codeql-action/pull/3569)
|
||||
|
||||
+7
-5
@@ -69,12 +69,14 @@ Once the mergeback and backport pull request have been merged, the release is co
|
||||
|
||||
## Keeping the PR checks up to date (admin access required)
|
||||
|
||||
Since the `codeql-action` runs most of its testing through individual Actions workflows, there are over two hundred required jobs that need to pass in order for a PR to turn green. It would be too tedious to maintain that list manually. You can regenerate the set of required checks automatically by running the [update-required-checks.sh](.github/workflows/script/update-required-checks.sh) script:
|
||||
Since the `codeql-action` runs most of its testing through individual Actions workflows, there are over two hundred required jobs that need to pass in order for a PR to turn green. It would be too tedious to maintain that list manually. You can regenerate the set of required checks automatically by running the [sync-checks.ts](pr-checks/sync-checks.ts) script:
|
||||
|
||||
- If you run the script without an argument, it will retrieve the set of workflows that ran for the latest commit on `main`. Make sure that your local `main` branch is up to date before running the script.
|
||||
- You can specify a commit SHA as argument to retrieve the set of workflows for that commit instead. You will likely want to use this if you have a PR that removes or adds PR checks.
|
||||
- At a minimum, you must provide an argument for the `--token` input. For example, `--token "$(gh auth token)"` to use the same token that `gh` uses. If no token is provided or the token has insufficient permissions, the script will fail.
|
||||
- By default, the script performs a dry run and outputs information about the changes it would make to the branch protection rules. To actually apply the changes, specify the `--apply` flag.
|
||||
- If you run the script without any other arguments, it will retrieve the set of workflows that ran for the latest commit on `main`.
|
||||
- You can specify a different git ref with the `--ref` input. You will likely want to use this if you have a PR that removes or adds PR checks. For example, `--ref "some/branch/name"` to use the HEAD of the `some/branch/name` branch.
|
||||
|
||||
After running, go to the [branch protection rules settings page](https://github.com/github/codeql-action/settings/branches) and validate that the rules for `main`, `v3`, and any other currently supported major versions have been updated.
|
||||
After running, go to the [branch protection rules settings page](https://github.com/github/codeql-action/settings/branches) and validate that the rules for `main`, `v4`, and any other currently supported major versions have been updated.
|
||||
|
||||
Note that any updates to checks on `main` need to be backported to all currently supported major version branches, in order to maintain the same set of names for required checks.
|
||||
|
||||
@@ -122,7 +124,7 @@ To deprecate an older version of the Action:
|
||||
- Implement an Actions warning for customers using the deprecated version.
|
||||
1. Wait for the deprecation period to pass.
|
||||
1. Upgrade the Actions warning for customers using the deprecated version to a non-fatal error, and mention that this version of the Action is no longer supported.
|
||||
1. Make a PR to bump the `OLDEST_SUPPORTED_MAJOR_VERSION` in [releases.ini](.github/releases.ini). Once this PR is merged, the release process will no longer backport changes to the deprecated release version.
|
||||
1. Make a PR to bump the `OLDEST_SUPPORTED_MAJOR_VERSION` in [config.ts](pr-checks/config.ts). Once this PR is merged, the release process will no longer backport changes to the deprecated release version.
|
||||
|
||||
## Resources
|
||||
|
||||
|
||||
@@ -72,6 +72,7 @@ We typically release new minor versions of the CodeQL Action and Bundle when a n
|
||||
|
||||
| Minimum CodeQL Action | Minimum CodeQL Bundle Version | GitHub Environment | Notes |
|
||||
|-----------------------|-------------------------------|--------------------|-------|
|
||||
| `v4.33.0` | `2.24.3` | Enterprise Server 3.21 | |
|
||||
| `v4.31.10` | `2.23.9` | Enterprise Server 3.20 | |
|
||||
| `v3.29.11` | `2.22.4` | Enterprise Server 3.19 | |
|
||||
| `v3.28.21` | `2.21.3` | Enterprise Server 3.18 | |
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { copyFile, rm } from "node:fs/promises";
|
||||
import { copyFile, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
@@ -64,7 +64,11 @@ const onEndPlugin = {
|
||||
|
||||
const context = await esbuild.context({
|
||||
// Include upload-lib.ts as an entry point for use in testing environments.
|
||||
entryPoints: globSync([`${SRC_DIR}/*-action.ts`, `${SRC_DIR}/*-action-post.ts`, "src/upload-lib.ts"]),
|
||||
entryPoints: globSync([
|
||||
`${SRC_DIR}/*-action.ts`,
|
||||
`${SRC_DIR}/*-action-post.ts`,
|
||||
"src/upload-lib.ts",
|
||||
]),
|
||||
bundle: true,
|
||||
format: "cjs",
|
||||
outdir: OUT_DIR,
|
||||
@@ -74,7 +78,10 @@ const context = await esbuild.context({
|
||||
define: {
|
||||
__CODEQL_ACTION_VERSION__: JSON.stringify(pkg.version),
|
||||
},
|
||||
metafile: true,
|
||||
});
|
||||
|
||||
await context.rebuild();
|
||||
const result = await context.rebuild();
|
||||
await writeFile(join(__dirname, "meta.json"), JSON.stringify(result.metafile));
|
||||
|
||||
await context.dispose();
|
||||
|
||||
+21
-6
@@ -7,7 +7,11 @@ import noAsyncForeach from "eslint-plugin-no-async-foreach";
|
||||
import jsdoc from "eslint-plugin-jsdoc";
|
||||
import tseslint from "typescript-eslint";
|
||||
import globals from "globals";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const githubFlatConfigs = github.getFlatConfigs();
|
||||
|
||||
export default [
|
||||
@@ -43,7 +47,7 @@ export default [
|
||||
plugins: {
|
||||
"import-x": importX,
|
||||
"no-async-foreach": fixupPluginRules(noAsyncForeach),
|
||||
"jsdoc": jsdoc,
|
||||
jsdoc: jsdoc,
|
||||
},
|
||||
|
||||
languageOptions: {
|
||||
@@ -67,7 +71,13 @@ export default [
|
||||
|
||||
typescript: {},
|
||||
},
|
||||
"import/ignore": ["sinon", "uuid", "@octokit/plugin-retry", "del", "get-folder-size"],
|
||||
"import/ignore": [
|
||||
"sinon",
|
||||
"uuid",
|
||||
"@octokit/plugin-retry",
|
||||
"del",
|
||||
"get-folder-size",
|
||||
],
|
||||
"import-x/resolver-next": [
|
||||
createTypeScriptImportResolver(),
|
||||
createNodeResolver({
|
||||
@@ -143,7 +153,7 @@ export default [
|
||||
// We don't currently require full JSDoc coverage, so this rule
|
||||
// should not error on missing @param annotations.
|
||||
disableMissingParamChecks: true,
|
||||
}
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
@@ -162,9 +172,9 @@ export default [
|
||||
"@typescript-eslint/no-unused-vars": [
|
||||
"error",
|
||||
{
|
||||
"args": "all",
|
||||
"argsIgnorePattern": "^_",
|
||||
}
|
||||
args: "all",
|
||||
argsIgnorePattern: "^_",
|
||||
},
|
||||
],
|
||||
"func-style": "off",
|
||||
},
|
||||
@@ -183,6 +193,11 @@ export default [
|
||||
// The scripts in `pr-checks` are expected to output to the console.
|
||||
"no-console": "off",
|
||||
|
||||
"import/no-extraneous-dependencies": [
|
||||
"error",
|
||||
{ packageDir: [__dirname, path.resolve(__dirname, "pr-checks")] },
|
||||
],
|
||||
|
||||
"@typescript-eslint/no-floating-promises": [
|
||||
"error",
|
||||
{
|
||||
|
||||
Generated
+2495
-2105
File diff suppressed because it is too large
Load Diff
Generated
+1435
-1197
File diff suppressed because it is too large
Load Diff
Generated
+1762
-1368
File diff suppressed because it is too large
Load Diff
+4
-4
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"bundleVersion": "codeql-bundle-v2.25.0",
|
||||
"cliVersion": "2.25.0",
|
||||
"priorBundleVersion": "codeql-bundle-v2.24.3",
|
||||
"priorCliVersion": "2.24.3"
|
||||
"bundleVersion": "codeql-bundle-v2.25.3",
|
||||
"cliVersion": "2.25.3",
|
||||
"priorBundleVersion": "codeql-bundle-v2.25.2",
|
||||
"priorCliVersion": "2.25.2"
|
||||
}
|
||||
|
||||
Generated
+1521
-1127
File diff suppressed because it is too large
Load Diff
Generated
+1866
-1337
File diff suppressed because it is too large
Load Diff
Generated
+1747
-1357
File diff suppressed because it is too large
Load Diff
Generated
+2781
-2390
File diff suppressed because it is too large
Load Diff
Generated
+1053
-743
File diff suppressed because it is too large
Load Diff
Generated
+3617
-3199
File diff suppressed because it is too large
Load Diff
Generated
+1261
-873
File diff suppressed because it is too large
Load Diff
Generated
+1053
-743
File diff suppressed because it is too large
Load Diff
Generated
+2591
-2199
File diff suppressed because it is too large
Load Diff
Generated
+919
-424
File diff suppressed because it is too large
Load Diff
+20
-21
@@ -1,18 +1,18 @@
|
||||
{
|
||||
"name": "codeql",
|
||||
"version": "4.34.0",
|
||||
"version": "4.35.4",
|
||||
"private": true,
|
||||
"description": "CodeQL action",
|
||||
"scripts": {
|
||||
"_build_comment": "echo 'Run the full build so we typecheck the project and can reuse the transpiled files in npm test'",
|
||||
"build": "./scripts/check-node-modules.sh && npm run transpile && node build.mjs",
|
||||
"build": "./scripts/check-node-modules.sh && npm run transpile && node build.mjs && npx tsx ./pr-checks/bundle-metadata.ts",
|
||||
"lint": "eslint --report-unused-disable-directives --max-warnings=0 .",
|
||||
"lint-ci": "SARIF_ESLINT_IGNORE_SUPPRESSED=true eslint --report-unused-disable-directives --max-warnings=0 . --format @microsoft/eslint-formatter-sarif --output-file=eslint.sarif",
|
||||
"lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --fix",
|
||||
"ava": "npm run transpile && ava --verbose",
|
||||
"test": "npm run ava -- src/",
|
||||
"test-debug": "npm run test -- --timeout=20m",
|
||||
"transpile": "tsc --build --verbose"
|
||||
"transpile": "tsc --build --verbose tsconfig.json"
|
||||
},
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
@@ -29,23 +29,22 @@
|
||||
"@actions/http-client": "^3.0.0",
|
||||
"@actions/io": "^2.0.0",
|
||||
"@actions/tool-cache": "^3.0.1",
|
||||
"@octokit/plugin-retry": "^8.0.0",
|
||||
"@schemastore/package": "0.0.10",
|
||||
"@octokit/plugin-retry": "^8.1.0",
|
||||
"archiver": "^7.0.1",
|
||||
"fast-deep-equal": "^3.1.3",
|
||||
"follow-redirects": "^1.15.11",
|
||||
"follow-redirects": "^1.16.0",
|
||||
"get-folder-size": "^5.0.0",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"js-yaml": "^4.1.1",
|
||||
"jsonschema": "1.4.1",
|
||||
"jsonschema": "1.5.0",
|
||||
"long": "^5.3.2",
|
||||
"node-forge": "^1.3.3",
|
||||
"node-forge": "^1.4.0",
|
||||
"semver": "^7.7.4",
|
||||
"uuid": "^13.0.0"
|
||||
"uuid": "^14.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ava/typescript": "6.0.0",
|
||||
"@eslint/compat": "^2.0.3",
|
||||
"@ava/typescript": "7.0.0",
|
||||
"@eslint/compat": "^2.0.5",
|
||||
"@microsoft/eslint-formatter-sarif": "^3.1.0",
|
||||
"@octokit/types": "^16.0.0",
|
||||
"@types/archiver": "^7.0.0",
|
||||
@@ -55,21 +54,21 @@
|
||||
"@types/node-forge": "^1.3.14",
|
||||
"@types/sarif": "^2.1.7",
|
||||
"@types/semver": "^7.7.1",
|
||||
"@types/sinon": "^21.0.0",
|
||||
"@types/sinon": "^21.0.1",
|
||||
"ava": "^7.0.0",
|
||||
"esbuild": "^0.27.3",
|
||||
"esbuild": "^0.28.0",
|
||||
"eslint": "^9.39.2",
|
||||
"eslint-import-resolver-typescript": "^3.8.7",
|
||||
"eslint-import-resolver-typescript": "^4.4.4",
|
||||
"eslint-plugin-github": "^6.0.0",
|
||||
"eslint-plugin-import-x": "^4.16.1",
|
||||
"eslint-plugin-jsdoc": "^62.7.1",
|
||||
"eslint-plugin-import-x": "^4.16.2",
|
||||
"eslint-plugin-jsdoc": "^62.9.0",
|
||||
"eslint-plugin-no-async-foreach": "^0.1.1",
|
||||
"glob": "^11.1.0",
|
||||
"globals": "^17.4.0",
|
||||
"nock": "^14.0.11",
|
||||
"sinon": "^21.0.2",
|
||||
"typescript": "^5.9.3",
|
||||
"typescript-eslint": "^8.57.0"
|
||||
"globals": "^17.5.0",
|
||||
"nock": "^14.0.12",
|
||||
"sinon": "^21.1.2",
|
||||
"typescript": "^6.0.2",
|
||||
"typescript-eslint": "^8.58.2"
|
||||
},
|
||||
"overrides": {
|
||||
"@actions/tool-cache": {
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import * as githubUtils from "@actions/github/lib/utils";
|
||||
import { type Octokit } from "@octokit/core";
|
||||
import { type PaginateInterface } from "@octokit/plugin-paginate-rest";
|
||||
import { type Api } from "@octokit/plugin-rest-endpoint-methods";
|
||||
|
||||
/** The type of the Octokit client. */
|
||||
export type ApiClient = Octokit & Api & { paginate: PaginateInterface };
|
||||
|
||||
/** Constructs an `ApiClient` using `token` for authentication. */
|
||||
export function getApiClient(token: string): ApiClient {
|
||||
const opts = githubUtils.getOctokitOptions(token);
|
||||
return new githubUtils.GitHub(opts);
|
||||
}
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
import * as fs from "node:fs/promises";
|
||||
|
||||
import { BUNDLE_METADATA_FILE } from "./config";
|
||||
|
||||
interface InputInfo {
|
||||
bytesInOutput: number;
|
||||
}
|
||||
|
||||
type Inputs = Record<string, InputInfo>;
|
||||
|
||||
interface Output {
|
||||
bytes: number;
|
||||
inputs: Inputs;
|
||||
}
|
||||
|
||||
interface Metadata {
|
||||
outputs: Record<string, Output>;
|
||||
}
|
||||
|
||||
function toMB(bytes: number): string {
|
||||
return `${(bytes / (1024 * 1024)).toFixed(2)}MB`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const fileContents = await fs.readFile(BUNDLE_METADATA_FILE);
|
||||
const metadata = JSON.parse(String(fileContents)) as Metadata;
|
||||
|
||||
for (const [outputFile, outputData] of Object.entries(
|
||||
metadata.outputs,
|
||||
).reverse()) {
|
||||
console.info(`${outputFile}: ${toMB(outputData.bytes)}`);
|
||||
|
||||
for (const [inputName, inputData] of Object.entries(outputData.inputs)) {
|
||||
// Ignore any inputs that make up less than 5% of the output.
|
||||
const percentage = (inputData.bytesInOutput / outputData.bytes) * 100.0;
|
||||
if (percentage < 5.0) continue;
|
||||
|
||||
console.info(` ${inputName}: ${toMB(inputData.bytesInOutput)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only call `main` if this script was run directly.
|
||||
if (require.main === module) {
|
||||
void main();
|
||||
}
|
||||
@@ -5,12 +5,12 @@ versions:
|
||||
steps:
|
||||
- uses: ./../action/init
|
||||
with:
|
||||
languages: C#,java-kotlin,swift,typescript
|
||||
languages: C#,java-kotlin,typescript
|
||||
tools: ${{ steps.prepare-test.outputs.tools-url }}
|
||||
|
||||
- name: "Check languages"
|
||||
run: |
|
||||
expected_languages="csharp,java,swift,javascript"
|
||||
expected_languages="csharp,java,javascript"
|
||||
actual_languages=$(jq -r '.languages | join(",")' "$RUNNER_TEMP"/config)
|
||||
|
||||
if [ "$expected_languages" != "$actual_languages" ]; then
|
||||
|
||||
@@ -5,7 +5,7 @@ versions:
|
||||
- default
|
||||
steps:
|
||||
- name: Set up Ruby
|
||||
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
||||
uses: ruby/setup-ruby@0cb964fd540e0a24c900370abf38a33466142735 # v1.305.0
|
||||
with:
|
||||
ruby-version: 2.6
|
||||
- name: Install Code Scanning integration
|
||||
|
||||
@@ -16,7 +16,17 @@ steps:
|
||||
id: proxy
|
||||
uses: ./../action/start-proxy
|
||||
with:
|
||||
registry_secrets: '[{ "type": "nuget_feed", "url": "https://api.nuget.org/v3/index.json" }]'
|
||||
registry_secrets: |
|
||||
[
|
||||
{
|
||||
"type": "maven_repository",
|
||||
"url": "https://repo.maven.apache.org/maven2/"
|
||||
},
|
||||
{
|
||||
"type": "maven_repository",
|
||||
"url": "https://repo1.maven.org/maven2"
|
||||
}
|
||||
]
|
||||
|
||||
- name: Print proxy outputs
|
||||
run: |
|
||||
@@ -27,3 +37,10 @@ steps:
|
||||
- name: Fail if proxy outputs are not set
|
||||
if: (!steps.proxy.outputs.proxy_host) || (!steps.proxy.outputs.proxy_port) || (!steps.proxy.outputs.proxy_ca_certificate) || (!steps.proxy.outputs.proxy_urls)
|
||||
run: exit 1
|
||||
|
||||
- name: Fail if proxy_urls does not contain all registries
|
||||
if: |
|
||||
join(fromJSON(steps.proxy.outputs.proxy_urls)[*].type, ',') != 'maven_repository,maven_repository'
|
||||
|| !contains(steps.proxy.outputs.proxy_urls, 'https://repo.maven.apache.org/maven2/')
|
||||
|| !contains(steps.proxy.outputs.proxy_urls, 'https://repo1.maven.org/maven2')
|
||||
run: exit 1
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import path from "path";
|
||||
|
||||
/** The oldest supported major version of the CodeQL Action. */
|
||||
export const OLDEST_SUPPORTED_MAJOR_VERSION = 3;
|
||||
|
||||
/** The `pr-checks` directory. */
|
||||
export const PR_CHECKS_DIR = __dirname;
|
||||
|
||||
/** The path of the file configuring which checks shouldn't be required. */
|
||||
export const PR_CHECK_EXCLUDED_FILE = path.join(PR_CHECKS_DIR, "excluded.yml");
|
||||
|
||||
/** The path to the esbuild metadata file. */
|
||||
export const BUNDLE_METADATA_FILE = path.join(PR_CHECKS_DIR, "..", "meta.json");
|
||||
|
||||
/** The `src` directory. */
|
||||
const SOURCE_ROOT = path.join(PR_CHECKS_DIR, "..", "src");
|
||||
|
||||
/** The path to the built-in languages file. */
|
||||
export const BUILTIN_LANGUAGES_FILE = path.join(
|
||||
SOURCE_ROOT,
|
||||
"languages",
|
||||
"builtin.json",
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
# PR checks to exclude from required checks
|
||||
contains:
|
||||
- "https://"
|
||||
- "Update"
|
||||
- "ESLint"
|
||||
- "update"
|
||||
- "test-setup-python-scripts"
|
||||
is:
|
||||
- "CodeQL"
|
||||
- "Dependabot"
|
||||
- "check-expected-release-files"
|
||||
- "Agent"
|
||||
- "Cleanup artifacts"
|
||||
- "Prepare"
|
||||
- "Upload results"
|
||||
- "Label PR with size"
|
||||
@@ -2,11 +2,15 @@
|
||||
"private": true,
|
||||
"description": "Dependencies for the sync.ts",
|
||||
"dependencies": {
|
||||
"yaml": "^2.8.2"
|
||||
"@actions/core": "^2.0.3",
|
||||
"@actions/github": "^8.0.1",
|
||||
"@octokit/core": "^7.0.6",
|
||||
"@octokit/plugin-paginate-rest": ">=9.2.2",
|
||||
"@octokit/plugin-rest-endpoint-methods": "^17.0.0",
|
||||
"yaml": "^2.8.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.19.9",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5.9.3"
|
||||
"tsx": "^4.21.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/*
|
||||
Tests for the release-branches.ts script
|
||||
*/
|
||||
|
||||
import * as assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
|
||||
import { computeBackportBranches } from "./release-branches";
|
||||
|
||||
describe("computeBackportBranches", async () => {
|
||||
await it("rejects invalid major versions", () => {
|
||||
// The majorVersion is expected to be in vN format.
|
||||
assert.throws(() => computeBackportBranches("3", "v4.28.0", 3));
|
||||
assert.throws(() => computeBackportBranches("v3.1", "v4.28.0", 3));
|
||||
});
|
||||
|
||||
await it("rejects invalid latest tags", () => {
|
||||
// The latestTag is expected to be in vN.M.P format.
|
||||
assert.throws(() => computeBackportBranches("v3", "v4", 3));
|
||||
assert.throws(() => computeBackportBranches("v3", "4", 3));
|
||||
assert.throws(() => computeBackportBranches("v3", "v4.28", 3));
|
||||
assert.throws(() => computeBackportBranches("v3", "4.28", 3));
|
||||
assert.throws(() => computeBackportBranches("v3", "4.28.0", 3));
|
||||
});
|
||||
|
||||
await it("sets backport source branch based on major version", () => {
|
||||
// Test that the backport source branch is releases/v{majorVersion}
|
||||
const result = computeBackportBranches("v3", "v4.28.0", 3);
|
||||
assert.equal(result.backportSourceBranch, "releases/v3");
|
||||
});
|
||||
|
||||
await it("no backport targets when major version is the oldest supported", () => {
|
||||
// When majorVersion equals the major version of latestTag and we do not support older major versions,
|
||||
// then there are no older supported branches to backport to.
|
||||
const result = computeBackportBranches("v3", "v3.28.0", 3);
|
||||
assert.deepEqual(result.backportTargetBranches, []);
|
||||
});
|
||||
|
||||
await it("backports to older supported major versions", () => {
|
||||
const result = computeBackportBranches("v4", "v4.1.0", 3);
|
||||
assert.equal(result.backportSourceBranch, "releases/v4");
|
||||
assert.deepEqual(result.backportTargetBranches, ["releases/v3"]);
|
||||
});
|
||||
|
||||
await it("backports to multiple older supported branches", () => {
|
||||
const result = computeBackportBranches("v5", "v5.0.0", 3);
|
||||
assert.equal(result.backportSourceBranch, "releases/v5");
|
||||
assert.deepEqual(result.backportTargetBranches, [
|
||||
"releases/v4",
|
||||
"releases/v3",
|
||||
]);
|
||||
});
|
||||
|
||||
await it("does not backport when major version is older than latest tag", () => {
|
||||
const result = computeBackportBranches("v2", "v3.28.0", 2);
|
||||
assert.equal(result.backportSourceBranch, "releases/v2");
|
||||
assert.deepEqual(result.backportTargetBranches, []);
|
||||
});
|
||||
});
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import * as core from "@actions/core";
|
||||
|
||||
import { OLDEST_SUPPORTED_MAJOR_VERSION } from "./config";
|
||||
|
||||
/** The results of checking which release branches to backport to. */
|
||||
export interface BackportInfo {
|
||||
/** The source release branch. */
|
||||
backportSourceBranch: string;
|
||||
/**
|
||||
* The computed release branches we should backport to.
|
||||
* Will be empty if there are no branches we need to backport to.
|
||||
*/
|
||||
backportTargetBranches: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the backport source and target branches for a release.
|
||||
*
|
||||
* @param majorVersion - The major version string (e.g. "v4").
|
||||
* @param latestTag - The most recent tag published to the repository (e.g. "v4.32.6").
|
||||
* @param oldestSupportedMajorVersion - The oldest supported major version number.
|
||||
* @returns The names of the source branch and target branches.
|
||||
*/
|
||||
export function computeBackportBranches(
|
||||
majorVersion: string,
|
||||
latestTag: string,
|
||||
oldestSupportedMajorVersion: number,
|
||||
): BackportInfo {
|
||||
// Perform some sanity checks on the inputs.
|
||||
// For `majorVersion`, we expect exactly `vN` for some `N`.
|
||||
const majorVersionMatch = majorVersion.match(/^v(\d+)$/);
|
||||
if (!majorVersionMatch) {
|
||||
throw new Error("--major-version value must be in `vN` format.");
|
||||
}
|
||||
|
||||
// For latestTag, we expect something starting with `vN.M.P`
|
||||
const latestTagMatch = latestTag.match(/^v(\d+)\.\d+\.\d+/);
|
||||
if (!latestTagMatch) {
|
||||
throw new Error(
|
||||
`--latest-tag value must be in 'vN.M.P' format, but '${latestTag}' is not.`,
|
||||
);
|
||||
}
|
||||
|
||||
const majorVersionNumber = Number.parseInt(majorVersionMatch[1]);
|
||||
const latestTagMajor = Number.parseInt(latestTagMatch[1]);
|
||||
|
||||
// If this is a primary release, we backport to all supported branches,
|
||||
// so we check whether the majorVersion taken from the package.json
|
||||
// is greater than or equal to the latest tag pulled from the repo.
|
||||
// For example...
|
||||
// 'v1' >= 'v2' is False # we're operating from an older release branch and should not backport
|
||||
// 'v2' >= 'v2' is True # the normal case where we're updating the current version
|
||||
// 'v3' >= 'v2' is True # in this case we are making the first release of a new major version
|
||||
const considerBackports = majorVersionNumber >= latestTagMajor;
|
||||
|
||||
const backportSourceBranch = `releases/v${majorVersionNumber}`;
|
||||
const backportTargetBranches: string[] = [];
|
||||
|
||||
if (considerBackports) {
|
||||
for (let i = majorVersionNumber - 1; i > 0; i--) {
|
||||
const branchName = `releases/v${i}`;
|
||||
if (i >= oldestSupportedMajorVersion) {
|
||||
backportTargetBranches.push(branchName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { backportSourceBranch, backportTargetBranches };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { values: options } = parseArgs({
|
||||
options: {
|
||||
// The major version of the release in `vN` format (e.g. `v4`).
|
||||
"major-version": {
|
||||
type: "string",
|
||||
},
|
||||
// The most recent tag published to the repository (e.g. `v4.28.0`).
|
||||
"latest-tag": {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
strict: true,
|
||||
});
|
||||
|
||||
if (options["major-version"] === undefined) {
|
||||
throw Error("--major-version is required");
|
||||
}
|
||||
if (options["latest-tag"] === undefined) {
|
||||
throw Error("--latest-tag is required");
|
||||
}
|
||||
|
||||
const majorVersion = options["major-version"];
|
||||
const latestTag = options["latest-tag"];
|
||||
|
||||
console.log(`Major version: ${majorVersion}`);
|
||||
console.log(`Latest tag: ${latestTag}`);
|
||||
|
||||
const result = computeBackportBranches(
|
||||
majorVersion,
|
||||
latestTag,
|
||||
OLDEST_SUPPORTED_MAJOR_VERSION,
|
||||
);
|
||||
|
||||
core.setOutput("backport_source_branch", result.backportSourceBranch);
|
||||
core.setOutput(
|
||||
"backport_target_branches",
|
||||
JSON.stringify(result.backportTargetBranches),
|
||||
);
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Only call `main` if this script was run directly.
|
||||
if (require.main === module) {
|
||||
void main();
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/*
|
||||
Tests for the sync-checks.ts script
|
||||
*/
|
||||
|
||||
import * as assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
|
||||
import { CheckInfo, Exclusions, Options, removeExcluded } from "./sync-checks";
|
||||
|
||||
const defaultOptions: Options = {
|
||||
apply: false,
|
||||
verbose: false,
|
||||
};
|
||||
|
||||
const toCheckInfo = (name: string) =>
|
||||
({ context: name, app_id: -1 }) satisfies CheckInfo;
|
||||
|
||||
const expectedPartialMatches = ["PR Check - Foo", "https://example.com"].map(
|
||||
toCheckInfo,
|
||||
);
|
||||
|
||||
const expectedExactMatches = ["CodeQL", "Update"].map(toCheckInfo);
|
||||
|
||||
const testChecks = expectedExactMatches.concat(expectedPartialMatches);
|
||||
|
||||
const emptyExclusions: Exclusions = {
|
||||
is: [],
|
||||
contains: [],
|
||||
};
|
||||
|
||||
describe("removeExcluded", async () => {
|
||||
await it("retains all checks if no exclusions are configured", () => {
|
||||
const retained = removeExcluded(
|
||||
defaultOptions,
|
||||
emptyExclusions,
|
||||
testChecks,
|
||||
);
|
||||
assert.deepEqual(retained, testChecks);
|
||||
});
|
||||
|
||||
await it("removes exact matches", () => {
|
||||
const retained = removeExcluded(
|
||||
defaultOptions,
|
||||
{ ...emptyExclusions, is: ["CodeQL", "Update"] },
|
||||
testChecks,
|
||||
);
|
||||
assert.deepEqual(retained, expectedPartialMatches);
|
||||
});
|
||||
|
||||
await it("removes partial matches", () => {
|
||||
const retained = removeExcluded(
|
||||
defaultOptions,
|
||||
{ ...emptyExclusions, contains: ["https://", "PR Check"] },
|
||||
testChecks,
|
||||
);
|
||||
assert.deepEqual(retained, expectedExactMatches);
|
||||
});
|
||||
});
|
||||
Executable
+287
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/** Update the required checks based on the current branch. */
|
||||
|
||||
import * as fs from "fs";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import * as yaml from "yaml";
|
||||
|
||||
import { type ApiClient, getApiClient } from "./api-client";
|
||||
import {
|
||||
OLDEST_SUPPORTED_MAJOR_VERSION,
|
||||
PR_CHECK_EXCLUDED_FILE,
|
||||
} from "./config";
|
||||
|
||||
/** Represents the command-line options. */
|
||||
export interface Options {
|
||||
/** The token to use to authenticate to the GitHub API. */
|
||||
token?: string;
|
||||
/** The git ref to use the checks for. */
|
||||
ref?: string;
|
||||
/** Whether to actually apply the changes or not. */
|
||||
apply: boolean;
|
||||
/** Whether to output additional information. */
|
||||
verbose: boolean;
|
||||
}
|
||||
|
||||
/** Identifies the CodeQL Action repository. */
|
||||
const codeqlActionRepo = {
|
||||
owner: "github",
|
||||
repo: "codeql-action",
|
||||
};
|
||||
|
||||
/** Represents a configuration of which checks should not be set up as required checks. */
|
||||
export interface Exclusions {
|
||||
/** A list of strings that, if contained in a check name, are excluded. */
|
||||
contains: string[];
|
||||
/** A list of check names that are excluded if their name is an exact match. */
|
||||
is: string[];
|
||||
}
|
||||
|
||||
/** Loads the configuration for which checks to exclude. */
|
||||
function loadExclusions(): Exclusions {
|
||||
return yaml.parse(
|
||||
fs.readFileSync(PR_CHECK_EXCLUDED_FILE, "utf-8"),
|
||||
) as Exclusions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents information about a check run. We track the `app_id` that generated the check,
|
||||
* because the API will require it in addition to the name in the future.
|
||||
*/
|
||||
export interface CheckInfo {
|
||||
/** The display name of the check. */
|
||||
context: string;
|
||||
/** The ID of the app that generated the check. */
|
||||
app_id: number;
|
||||
}
|
||||
|
||||
/** Removes entries from `checkInfos` based on the configuration. */
|
||||
export function removeExcluded(
|
||||
options: Options,
|
||||
exclusions: Exclusions,
|
||||
checkInfos: CheckInfo[],
|
||||
): CheckInfo[] {
|
||||
if (options.verbose) {
|
||||
console.log(exclusions);
|
||||
}
|
||||
|
||||
return checkInfos.filter((checkInfo) => {
|
||||
if (exclusions.is.includes(checkInfo.context)) {
|
||||
console.info(
|
||||
`Excluding '${checkInfo.context}' because it is an exact exclusion.`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const containsStr of exclusions.contains) {
|
||||
if (checkInfo.context.includes(containsStr)) {
|
||||
console.info(
|
||||
`Excluding '${checkInfo.context}' because it contains '${containsStr}'.`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep.
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
/** Gets a list of check run names for `ref`. */
|
||||
async function getChecksFor(
|
||||
options: Options,
|
||||
client: ApiClient,
|
||||
ref: string,
|
||||
): Promise<CheckInfo[]> {
|
||||
console.info(`Getting checks for '${ref}'`);
|
||||
|
||||
const response = await client.paginate(
|
||||
"GET /repos/{owner}/{repo}/commits/{ref}/check-runs",
|
||||
{
|
||||
...codeqlActionRepo,
|
||||
ref,
|
||||
},
|
||||
);
|
||||
|
||||
if (response.length === 0) {
|
||||
throw new Error(`No checks found for '${ref}'.`);
|
||||
}
|
||||
|
||||
console.info(`Retrieved ${response.length} check runs.`);
|
||||
|
||||
const notSkipped = response.filter(
|
||||
(checkRun) => checkRun.conclusion !== "skipped",
|
||||
);
|
||||
console.info(`Of those: ${notSkipped.length} were not skipped.`);
|
||||
|
||||
// We use the ID of the app that generated the check run when returned by the API,
|
||||
// but default to -1 to tell the API that any check with the given name should be
|
||||
// required.
|
||||
const checkInfos = notSkipped.map((check) => ({
|
||||
context: check.name,
|
||||
app_id: check.app?.id || -1,
|
||||
}));
|
||||
|
||||
// Load the configuration for which checks to exclude and apply it before
|
||||
// returning the checks.
|
||||
const exclusions = loadExclusions();
|
||||
return removeExcluded(options, exclusions, checkInfos);
|
||||
}
|
||||
|
||||
/** Gets the current list of release branches. */
|
||||
async function getReleaseBranches(client: ApiClient): Promise<string[]> {
|
||||
const refs = await client.rest.git.listMatchingRefs({
|
||||
...codeqlActionRepo,
|
||||
ref: "heads/releases/v",
|
||||
});
|
||||
return refs.data.map((ref) => ref.ref).sort();
|
||||
}
|
||||
|
||||
/** Updates the required status checks for `branch` to `checks`. */
|
||||
async function patchBranchProtectionRule(
|
||||
client: ApiClient,
|
||||
branch: string,
|
||||
checks: Set<string>,
|
||||
) {
|
||||
await client.rest.repos.setStatusCheckContexts({
|
||||
...codeqlActionRepo,
|
||||
branch,
|
||||
contexts: Array.from(checks),
|
||||
});
|
||||
}
|
||||
|
||||
/** Sets `checkNames` as required checks for `branch`. */
|
||||
async function updateBranch(
|
||||
options: Options,
|
||||
client: ApiClient,
|
||||
branch: string,
|
||||
checkNames: Set<string>,
|
||||
) {
|
||||
console.info(`Updating '${branch}'...`);
|
||||
|
||||
// Query the current set of required checks for this branch.
|
||||
const currentContexts = await client.rest.repos.getAllStatusCheckContexts({
|
||||
...codeqlActionRepo,
|
||||
branch,
|
||||
});
|
||||
|
||||
// Identify which required checks we will remove and which ones we will add.
|
||||
const currentCheckNames = new Set(currentContexts.data);
|
||||
let additions = 0;
|
||||
let removals = 0;
|
||||
let unchanged = 0;
|
||||
|
||||
for (const currentCheck of currentCheckNames) {
|
||||
if (!checkNames.has(currentCheck)) {
|
||||
console.info(`- Removing '${currentCheck}' for branch '${branch}'`);
|
||||
removals++;
|
||||
} else {
|
||||
unchanged++;
|
||||
}
|
||||
}
|
||||
for (const newCheck of checkNames) {
|
||||
if (!currentCheckNames.has(newCheck)) {
|
||||
console.info(`+ Adding '${newCheck}' for branch '${branch}'`);
|
||||
additions++;
|
||||
}
|
||||
}
|
||||
|
||||
console.info(
|
||||
`For '${branch}': ${removals} removals; ${additions} additions; ${unchanged} unchanged`,
|
||||
);
|
||||
|
||||
// Perform the update if there are changes and `--apply` was specified.
|
||||
if (unchanged === checkNames.size && removals === 0 && additions === 0) {
|
||||
console.info("Not applying changes because there is nothing to do.");
|
||||
} else if (options.apply) {
|
||||
await patchBranchProtectionRule(client, branch, checkNames);
|
||||
} else {
|
||||
console.info("Not applying changes because `--apply` was not specified.");
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { values: options } = parseArgs({
|
||||
options: {
|
||||
// The token to use to authenticate to the API.
|
||||
token: {
|
||||
type: "string",
|
||||
},
|
||||
// The git ref for which to retrieve the check runs.
|
||||
ref: {
|
||||
type: "string",
|
||||
default: "main",
|
||||
},
|
||||
// By default, we perform a dry-run. Setting `apply` to `true` actually applies the changes.
|
||||
apply: {
|
||||
type: "boolean",
|
||||
default: false,
|
||||
},
|
||||
// Whether to output additional information.
|
||||
verbose: {
|
||||
type: "boolean",
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
strict: true,
|
||||
});
|
||||
|
||||
if (options.token === undefined) {
|
||||
throw new Error("Missing --token");
|
||||
}
|
||||
|
||||
console.info(
|
||||
`Oldest supported major version is: ${OLDEST_SUPPORTED_MAJOR_VERSION}`,
|
||||
);
|
||||
|
||||
// Initialise the API client.
|
||||
const client = getApiClient(options.token);
|
||||
|
||||
// Find the check runs for the specified `ref` that we will later set as the required checks
|
||||
// for the main and release branches.
|
||||
const checkInfos = await getChecksFor(options, client, options.ref);
|
||||
const checkNames = new Set(checkInfos.map((info) => info.context));
|
||||
|
||||
// Update the main branch.
|
||||
await updateBranch(options, client, "main", checkNames);
|
||||
|
||||
// Retrieve the refs of the release branches.
|
||||
const releaseBranches = await getReleaseBranches(client);
|
||||
console.info(
|
||||
`Found ${releaseBranches.length} release branches: ${releaseBranches.join(", ")}`,
|
||||
);
|
||||
|
||||
for (const releaseBranchRef of releaseBranches) {
|
||||
// Sanity check that the ref name is in the expected format and extract the major version.
|
||||
const releaseBranchMatch = releaseBranchRef.match(
|
||||
/^refs\/heads\/(releases\/v(\d+))/,
|
||||
);
|
||||
if (!releaseBranchMatch) {
|
||||
console.warn(
|
||||
`Branch ref '${releaseBranchRef}' not in the expected format.`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const releaseBranch = releaseBranchMatch[1];
|
||||
const releaseBranchMajor = Number.parseInt(releaseBranchMatch[2]);
|
||||
|
||||
// Update the required checks for this major version if it is still supported.
|
||||
if (releaseBranchMajor < OLDEST_SUPPORTED_MAJOR_VERSION) {
|
||||
console.info(
|
||||
`Skipping '${releaseBranch}' since it is older than v${OLDEST_SUPPORTED_MAJOR_VERSION}`,
|
||||
);
|
||||
continue;
|
||||
} else {
|
||||
await updateBranch(options, client, releaseBranch, checkNames);
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Only call `main` if this script was run directly.
|
||||
if (require.main === module) {
|
||||
void main();
|
||||
}
|
||||
+5
-5
@@ -5,7 +5,7 @@ import * as path from "path";
|
||||
|
||||
import * as yaml from "yaml";
|
||||
|
||||
import { KnownLanguage } from "../src/languages";
|
||||
import { BuiltInLanguage } from "../src/languages";
|
||||
|
||||
/** Known workflow input names. */
|
||||
enum KnownInputName {
|
||||
@@ -91,8 +91,8 @@ interface LanguageSetup {
|
||||
steps: Step[];
|
||||
}
|
||||
|
||||
/** Describes partial mappings from known languages to their specific setup information. */
|
||||
type LanguageSetups = Partial<Record<KnownLanguage, LanguageSetup>>;
|
||||
/** Describes partial mappings from built-in languages to their specific setup information. */
|
||||
type LanguageSetups = Partial<Record<BuiltInLanguage, LanguageSetup>>;
|
||||
|
||||
// The default set of CodeQL Bundle versions to use for the PR checks.
|
||||
const defaultTestVersions = [
|
||||
@@ -125,7 +125,7 @@ const defaultLanguageVersions = {
|
||||
java: "17",
|
||||
python: "3.13",
|
||||
csharp: "9.x",
|
||||
} as const satisfies Partial<Record<KnownLanguage, string>>;
|
||||
} as const satisfies Partial<Record<BuiltInLanguage, string>>;
|
||||
|
||||
/** A mapping from known input names to their specifications. */
|
||||
const inputSpecs: WorkflowInputs = {
|
||||
@@ -364,7 +364,7 @@ function getSetupSteps(checkSpecification: JobSpecification): {
|
||||
const inputs: Array<Set<KnownInputName>> = [];
|
||||
const steps: Step[] = [];
|
||||
|
||||
for (const language of Object.values(KnownLanguage).sort()) {
|
||||
for (const language of Object.values(BuiltInLanguage).sort()) {
|
||||
const setupSpec = languageSetups[language];
|
||||
|
||||
if (
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"compilerOptions": {
|
||||
/* Basic Options */
|
||||
"lib": ["esnext"],
|
||||
"module": "preserve",
|
||||
"rootDir": "..",
|
||||
"sourceMap": false,
|
||||
"noEmit": true,
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/*
|
||||
* Updates src/languages/builtin.json by querying the CodeQL CLI for:
|
||||
* - Languages that have default queries (via codeql-extractor.yml)
|
||||
* - Language aliases (via `codeql resolve languages --format=betterjson --extractor-include-aliases`)
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx pr-checks/update-builtin-languages.ts [path-to-codeql]
|
||||
*
|
||||
* If no path is given, falls back to "codeql".
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
import * as yaml from "yaml";
|
||||
|
||||
import { EnvVar } from "../src/environment";
|
||||
|
||||
import { BUILTIN_LANGUAGES_FILE } from "./config";
|
||||
|
||||
/** Resolve all known language extractor directories. */
|
||||
function resolveLanguages(codeqlPath: string): Record<string, string[]> {
|
||||
return JSON.parse(
|
||||
execFileSync(codeqlPath, ["resolve", "languages", "--format=json"], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
[EnvVar.EXPERIMENTAL_FEATURES]: "true", // include experimental languages
|
||||
},
|
||||
}),
|
||||
) as Record<string, string[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the sorted list of languages whose extractors ship default queries.
|
||||
*
|
||||
* @param extractorDirs - Map from language to list of extractor directories
|
||||
*/
|
||||
function findLanguagesWithDefaultQueries(
|
||||
extractorDirs: Record<string, string[]>,
|
||||
): string[] {
|
||||
const languages: string[] = [];
|
||||
|
||||
for (const [language, dirs] of Object.entries(extractorDirs)) {
|
||||
if (dirs.length !== 1) {
|
||||
throw new Error(
|
||||
`Expected exactly one extractor directory for language '${language}', but found ${dirs.length}: ${dirs.join(
|
||||
", ",
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const extractorYmlPath = path.join(dirs[0], "codeql-extractor.yml");
|
||||
|
||||
if (!fs.existsSync(extractorYmlPath)) {
|
||||
throw new Error(
|
||||
`Extractor YAML not found for language '${language}' at expected path: ${extractorYmlPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
const extractorYml = yaml.parse(fs.readFileSync(extractorYmlPath, "utf8"));
|
||||
const defaultQueries: unknown[] | undefined = extractorYml.default_queries;
|
||||
|
||||
if (Array.isArray(defaultQueries) && defaultQueries.length > 0) {
|
||||
console.log(
|
||||
` ✅ ${language}: included (default queries: ${JSON.stringify(defaultQueries)})`,
|
||||
);
|
||||
languages.push(language);
|
||||
} else {
|
||||
console.log(` ❌ ${language}: excluded (no default queries)`);
|
||||
}
|
||||
}
|
||||
|
||||
return languages.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve language aliases from the CodeQL CLI, keeping only those whose
|
||||
* target is in the given set of included languages.
|
||||
*/
|
||||
function resolveAliases(
|
||||
codeqlPath: string,
|
||||
includedLanguages: Set<string>,
|
||||
): Record<string, string> {
|
||||
const betterjsonOutput = JSON.parse(
|
||||
execFileSync(
|
||||
codeqlPath,
|
||||
[
|
||||
"resolve",
|
||||
"languages",
|
||||
"--format=betterjson",
|
||||
"--extractor-include-aliases",
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
),
|
||||
);
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries((betterjsonOutput.aliases ?? {}) as Record<string, string>)
|
||||
.filter(([, target]) => includedLanguages.has(target))
|
||||
.sort(([a], [b]) => a.localeCompare(b)),
|
||||
);
|
||||
}
|
||||
|
||||
/** Write the built-in languages data to disk. */
|
||||
function writeBuiltinLanguages(
|
||||
languages: string[],
|
||||
aliases: Record<string, string>,
|
||||
): void {
|
||||
const content = `${JSON.stringify({ languages, aliases }, null, 2)}\n`;
|
||||
fs.mkdirSync(path.dirname(BUILTIN_LANGUAGES_FILE), { recursive: true });
|
||||
fs.writeFileSync(BUILTIN_LANGUAGES_FILE, content);
|
||||
|
||||
console.log(`\nWrote ${BUILTIN_LANGUAGES_FILE}`);
|
||||
console.log(` Languages: ${languages.join(", ")}`);
|
||||
console.log(` Aliases: ${Object.keys(aliases).join(", ")}`);
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const codeqlPath = process.argv[2] || "codeql";
|
||||
|
||||
const extractorDirs = resolveLanguages(codeqlPath);
|
||||
const languages = findLanguagesWithDefaultQueries(extractorDirs);
|
||||
const aliases = resolveAliases(codeqlPath, new Set(languages));
|
||||
writeBuiltinLanguages(languages, aliases);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -53,6 +53,12 @@ export function getTemporaryDirectory(): string {
|
||||
: getRequiredEnvParam("RUNNER_TEMP");
|
||||
}
|
||||
|
||||
const PR_DIFF_RANGE_JSON_FILENAME = "pr-diff-range.json";
|
||||
|
||||
export function getDiffRangesJsonFilePath(): string {
|
||||
return path.join(getTemporaryDirectory(), PR_DIFF_RANGE_JSON_FILENAME);
|
||||
}
|
||||
|
||||
export function getActionVersion(): string {
|
||||
return __CODEQL_ACTION_VERSION__;
|
||||
}
|
||||
|
||||
+13
-16
@@ -28,12 +28,11 @@ import {
|
||||
DependencyCacheUploadStatusReport,
|
||||
uploadDependencyCaches,
|
||||
} from "./dependency-caching";
|
||||
import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { initFeatures } from "./feature-flags";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { getActionsLogger, Logger } from "./logging";
|
||||
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay";
|
||||
import { cleanupAndUploadOverlayBaseDatabaseToCache } from "./overlay/caching";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import * as statusReport from "./status-report";
|
||||
import {
|
||||
@@ -136,9 +135,13 @@ function hasBadExpectErrorInput(): boolean {
|
||||
function doesGoExtractionOutputExist(config: Config): boolean {
|
||||
const golangDbDirectory = util.getCodeQLDatabasePath(
|
||||
config,
|
||||
KnownLanguage.go,
|
||||
BuiltInLanguage.go,
|
||||
);
|
||||
const trapDirectory = path.join(
|
||||
golangDbDirectory,
|
||||
"trap",
|
||||
BuiltInLanguage.go,
|
||||
);
|
||||
const trapDirectory = path.join(golangDbDirectory, "trap", KnownLanguage.go);
|
||||
return (
|
||||
fs.existsSync(trapDirectory) &&
|
||||
fs
|
||||
@@ -170,7 +173,7 @@ function doesGoExtractionOutputExist(config: Config): boolean {
|
||||
* whether any extraction output already exists for Go.
|
||||
*/
|
||||
async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
|
||||
if (!config.languages.includes(KnownLanguage.go)) {
|
||||
if (!config.languages.includes(BuiltInLanguage.go)) {
|
||||
return;
|
||||
}
|
||||
if (config.buildMode) {
|
||||
@@ -183,7 +186,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
|
||||
logger.debug("Won't run Go autobuild since it has already been run.");
|
||||
return;
|
||||
}
|
||||
if (dbIsFinalized(config, KnownLanguage.go, logger)) {
|
||||
if (dbIsFinalized(config, BuiltInLanguage.go, logger)) {
|
||||
logger.debug(
|
||||
"Won't run Go autobuild since there is already a finalized database for Go.",
|
||||
);
|
||||
@@ -206,7 +209,7 @@ async function runAutobuildIfLegacyGoWorkflow(config: Config, logger: Logger) {
|
||||
logger.debug(
|
||||
"Running Go autobuild because extraction output (TRAP files) for Go code has not been found.",
|
||||
);
|
||||
await runAutobuild(config, KnownLanguage.go, logger);
|
||||
await runAutobuild(config, BuiltInLanguage.go, logger);
|
||||
}
|
||||
|
||||
async function run(startedAt: Date) {
|
||||
@@ -305,14 +308,8 @@ async function run(startedAt: Date) {
|
||||
logger,
|
||||
);
|
||||
|
||||
const branches = await getDiffInformedAnalysisBranches(
|
||||
codeql,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
const diffRangePackDir = branches
|
||||
? await setupDiffInformedQueryRun(branches, logger)
|
||||
: undefined;
|
||||
// Setup diff informed analysis if needed (based on whether init created the file)
|
||||
const diffRangePackDir = await setupDiffInformedQueryRun(logger);
|
||||
|
||||
await warnIfGoInstalledAfterInit(config, logger);
|
||||
await runAutobuildIfLegacyGoWorkflow(config, logger);
|
||||
|
||||
+6
-6
@@ -14,7 +14,7 @@ import {
|
||||
} from "./analyze";
|
||||
import { createStubCodeQL } from "./codeql";
|
||||
import { Feature } from "./feature-flags";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import {
|
||||
setupTests,
|
||||
@@ -41,7 +41,7 @@ test.serial("status report fields", async (t) => {
|
||||
const threadsFlag = "";
|
||||
sinon.stub(uploadLib, "validateSarifFileSchema");
|
||||
|
||||
for (const language of Object.values(KnownLanguage)) {
|
||||
for (const language of Object.values(BuiltInLanguage)) {
|
||||
const codeql = createStubCodeQL({
|
||||
databaseRunQueries: async () => {},
|
||||
databaseInterpretResults: async (
|
||||
@@ -130,13 +130,13 @@ test.serial("status report fields", async (t) => {
|
||||
test("resolveQuerySuiteAlias", (t) => {
|
||||
// default query suite names should resolve to something language-specific ending in `.qls`.
|
||||
for (const suite of defaultSuites) {
|
||||
const resolved = resolveQuerySuiteAlias(KnownLanguage.go, suite);
|
||||
const resolved = resolveQuerySuiteAlias(BuiltInLanguage.go, suite);
|
||||
t.assert(
|
||||
path.extname(resolved) === ".qls",
|
||||
"Resolved default suite doesn't end in .qls",
|
||||
);
|
||||
t.assert(
|
||||
resolved.indexOf(KnownLanguage.go) >= 0,
|
||||
resolved.indexOf(BuiltInLanguage.go) >= 0,
|
||||
"Resolved default suite doesn't contain language name",
|
||||
);
|
||||
}
|
||||
@@ -145,12 +145,12 @@ test("resolveQuerySuiteAlias", (t) => {
|
||||
const names = ["foo", "bar", "codeql/go-queries@1.0"];
|
||||
|
||||
for (const name of names) {
|
||||
t.deepEqual(resolveQuerySuiteAlias(KnownLanguage.go, name), name);
|
||||
t.deepEqual(resolveQuerySuiteAlias(BuiltInLanguage.go, name), name);
|
||||
}
|
||||
});
|
||||
|
||||
test("addSarifExtension", (t) => {
|
||||
for (const language of Object.values(KnownLanguage)) {
|
||||
for (const language of Object.values(BuiltInLanguage)) {
|
||||
t.deepEqual(addSarifExtension(CodeScanning, language), `${language}.sarif`);
|
||||
t.deepEqual(
|
||||
addSarifExtension(CodeQuality, language),
|
||||
|
||||
+26
-44
@@ -5,11 +5,7 @@ import { performance } from "perf_hooks";
|
||||
import * as io from "@actions/io";
|
||||
import * as yaml from "js-yaml";
|
||||
|
||||
import {
|
||||
getTemporaryDirectory,
|
||||
getRequiredInput,
|
||||
PullRequestBranches,
|
||||
} from "./actions-util";
|
||||
import { getTemporaryDirectory, getRequiredInput } from "./actions-util";
|
||||
import * as analyses from "./analyses";
|
||||
import { setupCppAutobuild } from "./autobuild";
|
||||
import { type CodeQL } from "./codeql";
|
||||
@@ -21,14 +17,13 @@ import {
|
||||
import { addDiagnostic, makeDiagnostic } from "./diagnostics";
|
||||
import {
|
||||
DiffThunkRange,
|
||||
writeDiffRangesJsonFile,
|
||||
getPullRequestEditedDiffRanges,
|
||||
readDiffRangesJsonFile,
|
||||
} from "./diff-informed-analysis-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { FeatureEnablement, Feature } from "./feature-flags";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { BuiltInLanguage, Language } from "./languages";
|
||||
import { Logger, withGroupAsync } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import type * as sarif from "./sarif";
|
||||
import { DatabaseCreationTimings, EventReport } from "./status-report";
|
||||
import { endTracingForCluster } from "./tracer-config";
|
||||
@@ -46,7 +41,7 @@ export class CodeQLAnalysisError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
type KnownLanguageKey = keyof typeof KnownLanguage;
|
||||
type BuiltInLanguageKey = keyof typeof BuiltInLanguage;
|
||||
|
||||
type RunQueriesDurationStatusReport = {
|
||||
/**
|
||||
@@ -55,12 +50,12 @@ type RunQueriesDurationStatusReport = {
|
||||
* The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
|
||||
* taken to run _all_ the queries.
|
||||
*/
|
||||
[L in KnownLanguageKey as `analyze_builtin_queries_${L}_duration_ms`]?: number;
|
||||
[L in BuiltInLanguageKey as `analyze_builtin_queries_${L}_duration_ms`]?: number;
|
||||
};
|
||||
|
||||
type InterpretResultsDurationStatusReport = {
|
||||
/** Time taken in ms to interpret results for the language (or undefined if this language was not analyzed). */
|
||||
[L in KnownLanguageKey as `interpret_results_${L}_duration_ms`]?: number;
|
||||
[L in BuiltInLanguageKey as `interpret_results_${L}_duration_ms`]?: number;
|
||||
};
|
||||
|
||||
export interface QueriesStatusReport
|
||||
@@ -120,12 +115,12 @@ export async function runExtraction(
|
||||
|
||||
if (await shouldExtractLanguage(codeql, config, language)) {
|
||||
logger.startGroup(`Extracting ${language}`);
|
||||
if (language === KnownLanguage.python) {
|
||||
if (language === BuiltInLanguage.python) {
|
||||
await setupPythonExtractor(logger);
|
||||
}
|
||||
if (config.buildMode) {
|
||||
if (
|
||||
language === KnownLanguage.cpp &&
|
||||
language === BuiltInLanguage.cpp &&
|
||||
config.buildMode === BuildMode.Autobuild
|
||||
) {
|
||||
await setupCppAutobuild(codeql, logger);
|
||||
@@ -136,14 +131,14 @@ export async function runExtraction(
|
||||
// a stable path that caches can be restored into and that we can cache at the
|
||||
// end of the workflow (i.e. that does not get removed when the scratch directory is).
|
||||
if (
|
||||
language === KnownLanguage.java &&
|
||||
language === BuiltInLanguage.java &&
|
||||
config.buildMode === BuildMode.None
|
||||
) {
|
||||
process.env["CODEQL_EXTRACTOR_JAVA_OPTION_BUILDLESS_DEPENDENCY_DIR"] =
|
||||
getJavaTempDependencyDir();
|
||||
}
|
||||
if (
|
||||
language === KnownLanguage.csharp &&
|
||||
language === BuiltInLanguage.csharp &&
|
||||
config.buildMode === BuildMode.None &&
|
||||
(await features.getValue(Feature.CsharpCacheBuildModeNone))
|
||||
) {
|
||||
@@ -237,32 +232,28 @@ async function finalizeDatabaseCreation(
|
||||
* the diff range information, or `undefined` if the feature is disabled.
|
||||
*/
|
||||
export async function setupDiffInformedQueryRun(
|
||||
branches: PullRequestBranches,
|
||||
logger: Logger,
|
||||
): Promise<string | undefined> {
|
||||
return await withGroupAsync(
|
||||
"Generating diff range extension pack",
|
||||
async () => {
|
||||
logger.info(
|
||||
`Calculating diff ranges for ${branches.base}...${branches.head}`,
|
||||
);
|
||||
const diffRanges = await getPullRequestEditedDiffRanges(branches, logger);
|
||||
const diffRanges = readDiffRangesJsonFile(logger);
|
||||
if (diffRanges === undefined) {
|
||||
logger.info(
|
||||
"No precomputed diff ranges found; skipping diff-informed analysis stage.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const checkoutPath = getRequiredInput("checkout_path");
|
||||
const packDir = writeDiffRangeDataExtensionPack(
|
||||
logger,
|
||||
diffRanges,
|
||||
checkoutPath,
|
||||
);
|
||||
if (packDir === undefined) {
|
||||
logger.warning(
|
||||
"Cannot create diff range extension pack for diff-informed queries; " +
|
||||
"reverting to performing full analysis.",
|
||||
);
|
||||
} else {
|
||||
logger.info(
|
||||
`Successfully created diff range extension pack at ${packDir}.`,
|
||||
);
|
||||
}
|
||||
logger.info(
|
||||
`Successfully created diff range extension pack at ${packDir}.`,
|
||||
);
|
||||
return packDir;
|
||||
},
|
||||
);
|
||||
@@ -316,18 +307,13 @@ extensions:
|
||||
* @param ranges The file line ranges, as returned by
|
||||
* `getPullRequestEditedDiffRanges`.
|
||||
* @param checkoutPath The path at which the repository was checked out.
|
||||
* @returns The absolute path of the directory containing the extension pack, or
|
||||
* `undefined` if no extension pack was created.
|
||||
* @returns The absolute path of the directory containing the extension pack.
|
||||
*/
|
||||
function writeDiffRangeDataExtensionPack(
|
||||
logger: Logger,
|
||||
ranges: DiffThunkRange[] | undefined,
|
||||
ranges: DiffThunkRange[],
|
||||
checkoutPath: string,
|
||||
): string | undefined {
|
||||
if (ranges === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
): string {
|
||||
if (ranges.length === 0) {
|
||||
// An empty diff range means that there are no added or modified lines in
|
||||
// the pull request. But the `restrictAlertsTo` extensible predicate
|
||||
@@ -368,10 +354,6 @@ dataExtensions:
|
||||
`Wrote pr-diff-range extension pack to ${extensionFilePath}:\n${extensionContents}`,
|
||||
);
|
||||
|
||||
// Write the diff ranges to a JSON file, for action-side alert filtering by the
|
||||
// upload-lib module.
|
||||
writeDiffRangesJsonFile(logger, ranges);
|
||||
|
||||
return diffRangeDir;
|
||||
}
|
||||
|
||||
@@ -704,7 +686,7 @@ export async function warnIfGoInstalledAfterInit(
|
||||
|
||||
addDiagnostic(
|
||||
config,
|
||||
KnownLanguage.go,
|
||||
BuiltInLanguage.go,
|
||||
makeDiagnostic(
|
||||
"go/workflow/go-installed-after-codeql-init",
|
||||
"Go was installed after the `codeql-action/init` Action was run",
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"maximumVersion": "3.21", "minimumVersion": "3.14"}
|
||||
{"maximumVersion": "3.21", "minimumVersion": "3.16"}
|
||||
|
||||
+5
-5
@@ -7,7 +7,7 @@ import * as configUtils from "./config-utils";
|
||||
import { DocUrl } from "./doc-url";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature, featureConfig, initFeatures } from "./feature-flags";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { BuiltInLanguage, Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import { asyncFilter, BuildMode } from "./util";
|
||||
@@ -72,7 +72,7 @@ export async function determineAutobuildLanguages(
|
||||
* version of the CodeQL Action.
|
||||
*/
|
||||
const autobuildLanguagesWithoutGo = autobuildLanguages.filter(
|
||||
(l) => l !== KnownLanguage.go,
|
||||
(l) => l !== BuiltInLanguage.go,
|
||||
);
|
||||
|
||||
const languages: Language[] = [];
|
||||
@@ -84,7 +84,7 @@ export async function determineAutobuildLanguages(
|
||||
// If Go is requested, run the Go autobuilder last to ensure it doesn't
|
||||
// interfere with the other autobuilder.
|
||||
if (autobuildLanguages.length !== autobuildLanguagesWithoutGo.length) {
|
||||
languages.push(KnownLanguage.go);
|
||||
languages.push(BuiltInLanguage.go);
|
||||
}
|
||||
|
||||
logger.debug(`Will autobuild ${languages.join(" and ")}.`);
|
||||
@@ -156,7 +156,7 @@ export async function runAutobuild(
|
||||
) {
|
||||
logger.startGroup(`Attempting to automatically build ${language} code`);
|
||||
const codeQL = await getCodeQL(config.codeQLCmd);
|
||||
if (language === KnownLanguage.cpp) {
|
||||
if (language === BuiltInLanguage.cpp) {
|
||||
await setupCppAutobuild(codeQL, logger);
|
||||
}
|
||||
if (config.buildMode) {
|
||||
@@ -164,7 +164,7 @@ export async function runAutobuild(
|
||||
} else {
|
||||
await codeQL.runAutobuild(config, language);
|
||||
}
|
||||
if (language === KnownLanguage.go) {
|
||||
if (language === BuiltInLanguage.go) {
|
||||
core.exportVariable(EnvVar.DID_AUTOBUILD_GOLANG, "true");
|
||||
}
|
||||
logger.endGroup();
|
||||
|
||||
@@ -299,6 +299,20 @@ test("wrapCliConfigurationError - swift build failed", (t) => {
|
||||
t.true(wrappedError instanceof ConfigurationError);
|
||||
});
|
||||
|
||||
test("wrapCliConfigurationError - swift incompatible os", (t) => {
|
||||
const commandError = new CommandInvocationError(
|
||||
"codeql",
|
||||
["swift/tools/autobuild.sh"],
|
||||
1,
|
||||
"2026-04-01 18:35:00 EST ERRO [extractor/main] [incompatible-os] Currently, Swift analysis is only supported on macOS. (IncompatibleOs.cpp:26)",
|
||||
);
|
||||
const cliError = new CliError(commandError);
|
||||
|
||||
const wrappedError = wrapCliConfigurationError(cliError);
|
||||
|
||||
t.true(wrappedError instanceof ConfigurationError);
|
||||
});
|
||||
|
||||
test("wrapCliConfigurationError - pack cannot be found", (t) => {
|
||||
const commandError = new CommandInvocationError(
|
||||
"codeql",
|
||||
|
||||
@@ -144,6 +144,7 @@ export enum CliConfigErrorCategory {
|
||||
OutOfMemoryOrDisk = "OutOfMemoryOrDisk",
|
||||
PackCannotBeFound = "PackCannotBeFound",
|
||||
PackMissingAuth = "PackMissingAuth",
|
||||
SwiftIncompatibleOs = "SwiftIncompatibleOs",
|
||||
SwiftBuildFailed = "SwiftBuildFailed",
|
||||
UnsupportedBuildMode = "UnsupportedBuildMode",
|
||||
}
|
||||
@@ -281,6 +282,12 @@ const cliErrorsConfig: Record<CliConfigErrorCategory, CliErrorConfiguration> = {
|
||||
),
|
||||
],
|
||||
},
|
||||
[CliConfigErrorCategory.SwiftIncompatibleOs]: {
|
||||
cliErrorMessageCandidates: [
|
||||
new RegExp("\\[incompatible-os\\]"),
|
||||
new RegExp("Swift analysis is only supported on macOS"),
|
||||
],
|
||||
},
|
||||
[CliConfigErrorCategory.UnsupportedBuildMode]: {
|
||||
cliErrorMessageCandidates: [
|
||||
new RegExp(
|
||||
|
||||
+7
-5
@@ -21,7 +21,7 @@ import {
|
||||
import type { Config } from "./config-utils";
|
||||
import * as defaults from "./defaults.json";
|
||||
import { DocUrl } from "./doc-url";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { ToolsSource } from "./setup-codeql";
|
||||
import {
|
||||
@@ -46,7 +46,7 @@ test.beforeEach(() => {
|
||||
initializeEnvironment("1.2.3");
|
||||
|
||||
stubConfig = createTestConfig({
|
||||
languages: [KnownLanguage.cpp],
|
||||
languages: [BuiltInLanguage.cpp],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -115,7 +115,7 @@ async function stubCodeql(): Promise<codeql.CodeQL> {
|
||||
sinon.stub(codeqlObject, "getVersion").resolves(makeVersionInfo("2.17.6"));
|
||||
sinon
|
||||
.stub(codeqlObject, "isTracedLanguage")
|
||||
.withArgs(KnownLanguage.cpp)
|
||||
.withArgs(BuiltInLanguage.cpp)
|
||||
.resolves(true);
|
||||
return codeqlObject;
|
||||
}
|
||||
@@ -956,7 +956,8 @@ test.serial("runTool summarizes autobuilder errors", async (t) => {
|
||||
sinon.stub(io, "which").resolves("");
|
||||
|
||||
await t.throwsAsync(
|
||||
async () => await codeqlObject.runAutobuild(stubConfig, KnownLanguage.java),
|
||||
async () =>
|
||||
await codeqlObject.runAutobuild(stubConfig, BuiltInLanguage.java),
|
||||
{
|
||||
instanceOf: util.ConfigurationError,
|
||||
message:
|
||||
@@ -982,7 +983,8 @@ test.serial("runTool truncates long autobuilder errors", async (t) => {
|
||||
sinon.stub(io, "which").resolves("");
|
||||
|
||||
await t.throwsAsync(
|
||||
async () => await codeqlObject.runAutobuild(stubConfig, KnownLanguage.java),
|
||||
async () =>
|
||||
await codeqlObject.runAutobuild(stubConfig, BuiltInLanguage.java),
|
||||
{
|
||||
instanceOf: util.ConfigurationError,
|
||||
message:
|
||||
|
||||
+5
-8
@@ -24,11 +24,8 @@ import {
|
||||
import { isAnalyzingDefaultBranch } from "./git-utils";
|
||||
import { Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import {
|
||||
OverlayDatabaseMode,
|
||||
writeBaseDatabaseOidsFile,
|
||||
writeOverlayChangesFile,
|
||||
} from "./overlay";
|
||||
import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from "./overlay";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import * as setupCodeql from "./setup-codeql";
|
||||
import { ZstdAvailability } from "./tar";
|
||||
import { ToolsDownloadStatusReport } from "./tools-download";
|
||||
@@ -285,17 +282,17 @@ const CODEQL_MINIMUM_VERSION = "2.17.6";
|
||||
/**
|
||||
* This version will shortly become the oldest version of CodeQL that the Action will run with.
|
||||
*/
|
||||
const CODEQL_NEXT_MINIMUM_VERSION = "2.17.6";
|
||||
const CODEQL_NEXT_MINIMUM_VERSION = "2.19.4";
|
||||
|
||||
/**
|
||||
* This is the version of GHES that was most recently deprecated.
|
||||
*/
|
||||
const GHES_VERSION_MOST_RECENTLY_DEPRECATED = "3.13";
|
||||
const GHES_VERSION_MOST_RECENTLY_DEPRECATED = "3.15";
|
||||
|
||||
/**
|
||||
* This is the deprecation date for the version of GHES that was most recently deprecated.
|
||||
*/
|
||||
const GHES_MOST_RECENT_DEPRECATION_DATE = "2025-06-19";
|
||||
const GHES_MOST_RECENT_DEPRECATION_DATE = "2026-04-09";
|
||||
|
||||
/** The CLI verbosity level to use for extraction in debug mode. */
|
||||
const EXTRACTION_DEBUG_MODE_VERBOSITY = "progress++";
|
||||
|
||||
+85
-66
@@ -18,10 +18,11 @@ import { Feature } from "./feature-flags";
|
||||
import { RepositoryProperties } from "./feature-flags/properties";
|
||||
import * as gitUtils from "./git-utils";
|
||||
import { GitVersionInfo } from "./git-utils";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { BuiltInLanguage, Language } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION, OverlayDatabaseMode } from "./overlay";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION } from "./overlay";
|
||||
import { OverlayDisabledReason } from "./overlay/diagnostics";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import * as overlayStatus from "./overlay/status";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import {
|
||||
@@ -214,7 +215,7 @@ test.serial("load code quality config", async (t) => {
|
||||
// And the config we expect it to result in
|
||||
const expectedConfig = createTestConfig({
|
||||
analysisKinds: [AnalysisKind.CodeQuality],
|
||||
languages: [KnownLanguage.actions],
|
||||
languages: [BuiltInLanguage.actions],
|
||||
// This gets set because we only have `AnalysisKind.CodeQuality`
|
||||
computedConfig: {
|
||||
"disable-default-queries": true,
|
||||
@@ -267,7 +268,7 @@ test.serial(
|
||||
|
||||
const expectedConfig = createTestConfig({
|
||||
analysisKinds: [AnalysisKind.CodeQuality],
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
codeQLCmd: codeql.getPath(),
|
||||
computedConfig,
|
||||
dbLocation: path.resolve(tempDir, "codeql_databases"),
|
||||
@@ -517,7 +518,7 @@ test.serial("load non-empty input", async (t) => {
|
||||
|
||||
// And the config we expect it to parse to
|
||||
const expectedConfig = createTestConfig({
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
buildMode: BuildMode.None,
|
||||
originalUserInput: userConfig,
|
||||
computedConfig: userConfig,
|
||||
@@ -891,10 +892,10 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
|
||||
betterResolveLanguages: (options) =>
|
||||
Promise.resolve({
|
||||
aliases: {
|
||||
"c#": KnownLanguage.csharp,
|
||||
c: KnownLanguage.cpp,
|
||||
kotlin: KnownLanguage.java,
|
||||
typescript: KnownLanguage.javascript,
|
||||
"c#": BuiltInLanguage.csharp,
|
||||
c: BuiltInLanguage.cpp,
|
||||
kotlin: BuiltInLanguage.java,
|
||||
typescript: BuiltInLanguage.javascript,
|
||||
},
|
||||
extractors: {
|
||||
cpp: [stubExtractorEntry],
|
||||
@@ -943,12 +944,12 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo");
|
||||
for (const { displayName, language, feature } of [
|
||||
{
|
||||
displayName: "Java",
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
feature: Feature.DisableJavaBuildlessEnabled,
|
||||
},
|
||||
{
|
||||
displayName: "C#",
|
||||
language: KnownLanguage.csharp,
|
||||
language: BuiltInLanguage.csharp,
|
||||
feature: Feature.DisableCsharpBuildless,
|
||||
},
|
||||
]) {
|
||||
@@ -968,7 +969,7 @@ for (const { displayName, language, feature } of [
|
||||
const messages: LoggedMessage[] = [];
|
||||
const buildMode = await configUtils.parseBuildModeInput(
|
||||
"none",
|
||||
[KnownLanguage.python],
|
||||
[BuiltInLanguage.python],
|
||||
createFeatures([feature]),
|
||||
getRecordingLogger(messages),
|
||||
);
|
||||
@@ -1004,6 +1005,7 @@ interface OverlayDatabaseModeTestSetup {
|
||||
codeqlVersion: string;
|
||||
gitRoot: string | undefined;
|
||||
gitVersion: GitVersionInfo | undefined;
|
||||
hasSubmodules: boolean;
|
||||
codeScanningConfig: UserConfig;
|
||||
diskUsage: DiskUsage | undefined;
|
||||
memoryFlagValue: number;
|
||||
@@ -1017,13 +1019,11 @@ const defaultOverlayDatabaseModeTestSetup: OverlayDatabaseModeTestSetup = {
|
||||
isPullRequest: false,
|
||||
isDefaultBranch: false,
|
||||
buildMode: BuildMode.None,
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
codeqlVersion: CODEQL_OVERLAY_MINIMUM_VERSION,
|
||||
gitRoot: "/some/git/root",
|
||||
gitVersion: new GitVersionInfo(
|
||||
gitUtils.GIT_MINIMUM_VERSION_FOR_OVERLAY,
|
||||
gitUtils.GIT_MINIMUM_VERSION_FOR_OVERLAY,
|
||||
),
|
||||
gitVersion: new GitVersionInfo("2.39.0", "2.39.0"),
|
||||
hasSubmodules: false,
|
||||
codeScanningConfig: {},
|
||||
diskUsage: {
|
||||
numAvailableBytes: 50_000_000_000,
|
||||
@@ -1091,7 +1091,7 @@ const checkOverlayEnablementMacro = test.macro({
|
||||
sinon
|
||||
.stub(codeql, "isTracedLanguage")
|
||||
.callsFake(async (lang: Language) => {
|
||||
return [KnownLanguage.java].includes(lang as KnownLanguage);
|
||||
return lang === BuiltInLanguage.java;
|
||||
});
|
||||
|
||||
// Mock git root detection
|
||||
@@ -1099,6 +1099,9 @@ const checkOverlayEnablementMacro = test.macro({
|
||||
sinon.stub(gitUtils, "getGitRoot").resolves(setup.gitRoot);
|
||||
}
|
||||
|
||||
// Mock submodule detection
|
||||
sinon.stub(gitUtils, "hasSubmodules").returns(setup.hasSubmodules);
|
||||
|
||||
// Mock default branch detection
|
||||
sinon
|
||||
.stub(gitUtils, "isAnalyzingDefaultBranch")
|
||||
@@ -1181,7 +1184,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Ignore feature flag when analyzing non-default branch",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
},
|
||||
{
|
||||
@@ -1193,7 +1196,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay-base database on default branch when feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
isDefaultBranch: true,
|
||||
},
|
||||
@@ -1207,7 +1210,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay-base database on default branch when feature enabled with custom analysis",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
codeScanningConfig: {
|
||||
packs: ["some-custom-pack@1.0.0"],
|
||||
@@ -1224,7 +1227,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay-base database on default branch when code-scanning feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1241,7 +1244,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch if runner disk space is too low",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1261,7 +1264,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch if we can't determine runner disk space",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1278,7 +1281,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay-base database on default branch if runner disk space is too low and skip resource checks flag is enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1300,7 +1303,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch if runner disk space is below v2 limit and v2 resource checks enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1321,7 +1324,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1343,7 +1346,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks not enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1363,7 +1366,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch if memory flag is too low",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1380,7 +1383,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay-base database on default branch if memory flag is too low but CodeQL >= 2.24.3",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1399,7 +1402,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay-base database on default branch if memory flag is too low and skip resource checks flag is enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1418,7 +1421,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when cached status indicates previous failure",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisJavascript,
|
||||
@@ -1436,7 +1439,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when cached status indicates previous failure",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisJavascript,
|
||||
@@ -1454,7 +1457,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when code-scanning feature enabled with disable-default-queries",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1473,7 +1476,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when code-scanning feature enabled with packs",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1492,7 +1495,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when code-scanning feature enabled with queries",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1511,7 +1514,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when code-scanning feature enabled with query-filters",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1530,7 +1533,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when only language-specific feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysisJavascript],
|
||||
isDefaultBranch: true,
|
||||
},
|
||||
@@ -1543,7 +1546,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when only code-scanning feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysisCodeScanningJavascript],
|
||||
isDefaultBranch: true,
|
||||
},
|
||||
@@ -1556,7 +1559,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay-base database on default branch when language-specific feature disabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis],
|
||||
isDefaultBranch: true,
|
||||
},
|
||||
@@ -1569,7 +1572,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay analysis on PR when feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
isPullRequest: true,
|
||||
},
|
||||
@@ -1583,7 +1586,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay analysis on PR when feature enabled with custom analysis",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
codeScanningConfig: {
|
||||
packs: ["some-custom-pack@1.0.0"],
|
||||
@@ -1600,7 +1603,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay analysis on PR when code-scanning feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1617,7 +1620,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR if runner disk space is too low",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1637,7 +1640,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay analysis on PR if runner disk space is too low and skip resource checks flag is enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1659,7 +1662,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR if we can't determine runner disk space",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1676,7 +1679,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR if memory flag is too low",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1693,7 +1696,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay analysis on PR if memory flag is too low but CodeQL >= 2.24.3",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1712,7 +1715,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay analysis on PR if memory flag is too low and skip resource checks flag is enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1731,7 +1734,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when code-scanning feature enabled with disable-default-queries",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1750,7 +1753,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when code-scanning feature enabled with packs",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1769,7 +1772,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when code-scanning feature enabled with queries",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1788,7 +1791,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when code-scanning feature enabled with query-filters",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
@@ -1807,7 +1810,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when only language-specific feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysisJavascript],
|
||||
isPullRequest: true,
|
||||
},
|
||||
@@ -1820,7 +1823,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when only code-scanning feature enabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysisCodeScanningJavascript],
|
||||
isPullRequest: true,
|
||||
},
|
||||
@@ -1833,7 +1836,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis on PR when language-specific feature disabled",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis],
|
||||
isPullRequest: true,
|
||||
},
|
||||
@@ -1871,7 +1874,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay PR analysis by feature flag",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
isPullRequest: true,
|
||||
},
|
||||
@@ -1887,7 +1890,7 @@ test.serial(
|
||||
{
|
||||
overlayDatabaseEnvVar: "overlay",
|
||||
buildMode: BuildMode.Autobuild,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
},
|
||||
{
|
||||
disabledReason: OverlayDisabledReason.IncompatibleBuildMode,
|
||||
@@ -1900,7 +1903,7 @@ test.serial(
|
||||
{
|
||||
overlayDatabaseEnvVar: "overlay",
|
||||
buildMode: undefined,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
},
|
||||
{
|
||||
disabledReason: OverlayDisabledReason.IncompatibleBuildMode,
|
||||
@@ -1933,10 +1936,11 @@ test.serial(
|
||||
|
||||
test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Fallback due to old git version",
|
||||
"Fallback due to old git version with submodules",
|
||||
{
|
||||
overlayDatabaseEnvVar: "overlay",
|
||||
gitVersion: new GitVersionInfo("2.30.0", "2.30.0"), // Version below required 2.38.0
|
||||
gitVersion: new GitVersionInfo("2.34.1", "2.34.1"), // Above 2.11.0 but below 2.36.0
|
||||
hasSubmodules: true,
|
||||
},
|
||||
{
|
||||
disabledReason: OverlayDisabledReason.IncompatibleGit,
|
||||
@@ -1945,21 +1949,36 @@ test.serial(
|
||||
|
||||
test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Fallback when git version cannot be determined",
|
||||
"Fallback when git version cannot be determined and repo has submodules",
|
||||
{
|
||||
overlayDatabaseEnvVar: "overlay",
|
||||
gitVersion: undefined,
|
||||
hasSubmodules: true,
|
||||
},
|
||||
{
|
||||
disabledReason: OverlayDisabledReason.IncompatibleGit,
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay enabled when git version cannot be determined and repo has no submodules",
|
||||
{
|
||||
overlayDatabaseEnvVar: "overlay",
|
||||
gitVersion: undefined,
|
||||
hasSubmodules: false,
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
|
||||
useOverlayDatabaseCaching: false,
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay when disabled via repository property",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
isPullRequest: true,
|
||||
repositoryProperties: {
|
||||
@@ -1975,7 +1994,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"Overlay not disabled when repository property is false",
|
||||
{
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [Feature.OverlayAnalysis, Feature.OverlayAnalysisJavascript],
|
||||
isPullRequest: true,
|
||||
repositoryProperties: {
|
||||
@@ -2004,7 +2023,7 @@ test.serial(
|
||||
);
|
||||
|
||||
// Exercise language-specific overlay analysis features code paths
|
||||
for (const language in KnownLanguage) {
|
||||
for (const language in BuiltInLanguage) {
|
||||
test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
`Check default overlay analysis feature for ${language}`,
|
||||
@@ -2027,7 +2046,7 @@ test.serial(
|
||||
checkOverlayEnablementMacro,
|
||||
"No overlay analysis for language without per-language overlay feature flag",
|
||||
{
|
||||
languages: [KnownLanguage.swift],
|
||||
languages: [BuiltInLanguage.swift],
|
||||
features: [Feature.OverlayAnalysis],
|
||||
isPullRequest: true,
|
||||
},
|
||||
|
||||
+34
-26
@@ -43,17 +43,19 @@ import {
|
||||
getGeneratedFiles,
|
||||
getGitRoot,
|
||||
getGitVersionOrThrow,
|
||||
GIT_MINIMUM_VERSION_FOR_OVERLAY,
|
||||
GIT_MINIMUM_VERSION_FOR_OVERLAY_WITH_SUBMODULES,
|
||||
GitVersionInfo,
|
||||
hasSubmodules,
|
||||
isAnalyzingDefaultBranch,
|
||||
} from "./git-utils";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { BuiltInLanguage, Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION, OverlayDatabaseMode } from "./overlay";
|
||||
import { CODEQL_OVERLAY_MINIMUM_VERSION } from "./overlay";
|
||||
import {
|
||||
addOverlayDisablementDiagnostics,
|
||||
OverlayDisabledReason,
|
||||
} from "./overlay/diagnostics";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import { shouldSkipOverlayAnalysis } from "./overlay/status";
|
||||
import { RepositoryNwo } from "./repository";
|
||||
import { ToolsFeature } from "./tools-features";
|
||||
@@ -272,10 +274,10 @@ async function getSupportedLanguageMap(
|
||||
for (const extractor of Object.keys(resolveResult.extractors)) {
|
||||
// If the CLI supports resolving languages with default queries, use these
|
||||
// as the set of supported languages. Otherwise, require the language to be
|
||||
// a known language.
|
||||
// a built-in language.
|
||||
if (
|
||||
resolveSupportedLanguagesUsingCli ||
|
||||
KnownLanguage[extractor] !== undefined
|
||||
BuiltInLanguage[extractor] !== undefined
|
||||
) {
|
||||
supportedLanguages[extractor] = extractor;
|
||||
}
|
||||
@@ -945,7 +947,7 @@ async function validateOverlayDatabaseMode(
|
||||
await Promise.all(
|
||||
languages.map(
|
||||
async (l) =>
|
||||
l !== KnownLanguage.go && // Workaround to allow overlay analysis for Go with any build
|
||||
l !== BuiltInLanguage.go && // Workaround to allow overlay analysis for Go with any build
|
||||
// mode, since it does not yet support BMN. The Go autobuilder and/or extractor will
|
||||
// ensure that overlay-base databases are only created for supported Go build setups,
|
||||
// and that we'll fall back to full databases in other cases.
|
||||
@@ -969,7 +971,8 @@ async function validateOverlayDatabaseMode(
|
||||
);
|
||||
return new Failure(OverlayDisabledReason.IncompatibleCodeQl);
|
||||
}
|
||||
if ((await getGitRoot(sourceRoot)) === undefined) {
|
||||
const gitRoot = await getGitRoot(sourceRoot);
|
||||
if (gitRoot === undefined) {
|
||||
logger.warning(
|
||||
`Cannot build an ${overlayDatabaseMode} database because ` +
|
||||
`the source root "${sourceRoot}" is not inside a git repository. ` +
|
||||
@@ -977,21 +980,26 @@ async function validateOverlayDatabaseMode(
|
||||
);
|
||||
return new Failure(OverlayDisabledReason.NoGitRoot);
|
||||
}
|
||||
if (gitVersion === undefined) {
|
||||
logger.warning(
|
||||
`Cannot build an ${overlayDatabaseMode} database because ` +
|
||||
"the Git version could not be determined. " +
|
||||
"Falling back to creating a normal full database instead.",
|
||||
);
|
||||
return new Failure(OverlayDisabledReason.IncompatibleGit);
|
||||
}
|
||||
if (!gitVersion.isAtLeast(GIT_MINIMUM_VERSION_FOR_OVERLAY)) {
|
||||
logger.warning(
|
||||
`Cannot build an ${overlayDatabaseMode} database because ` +
|
||||
`the installed Git version is older than ${GIT_MINIMUM_VERSION_FOR_OVERLAY}. ` +
|
||||
"Falling back to creating a normal full database instead.",
|
||||
);
|
||||
return new Failure(OverlayDisabledReason.IncompatibleGit);
|
||||
if (hasSubmodules(gitRoot)) {
|
||||
if (gitVersion === undefined) {
|
||||
logger.warning(
|
||||
`Cannot build an ${overlayDatabaseMode} database because ` +
|
||||
"the repository has submodules and the Git version could not be determined. " +
|
||||
"Falling back to creating a normal full database instead.",
|
||||
);
|
||||
return new Failure(OverlayDisabledReason.IncompatibleGit);
|
||||
}
|
||||
if (
|
||||
!gitVersion.isAtLeast(GIT_MINIMUM_VERSION_FOR_OVERLAY_WITH_SUBMODULES)
|
||||
) {
|
||||
logger.warning(
|
||||
`Cannot build an ${overlayDatabaseMode} database because ` +
|
||||
"the repository has submodules and the installed Git version is older " +
|
||||
`than ${GIT_MINIMUM_VERSION_FOR_OVERLAY_WITH_SUBMODULES}. ` +
|
||||
"Falling back to creating a normal full database instead.",
|
||||
);
|
||||
return new Failure(OverlayDisabledReason.IncompatibleGit);
|
||||
}
|
||||
}
|
||||
|
||||
return new Success({
|
||||
@@ -1028,13 +1036,13 @@ async function setCppTrapCachingEnvironmentVariables(
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<void> {
|
||||
if (config.languages.includes(KnownLanguage.cpp)) {
|
||||
if (config.languages.includes(BuiltInLanguage.cpp)) {
|
||||
const envVar = "CODEQL_EXTRACTOR_CPP_TRAP_CACHING";
|
||||
if (process.env[envVar]) {
|
||||
logger.info(
|
||||
`Environment variable ${envVar} already set, leaving it unchanged.`,
|
||||
);
|
||||
} else if (config.trapCaches[KnownLanguage.cpp]) {
|
||||
} else if (config.trapCaches[BuiltInLanguage.cpp]) {
|
||||
logger.info("Enabling TRAP caching for C/C++.");
|
||||
core.exportVariable(envVar, "true");
|
||||
} else {
|
||||
@@ -1531,7 +1539,7 @@ export async function parseBuildModeInput(
|
||||
}
|
||||
|
||||
if (
|
||||
languages.includes(KnownLanguage.csharp) &&
|
||||
languages.includes(BuiltInLanguage.csharp) &&
|
||||
(await features.getValue(Feature.DisableCsharpBuildless))
|
||||
) {
|
||||
logger.warning(
|
||||
@@ -1541,7 +1549,7 @@ export async function parseBuildModeInput(
|
||||
}
|
||||
|
||||
if (
|
||||
languages.includes(KnownLanguage.java) &&
|
||||
languages.includes(BuiltInLanguage.java) &&
|
||||
(await features.getValue(Feature.DisableJavaBuildlessEnabled))
|
||||
) {
|
||||
logger.warning(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import test, { ExecutionContext } from "ava";
|
||||
|
||||
import { RepositoryProperties } from "../feature-flags/properties";
|
||||
import { KnownLanguage, Language } from "../languages";
|
||||
import { BuiltInLanguage, Language } from "../languages";
|
||||
import { getRunnerLogger } from "../logging";
|
||||
import {
|
||||
checkExpectedLogMessages,
|
||||
@@ -54,7 +54,7 @@ const invalidPackNameMacro = test.macro({
|
||||
parsePacksErrorMacro.exec(
|
||||
t,
|
||||
name,
|
||||
[KnownLanguage.cpp],
|
||||
[BuiltInLanguage.cpp],
|
||||
new RegExp(`^"${name}" is not a valid pack$`),
|
||||
),
|
||||
title: (_providedTitle: string | undefined, arg: string | undefined) =>
|
||||
@@ -62,23 +62,23 @@ const invalidPackNameMacro = test.macro({
|
||||
});
|
||||
|
||||
test("no packs", parsePacksMacro, "", [], undefined);
|
||||
test("two packs", parsePacksMacro, "a/b,c/d@1.2.3", [KnownLanguage.cpp], {
|
||||
[KnownLanguage.cpp]: ["a/b", "c/d@1.2.3"],
|
||||
test("two packs", parsePacksMacro, "a/b,c/d@1.2.3", [BuiltInLanguage.cpp], {
|
||||
[BuiltInLanguage.cpp]: ["a/b", "c/d@1.2.3"],
|
||||
});
|
||||
test(
|
||||
"two packs with spaces",
|
||||
parsePacksMacro,
|
||||
" a/b , c/d@1.2.3 ",
|
||||
[KnownLanguage.cpp],
|
||||
[BuiltInLanguage.cpp],
|
||||
{
|
||||
[KnownLanguage.cpp]: ["a/b", "c/d@1.2.3"],
|
||||
[BuiltInLanguage.cpp]: ["a/b", "c/d@1.2.3"],
|
||||
},
|
||||
);
|
||||
test(
|
||||
"two packs with language",
|
||||
parsePacksErrorMacro,
|
||||
"a/b,c/d@1.2.3",
|
||||
[KnownLanguage.cpp, KnownLanguage.java],
|
||||
[BuiltInLanguage.cpp, BuiltInLanguage.java],
|
||||
new RegExp(
|
||||
"Cannot specify a 'packs' input in a multi-language analysis. " +
|
||||
"Use a codeql-config.yml file instead and specify packs by language.",
|
||||
@@ -106,9 +106,9 @@ test(
|
||||
// (globbing is not done)
|
||||
"c/d@1.2.3:+*)_(",
|
||||
].join(","),
|
||||
[KnownLanguage.cpp],
|
||||
[BuiltInLanguage.cpp],
|
||||
{
|
||||
[KnownLanguage.cpp]: [
|
||||
[BuiltInLanguage.cpp]: [
|
||||
"c/d@1.0",
|
||||
"c/d@~1.0.0",
|
||||
"c/d@~1.0.0:a/b",
|
||||
@@ -215,7 +215,7 @@ test(
|
||||
"All empty",
|
||||
undefined,
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
{
|
||||
...dbConfig.defaultAugmentationProperties,
|
||||
@@ -227,7 +227,7 @@ test(
|
||||
"With queries",
|
||||
undefined,
|
||||
" a, b , c, d",
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
{
|
||||
...dbConfig.defaultAugmentationProperties,
|
||||
@@ -240,7 +240,7 @@ test(
|
||||
"With queries combining",
|
||||
undefined,
|
||||
" + a, b , c, d ",
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
{
|
||||
...dbConfig.defaultAugmentationProperties,
|
||||
@@ -254,7 +254,7 @@ test(
|
||||
"With packs",
|
||||
" codeql/a , codeql/b , codeql/c , codeql/d ",
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
{
|
||||
...dbConfig.defaultAugmentationProperties,
|
||||
@@ -267,7 +267,7 @@ test(
|
||||
"With packs combining",
|
||||
" + codeql/a, codeql/b, codeql/c, codeql/d",
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
{
|
||||
...dbConfig.defaultAugmentationProperties,
|
||||
@@ -281,7 +281,7 @@ test(
|
||||
"With repo property queries",
|
||||
undefined,
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{
|
||||
"github-codeql-extra-queries": "a, b, c, d",
|
||||
},
|
||||
@@ -299,7 +299,7 @@ test(
|
||||
"With repo property queries combining",
|
||||
undefined,
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{
|
||||
"github-codeql-extra-queries": "+ a, b, c, d",
|
||||
},
|
||||
@@ -341,7 +341,7 @@ test(
|
||||
"Plus (+) with nothing else (queries)",
|
||||
undefined,
|
||||
" + ",
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
/The workflow property "queries" is invalid/,
|
||||
);
|
||||
@@ -351,7 +351,7 @@ test(
|
||||
"Plus (+) with nothing else (packs)",
|
||||
" + ",
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
/The workflow property "packs" is invalid/,
|
||||
);
|
||||
@@ -361,7 +361,7 @@ test(
|
||||
"Plus (+) with nothing else (repo property queries)",
|
||||
undefined,
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{
|
||||
"github-codeql-extra-queries": " + ",
|
||||
},
|
||||
@@ -373,7 +373,7 @@ test(
|
||||
"Packs input with multiple languages",
|
||||
" + a/b, c/d ",
|
||||
undefined,
|
||||
[KnownLanguage.javascript, KnownLanguage.java],
|
||||
[BuiltInLanguage.javascript, BuiltInLanguage.java],
|
||||
{},
|
||||
/Cannot specify a 'packs' input in a multi-language analysis/,
|
||||
);
|
||||
@@ -393,7 +393,7 @@ test(
|
||||
"Invalid packs",
|
||||
" a-pack-without-a-scope ",
|
||||
undefined,
|
||||
[KnownLanguage.javascript],
|
||||
[BuiltInLanguage.javascript],
|
||||
{},
|
||||
/"a-pack-without-a-scope" is not a valid pack/,
|
||||
);
|
||||
|
||||
@@ -12,7 +12,7 @@ import { createStubCodeQL } from "./codeql";
|
||||
import { Config } from "./config-utils";
|
||||
import { cleanupAndUploadDatabases } from "./database-upload";
|
||||
import * as gitUtils from "./git-utils";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { RepositoryNwo } from "./repository";
|
||||
import {
|
||||
checkExpectedLogMessages,
|
||||
@@ -45,7 +45,7 @@ const testApiDetails: GitHubApiDetails = {
|
||||
|
||||
function getTestConfig(tmpDir: string): Config {
|
||||
return createTestConfig({
|
||||
languages: [KnownLanguage.javascript],
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
dbLocation: tmpDir,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import { Config } from "./config-utils";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import * as gitUtils from "./git-utils";
|
||||
import { Logger, withGroupAsync } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import { RepositoryNwo } from "./repository";
|
||||
import * as util from "./util";
|
||||
import { asHTTPError, bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"bundleVersion": "codeql-bundle-v2.25.0",
|
||||
"cliVersion": "2.25.0",
|
||||
"priorBundleVersion": "codeql-bundle-v2.24.3",
|
||||
"priorCliVersion": "2.24.3"
|
||||
"bundleVersion": "codeql-bundle-v2.25.3",
|
||||
"cliVersion": "2.25.3",
|
||||
"priorBundleVersion": "codeql-bundle-v2.25.2",
|
||||
"priorCliVersion": "2.25.2"
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
CacheStoreResult,
|
||||
} from "./dependency-caching";
|
||||
import { Feature } from "./feature-flags";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import {
|
||||
setupTests,
|
||||
createFeatures,
|
||||
@@ -179,7 +179,7 @@ test("checkHashPatterns - logs when no patterns match", async (t) => {
|
||||
const result = await checkHashPatterns(
|
||||
codeql,
|
||||
features,
|
||||
KnownLanguage.csharp,
|
||||
BuiltInLanguage.csharp,
|
||||
config,
|
||||
"download",
|
||||
getRecordingLogger(messages),
|
||||
@@ -208,7 +208,7 @@ test("checkHashPatterns - returns patterns when patterns match", async (t) => {
|
||||
const result = await checkHashPatterns(
|
||||
codeql,
|
||||
features,
|
||||
KnownLanguage.csharp,
|
||||
BuiltInLanguage.csharp,
|
||||
config,
|
||||
"upload",
|
||||
getRecordingLogger(messages),
|
||||
@@ -270,7 +270,7 @@ test.serial(
|
||||
const keyWithFeature = await cacheKey(
|
||||
codeql,
|
||||
createFeatures([Feature.CsharpNewCacheKey]),
|
||||
KnownLanguage.csharp,
|
||||
BuiltInLanguage.csharp,
|
||||
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
|
||||
[],
|
||||
);
|
||||
@@ -288,12 +288,12 @@ test.serial(
|
||||
const result = await downloadDependencyCaches(
|
||||
codeql,
|
||||
createFeatures([]),
|
||||
[KnownLanguage.csharp],
|
||||
[BuiltInLanguage.csharp],
|
||||
logger,
|
||||
);
|
||||
const statusReport = result.statusReport;
|
||||
t.is(statusReport.length, 1);
|
||||
t.is(statusReport[0].language, KnownLanguage.csharp);
|
||||
t.is(statusReport[0].language, BuiltInLanguage.csharp);
|
||||
t.is(statusReport[0].hit_kind, CacheHitKind.Miss);
|
||||
t.deepEqual(result.restoredKeys, []);
|
||||
t.assert(restoreCacheStub.calledOnce);
|
||||
@@ -316,7 +316,7 @@ test.serial(
|
||||
const keyWithFeature = await cacheKey(
|
||||
codeql,
|
||||
features,
|
||||
KnownLanguage.csharp,
|
||||
BuiltInLanguage.csharp,
|
||||
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
|
||||
[],
|
||||
);
|
||||
@@ -334,14 +334,14 @@ test.serial(
|
||||
const result = await downloadDependencyCaches(
|
||||
codeql,
|
||||
features,
|
||||
[KnownLanguage.csharp],
|
||||
[BuiltInLanguage.csharp],
|
||||
logger,
|
||||
);
|
||||
|
||||
// Check that the status report for telemetry indicates that one cache was restored with an exact match.
|
||||
const statusReport = result.statusReport;
|
||||
t.is(statusReport.length, 1);
|
||||
t.is(statusReport[0].language, KnownLanguage.csharp);
|
||||
t.is(statusReport[0].language, BuiltInLanguage.csharp);
|
||||
t.is(statusReport[0].hit_kind, CacheHitKind.Exact);
|
||||
|
||||
// Check that the restored key has been returned.
|
||||
@@ -380,7 +380,7 @@ test.serial(
|
||||
const keyWithFeature = await cacheKey(
|
||||
codeql,
|
||||
features,
|
||||
KnownLanguage.csharp,
|
||||
BuiltInLanguage.csharp,
|
||||
// Patterns don't matter here because we have stubbed `hashFiles` to always return a specific hash above.
|
||||
[],
|
||||
);
|
||||
@@ -398,14 +398,14 @@ test.serial(
|
||||
const result = await downloadDependencyCaches(
|
||||
codeql,
|
||||
features,
|
||||
[KnownLanguage.csharp],
|
||||
[BuiltInLanguage.csharp],
|
||||
logger,
|
||||
);
|
||||
|
||||
// Check that the status report for telemetry indicates that one cache was restored with a partial match.
|
||||
const statusReport = result.statusReport;
|
||||
t.is(statusReport.length, 1);
|
||||
t.is(statusReport[0].language, KnownLanguage.csharp);
|
||||
t.is(statusReport[0].language, BuiltInLanguage.csharp);
|
||||
t.is(statusReport[0].hit_kind, CacheHitKind.Partial);
|
||||
|
||||
// Check that the restored key has been returned.
|
||||
@@ -426,7 +426,7 @@ test("uploadDependencyCaches - skips upload for a language with no cache config"
|
||||
const logger = getRecordingLogger(messages);
|
||||
const features = createFeatures([]);
|
||||
const config = createTestConfig({
|
||||
languages: [KnownLanguage.actions],
|
||||
languages: [BuiltInLanguage.actions],
|
||||
});
|
||||
|
||||
const result = await uploadDependencyCaches(codeql, features, config, logger);
|
||||
@@ -444,7 +444,7 @@ test.serial(
|
||||
const logger = getRecordingLogger(messages);
|
||||
const features = createFeatures([]);
|
||||
const config = createTestConfig({
|
||||
languages: [KnownLanguage.go],
|
||||
languages: [BuiltInLanguage.go],
|
||||
});
|
||||
|
||||
const makePatternCheckStub = sinon.stub(internal, "makePatternCheck");
|
||||
@@ -457,7 +457,7 @@ test.serial(
|
||||
logger,
|
||||
);
|
||||
t.is(result.length, 1);
|
||||
t.is(result[0].language, KnownLanguage.go);
|
||||
t.is(result[0].language, BuiltInLanguage.go);
|
||||
t.is(result[0].result, CacheStoreResult.NoHash);
|
||||
},
|
||||
);
|
||||
@@ -483,12 +483,12 @@ test.serial(
|
||||
const primaryCacheKey = await cacheKey(
|
||||
codeql,
|
||||
features,
|
||||
KnownLanguage.csharp,
|
||||
BuiltInLanguage.csharp,
|
||||
CSHARP_BASE_PATTERNS,
|
||||
);
|
||||
|
||||
const config = createTestConfig({
|
||||
languages: [KnownLanguage.csharp],
|
||||
languages: [BuiltInLanguage.csharp],
|
||||
dependencyCachingRestoredKeys: [primaryCacheKey],
|
||||
});
|
||||
|
||||
@@ -499,7 +499,7 @@ test.serial(
|
||||
logger,
|
||||
);
|
||||
t.is(result.length, 1);
|
||||
t.is(result[0].language, KnownLanguage.csharp);
|
||||
t.is(result[0].language, BuiltInLanguage.csharp);
|
||||
t.is(result[0].result, CacheStoreResult.Duplicate);
|
||||
},
|
||||
);
|
||||
@@ -525,7 +525,7 @@ test.serial(
|
||||
sinon.stub(cachingUtils, "getTotalCacheSize").resolves(0);
|
||||
|
||||
const config = createTestConfig({
|
||||
languages: [KnownLanguage.csharp],
|
||||
languages: [BuiltInLanguage.csharp],
|
||||
});
|
||||
|
||||
const result = await uploadDependencyCaches(
|
||||
@@ -535,7 +535,7 @@ test.serial(
|
||||
logger,
|
||||
);
|
||||
t.is(result.length, 1);
|
||||
t.is(result[0].language, KnownLanguage.csharp);
|
||||
t.is(result[0].language, BuiltInLanguage.csharp);
|
||||
t.is(result[0].result, CacheStoreResult.Empty);
|
||||
|
||||
checkExpectedLogMessages(t, messages, [
|
||||
@@ -566,7 +566,7 @@ test.serial(
|
||||
sinon.stub(actionsCache, "saveCache").resolves();
|
||||
|
||||
const config = createTestConfig({
|
||||
languages: [KnownLanguage.csharp],
|
||||
languages: [BuiltInLanguage.csharp],
|
||||
});
|
||||
|
||||
const result = await uploadDependencyCaches(
|
||||
@@ -576,7 +576,7 @@ test.serial(
|
||||
logger,
|
||||
);
|
||||
t.is(result.length, 1);
|
||||
t.is(result[0].language, KnownLanguage.csharp);
|
||||
t.is(result[0].language, BuiltInLanguage.csharp);
|
||||
t.is(result[0].result, CacheStoreResult.Stored);
|
||||
t.is(result[0].upload_size_bytes, 1024);
|
||||
|
||||
@@ -608,7 +608,7 @@ test.serial(
|
||||
.throws(new actionsCache.ReserveCacheError("Already in use"));
|
||||
|
||||
const config = createTestConfig({
|
||||
languages: [KnownLanguage.csharp],
|
||||
languages: [BuiltInLanguage.csharp],
|
||||
});
|
||||
|
||||
await t.notThrowsAsync(async () => {
|
||||
@@ -619,7 +619,7 @@ test.serial(
|
||||
logger,
|
||||
);
|
||||
t.is(result.length, 1);
|
||||
t.is(result[0].language, KnownLanguage.csharp);
|
||||
t.is(result[0].language, BuiltInLanguage.csharp);
|
||||
t.is(result[0].result, CacheStoreResult.Duplicate);
|
||||
|
||||
checkExpectedLogMessages(t, messages, ["Not uploading cache for"]);
|
||||
@@ -647,7 +647,7 @@ test.serial("uploadDependencyCaches - throws other exceptions", async (t) => {
|
||||
sinon.stub(actionsCache, "saveCache").throws();
|
||||
|
||||
const config = createTestConfig({
|
||||
languages: [KnownLanguage.csharp],
|
||||
languages: [BuiltInLanguage.csharp],
|
||||
});
|
||||
|
||||
await t.throwsAsync(async () => {
|
||||
@@ -659,7 +659,7 @@ test("getFeaturePrefix - returns empty string if no features are enabled", async
|
||||
const codeql = createStubCodeQL({});
|
||||
const features = createFeatures([]);
|
||||
|
||||
for (const knownLanguage of Object.values(KnownLanguage)) {
|
||||
for (const knownLanguage of Object.values(BuiltInLanguage)) {
|
||||
const result = await getFeaturePrefix(codeql, features, knownLanguage);
|
||||
t.deepEqual(result, "", `Expected no feature prefix for ${knownLanguage}`);
|
||||
}
|
||||
@@ -669,7 +669,11 @@ test("getFeaturePrefix - C# - returns prefix if CsharpNewCacheKey is enabled", a
|
||||
const codeql = createStubCodeQL({});
|
||||
const features = createFeatures([Feature.CsharpNewCacheKey]);
|
||||
|
||||
const result = await getFeaturePrefix(codeql, features, KnownLanguage.csharp);
|
||||
const result = await getFeaturePrefix(
|
||||
codeql,
|
||||
features,
|
||||
BuiltInLanguage.csharp,
|
||||
);
|
||||
t.notDeepEqual(result, "");
|
||||
t.assert(result.endsWith("-"));
|
||||
// Check the length of the prefix, which should correspond to `cacheKeyHashLength` + 1 for the trailing `-`.
|
||||
@@ -680,9 +684,9 @@ test("getFeaturePrefix - non-C# - returns '' if CsharpNewCacheKey is enabled", a
|
||||
const codeql = createStubCodeQL({});
|
||||
const features = createFeatures([Feature.CsharpNewCacheKey]);
|
||||
|
||||
for (const knownLanguage of Object.values(KnownLanguage)) {
|
||||
for (const knownLanguage of Object.values(BuiltInLanguage)) {
|
||||
// Skip C# since we expect a result for it, which is tested in the previous test.
|
||||
if (knownLanguage === KnownLanguage.csharp) {
|
||||
if (knownLanguage === BuiltInLanguage.csharp) {
|
||||
continue;
|
||||
}
|
||||
const result = await getFeaturePrefix(codeql, features, knownLanguage);
|
||||
@@ -694,7 +698,11 @@ test("getFeaturePrefix - C# - returns prefix if CsharpCacheBuildModeNone is enab
|
||||
const codeql = createStubCodeQL({});
|
||||
const features = createFeatures([Feature.CsharpCacheBuildModeNone]);
|
||||
|
||||
const result = await getFeaturePrefix(codeql, features, KnownLanguage.csharp);
|
||||
const result = await getFeaturePrefix(
|
||||
codeql,
|
||||
features,
|
||||
BuiltInLanguage.csharp,
|
||||
);
|
||||
t.notDeepEqual(result, "");
|
||||
t.assert(result.endsWith("-"));
|
||||
// Check the length of the prefix, which should correspond to `cacheKeyHashLength` + 1 for the trailing `-`.
|
||||
@@ -705,9 +713,9 @@ test("getFeaturePrefix - non-C# - returns '' if CsharpCacheBuildModeNone is enab
|
||||
const codeql = createStubCodeQL({});
|
||||
const features = createFeatures([Feature.CsharpCacheBuildModeNone]);
|
||||
|
||||
for (const knownLanguage of Object.values(KnownLanguage)) {
|
||||
for (const knownLanguage of Object.values(BuiltInLanguage)) {
|
||||
// Skip C# since we expect a result for it, which is tested in the previous test.
|
||||
if (knownLanguage === KnownLanguage.csharp) {
|
||||
if (knownLanguage === BuiltInLanguage.csharp) {
|
||||
continue;
|
||||
}
|
||||
const result = await getFeaturePrefix(codeql, features, knownLanguage);
|
||||
|
||||
@@ -11,7 +11,7 @@ import { CodeQL } from "./codeql";
|
||||
import { Config } from "./config-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { BuiltInLanguage, Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { getErrorMessage, getRequiredEnvParam } from "./util";
|
||||
|
||||
@@ -541,7 +541,7 @@ export async function getFeaturePrefix(
|
||||
}
|
||||
};
|
||||
|
||||
if (language === KnownLanguage.csharp) {
|
||||
if (language === BuiltInLanguage.csharp) {
|
||||
await addFeatureIfEnabled(Feature.CsharpNewCacheKey);
|
||||
await addFeatureIfEnabled(Feature.CsharpCacheBuildModeNone);
|
||||
}
|
||||
|
||||
+17
-2
@@ -72,6 +72,13 @@ let unwrittenDiagnostics: UnwrittenDiagnostic[] = [];
|
||||
*/
|
||||
let unwrittenDefaultLanguageDiagnostics: DiagnosticMessage[] = [];
|
||||
|
||||
/**
|
||||
* Counter used to generate a unique suffix for each diagnostic filename, so that
|
||||
* two diagnostics produced within the same millisecond do not overwrite each
|
||||
* other on disk.
|
||||
*/
|
||||
let diagnosticCounter = 0;
|
||||
|
||||
/**
|
||||
* Constructs a new diagnostic message with the specified id and name, as well as optional additional data.
|
||||
*
|
||||
@@ -167,10 +174,18 @@ function writeDiagnostic(
|
||||
// Create the directory if it doesn't exist yet.
|
||||
mkdirSync(diagnosticsPath, { recursive: true });
|
||||
|
||||
// Include a monotonically increasing suffix to avoid filename collisions
|
||||
// between diagnostics produced within the same millisecond.
|
||||
const uniqueSuffix = (diagnosticCounter++).toString();
|
||||
// We should only need to remove colons, but to be defensive, only allow a restricted set of
|
||||
// characters.
|
||||
const sanitizedTimestamp = diagnostic.timestamp.replace(
|
||||
/[^a-zA-Z0-9.-]/g,
|
||||
"",
|
||||
);
|
||||
const jsonPath = path.resolve(
|
||||
diagnosticsPath,
|
||||
// Remove colons from the timestamp as these are not allowed in Windows filenames.
|
||||
`codeql-action-${diagnostic.timestamp.replaceAll(":", "")}.json`,
|
||||
`codeql-action-${sanitizedTimestamp}-${uniqueSuffix}.json`,
|
||||
);
|
||||
|
||||
writeFileSync(jsonPath, JSON.stringify(diagnostic));
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import type { PullRequestBranches } from "./actions-util";
|
||||
@@ -77,16 +76,12 @@ export interface DiffThunkRange {
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
function getDiffRangesJsonFilePath(): string {
|
||||
return path.join(actionsUtil.getTemporaryDirectory(), "pr-diff-range.json");
|
||||
}
|
||||
|
||||
export function writeDiffRangesJsonFile(
|
||||
logger: Logger,
|
||||
ranges: DiffThunkRange[],
|
||||
): void {
|
||||
const jsonContents = JSON.stringify(ranges, null, 2);
|
||||
const jsonFilePath = getDiffRangesJsonFilePath();
|
||||
const jsonFilePath = actionsUtil.getDiffRangesJsonFilePath();
|
||||
fs.writeFileSync(jsonFilePath, jsonContents);
|
||||
logger.debug(
|
||||
`Wrote pr-diff-range JSON file to ${jsonFilePath}:\n${jsonContents}`,
|
||||
@@ -96,7 +91,7 @@ export function writeDiffRangesJsonFile(
|
||||
export function readDiffRangesJsonFile(
|
||||
logger: Logger,
|
||||
): DiffThunkRange[] | undefined {
|
||||
const jsonFilePath = getDiffRangesJsonFilePath();
|
||||
const jsonFilePath = actionsUtil.getDiffRangesJsonFilePath();
|
||||
if (!fs.existsSync(jsonFilePath)) {
|
||||
logger.debug(`Diff ranges JSON file does not exist at ${jsonFilePath}`);
|
||||
return undefined;
|
||||
@@ -105,7 +100,14 @@ export function readDiffRangesJsonFile(
|
||||
logger.debug(
|
||||
`Read pr-diff-range JSON file from ${jsonFilePath}:\n${jsonContents}`,
|
||||
);
|
||||
return JSON.parse(jsonContents) as DiffThunkRange[];
|
||||
try {
|
||||
return JSON.parse(jsonContents) as DiffThunkRange[];
|
||||
} catch (e) {
|
||||
logger.warning(
|
||||
`Failed to parse diff ranges JSON file at ${jsonFilePath}: ${e}`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -8,6 +8,7 @@ export enum DocUrl {
|
||||
CODEQL_BUILD_MODES = "https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#codeql-build-modes",
|
||||
DEFINE_ENV_VARIABLES = "https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow",
|
||||
DELETE_ACTIONS_CACHE_ENTRIES = "https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-caches#deleting-cache-entries",
|
||||
PRIVATE_REGISTRY_LOGS = "https://docs.github.com/en/code-security/reference/code-scanning/code-scanning-logs#diagnostic-information-for-private-package-registries",
|
||||
SCANNING_ON_PUSH = "https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#scanning-on-push",
|
||||
SPECIFY_BUILD_STEPS_MANUALLY = "https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#about-specifying-build-steps-manually",
|
||||
SYSTEM_REQUIREMENTS = "https://codeql.github.com/docs/codeql-overview/system-requirements/",
|
||||
|
||||
@@ -85,7 +85,6 @@ export enum Feature {
|
||||
OverlayAnalysisStatusCheck = "overlay_analysis_status_check",
|
||||
/** Controls whether overlay build failures on the default branch are stored in the Actions cache. */
|
||||
OverlayAnalysisStatusSave = "overlay_analysis_status_save",
|
||||
PythonDefaultIsToNotExtractStdlib = "python_default_is_to_not_extract_stdlib",
|
||||
QaTelemetryEnabled = "qa_telemetry_enabled",
|
||||
/** Note that this currently only disables baseline file coverage information. */
|
||||
SkipFileCoverageOnPrs = "skip_file_coverage_on_prs",
|
||||
@@ -298,12 +297,6 @@ export const featureConfig = {
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_SKIP_RESOURCE_CHECKS",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.PythonDefaultIsToNotExtractStdlib]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_DISABLE_PYTHON_STANDARD_LIBRARY_EXTRACTION",
|
||||
minimumVersion: undefined,
|
||||
toolsFeature: ToolsFeature.PythonDefaultIsToNotExtractStdlib,
|
||||
},
|
||||
[Feature.QaTelemetryEnabled]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_QA_TELEMETRY",
|
||||
|
||||
+118
-51
@@ -343,75 +343,142 @@ test.serial("decodeGitFilePath quoted strings", async (t) => {
|
||||
);
|
||||
});
|
||||
|
||||
test.serial("getFileOidsUnderPath returns correct file mapping", async (t) => {
|
||||
const runGitCommandStub = sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.resolves(
|
||||
"30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
|
||||
"d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map\n" +
|
||||
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
|
||||
);
|
||||
test.serial(
|
||||
"getFileOidsUnderPath uses --recurse-submodules when submodules exist",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
fs.writeFileSync(path.join(tmpDir, ".gitmodules"), "");
|
||||
const runGitCommandStub = sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.callsFake(async (_cwd: any, args: any) => {
|
||||
if (args[0] === "rev-parse") {
|
||||
return `${tmpDir}\n`;
|
||||
}
|
||||
return (
|
||||
"100644 30d998ded095371488be3a729eb61d86ed721a18 0\tlib/git-utils.js\n" +
|
||||
"100644 d89514599a9a99f22b4085766d40af7b99974827 0\tlib/git-utils.js.map\n" +
|
||||
"100644 a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96 0\tsrc/git-utils.ts"
|
||||
);
|
||||
});
|
||||
|
||||
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
|
||||
t.deepEqual(result, {
|
||||
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
|
||||
"lib/git-utils.js.map": "d89514599a9a99f22b4085766d40af7b99974827",
|
||||
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
|
||||
});
|
||||
t.deepEqual(result, {
|
||||
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
|
||||
"lib/git-utils.js.map": "d89514599a9a99f22b4085766d40af7b99974827",
|
||||
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
|
||||
});
|
||||
|
||||
t.deepEqual(runGitCommandStub.firstCall.args, [
|
||||
"/fake/path",
|
||||
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
|
||||
"Cannot list Git OIDs of tracked files.",
|
||||
]);
|
||||
});
|
||||
// Second call (after getGitRoot) should include --recurse-submodules
|
||||
t.deepEqual(runGitCommandStub.secondCall.args[1], [
|
||||
"ls-files",
|
||||
"--recurse-submodules",
|
||||
"--stage",
|
||||
]);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"getFileOidsUnderPath omits --recurse-submodules when no submodules exist",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const runGitCommandStub = sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.callsFake(async (_cwd: any, args: any) => {
|
||||
if (args[0] === "rev-parse") {
|
||||
return `${tmpDir}\n`;
|
||||
}
|
||||
return (
|
||||
"100644 30d998ded095371488be3a729eb61d86ed721a18 0\tlib/git-utils.js\n" +
|
||||
"100644 a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96 0\tsrc/git-utils.ts"
|
||||
);
|
||||
});
|
||||
|
||||
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
|
||||
t.deepEqual(result, {
|
||||
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
|
||||
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
|
||||
});
|
||||
|
||||
// Second call (after getGitRoot) should NOT include --recurse-submodules
|
||||
t.deepEqual(runGitCommandStub.secondCall.args[1], [
|
||||
"ls-files",
|
||||
"--stage",
|
||||
]);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial("getFileOidsUnderPath handles quoted paths", async (t) => {
|
||||
sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.resolves(
|
||||
"30d998ded095371488be3a729eb61d86ed721a18_lib/normal-file.js\n" +
|
||||
'd89514599a9a99f22b4085766d40af7b99974827_"lib/file with spaces.js"\n' +
|
||||
'a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_"lib/file\\twith\\ttabs.js"',
|
||||
);
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.callsFake(async (_cwd: any, args: any) => {
|
||||
if (args[0] === "rev-parse") {
|
||||
return `${tmpDir}\n`;
|
||||
}
|
||||
return (
|
||||
"100644 30d998ded095371488be3a729eb61d86ed721a18 0\tlib/normal-file.js\n" +
|
||||
'100644 d89514599a9a99f22b4085766d40af7b99974827 0\t"lib/file with spaces.js"\n' +
|
||||
'100644 a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96 0\t"lib/file\\twith\\ttabs.js"'
|
||||
);
|
||||
});
|
||||
|
||||
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
|
||||
t.deepEqual(result, {
|
||||
"lib/normal-file.js": "30d998ded095371488be3a729eb61d86ed721a18",
|
||||
"lib/file with spaces.js": "d89514599a9a99f22b4085766d40af7b99974827",
|
||||
"lib/file\twith\ttabs.js": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
|
||||
t.deepEqual(result, {
|
||||
"lib/normal-file.js": "30d998ded095371488be3a729eb61d86ed721a18",
|
||||
"lib/file with spaces.js": "d89514599a9a99f22b4085766d40af7b99974827",
|
||||
"lib/file\twith\ttabs.js": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test.serial("getFileOidsUnderPath handles empty output", async (t) => {
|
||||
sinon.stub(gitUtils as any, "runGitCommand").resolves("");
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.callsFake(async (_cwd: any, args: any) => {
|
||||
if (args[0] === "rev-parse") {
|
||||
return `${tmpDir}\n`;
|
||||
}
|
||||
return "";
|
||||
});
|
||||
|
||||
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
t.deepEqual(result, {});
|
||||
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
t.deepEqual(result, {});
|
||||
});
|
||||
});
|
||||
|
||||
test.serial(
|
||||
"getFileOidsUnderPath throws on unexpected output format",
|
||||
async (t) => {
|
||||
sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.resolves(
|
||||
"30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
|
||||
"invalid-line-format\n" +
|
||||
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
|
||||
);
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
sinon
|
||||
.stub(gitUtils as any, "runGitCommand")
|
||||
.callsFake(async (_cwd: any, args: any) => {
|
||||
if (args[0] === "rev-parse") {
|
||||
return `${tmpDir}\n`;
|
||||
}
|
||||
return (
|
||||
"100644 30d998ded095371488be3a729eb61d86ed721a18 0\tlib/git-utils.js\n" +
|
||||
"invalid-line-format\n" +
|
||||
"100644 a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96 0\tsrc/git-utils.ts"
|
||||
);
|
||||
});
|
||||
|
||||
await t.throwsAsync(
|
||||
async () => {
|
||||
await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
},
|
||||
{
|
||||
instanceOf: Error,
|
||||
message: 'Unexpected "git ls-files" output: invalid-line-format',
|
||||
},
|
||||
);
|
||||
await t.throwsAsync(
|
||||
async () => {
|
||||
await gitUtils.getFileOidsUnderPath("/fake/path");
|
||||
},
|
||||
{
|
||||
instanceOf: Error,
|
||||
message: 'Unexpected "git ls-files" output: invalid-line-format',
|
||||
},
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
+38
-11
@@ -1,4 +1,6 @@
|
||||
import * as fs from "fs";
|
||||
import * as os from "os";
|
||||
import * as path from "path";
|
||||
|
||||
import * as core from "@actions/core";
|
||||
import { ExecOptions } from "@actions/exec";
|
||||
@@ -14,10 +16,11 @@ import {
|
||||
import { ConfigurationError, getRequiredEnvParam } from "./util";
|
||||
|
||||
/**
|
||||
* Minimum Git version required for overlay analysis. The `git ls-files --format`
|
||||
* option, which is used by `getFileOidsUnderPath`, was introduced in Git 2.38.0.
|
||||
* Minimum Git version required for overlay analysis in repositories that
|
||||
* contain submodules. Support for using the `git ls-files
|
||||
* --recurse-submodules` option with `--stage` was added in Git 2.36.0.
|
||||
*/
|
||||
export const GIT_MINIMUM_VERSION_FOR_OVERLAY = "2.38.0";
|
||||
export const GIT_MINIMUM_VERSION_FOR_OVERLAY_WITH_SUBMODULES = "2.36.0";
|
||||
|
||||
/**
|
||||
* Git version information
|
||||
@@ -244,6 +247,16 @@ export const getGitRoot = async function (
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns true if the Git repository has submodules registered (i.e. a
|
||||
* `.gitmodules` file exists at the repository root).
|
||||
*
|
||||
* @param gitRoot The root of the Git repository.
|
||||
*/
|
||||
export function hasSubmodules(gitRoot: string): boolean {
|
||||
return fs.existsSync(path.join(gitRoot, ".gitmodules"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Git OIDs of all tracked files (in the index and in the working
|
||||
* tree) that are under the given base path, including files in active
|
||||
@@ -252,31 +265,45 @@ export const getGitRoot = async function (
|
||||
*
|
||||
* @param basePath A path into the Git repository.
|
||||
* @returns a map from file paths (relative to `basePath`) to Git OIDs.
|
||||
* @throws {Error} if "git ls-tree" produces unexpected output.
|
||||
* @throws {Error} if "git ls-files" produces unexpected output.
|
||||
*/
|
||||
export const getFileOidsUnderPath = async function (
|
||||
basePath: string,
|
||||
): Promise<{ [key: string]: string }> {
|
||||
// Without the --full-name flag, the path is relative to the current working
|
||||
// directory of the git command, which is basePath.
|
||||
//
|
||||
// We use --stage rather than --format here because --format was only
|
||||
// introduced in Git 2.38.0, which would limit overlay rollout.
|
||||
//
|
||||
// We only pass --recurse-submodules when the repository actually has
|
||||
// submodules, because the combination of --recurse-submodules and --stage is
|
||||
// only supported since Git 2.36.0.
|
||||
const gitRoot = await getGitRoot(basePath);
|
||||
const mayHaveSubmodules =
|
||||
gitRoot === undefined ? true : hasSubmodules(gitRoot);
|
||||
const args = mayHaveSubmodules
|
||||
? ["ls-files", "--recurse-submodules", "--stage"]
|
||||
: ["ls-files", "--stage"];
|
||||
const stdout = await runGitCommand(
|
||||
basePath,
|
||||
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
|
||||
args,
|
||||
"Cannot list Git OIDs of tracked files.",
|
||||
);
|
||||
|
||||
const fileOidMap: { [key: string]: string } = {};
|
||||
// With --format=%(objectname)_%(path), the output is a list of lines like:
|
||||
// 30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js
|
||||
// d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map
|
||||
const regex = /^([0-9a-f]{40})_(.+)$/;
|
||||
// With --stage, the output is a list of lines like:
|
||||
// 100644 4c51bc1d9e86cd86e01b0f340cb8ce095c33b283 0\tsrc/git-utils.test.ts
|
||||
// 100644 6b792ea543ce75d7a8a03df591e3c85311ecb64f 0\tsrc/git-utils.ts
|
||||
// The fields are: <mode> <oid> <stage>\t<path>
|
||||
const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
|
||||
for (const line of stdout.split("\n")) {
|
||||
if (line) {
|
||||
const match = line.match(regex);
|
||||
if (match) {
|
||||
const oid = match[1];
|
||||
const path = decodeGitFilePath(match[2]);
|
||||
fileOidMap[path] = oid;
|
||||
const filePath = decodeGitFilePath(match[2]);
|
||||
fileOidMap[filePath] = oid;
|
||||
} else {
|
||||
throw new Error(`Unexpected "git ls-files" output: ${line}`);
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import { EnvVar } from "./environment";
|
||||
import { Feature } from "./feature-flags";
|
||||
import * as initActionPostHelper from "./init-action-post-helper";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import * as overlayStatus from "./overlay/status";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import {
|
||||
|
||||
@@ -21,7 +21,7 @@ import * as dependencyCaching from "./dependency-caching";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { Logger } from "./logging";
|
||||
import { OverlayDatabaseMode } from "./overlay";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import {
|
||||
createOverlayStatus,
|
||||
OverlayStatus,
|
||||
|
||||
+75
-49
@@ -37,6 +37,11 @@ import {
|
||||
makeDiagnostic,
|
||||
makeTelemetryDiagnostic,
|
||||
} from "./diagnostics";
|
||||
import {
|
||||
getDiffInformedAnalysisBranches,
|
||||
getPullRequestEditedDiffRanges,
|
||||
writeDiffRangesJsonFile,
|
||||
} from "./diff-informed-analysis-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { Feature, FeatureEnablement, initFeatures } from "./feature-flags";
|
||||
import {
|
||||
@@ -53,13 +58,13 @@ import {
|
||||
initConfig,
|
||||
runDatabaseInitCluster,
|
||||
} from "./init";
|
||||
import { JavaEnvVars, KnownLanguage } from "./languages";
|
||||
import { getActionsLogger, Logger } from "./logging";
|
||||
import { JavaEnvVars, BuiltInLanguage } from "./languages";
|
||||
import { getActionsLogger, Logger, withGroupAsync } from "./logging";
|
||||
import {
|
||||
downloadOverlayBaseDatabaseFromCache,
|
||||
OverlayBaseDatabaseDownloadStats,
|
||||
OverlayDatabaseMode,
|
||||
} from "./overlay";
|
||||
} from "./overlay/caching";
|
||||
import { OverlayDatabaseMode } from "./overlay/overlay-database-mode";
|
||||
import { getRepositoryNwo, RepositoryNwo } from "./repository";
|
||||
import { ToolsSource } from "./setup-codeql";
|
||||
import {
|
||||
@@ -325,7 +330,7 @@ async function run(startedAt: Date) {
|
||||
// requested rust - don't enable it via language autodetection.
|
||||
configUtils
|
||||
.getRawLanguagesNoAutodetect(getOptionalInput("languages"))
|
||||
.includes(KnownLanguage.rust)
|
||||
.includes(BuiltInLanguage.rust)
|
||||
) {
|
||||
const experimental = "2.19.3";
|
||||
const publicPreview = "2.22.1";
|
||||
@@ -384,6 +389,15 @@ async function run(startedAt: Date) {
|
||||
logger,
|
||||
});
|
||||
|
||||
if (
|
||||
config.languages.includes(BuiltInLanguage.swift) &&
|
||||
process.platform !== "darwin"
|
||||
) {
|
||||
throw new ConfigurationError(
|
||||
`Swift analysis is only supported on macOS runner images. Please migrate to a macOS runner.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (repositoryPropertiesResult.isFailure()) {
|
||||
addNoLanguageDiagnostic(
|
||||
config,
|
||||
@@ -413,6 +427,7 @@ async function run(startedAt: Date) {
|
||||
}
|
||||
|
||||
await checkInstallPython311(config.languages, codeql);
|
||||
await computeAndPersistDiffRanges(codeql, features, logger);
|
||||
} catch (unwrappedError) {
|
||||
const error = wrapError(unwrappedError);
|
||||
core.setFailed(error.message);
|
||||
@@ -450,18 +465,23 @@ async function run(startedAt: Date) {
|
||||
// necessary preparations. So, in that mode, we would assume that
|
||||
// everything is in order and let the analysis fail if that turns out not
|
||||
// to be the case.
|
||||
overlayBaseDatabaseStats = await downloadOverlayBaseDatabaseFromCache(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
await withGroupAsync(
|
||||
"Checking cache for overlay-base database",
|
||||
async () => {
|
||||
overlayBaseDatabaseStats = await downloadOverlayBaseDatabaseFromCache(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
);
|
||||
if (!overlayBaseDatabaseStats) {
|
||||
config.overlayDatabaseMode = OverlayDatabaseMode.None;
|
||||
logger.info(
|
||||
"No overlay-base database found in cache, " +
|
||||
`reverting overlay database mode to ${OverlayDatabaseMode.None}.`,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
if (!overlayBaseDatabaseStats) {
|
||||
config.overlayDatabaseMode = OverlayDatabaseMode.None;
|
||||
logger.info(
|
||||
"No overlay-base database found in cache, " +
|
||||
`reverting overlay database mode to ${OverlayDatabaseMode.None}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (config.overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
|
||||
@@ -494,16 +514,7 @@ async function run(startedAt: Date) {
|
||||
}
|
||||
|
||||
if (
|
||||
config.languages.includes(KnownLanguage.swift) &&
|
||||
process.platform === "linux"
|
||||
) {
|
||||
logger.warning(
|
||||
`Swift analysis on Ubuntu runner images is no longer supported. Please migrate to a macOS runner if this affects you.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
config.languages.includes(KnownLanguage.go) &&
|
||||
config.languages.includes(BuiltInLanguage.go) &&
|
||||
process.platform === "linux"
|
||||
) {
|
||||
try {
|
||||
@@ -561,7 +572,7 @@ async function run(startedAt: Date) {
|
||||
if (e instanceof FileCmdNotFoundError) {
|
||||
addDiagnostic(
|
||||
config,
|
||||
KnownLanguage.go,
|
||||
BuiltInLanguage.go,
|
||||
makeDiagnostic(
|
||||
"go/workflow/file-program-unavailable",
|
||||
"The `file` program is required on Linux, but does not appear to be installed",
|
||||
@@ -639,27 +650,6 @@ async function run(startedAt: Date) {
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
await codeql.supportsFeature(
|
||||
ToolsFeature.PythonDefaultIsToNotExtractStdlib,
|
||||
)
|
||||
) {
|
||||
if (process.env["CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB"]) {
|
||||
logger.debug(
|
||||
"CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB is already set, so the Action will not override it.",
|
||||
);
|
||||
} else if (
|
||||
!(await features.getValue(
|
||||
Feature.PythonDefaultIsToNotExtractStdlib,
|
||||
codeql,
|
||||
))
|
||||
) {
|
||||
// We are in a situation where the feature flag is not rolled out,
|
||||
// so we need to suppress the new default CLI behavior.
|
||||
core.exportVariable("CODEQL_EXTRACTOR_PYTHON_EXTRACT_STDLIB", "true");
|
||||
}
|
||||
}
|
||||
|
||||
// If we are doing a Java `build-mode: none` analysis, then set the environment variable that
|
||||
// enables the option in the Java extractor to minimize dependency jars. We also only do this if
|
||||
// dependency caching is enabled, since the option is intended to reduce the size of dependency
|
||||
@@ -676,7 +666,7 @@ async function run(startedAt: Date) {
|
||||
(await codeQlVersionAtLeast(codeql, CODEQL_VERSION_JAR_MINIMIZATION)) &&
|
||||
config.dependencyCachingEnabled &&
|
||||
config.buildMode === BuildMode.None &&
|
||||
config.languages.includes(KnownLanguage.java)
|
||||
config.languages.includes(BuiltInLanguage.java)
|
||||
) {
|
||||
core.exportVariable(
|
||||
EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS,
|
||||
@@ -833,6 +823,42 @@ async function loadRepositoryProperties(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute and persist diff ranges when diff-informed analysis is enabled
|
||||
* (feature flag + PR context). This writes the standard pr-diff-range.json
|
||||
* file for later reuse in the analyze step. Failures are logged but non-fatal.
|
||||
*/
|
||||
async function computeAndPersistDiffRanges(
|
||||
codeql: CodeQL,
|
||||
features: FeatureEnablement,
|
||||
logger: Logger,
|
||||
): Promise<void> {
|
||||
await withGroupAsync("Computing PR diff ranges", async () => {
|
||||
try {
|
||||
const branches = await getDiffInformedAnalysisBranches(
|
||||
codeql,
|
||||
features,
|
||||
logger,
|
||||
);
|
||||
if (!branches) {
|
||||
return;
|
||||
}
|
||||
const ranges = await getPullRequestEditedDiffRanges(branches, logger);
|
||||
if (ranges === undefined) {
|
||||
return;
|
||||
}
|
||||
writeDiffRangesJsonFile(logger, ranges);
|
||||
const distinctFiles = new Set(ranges.map((r) => r.path)).size;
|
||||
logger.info(
|
||||
`Persisted ${ranges.length} diff range(s) across ${distinctFiles} file(s).`,
|
||||
);
|
||||
} catch (e) {
|
||||
logger.warning(
|
||||
`Failed to compute and persist PR diff ranges: ${getErrorMessage(e)}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
async function recordZstdAvailability(
|
||||
config: configUtils.Config,
|
||||
zstdAvailability: ZstdAvailability,
|
||||
|
||||
+34
-34
@@ -15,7 +15,7 @@ import {
|
||||
getFileCoverageInformationEnabled,
|
||||
logFileCoverageOnPrsDeprecationWarning,
|
||||
} from "./init";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import {
|
||||
createFeatures,
|
||||
LoggedMessage,
|
||||
@@ -152,7 +152,7 @@ test("cleanupDatabaseClusterDirectory can disable warning with options", async (
|
||||
});
|
||||
|
||||
type PackInfo = {
|
||||
language: KnownLanguage;
|
||||
language: BuiltInLanguage;
|
||||
packinfoContents: string | undefined;
|
||||
sourceOnlyPack?: boolean;
|
||||
qlpackFileName?: string;
|
||||
@@ -169,13 +169,13 @@ const testCheckPacksForOverlayCompatibility = test.macro({
|
||||
expectedResult,
|
||||
}: {
|
||||
cliOverlayVersion: number | undefined;
|
||||
languages: KnownLanguage[];
|
||||
languages: BuiltInLanguage[];
|
||||
packs: Record<string, PackInfo>;
|
||||
expectedResult: boolean;
|
||||
},
|
||||
) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const packDirsByLanguage = new Map<KnownLanguage, string[]>();
|
||||
const packDirsByLanguage = new Map<BuiltInLanguage, string[]>();
|
||||
|
||||
for (const [packName, packInfo] of Object.entries(packs)) {
|
||||
const packPath = path.join(tmpDir, packName);
|
||||
@@ -242,10 +242,10 @@ test(
|
||||
"returns false when CLI does not support overlay",
|
||||
{
|
||||
cliOverlayVersion: undefined,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
},
|
||||
@@ -258,7 +258,7 @@ test(
|
||||
"returns true when there are no query packs",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {},
|
||||
expectedResult: true,
|
||||
},
|
||||
@@ -269,10 +269,10 @@ test(
|
||||
"returns true when query pack has not been compiled",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: undefined,
|
||||
sourceOnlyPack: true,
|
||||
},
|
||||
@@ -286,10 +286,10 @@ test(
|
||||
"returns true when query pack has expected overlay version",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
},
|
||||
@@ -302,14 +302,14 @@ test(
|
||||
"returns true when query packs for all languages to analyze are compatible",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.cpp, KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.cpp, BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/cpp-queries": {
|
||||
language: KnownLanguage.cpp,
|
||||
language: BuiltInLanguage.cpp,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
},
|
||||
@@ -322,14 +322,14 @@ test(
|
||||
"returns true when query pack for a language not analyzed is incompatible",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/cpp-queries": {
|
||||
language: KnownLanguage.cpp,
|
||||
language: BuiltInLanguage.cpp,
|
||||
packinfoContents: undefined,
|
||||
},
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
},
|
||||
@@ -342,14 +342,14 @@ test(
|
||||
"returns false when query pack for a language to analyze is incompatible",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.cpp, KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.cpp, BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/cpp-queries": {
|
||||
language: KnownLanguage.cpp,
|
||||
language: BuiltInLanguage.cpp,
|
||||
packinfoContents: '{"overlayVersion":1}',
|
||||
},
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
},
|
||||
@@ -362,14 +362,14 @@ test(
|
||||
"returns false when query pack is missing .packinfo",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
"custom/queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: undefined,
|
||||
},
|
||||
},
|
||||
@@ -382,14 +382,14 @@ test(
|
||||
"returns false when query pack has different overlay version",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
"custom/queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":1}',
|
||||
},
|
||||
},
|
||||
@@ -402,14 +402,14 @@ test(
|
||||
"returns false when query pack is missing overlayVersion in .packinfo",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
"custom/queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: "{}",
|
||||
},
|
||||
},
|
||||
@@ -422,14 +422,14 @@ test(
|
||||
"returns false when .packinfo is not valid JSON",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
},
|
||||
"custom/queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: "this_is_not_valid_json",
|
||||
},
|
||||
},
|
||||
@@ -442,10 +442,10 @@ test(
|
||||
"returns true when query pack uses codeql-pack.yml filename",
|
||||
{
|
||||
cliOverlayVersion: 2,
|
||||
languages: [KnownLanguage.java],
|
||||
languages: [BuiltInLanguage.java],
|
||||
packs: {
|
||||
"codeql/java-queries": {
|
||||
language: KnownLanguage.java,
|
||||
language: BuiltInLanguage.java,
|
||||
packinfoContents: '{"overlayVersion":2}',
|
||||
qlpackFileName: "codeql-pack.yml",
|
||||
},
|
||||
|
||||
+2
-2
@@ -26,7 +26,7 @@ import {
|
||||
RepositoryProperties,
|
||||
RepositoryPropertyName,
|
||||
} from "./feature-flags/properties";
|
||||
import { KnownLanguage, Language } from "./languages";
|
||||
import { BuiltInLanguage, Language } from "./languages";
|
||||
import { Logger, withGroupAsync } from "./logging";
|
||||
import { ToolsSource } from "./setup-codeql";
|
||||
import { ZstdAvailability } from "./tar";
|
||||
@@ -235,7 +235,7 @@ export async function checkInstallPython311(
|
||||
codeql: CodeQL,
|
||||
) {
|
||||
if (
|
||||
languages.includes(KnownLanguage.python) &&
|
||||
languages.includes(BuiltInLanguage.python) &&
|
||||
process.platform === "win32" &&
|
||||
!(await codeql.getVersion()).features?.supportsPython312
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import test from "ava";
|
||||
|
||||
import { setupTests } from "../testing-utils";
|
||||
|
||||
import * as json from ".";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
const testSchema = {
|
||||
requiredKey: json.string,
|
||||
};
|
||||
|
||||
const optionalSchema = {
|
||||
optionalKey: json.optional(json.string),
|
||||
};
|
||||
|
||||
test("validateSchema - required properties are required", async (t) => {
|
||||
t.false(json.validateSchema(testSchema, {}));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: undefined }));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: null }));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: 0 }));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: 123 }));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: false }));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: true }));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: [] }));
|
||||
t.false(json.validateSchema(testSchema, { requiredKey: {} }));
|
||||
t.true(json.validateSchema(testSchema, { requiredKey: "" }));
|
||||
t.true(json.validateSchema(testSchema, { requiredKey: "foo" }));
|
||||
});
|
||||
|
||||
test("validateSchema - optional properties are optional", async (t) => {
|
||||
// Optional fields may be absent
|
||||
t.true(json.validateSchema(optionalSchema, {}));
|
||||
t.true(json.validateSchema(optionalSchema, { optionalKey: undefined }));
|
||||
t.true(json.validateSchema(optionalSchema, { optionalKey: null }));
|
||||
|
||||
// But, if present, should have the expected type
|
||||
t.false(json.validateSchema(optionalSchema, { optionalKey: 0 }));
|
||||
t.false(json.validateSchema(optionalSchema, { optionalKey: 123 }));
|
||||
t.false(json.validateSchema(optionalSchema, { optionalKey: false }));
|
||||
t.false(json.validateSchema(optionalSchema, { optionalKey: true }));
|
||||
t.false(json.validateSchema(optionalSchema, { optionalKey: [] }));
|
||||
t.false(json.validateSchema(optionalSchema, { optionalKey: {} }));
|
||||
t.true(json.validateSchema(optionalSchema, { optionalKey: "" }));
|
||||
t.true(json.validateSchema(optionalSchema, { optionalKey: "foo" }));
|
||||
});
|
||||
@@ -36,3 +36,82 @@ export function isStringOrUndefined(
|
||||
): value is string | undefined {
|
||||
return value === undefined || isString(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a field of type `T` in a schema.
|
||||
* Carries a validation function and flag indicating whether the field is required or not.
|
||||
*/
|
||||
export type Validator<T> = {
|
||||
validate: (val: unknown) => val is T;
|
||||
required: boolean;
|
||||
};
|
||||
|
||||
/** Extracts `T` from `Validator<T>`. */
|
||||
export type UnwrapValidator<V> = V extends Validator<infer A> ? A : never;
|
||||
|
||||
/** A validator for string fields in schemas. */
|
||||
export const string = {
|
||||
validate: isString,
|
||||
required: true,
|
||||
} as const satisfies Validator<string>;
|
||||
|
||||
/** Transforms a validator to be optional. */
|
||||
export function optional<T>(validator: Validator<T>) {
|
||||
return {
|
||||
validate: (val: unknown) => {
|
||||
return val === undefined || val === null || validator.validate(val);
|
||||
},
|
||||
required: false,
|
||||
} as const satisfies Validator<T | undefined | null>;
|
||||
}
|
||||
|
||||
/** Represents an arbitrary object schema. */
|
||||
export type Schema = Record<string, Validator<any>>;
|
||||
|
||||
/** Extracts the required keys from `S`. */
|
||||
export type RequiredKeys<S extends Schema> = {
|
||||
[K in keyof S]: S[K]["required"] extends true ? K : never;
|
||||
}[keyof S];
|
||||
|
||||
/** Extracts optional keys from `S`. */
|
||||
export type OptionalKeys<S extends Schema> = {
|
||||
[K in keyof S]: S[K]["required"] extends true ? never : K;
|
||||
}[keyof S];
|
||||
|
||||
/** Constructs an object type corresponding to a schema. */
|
||||
export type FromSchema<S extends Schema> = {
|
||||
[K in RequiredKeys<S>]: UnwrapValidator<S[K]>;
|
||||
} & { [K in OptionalKeys<S>]?: UnwrapValidator<S[K]> };
|
||||
|
||||
/**
|
||||
* Validates that `obj` satisfies at least `schema`. Additional keys are accepted.
|
||||
*
|
||||
* @param schema The schema to validate against.
|
||||
* @param obj The object to validate.
|
||||
* @returns Asserts that `obj` is of the `schema`'s type if validation is successful.
|
||||
*/
|
||||
export function validateSchema<S extends Schema>(
|
||||
schema: S,
|
||||
obj: UnvalidatedObject<any>,
|
||||
): obj is FromSchema<S> {
|
||||
for (const [key, validator] of Object.entries(schema)) {
|
||||
const hasKey = key in obj;
|
||||
|
||||
// If the property is required, but absent, fail.
|
||||
if (validator.required && !hasKey) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If the property is required, but undefined or null, fail.
|
||||
if (validator.required && (obj[key] === undefined || obj[key] === null)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// If the property is present, validate it.
|
||||
if (hasKey && !validator.validate(obj[key])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { ExecutionContext } from "ava";
|
||||
|
||||
import * as json from ".";
|
||||
|
||||
/**
|
||||
* Constructs an object based on `schema` for unit tests.
|
||||
* Assumes that all keys in `schema` have string values.
|
||||
*
|
||||
* @param includeOptional Whether to include optional properties.
|
||||
* @param schema The schema to base the object on.
|
||||
* @returns An object that satisfies `schema`.
|
||||
*/
|
||||
export function makeFromSchema<S extends json.Schema>(
|
||||
includeOptional: boolean,
|
||||
schema: S,
|
||||
): json.FromSchema<S> {
|
||||
const result = {};
|
||||
for (const [key, validator] of Object.entries(schema)) {
|
||||
if (!validator.required && !includeOptional) {
|
||||
continue;
|
||||
}
|
||||
result[key] = `value-for-${key}`;
|
||||
}
|
||||
return result as json.FromSchema<S>;
|
||||
}
|
||||
|
||||
/** Options for `withSchemaMatrix`. */
|
||||
export interface SchemaMatrixOptions {
|
||||
/** Whether cases where the properties are entirely absent should be excluded. */
|
||||
excludeAbsent?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a test matrix of possible objects for `schema`: all required properties
|
||||
* plus all permutations of possible states for the optional properties.
|
||||
*
|
||||
* @param schema The schema to construct a test matrix for.
|
||||
* @param body The test body to call with each value from the test matrix.
|
||||
*/
|
||||
export function withSchemaMatrix<S extends json.Schema>(
|
||||
t: ExecutionContext<any>,
|
||||
schema: S,
|
||||
opts: SchemaMatrixOptions,
|
||||
body: (value: json.FromSchema<S>) => void,
|
||||
): void {
|
||||
// Construct a base object that includes all required properties.
|
||||
const required = makeFromSchema(false, schema);
|
||||
|
||||
// Identify optional properties.
|
||||
const optionalKeys: Array<keyof S> = [];
|
||||
|
||||
for (const [key, validator] of Object.entries(schema)) {
|
||||
if (!validator.required) {
|
||||
optionalKeys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
const optionalValues = (key: keyof S) => [
|
||||
null,
|
||||
undefined,
|
||||
`value-for-${String(key)}`,
|
||||
];
|
||||
|
||||
// Constructs an array of test objects, starting with `required` and combining it with all
|
||||
// possible states of each optional property. For example, with default settings:
|
||||
//
|
||||
// For { requiredKey: string }, we get: `[{ requiredKey: "some-string-value" }]`
|
||||
//
|
||||
// For { requiredKey: string, optionalKey?: string }, we get:
|
||||
// [ { requiredKey: "some-string-value" },
|
||||
// { requiredKey: "some-string-value", optionalKey: undefined },
|
||||
// { requiredKey: "some-string-value", optionalKey: null },
|
||||
// { requiredKey: "some-string-value", optionalKey: "some-value" },
|
||||
// ]
|
||||
const permutations = (keys: Array<keyof S>) => {
|
||||
if (keys.length === 0) return [required];
|
||||
|
||||
const bases = permutations(keys.slice(1));
|
||||
const result: Array<json.FromSchema<S>> = [];
|
||||
|
||||
const optionalKey = keys[0];
|
||||
for (const base of bases) {
|
||||
if (!opts.excludeAbsent) {
|
||||
// Optional keys can be absent entirely.
|
||||
result.push(base);
|
||||
}
|
||||
|
||||
// Or be present and have one of the `optionalValues`.
|
||||
for (const optionalValue of optionalValues(optionalKey)) {
|
||||
result.push({ ...base, [optionalKey]: optionalValue });
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
// Call `body` for all test cases.
|
||||
const testCases = permutations(optionalKeys);
|
||||
for (const testCase of testCases) {
|
||||
try {
|
||||
body(testCase);
|
||||
} catch (err) {
|
||||
t.log(testCase);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
/** A language to analyze with CodeQL. */
|
||||
export type Language = string;
|
||||
|
||||
/**
|
||||
* A language supported by CodeQL that is treated specially by the Action.
|
||||
*
|
||||
* This is not an exhaustive list of languages supported by CodeQL and new
|
||||
* languages do not need to be added here.
|
||||
*/
|
||||
export enum KnownLanguage {
|
||||
actions = "actions",
|
||||
cpp = "cpp",
|
||||
csharp = "csharp",
|
||||
go = "go",
|
||||
java = "java",
|
||||
javascript = "javascript",
|
||||
python = "python",
|
||||
ruby = "ruby",
|
||||
rust = "rust",
|
||||
swift = "swift",
|
||||
}
|
||||
|
||||
/** Java-specific environment variable names that we may care about. */
|
||||
export enum JavaEnvVars {
|
||||
JAVA_HOME = "JAVA_HOME",
|
||||
JAVA_TOOL_OPTIONS = "JAVA_TOOL_OPTIONS",
|
||||
JDK_JAVA_OPTIONS = "JDK_JAVA_OPTIONS",
|
||||
_JAVA_OPTIONS = "_JAVA_OPTIONS",
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"languages": [
|
||||
"actions",
|
||||
"cpp",
|
||||
"csharp",
|
||||
"go",
|
||||
"java",
|
||||
"javascript",
|
||||
"python",
|
||||
"ruby",
|
||||
"rust",
|
||||
"swift"
|
||||
],
|
||||
"aliases": {
|
||||
"c": "cpp",
|
||||
"c-c++": "cpp",
|
||||
"c-cpp": "cpp",
|
||||
"c#": "csharp",
|
||||
"c++": "cpp",
|
||||
"java-kotlin": "java",
|
||||
"javascript-typescript": "javascript",
|
||||
"kotlin": "java",
|
||||
"typescript": "javascript"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import test from "ava";
|
||||
|
||||
import { setupTests } from "../testing-utils";
|
||||
|
||||
import knownLanguagesData from "./builtin.json";
|
||||
|
||||
import { isBuiltInLanguage, BuiltInLanguage, parseBuiltInLanguage } from ".";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
test("parseBuiltInLanguage", (t) => {
|
||||
// Exact matches
|
||||
t.is(parseBuiltInLanguage("csharp"), BuiltInLanguage.csharp);
|
||||
t.is(parseBuiltInLanguage("cpp"), BuiltInLanguage.cpp);
|
||||
t.is(parseBuiltInLanguage("go"), BuiltInLanguage.go);
|
||||
t.is(parseBuiltInLanguage("java"), BuiltInLanguage.java);
|
||||
t.is(parseBuiltInLanguage("javascript"), BuiltInLanguage.javascript);
|
||||
t.is(parseBuiltInLanguage("python"), BuiltInLanguage.python);
|
||||
t.is(parseBuiltInLanguage("rust"), BuiltInLanguage.rust);
|
||||
|
||||
// Aliases
|
||||
t.is(parseBuiltInLanguage(" \t\nCsHaRp\t\t"), BuiltInLanguage.csharp);
|
||||
t.is(parseBuiltInLanguage("c"), BuiltInLanguage.cpp);
|
||||
t.is(parseBuiltInLanguage("c++"), BuiltInLanguage.cpp);
|
||||
t.is(parseBuiltInLanguage("kotlin"), BuiltInLanguage.java);
|
||||
t.is(parseBuiltInLanguage("typescript"), BuiltInLanguage.javascript);
|
||||
|
||||
// spaces and case-insensitivity
|
||||
t.is(parseBuiltInLanguage(" \t\nkOtLin\t\t"), BuiltInLanguage.java);
|
||||
|
||||
// Not matches
|
||||
t.is(parseBuiltInLanguage(BuiltInLanguage.python), BuiltInLanguage.python);
|
||||
t.is(parseBuiltInLanguage("foo"), undefined);
|
||||
t.is(parseBuiltInLanguage(" "), undefined);
|
||||
t.is(parseBuiltInLanguage(""), undefined);
|
||||
});
|
||||
|
||||
test("isBuiltInLanguage matches the curated built-in language set", (t) => {
|
||||
t.true(isBuiltInLanguage(BuiltInLanguage.actions));
|
||||
t.true(isBuiltInLanguage(BuiltInLanguage.swift));
|
||||
t.false(isBuiltInLanguage("typescript"));
|
||||
});
|
||||
|
||||
test("BuiltInLanguage enum matches builtin.json", (t) => {
|
||||
t.deepEqual(Object.values(BuiltInLanguage), knownLanguagesData.languages);
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import knownLanguagesData from "./builtin.json";
|
||||
|
||||
/** A language to analyze with CodeQL. */
|
||||
export type Language = string;
|
||||
|
||||
/** A language built into the `defaults.json` CodeQL distribution. */
|
||||
export enum BuiltInLanguage {
|
||||
actions = "actions",
|
||||
cpp = "cpp",
|
||||
csharp = "csharp",
|
||||
go = "go",
|
||||
java = "java",
|
||||
javascript = "javascript",
|
||||
python = "python",
|
||||
ruby = "ruby",
|
||||
rust = "rust",
|
||||
swift = "swift",
|
||||
}
|
||||
|
||||
/** Java-specific environment variable names that we may care about. */
|
||||
export enum JavaEnvVars {
|
||||
JAVA_HOME = "JAVA_HOME",
|
||||
JAVA_TOOL_OPTIONS = "JAVA_TOOL_OPTIONS",
|
||||
JDK_JAVA_OPTIONS = "JDK_JAVA_OPTIONS",
|
||||
_JAVA_OPTIONS = "_JAVA_OPTIONS",
|
||||
}
|
||||
|
||||
const builtInLanguageSet = new Set<string>(knownLanguagesData.languages);
|
||||
|
||||
export function isBuiltInLanguage(
|
||||
language: string,
|
||||
): language is BuiltInLanguage {
|
||||
return builtInLanguageSet.has(language);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a language input corresponding to a built-in language into its canonical CodeQL language
|
||||
* name.
|
||||
*
|
||||
* This uses the language aliases shipped with the Action and will not be able to resolve aliases
|
||||
* added by third-party CodeQL language support or versions of the CodeQL CLI newer than the one
|
||||
* mentioned in `defaults.json`. Therefore, this function should only be used when the CodeQL CLI is
|
||||
* not available.
|
||||
*/
|
||||
export function parseBuiltInLanguage(
|
||||
language: string,
|
||||
): BuiltInLanguage | undefined {
|
||||
language = language.trim().toLowerCase();
|
||||
language =
|
||||
knownLanguagesData.aliases[
|
||||
language as keyof typeof knownLanguagesData.aliases
|
||||
] ?? language;
|
||||
if (isBuiltInLanguage(language)) {
|
||||
return language;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as actionsCache from "@actions/cache";
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "../actions-util";
|
||||
import * as apiClient from "../api-client";
|
||||
import { ResolveDatabaseOutput } from "../codeql";
|
||||
import * as gitUtils from "../git-utils";
|
||||
import { BuiltInLanguage } from "../languages";
|
||||
import { getRunnerLogger } from "../logging";
|
||||
import {
|
||||
createTestConfig,
|
||||
mockCodeQLVersion,
|
||||
setupTests,
|
||||
} from "../testing-utils";
|
||||
import * as utils from "../util";
|
||||
import { withTmpDir } from "../util";
|
||||
|
||||
import {
|
||||
downloadOverlayBaseDatabaseFromCache,
|
||||
getCacheRestoreKeyPrefix,
|
||||
getCacheSaveKey,
|
||||
} from "./caching";
|
||||
import { OverlayDatabaseMode } from "./overlay-database-mode";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
interface DownloadOverlayBaseDatabaseTestCase {
|
||||
overlayDatabaseMode: OverlayDatabaseMode;
|
||||
useOverlayDatabaseCaching: boolean;
|
||||
isInTestMode: boolean;
|
||||
restoreCacheResult: string | undefined | Error;
|
||||
hasBaseDatabaseOidsFile: boolean;
|
||||
tryGetFolderBytesSucceeds: boolean;
|
||||
codeQLVersion: string;
|
||||
resolveDatabaseOutput: ResolveDatabaseOutput | Error;
|
||||
}
|
||||
|
||||
const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
|
||||
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
|
||||
useOverlayDatabaseCaching: true,
|
||||
isInTestMode: false,
|
||||
restoreCacheResult: "cache-key",
|
||||
hasBaseDatabaseOidsFile: true,
|
||||
tryGetFolderBytesSucceeds: true,
|
||||
codeQLVersion: "2.20.5",
|
||||
resolveDatabaseOutput: { overlayBaseSpecifier: "20250626:XXX" },
|
||||
};
|
||||
|
||||
const testDownloadOverlayBaseDatabaseFromCache = test.macro({
|
||||
exec: async (
|
||||
t,
|
||||
_title: string,
|
||||
partialTestCase: Partial<DownloadOverlayBaseDatabaseTestCase>,
|
||||
expectDownloadSuccess: boolean,
|
||||
) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const dbLocation = path.join(tmpDir, "db");
|
||||
await fs.promises.mkdir(dbLocation, { recursive: true });
|
||||
|
||||
const logger = getRunnerLogger(true);
|
||||
const testCase = { ...defaultDownloadTestCase, ...partialTestCase };
|
||||
const config = createTestConfig({
|
||||
dbLocation,
|
||||
languages: [BuiltInLanguage.java],
|
||||
});
|
||||
|
||||
config.overlayDatabaseMode = testCase.overlayDatabaseMode;
|
||||
config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching;
|
||||
|
||||
if (testCase.hasBaseDatabaseOidsFile) {
|
||||
const baseDatabaseOidsFile = path.join(
|
||||
dbLocation,
|
||||
"base-database-oids.json",
|
||||
);
|
||||
await fs.promises.writeFile(baseDatabaseOidsFile, JSON.stringify({}));
|
||||
}
|
||||
|
||||
const stubs: sinon.SinonStub[] = [];
|
||||
|
||||
const getAutomationIDStub = sinon
|
||||
.stub(apiClient, "getAutomationID")
|
||||
.resolves("test-automation-id/");
|
||||
stubs.push(getAutomationIDStub);
|
||||
|
||||
const isInTestModeStub = sinon
|
||||
.stub(utils, "isInTestMode")
|
||||
.returns(testCase.isInTestMode);
|
||||
stubs.push(isInTestModeStub);
|
||||
|
||||
if (testCase.restoreCacheResult instanceof Error) {
|
||||
const restoreCacheStub = sinon
|
||||
.stub(actionsCache, "restoreCache")
|
||||
.rejects(testCase.restoreCacheResult);
|
||||
stubs.push(restoreCacheStub);
|
||||
} else {
|
||||
const restoreCacheStub = sinon
|
||||
.stub(actionsCache, "restoreCache")
|
||||
.resolves(testCase.restoreCacheResult);
|
||||
stubs.push(restoreCacheStub);
|
||||
}
|
||||
|
||||
const tryGetFolderBytesStub = sinon
|
||||
.stub(utils, "tryGetFolderBytes")
|
||||
.resolves(testCase.tryGetFolderBytesSucceeds ? 1024 * 1024 : undefined);
|
||||
stubs.push(tryGetFolderBytesStub);
|
||||
|
||||
const codeql = mockCodeQLVersion(testCase.codeQLVersion);
|
||||
|
||||
if (testCase.resolveDatabaseOutput instanceof Error) {
|
||||
const resolveDatabaseStub = sinon
|
||||
.stub(codeql, "resolveDatabase")
|
||||
.rejects(testCase.resolveDatabaseOutput);
|
||||
stubs.push(resolveDatabaseStub);
|
||||
} else {
|
||||
const resolveDatabaseStub = sinon
|
||||
.stub(codeql, "resolveDatabase")
|
||||
.resolves(testCase.resolveDatabaseOutput);
|
||||
stubs.push(resolveDatabaseStub);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await downloadOverlayBaseDatabaseFromCache(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
);
|
||||
|
||||
if (expectDownloadSuccess) {
|
||||
t.truthy(result);
|
||||
} else {
|
||||
t.is(result, undefined);
|
||||
}
|
||||
} finally {
|
||||
for (const stub of stubs) {
|
||||
stub.restore();
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
title: (_, title) => `downloadOverlayBaseDatabaseFromCache: ${title}`,
|
||||
});
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns stats when successful",
|
||||
{},
|
||||
true,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when mode is OverlayDatabaseMode.OverlayBase",
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when mode is OverlayDatabaseMode.None",
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when caching is disabled",
|
||||
{
|
||||
useOverlayDatabaseCaching: false,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined in test mode",
|
||||
{
|
||||
isInTestMode: true,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when cache miss",
|
||||
{
|
||||
restoreCacheResult: undefined,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when download fails",
|
||||
{
|
||||
restoreCacheResult: new Error("Download failed"),
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when downloaded database is invalid",
|
||||
{
|
||||
hasBaseDatabaseOidsFile: false,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when downloaded database doesn't have an overlayBaseSpecifier",
|
||||
{
|
||||
resolveDatabaseOutput: {},
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when resolving database metadata fails",
|
||||
{
|
||||
resolveDatabaseOutput: new Error("Failed to resolve database metadata"),
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when filesystem error occurs",
|
||||
{
|
||||
tryGetFolderBytesSucceeds: false,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial("overlay-base database cache keys remain stable", async (t) => {
|
||||
const logger = getRunnerLogger(true);
|
||||
const config = createTestConfig({ languages: ["python", "javascript"] });
|
||||
const codeQlVersion = "2.23.0";
|
||||
const commitOid = "abc123def456";
|
||||
|
||||
sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
|
||||
sinon.stub(gitUtils, "getCommitOid").resolves(commitOid);
|
||||
sinon.stub(actionsUtil, "getWorkflowRunID").returns(12345);
|
||||
sinon.stub(actionsUtil, "getWorkflowRunAttempt").returns(1);
|
||||
|
||||
const saveKey = await getCacheSaveKey(
|
||||
config,
|
||||
codeQlVersion,
|
||||
"checkout-path",
|
||||
logger,
|
||||
);
|
||||
const expectedSaveKey =
|
||||
"codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456-12345-1";
|
||||
t.is(
|
||||
saveKey,
|
||||
expectedSaveKey,
|
||||
"Cache save key changed unexpectedly. " +
|
||||
"This may indicate breaking changes in the cache key generation logic.",
|
||||
);
|
||||
|
||||
const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
|
||||
config,
|
||||
codeQlVersion,
|
||||
);
|
||||
const expectedRestoreKeyPrefix =
|
||||
"codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-";
|
||||
t.is(
|
||||
restoreKeyPrefix,
|
||||
expectedRestoreKeyPrefix,
|
||||
"Cache restore key prefix changed unexpectedly. " +
|
||||
"This may indicate breaking changes in the cache key generation logic.",
|
||||
);
|
||||
|
||||
t.true(
|
||||
saveKey.startsWith(restoreKeyPrefix),
|
||||
`Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,428 @@
|
||||
import * as fs from "fs";
|
||||
|
||||
import * as actionsCache from "@actions/cache";
|
||||
|
||||
import {
|
||||
getRequiredInput,
|
||||
getWorkflowRunAttempt,
|
||||
getWorkflowRunID,
|
||||
} from "../actions-util";
|
||||
import { getAutomationID } from "../api-client";
|
||||
import { createCacheKeyHash } from "../caching-utils";
|
||||
import { type CodeQL } from "../codeql";
|
||||
import { type Config } from "../config-utils";
|
||||
import { getCommitOid } from "../git-utils";
|
||||
import { Logger, withGroupAsync } from "../logging";
|
||||
import {
|
||||
CleanupLevel,
|
||||
getBaseDatabaseOidsFilePath,
|
||||
getCodeQLDatabasePath,
|
||||
getErrorMessage,
|
||||
isInTestMode,
|
||||
tryGetFolderBytes,
|
||||
waitForResultWithTimeLimit,
|
||||
} from "../util";
|
||||
|
||||
import { OverlayDatabaseMode } from "./overlay-database-mode";
|
||||
|
||||
/**
|
||||
* The maximum (uncompressed) size of the overlay base database that we will
|
||||
* upload. By default, the Actions Cache has an overall capacity of 10 GB, and
|
||||
* the Actions Cache client library uses zstd compression.
|
||||
*
|
||||
* Ideally we would apply a size limit to the compressed overlay-base database,
|
||||
* but we cannot do so because compression is handled transparently by the
|
||||
* Actions Cache client library. Instead we place a limit on the uncompressed
|
||||
* size of the overlay-base database.
|
||||
*
|
||||
* Assuming 2.5:1 compression ratio, the 7.5 GB limit on uncompressed data would
|
||||
* translate to a limit of around 3 GB after compression.
|
||||
*/
|
||||
const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
|
||||
const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
|
||||
OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000;
|
||||
|
||||
// Constants for database caching
|
||||
const CACHE_VERSION = 1;
|
||||
const CACHE_PREFIX = "codeql-overlay-base-database";
|
||||
|
||||
// The purpose of this ten-minute limit is to guard against the possibility
|
||||
// that the cache service is unresponsive, which would otherwise cause the
|
||||
// entire action to hang. Normally we expect cache operations to complete
|
||||
// within two minutes.
|
||||
const MAX_CACHE_OPERATION_MS = 600_000;
|
||||
|
||||
/**
|
||||
* Checks that the overlay-base database is valid by checking for the
|
||||
* existence of the base database OIDs file.
|
||||
*
|
||||
* @param config The configuration object
|
||||
* @param logger The logger instance
|
||||
* @param warningPrefix Prefix for the check failure warning message
|
||||
* @returns True if the verification succeeded, false otherwise
|
||||
*/
|
||||
async function checkOverlayBaseDatabase(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
warningPrefix: string,
|
||||
): Promise<boolean> {
|
||||
// An overlay-base database should contain the base database OIDs file.
|
||||
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
|
||||
if (!fs.existsSync(baseDatabaseOidsFilePath)) {
|
||||
logger.warning(
|
||||
`${warningPrefix}: ${baseDatabaseOidsFilePath} does not exist`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const language of config.languages) {
|
||||
const dbPath = getCodeQLDatabasePath(config, language);
|
||||
try {
|
||||
const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
|
||||
if (
|
||||
resolveDatabaseOutput === undefined ||
|
||||
!("overlayBaseSpecifier" in resolveDatabaseOutput)
|
||||
) {
|
||||
logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
|
||||
return false;
|
||||
} else {
|
||||
logger.debug(
|
||||
`Overlay base specifier for ${language} overlay-base database found: ` +
|
||||
`${resolveDatabaseOutput.overlayBaseSpecifier}`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads the overlay-base database to the GitHub Actions cache. If conditions
|
||||
* for uploading are not met, the function does nothing and returns false.
|
||||
*
|
||||
* This function uses the `checkout_path` input to determine the repository path
|
||||
* and works only when called from `analyze` or `upload-sarif`.
|
||||
*
|
||||
* @param codeql The CodeQL instance
|
||||
* @param config The configuration object
|
||||
* @param logger The logger instance
|
||||
* @returns A promise that resolves to true if the upload was performed and
|
||||
* successfully completed, or false otherwise
|
||||
*/
|
||||
export async function cleanupAndUploadOverlayBaseDatabaseToCache(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<boolean> {
|
||||
const overlayDatabaseMode = config.overlayDatabaseMode;
|
||||
if (overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase) {
|
||||
logger.debug(
|
||||
`Overlay database mode is ${overlayDatabaseMode}. ` +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (!config.useOverlayDatabaseCaching) {
|
||||
logger.debug(
|
||||
"Overlay database caching is disabled. " +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (isInTestMode()) {
|
||||
logger.debug(
|
||||
"In test mode. Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const databaseIsValid = await checkOverlayBaseDatabase(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
"Abort uploading overlay-base database to cache",
|
||||
);
|
||||
if (!databaseIsValid) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Clean up the database using the overlay cleanup level.
|
||||
await withGroupAsync("Cleaning up databases", async () => {
|
||||
await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
|
||||
});
|
||||
|
||||
const dbLocation = config.dbLocation;
|
||||
|
||||
const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
|
||||
if (databaseSizeBytes === undefined) {
|
||||
logger.warning(
|
||||
"Failed to determine database size. " +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (databaseSizeBytes > OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES) {
|
||||
const databaseSizeMB = Math.round(databaseSizeBytes / 1_000_000);
|
||||
logger.warning(
|
||||
`Database size (${databaseSizeMB} MB) ` +
|
||||
`exceeds maximum upload size (${OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB} MB). ` +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const codeQlVersion = (await codeql.getVersion()).version;
|
||||
const checkoutPath = getRequiredInput("checkout_path");
|
||||
const cacheSaveKey = await getCacheSaveKey(
|
||||
config,
|
||||
codeQlVersion,
|
||||
checkoutPath,
|
||||
logger,
|
||||
);
|
||||
logger.info(
|
||||
`Uploading overlay-base database to Actions cache with key ${cacheSaveKey}`,
|
||||
);
|
||||
|
||||
try {
|
||||
const cacheId = await waitForResultWithTimeLimit(
|
||||
MAX_CACHE_OPERATION_MS,
|
||||
actionsCache.saveCache([dbLocation], cacheSaveKey),
|
||||
() => {},
|
||||
);
|
||||
if (cacheId === undefined) {
|
||||
logger.warning("Timed out while uploading overlay-base database");
|
||||
return false;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warning(
|
||||
"Failed to upload overlay-base database to cache: " +
|
||||
`${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
logger.info(`Successfully uploaded overlay-base database from ${dbLocation}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
export interface OverlayBaseDatabaseDownloadStats {
|
||||
databaseSizeBytes: number;
|
||||
databaseDownloadDurationMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads the overlay-base database from the GitHub Actions cache. If conditions
|
||||
* for downloading are not met, the function does nothing and returns false.
|
||||
*
|
||||
* @param codeql The CodeQL instance
|
||||
* @param config The configuration object
|
||||
* @param logger The logger instance
|
||||
* @returns A promise that resolves to download statistics if an overlay-base
|
||||
* database was successfully downloaded, or undefined if the download was
|
||||
* either not performed or failed.
|
||||
*/
|
||||
export async function downloadOverlayBaseDatabaseFromCache(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<OverlayBaseDatabaseDownloadStats | undefined> {
|
||||
const overlayDatabaseMode = config.overlayDatabaseMode;
|
||||
if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
|
||||
logger.debug(
|
||||
`Overlay database mode is ${overlayDatabaseMode}. ` +
|
||||
"Skip downloading overlay-base database from cache.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (!config.useOverlayDatabaseCaching) {
|
||||
logger.debug(
|
||||
"Overlay database caching is disabled. " +
|
||||
"Skip downloading overlay-base database from cache.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (isInTestMode()) {
|
||||
logger.debug(
|
||||
"In test mode. Skip downloading overlay-base database from cache.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const dbLocation = config.dbLocation;
|
||||
const codeQlVersion = (await codeql.getVersion()).version;
|
||||
const cacheRestoreKeyPrefix = await getCacheRestoreKeyPrefix(
|
||||
config,
|
||||
codeQlVersion,
|
||||
);
|
||||
|
||||
logger.info(
|
||||
"Looking in Actions cache for overlay-base database with " +
|
||||
`restore key ${cacheRestoreKeyPrefix}`,
|
||||
);
|
||||
|
||||
let databaseDownloadDurationMs = 0;
|
||||
try {
|
||||
const databaseDownloadStart = performance.now();
|
||||
const foundKey = await waitForResultWithTimeLimit(
|
||||
// This ten-minute limit for the cache restore operation is mainly to
|
||||
// guard against the possibility that the cache service is unresponsive
|
||||
// and hangs outside the data download.
|
||||
//
|
||||
// Data download (which is normally the most time-consuming part of the
|
||||
// restore operation) should not run long enough to hit this limit. Even
|
||||
// for an extremely large 10GB database, at a download speed of 40MB/s
|
||||
// (see below), the download should complete within five minutes. If we
|
||||
// do hit this limit, there are likely more serious problems other than
|
||||
// mere slow download speed.
|
||||
//
|
||||
// This is important because we don't want any ongoing file operations
|
||||
// on the database directory when we do hit this limit. Hitting this
|
||||
// time limit takes us to a fallback path where we re-initialize the
|
||||
// database from scratch at dbLocation, and having the cache restore
|
||||
// operation continue to write into dbLocation in the background would
|
||||
// really mess things up. We want to hit this limit only in the case
|
||||
// of a hung cache service, not just slow download speed.
|
||||
MAX_CACHE_OPERATION_MS,
|
||||
actionsCache.restoreCache(
|
||||
[dbLocation],
|
||||
cacheRestoreKeyPrefix,
|
||||
undefined,
|
||||
{
|
||||
// Azure SDK download (which is the default) uses 128MB segments; see
|
||||
// https://github.com/actions/toolkit/blob/main/packages/cache/README.md.
|
||||
// Setting segmentTimeoutInMs to 3000 translates to segment download
|
||||
// speed of about 40 MB/s, which should be achievable unless the
|
||||
// download is unreliable (in which case we do want to abort).
|
||||
segmentTimeoutInMs: 3000,
|
||||
},
|
||||
),
|
||||
() => {
|
||||
logger.info("Timed out downloading overlay-base database from cache");
|
||||
},
|
||||
);
|
||||
databaseDownloadDurationMs = Math.round(
|
||||
performance.now() - databaseDownloadStart,
|
||||
);
|
||||
|
||||
if (foundKey === undefined) {
|
||||
logger.info("No overlay-base database found in Actions cache");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Downloaded overlay-base database in cache with key ${foundKey}`,
|
||||
);
|
||||
} catch (error) {
|
||||
logger.warning(
|
||||
"Failed to download overlay-base database from cache: " +
|
||||
`${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const databaseIsValid = await checkOverlayBaseDatabase(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
"Downloaded overlay-base database is invalid",
|
||||
);
|
||||
if (!databaseIsValid) {
|
||||
logger.warning("Downloaded overlay-base database failed validation");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
|
||||
if (databaseSizeBytes === undefined) {
|
||||
logger.info(
|
||||
"Filesystem error while accessing downloaded overlay-base database",
|
||||
);
|
||||
// The problem that warrants reporting download failure is not that we are
|
||||
// unable to determine the size of the database. Rather, it is that we
|
||||
// encountered a filesystem error while accessing the database, which
|
||||
// indicates that an overlay analysis will likely fail.
|
||||
return undefined;
|
||||
}
|
||||
|
||||
logger.info(`Successfully downloaded overlay-base database to ${dbLocation}`);
|
||||
return {
|
||||
databaseSizeBytes: Math.round(databaseSizeBytes),
|
||||
databaseDownloadDurationMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cache key for saving the overlay-base database to the GitHub
|
||||
* Actions cache.
|
||||
*
|
||||
* The key consists of the restore key prefix (which does not include the
|
||||
* commit SHA) and the commit SHA of the current checkout.
|
||||
*/
|
||||
export async function getCacheSaveKey(
|
||||
config: Config,
|
||||
codeQlVersion: string,
|
||||
checkoutPath: string,
|
||||
logger: Logger,
|
||||
): Promise<string> {
|
||||
let runId = 1;
|
||||
let attemptId = 1;
|
||||
try {
|
||||
runId = getWorkflowRunID();
|
||||
attemptId = getWorkflowRunAttempt();
|
||||
} catch (e) {
|
||||
logger.warning(
|
||||
`Failed to get workflow run ID or attempt ID. Reason: ${getErrorMessage(e)}`,
|
||||
);
|
||||
}
|
||||
const sha = await getCommitOid(checkoutPath);
|
||||
const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
|
||||
config,
|
||||
codeQlVersion,
|
||||
);
|
||||
return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cache key prefix for restoring the overlay-base database from
|
||||
* the GitHub Actions cache.
|
||||
*
|
||||
* Actions cache supports using multiple restore keys to indicate preference,
|
||||
* and this function could in principle take advantage of that feature by
|
||||
* returning a list of restore key prefixes. However, since overlay-base
|
||||
* databases are built from the default branch and used in PR analysis, it is
|
||||
* exceedingly unlikely that the commit SHA will ever be the same.
|
||||
*
|
||||
* Therefore, this function returns only a single restore key prefix, which does
|
||||
* not include the commit SHA. This allows us to restore the most recent
|
||||
* compatible overlay-base database.
|
||||
*/
|
||||
export async function getCacheRestoreKeyPrefix(
|
||||
config: Config,
|
||||
codeQlVersion: string,
|
||||
): Promise<string> {
|
||||
const languages = [...config.languages].sort().join("_");
|
||||
|
||||
const cacheKeyComponents = {
|
||||
automationID: await getAutomationID(),
|
||||
// Add more components here as needed in the future
|
||||
};
|
||||
const componentsHash = createCacheKeyHash(cacheKeyComponents);
|
||||
|
||||
// For a cached overlay-base database to be considered compatible for overlay
|
||||
// analysis, all components in the cache restore key must match:
|
||||
//
|
||||
// CACHE_PREFIX: distinguishes overlay-base databases from other cache objects
|
||||
// CACHE_VERSION: cache format version
|
||||
// componentsHash: hash of additional components (see above for details)
|
||||
// languages: the languages included in the overlay-base database
|
||||
// codeQlVersion: CodeQL bundle version
|
||||
//
|
||||
// Technically we can also include languages and codeQlVersion in the
|
||||
// componentsHash, but including them explicitly in the cache key makes it
|
||||
// easier to debug and understand the cache key structure.
|
||||
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
|
||||
}
|
||||
+237
-259
@@ -1,32 +1,16 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as actionsCache from "@actions/cache";
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "../actions-util";
|
||||
import * as apiClient from "../api-client";
|
||||
import { ResolveDatabaseOutput } from "../codeql";
|
||||
import * as gitUtils from "../git-utils";
|
||||
import { KnownLanguage } from "../languages";
|
||||
import { getRunnerLogger } from "../logging";
|
||||
import {
|
||||
createTestConfig,
|
||||
mockCodeQLVersion,
|
||||
setupTests,
|
||||
} from "../testing-utils";
|
||||
import * as utils from "../util";
|
||||
import { createTestConfig, setupTests } from "../testing-utils";
|
||||
import { withTmpDir } from "../util";
|
||||
|
||||
import {
|
||||
downloadOverlayBaseDatabaseFromCache,
|
||||
getCacheRestoreKeyPrefix,
|
||||
getCacheSaveKey,
|
||||
OverlayDatabaseMode,
|
||||
writeBaseDatabaseOidsFile,
|
||||
writeOverlayChangesFile,
|
||||
} from ".";
|
||||
import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from ".";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
@@ -34,12 +18,14 @@ test.serial(
|
||||
"writeOverlayChangesFile generates correct changes file",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const dbLocation = path.join(tmpDir, "db");
|
||||
await fs.promises.mkdir(dbLocation, { recursive: true });
|
||||
const sourceRoot = path.join(tmpDir, "src");
|
||||
await fs.promises.mkdir(sourceRoot, { recursive: true });
|
||||
const tempDir = path.join(tmpDir, "temp");
|
||||
await fs.promises.mkdir(tempDir, { recursive: true });
|
||||
const [dbLocation, sourceRoot, tempDir] = ["db", "src", "temp"].map((d) =>
|
||||
path.join(tmpDir, d),
|
||||
);
|
||||
await Promise.all(
|
||||
[dbLocation, sourceRoot, tempDir].map((d) =>
|
||||
fs.promises.mkdir(d, { recursive: true }),
|
||||
),
|
||||
);
|
||||
|
||||
const logger = getRunnerLogger(true);
|
||||
const config = createTestConfig({ dbLocation });
|
||||
@@ -70,9 +56,16 @@ test.serial(
|
||||
|
||||
// Write the overlay changes file, which uses the mocked overlay OIDs
|
||||
// and the base database OIDs file
|
||||
const diffRangeFilePath = path.join(tempDir, "pr-diff-range.json");
|
||||
const getTempDirStub = sinon
|
||||
.stub(actionsUtil, "getTemporaryDirectory")
|
||||
.returns(tempDir);
|
||||
const getDiffRangesStub = sinon
|
||||
.stub(actionsUtil, "getDiffRangesJsonFilePath")
|
||||
.returns(diffRangeFilePath);
|
||||
const getGitRootStub = sinon
|
||||
.stub(gitUtils, "getGitRoot")
|
||||
.resolves(sourceRoot);
|
||||
const changesFilePath = await writeOverlayChangesFile(
|
||||
config,
|
||||
sourceRoot,
|
||||
@@ -80,6 +73,8 @@ test.serial(
|
||||
);
|
||||
getFileOidsStubForOverlay.restore();
|
||||
getTempDirStub.restore();
|
||||
getDiffRangesStub.restore();
|
||||
getGitRootStub.restore();
|
||||
|
||||
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
|
||||
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
|
||||
@@ -93,260 +88,243 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
interface DownloadOverlayBaseDatabaseTestCase {
|
||||
overlayDatabaseMode: OverlayDatabaseMode;
|
||||
useOverlayDatabaseCaching: boolean;
|
||||
isInTestMode: boolean;
|
||||
restoreCacheResult: string | undefined | Error;
|
||||
hasBaseDatabaseOidsFile: boolean;
|
||||
tryGetFolderBytesSucceeds: boolean;
|
||||
codeQLVersion: string;
|
||||
resolveDatabaseOutput: ResolveDatabaseOutput | Error;
|
||||
}
|
||||
|
||||
const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
|
||||
overlayDatabaseMode: OverlayDatabaseMode.Overlay,
|
||||
useOverlayDatabaseCaching: true,
|
||||
isInTestMode: false,
|
||||
restoreCacheResult: "cache-key",
|
||||
hasBaseDatabaseOidsFile: true,
|
||||
tryGetFolderBytesSucceeds: true,
|
||||
codeQLVersion: "2.20.5",
|
||||
resolveDatabaseOutput: { overlayBaseSpecifier: "20250626:XXX" },
|
||||
};
|
||||
|
||||
const testDownloadOverlayBaseDatabaseFromCache = test.macro({
|
||||
exec: async (
|
||||
t,
|
||||
_title: string,
|
||||
partialTestCase: Partial<DownloadOverlayBaseDatabaseTestCase>,
|
||||
expectDownloadSuccess: boolean,
|
||||
) => {
|
||||
test.serial(
|
||||
"writeOverlayChangesFile merges additional diff files into overlay changes",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const dbLocation = path.join(tmpDir, "db");
|
||||
await fs.promises.mkdir(dbLocation, { recursive: true });
|
||||
const [dbLocation, sourceRoot, tempDir] = ["db", "src", "temp"].map((d) =>
|
||||
path.join(tmpDir, d),
|
||||
);
|
||||
await Promise.all(
|
||||
[dbLocation, sourceRoot, tempDir].map((d) =>
|
||||
fs.promises.mkdir(d, { recursive: true }),
|
||||
),
|
||||
);
|
||||
|
||||
const logger = getRunnerLogger(true);
|
||||
const testCase = { ...defaultDownloadTestCase, ...partialTestCase };
|
||||
const config = createTestConfig({
|
||||
dbLocation,
|
||||
languages: [KnownLanguage.java],
|
||||
});
|
||||
const config = createTestConfig({ dbLocation });
|
||||
|
||||
config.overlayDatabaseMode = testCase.overlayDatabaseMode;
|
||||
config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching;
|
||||
// Mock the getFileOidsUnderPath function to return base OIDs
|
||||
// "reverted.js" has the same OID in both base and current, simulating
|
||||
// a revert PR where the file content matches the overlay-base
|
||||
const baseOids = {
|
||||
"unchanged.js": "aaa111",
|
||||
"modified.js": "bbb222",
|
||||
"reverted.js": "eee555",
|
||||
};
|
||||
const getFileOidsStubForBase = sinon
|
||||
.stub(gitUtils, "getFileOidsUnderPath")
|
||||
.resolves(baseOids);
|
||||
|
||||
if (testCase.hasBaseDatabaseOidsFile) {
|
||||
const baseDatabaseOidsFile = path.join(
|
||||
dbLocation,
|
||||
"base-database-oids.json",
|
||||
);
|
||||
await fs.promises.writeFile(baseDatabaseOidsFile, JSON.stringify({}));
|
||||
}
|
||||
// Write the base database OIDs file
|
||||
await writeBaseDatabaseOidsFile(config, sourceRoot);
|
||||
getFileOidsStubForBase.restore();
|
||||
|
||||
const stubs: sinon.SinonStub[] = [];
|
||||
// Mock the getFileOidsUnderPath function to return overlay OIDs
|
||||
// "reverted.js" has the same OID as the base -- OID comparison alone
|
||||
// would NOT include it, only additionalChangedFiles causes it to appear
|
||||
const currentOids = {
|
||||
"unchanged.js": "aaa111",
|
||||
"modified.js": "ddd444", // Changed OID
|
||||
"reverted.js": "eee555", // Same OID as base -- not detected by OID comparison
|
||||
};
|
||||
const getFileOidsStubForOverlay = sinon
|
||||
.stub(gitUtils, "getFileOidsUnderPath")
|
||||
.resolves(currentOids);
|
||||
|
||||
const getAutomationIDStub = sinon
|
||||
.stub(apiClient, "getAutomationID")
|
||||
.resolves("test-automation-id/");
|
||||
stubs.push(getAutomationIDStub);
|
||||
const diffRangeFilePath = path.join(tempDir, "pr-diff-range.json");
|
||||
const getTempDirStub = sinon
|
||||
.stub(actionsUtil, "getTemporaryDirectory")
|
||||
.returns(tempDir);
|
||||
const getDiffRangesStub = sinon
|
||||
.stub(actionsUtil, "getDiffRangesJsonFilePath")
|
||||
.returns(diffRangeFilePath);
|
||||
const getGitRootStub = sinon
|
||||
.stub(gitUtils, "getGitRoot")
|
||||
.resolves(sourceRoot);
|
||||
|
||||
const isInTestModeStub = sinon
|
||||
.stub(utils, "isInTestMode")
|
||||
.returns(testCase.isInTestMode);
|
||||
stubs.push(isInTestModeStub);
|
||||
// Write a pr-diff-range.json file with diff ranges including
|
||||
// "reverted.js" (unchanged OIDs) and "modified.js" (already in OID changes)
|
||||
await fs.promises.writeFile(
|
||||
diffRangeFilePath,
|
||||
JSON.stringify([
|
||||
{ path: "reverted.js", startLine: 1, endLine: 10 },
|
||||
{ path: "modified.js", startLine: 1, endLine: 5 },
|
||||
{ path: "diff-only.js", startLine: 1, endLine: 3 },
|
||||
]),
|
||||
);
|
||||
|
||||
if (testCase.restoreCacheResult instanceof Error) {
|
||||
const restoreCacheStub = sinon
|
||||
.stub(actionsCache, "restoreCache")
|
||||
.rejects(testCase.restoreCacheResult);
|
||||
stubs.push(restoreCacheStub);
|
||||
} else {
|
||||
const restoreCacheStub = sinon
|
||||
.stub(actionsCache, "restoreCache")
|
||||
.resolves(testCase.restoreCacheResult);
|
||||
stubs.push(restoreCacheStub);
|
||||
}
|
||||
const changesFilePath = await writeOverlayChangesFile(
|
||||
config,
|
||||
sourceRoot,
|
||||
logger,
|
||||
);
|
||||
getFileOidsStubForOverlay.restore();
|
||||
getTempDirStub.restore();
|
||||
getDiffRangesStub.restore();
|
||||
getGitRootStub.restore();
|
||||
|
||||
const tryGetFolderBytesStub = sinon
|
||||
.stub(utils, "tryGetFolderBytes")
|
||||
.resolves(testCase.tryGetFolderBytesSucceeds ? 1024 * 1024 : undefined);
|
||||
stubs.push(tryGetFolderBytesStub);
|
||||
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
|
||||
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
|
||||
|
||||
const codeql = mockCodeQLVersion(testCase.codeQLVersion);
|
||||
|
||||
if (testCase.resolveDatabaseOutput instanceof Error) {
|
||||
const resolveDatabaseStub = sinon
|
||||
.stub(codeql, "resolveDatabase")
|
||||
.rejects(testCase.resolveDatabaseOutput);
|
||||
stubs.push(resolveDatabaseStub);
|
||||
} else {
|
||||
const resolveDatabaseStub = sinon
|
||||
.stub(codeql, "resolveDatabase")
|
||||
.resolves(testCase.resolveDatabaseOutput);
|
||||
stubs.push(resolveDatabaseStub);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await downloadOverlayBaseDatabaseFromCache(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
);
|
||||
|
||||
if (expectDownloadSuccess) {
|
||||
t.truthy(result);
|
||||
} else {
|
||||
t.is(result, undefined);
|
||||
}
|
||||
} finally {
|
||||
for (const stub of stubs) {
|
||||
stub.restore();
|
||||
}
|
||||
}
|
||||
t.deepEqual(
|
||||
parsedContent.changes.sort(),
|
||||
["diff-only.js", "modified.js", "reverted.js"],
|
||||
"Should include OID-changed files, diff-only files, and deduplicate overlapping files",
|
||||
);
|
||||
});
|
||||
},
|
||||
title: (_, title) => `downloadOverlayBaseDatabaseFromCache: ${title}`,
|
||||
});
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns stats when successful",
|
||||
{},
|
||||
true,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when mode is OverlayDatabaseMode.OverlayBase",
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
"writeOverlayChangesFile works without additional diff files",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const [dbLocation, sourceRoot, tempDir] = ["db", "src", "temp"].map((d) =>
|
||||
path.join(tmpDir, d),
|
||||
);
|
||||
await Promise.all(
|
||||
[dbLocation, sourceRoot, tempDir].map((d) =>
|
||||
fs.promises.mkdir(d, { recursive: true }),
|
||||
),
|
||||
);
|
||||
|
||||
const logger = getRunnerLogger(true);
|
||||
const config = createTestConfig({ dbLocation });
|
||||
|
||||
// Mock the getFileOidsUnderPath function to return base OIDs
|
||||
const baseOids = {
|
||||
"unchanged.js": "aaa111",
|
||||
"modified.js": "bbb222",
|
||||
};
|
||||
const getFileOidsStubForBase = sinon
|
||||
.stub(gitUtils, "getFileOidsUnderPath")
|
||||
.resolves(baseOids);
|
||||
|
||||
await writeBaseDatabaseOidsFile(config, sourceRoot);
|
||||
getFileOidsStubForBase.restore();
|
||||
|
||||
const currentOids = {
|
||||
"unchanged.js": "aaa111",
|
||||
"modified.js": "ddd444",
|
||||
};
|
||||
const getFileOidsStubForOverlay = sinon
|
||||
.stub(gitUtils, "getFileOidsUnderPath")
|
||||
.resolves(currentOids);
|
||||
|
||||
const diffRangeFilePath = path.join(tempDir, "pr-diff-range.json");
|
||||
const getTempDirStub = sinon
|
||||
.stub(actionsUtil, "getTemporaryDirectory")
|
||||
.returns(tempDir);
|
||||
const getDiffRangesStub = sinon
|
||||
.stub(actionsUtil, "getDiffRangesJsonFilePath")
|
||||
.returns(diffRangeFilePath);
|
||||
const getGitRootStub = sinon
|
||||
.stub(gitUtils, "getGitRoot")
|
||||
.resolves(sourceRoot);
|
||||
|
||||
// No pr-diff-range.json file exists - should work the same as before
|
||||
const changesFilePath = await writeOverlayChangesFile(
|
||||
config,
|
||||
sourceRoot,
|
||||
logger,
|
||||
);
|
||||
getFileOidsStubForOverlay.restore();
|
||||
getTempDirStub.restore();
|
||||
getDiffRangesStub.restore();
|
||||
getGitRootStub.restore();
|
||||
|
||||
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
|
||||
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
|
||||
|
||||
t.deepEqual(
|
||||
parsedContent.changes.sort(),
|
||||
["modified.js"],
|
||||
"Should only include OID-changed files when no additional files provided",
|
||||
);
|
||||
});
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when mode is OverlayDatabaseMode.None",
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.None,
|
||||
"writeOverlayChangesFile converts diff range paths to sourceRoot-relative when sourceRoot is a subdirectory",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
// Simulate: repo root = tmpDir, sourceRoot = tmpDir/src
|
||||
const repoRoot = tmpDir;
|
||||
const sourceRoot = path.join(tmpDir, "src");
|
||||
const [dbLocation, tempDir] = ["db", "temp"].map((d) =>
|
||||
path.join(tmpDir, d),
|
||||
);
|
||||
await Promise.all(
|
||||
[dbLocation, sourceRoot, tempDir].map((d) =>
|
||||
fs.promises.mkdir(d, { recursive: true }),
|
||||
),
|
||||
);
|
||||
|
||||
const logger = getRunnerLogger(true);
|
||||
const config = createTestConfig({ dbLocation });
|
||||
|
||||
// Base OIDs (sourceRoot-relative paths)
|
||||
const baseOids = {
|
||||
"app.js": "aaa111",
|
||||
"lib/util.js": "bbb222",
|
||||
};
|
||||
const getFileOidsStubForBase = sinon
|
||||
.stub(gitUtils, "getFileOidsUnderPath")
|
||||
.resolves(baseOids);
|
||||
await writeBaseDatabaseOidsFile(config, sourceRoot);
|
||||
getFileOidsStubForBase.restore();
|
||||
|
||||
// Current OIDs — same as base (no OID changes)
|
||||
const currentOids = {
|
||||
"app.js": "aaa111",
|
||||
"lib/util.js": "bbb222",
|
||||
};
|
||||
const getFileOidsStubForOverlay = sinon
|
||||
.stub(gitUtils, "getFileOidsUnderPath")
|
||||
.resolves(currentOids);
|
||||
|
||||
const diffRangeFilePath = path.join(tempDir, "pr-diff-range.json");
|
||||
const getTempDirStub = sinon
|
||||
.stub(actionsUtil, "getTemporaryDirectory")
|
||||
.returns(tempDir);
|
||||
const getDiffRangesStub = sinon
|
||||
.stub(actionsUtil, "getDiffRangesJsonFilePath")
|
||||
.returns(diffRangeFilePath);
|
||||
// getGitRoot returns the repo root (parent of sourceRoot)
|
||||
const getGitRootStub = sinon
|
||||
.stub(gitUtils, "getGitRoot")
|
||||
.resolves(repoRoot);
|
||||
|
||||
// Diff ranges use repo-root-relative paths (as returned by the GitHub compare API)
|
||||
await fs.promises.writeFile(
|
||||
diffRangeFilePath,
|
||||
JSON.stringify([
|
||||
{ path: "src/app.js", startLine: 1, endLine: 10 },
|
||||
{ path: "src/lib/util.js", startLine: 5, endLine: 8 },
|
||||
{ path: "other/outside.js", startLine: 1, endLine: 3 }, // not under sourceRoot
|
||||
]),
|
||||
);
|
||||
|
||||
const changesFilePath = await writeOverlayChangesFile(
|
||||
config,
|
||||
sourceRoot,
|
||||
logger,
|
||||
);
|
||||
getFileOidsStubForOverlay.restore();
|
||||
getTempDirStub.restore();
|
||||
getDiffRangesStub.restore();
|
||||
getGitRootStub.restore();
|
||||
|
||||
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
|
||||
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
|
||||
|
||||
t.deepEqual(
|
||||
parsedContent.changes.sort(),
|
||||
["app.js", "lib/util.js"],
|
||||
"Should convert repo-root-relative paths to sourceRoot-relative and filter out files outside sourceRoot",
|
||||
);
|
||||
});
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when caching is disabled",
|
||||
{
|
||||
useOverlayDatabaseCaching: false,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined in test mode",
|
||||
{
|
||||
isInTestMode: true,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when cache miss",
|
||||
{
|
||||
restoreCacheResult: undefined,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when download fails",
|
||||
{
|
||||
restoreCacheResult: new Error("Download failed"),
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when downloaded database is invalid",
|
||||
{
|
||||
hasBaseDatabaseOidsFile: false,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when downloaded database doesn't have an overlayBaseSpecifier",
|
||||
{
|
||||
resolveDatabaseOutput: {},
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when resolving database metadata fails",
|
||||
{
|
||||
resolveDatabaseOutput: new Error("Failed to resolve database metadata"),
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial(
|
||||
testDownloadOverlayBaseDatabaseFromCache,
|
||||
"returns undefined when filesystem error occurs",
|
||||
{
|
||||
tryGetFolderBytesSucceeds: false,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
test.serial("overlay-base database cache keys remain stable", async (t) => {
|
||||
const logger = getRunnerLogger(true);
|
||||
const config = createTestConfig({ languages: ["python", "javascript"] });
|
||||
const codeQlVersion = "2.23.0";
|
||||
const commitOid = "abc123def456";
|
||||
|
||||
sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
|
||||
sinon.stub(gitUtils, "getCommitOid").resolves(commitOid);
|
||||
sinon.stub(actionsUtil, "getWorkflowRunID").returns(12345);
|
||||
sinon.stub(actionsUtil, "getWorkflowRunAttempt").returns(1);
|
||||
|
||||
const saveKey = await getCacheSaveKey(
|
||||
config,
|
||||
codeQlVersion,
|
||||
"checkout-path",
|
||||
logger,
|
||||
);
|
||||
const expectedSaveKey =
|
||||
"codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-abc123def456-12345-1";
|
||||
t.is(
|
||||
saveKey,
|
||||
expectedSaveKey,
|
||||
"Cache save key changed unexpectedly. " +
|
||||
"This may indicate breaking changes in the cache key generation logic.",
|
||||
);
|
||||
|
||||
const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
|
||||
config,
|
||||
codeQlVersion,
|
||||
);
|
||||
const expectedRestoreKeyPrefix =
|
||||
"codeql-overlay-base-database-1-c5666c509a2d9895-javascript_python-2.23.0-";
|
||||
t.is(
|
||||
restoreKeyPrefix,
|
||||
expectedRestoreKeyPrefix,
|
||||
"Cache restore key prefix changed unexpectedly. " +
|
||||
"This may indicate breaking changes in the cache key generation logic.",
|
||||
);
|
||||
|
||||
t.true(
|
||||
saveKey.startsWith(restoreKeyPrefix),
|
||||
`Expected save key "${saveKey}" to start with restore key prefix "${restoreKeyPrefix}"`,
|
||||
);
|
||||
});
|
||||
|
||||
+62
-418
@@ -1,35 +1,12 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as actionsCache from "@actions/cache";
|
||||
|
||||
import {
|
||||
getRequiredInput,
|
||||
getTemporaryDirectory,
|
||||
getWorkflowRunAttempt,
|
||||
getWorkflowRunID,
|
||||
} from "../actions-util";
|
||||
import { getAutomationID } from "../api-client";
|
||||
import { createCacheKeyHash } from "../caching-utils";
|
||||
import { type CodeQL } from "../codeql";
|
||||
import * as actionsUtil from "../actions-util";
|
||||
import { getOptionalInput, getTemporaryDirectory } from "../actions-util";
|
||||
import { type Config } from "../config-utils";
|
||||
import { getCommitOid, getFileOidsUnderPath } from "../git-utils";
|
||||
import { Logger, withGroupAsync } from "../logging";
|
||||
import {
|
||||
CleanupLevel,
|
||||
getBaseDatabaseOidsFilePath,
|
||||
getCodeQLDatabasePath,
|
||||
getErrorMessage,
|
||||
isInTestMode,
|
||||
tryGetFolderBytes,
|
||||
waitForResultWithTimeLimit,
|
||||
} from "../util";
|
||||
|
||||
export enum OverlayDatabaseMode {
|
||||
Overlay = "overlay",
|
||||
OverlayBase = "overlay-base",
|
||||
None = "none",
|
||||
}
|
||||
import { getFileOidsUnderPath, getGitRoot } from "../git-utils";
|
||||
import { Logger } from "../logging";
|
||||
import { getBaseDatabaseOidsFilePath } from "../util";
|
||||
|
||||
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.23.8";
|
||||
|
||||
@@ -43,23 +20,6 @@ export const CODEQL_OVERLAY_MINIMUM_VERSION_JAVASCRIPT = "2.23.9";
|
||||
export const CODEQL_OVERLAY_MINIMUM_VERSION_PYTHON = "2.23.9";
|
||||
export const CODEQL_OVERLAY_MINIMUM_VERSION_RUBY = "2.23.9";
|
||||
|
||||
/**
|
||||
* The maximum (uncompressed) size of the overlay base database that we will
|
||||
* upload. By default, the Actions Cache has an overall capacity of 10 GB, and
|
||||
* the Actions Cache client library uses zstd compression.
|
||||
*
|
||||
* Ideally we would apply a size limit to the compressed overlay-base database,
|
||||
* but we cannot do so because compression is handled transparently by the
|
||||
* Actions Cache client library. Instead we place a limit on the uncompressed
|
||||
* size of the overlay-base database.
|
||||
*
|
||||
* Assuming 2.5:1 compression ratio, the 7.5 GB limit on uncompressed data would
|
||||
* translate to a limit of around 3 GB after compression.
|
||||
*/
|
||||
const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500;
|
||||
const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
|
||||
OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000;
|
||||
|
||||
/**
|
||||
* Writes a JSON file containing Git OIDs for all tracked files (represented
|
||||
* by path relative to the source root) under the source root. The file is
|
||||
@@ -130,11 +90,17 @@ export async function writeOverlayChangesFile(
|
||||
): Promise<string> {
|
||||
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
|
||||
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
|
||||
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
|
||||
const oidChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
|
||||
logger.info(
|
||||
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`,
|
||||
`Found ${oidChangedFiles.length} changed file(s) under ${sourceRoot} from OID comparison.`,
|
||||
);
|
||||
|
||||
// Merge in any file paths from precomputed PR diff ranges to ensure the
|
||||
// overlay always includes all files from the PR diff, even in edge cases
|
||||
// like revert PRs where OID comparison shows no change.
|
||||
const diffRangeFiles = await getDiffRangeFilePaths(sourceRoot, logger);
|
||||
const changedFiles = [...new Set([...oidChangedFiles, ...diffRangeFiles])];
|
||||
|
||||
const changedFilesJson = JSON.stringify({ changes: changedFiles });
|
||||
const overlayChangesFile = path.join(
|
||||
getTemporaryDirectory(),
|
||||
@@ -165,387 +131,65 @@ function computeChangedFiles(
|
||||
return changes;
|
||||
}
|
||||
|
||||
// Constants for database caching
|
||||
const CACHE_VERSION = 1;
|
||||
const CACHE_PREFIX = "codeql-overlay-base-database";
|
||||
|
||||
// The purpose of this ten-minute limit is to guard against the possibility
|
||||
// that the cache service is unresponsive, which would otherwise cause the
|
||||
// entire action to hang. Normally we expect cache operations to complete
|
||||
// within two minutes.
|
||||
const MAX_CACHE_OPERATION_MS = 600_000;
|
||||
|
||||
/**
|
||||
* Checks that the overlay-base database is valid by checking for the
|
||||
* existence of the base database OIDs file.
|
||||
*
|
||||
* @param config The configuration object
|
||||
* @param logger The logger instance
|
||||
* @param warningPrefix Prefix for the check failure warning message
|
||||
* @returns True if the verification succeeded, false otherwise
|
||||
*/
|
||||
async function checkOverlayBaseDatabase(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
async function getDiffRangeFilePaths(
|
||||
sourceRoot: string,
|
||||
logger: Logger,
|
||||
warningPrefix: string,
|
||||
): Promise<boolean> {
|
||||
// An overlay-base database should contain the base database OIDs file.
|
||||
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
|
||||
if (!fs.existsSync(baseDatabaseOidsFilePath)) {
|
||||
logger.warning(
|
||||
`${warningPrefix}: ${baseDatabaseOidsFilePath} does not exist`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
): Promise<string[]> {
|
||||
const jsonFilePath = actionsUtil.getDiffRangesJsonFilePath();
|
||||
|
||||
for (const language of config.languages) {
|
||||
const dbPath = getCodeQLDatabasePath(config, language);
|
||||
try {
|
||||
const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
|
||||
if (
|
||||
resolveDatabaseOutput === undefined ||
|
||||
!("overlayBaseSpecifier" in resolveDatabaseOutput)
|
||||
) {
|
||||
logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
|
||||
return false;
|
||||
} else {
|
||||
logger.debug(
|
||||
`Overlay base specifier for ${language} overlay-base database found: ` +
|
||||
`${resolveDatabaseOutput.overlayBaseSpecifier}`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads the overlay-base database to the GitHub Actions cache. If conditions
|
||||
* for uploading are not met, the function does nothing and returns false.
|
||||
*
|
||||
* This function uses the `checkout_path` input to determine the repository path
|
||||
* and works only when called from `analyze` or `upload-sarif`.
|
||||
*
|
||||
* @param codeql The CodeQL instance
|
||||
* @param config The configuration object
|
||||
* @param logger The logger instance
|
||||
* @returns A promise that resolves to true if the upload was performed and
|
||||
* successfully completed, or false otherwise
|
||||
*/
|
||||
export async function cleanupAndUploadOverlayBaseDatabaseToCache(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<boolean> {
|
||||
const overlayDatabaseMode = config.overlayDatabaseMode;
|
||||
if (overlayDatabaseMode !== OverlayDatabaseMode.OverlayBase) {
|
||||
if (!fs.existsSync(jsonFilePath)) {
|
||||
logger.debug(
|
||||
`Overlay database mode is ${overlayDatabaseMode}. ` +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
`No diff ranges JSON file found at ${jsonFilePath}; skipping.`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (!config.useOverlayDatabaseCaching) {
|
||||
logger.debug(
|
||||
"Overlay database caching is disabled. " +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (isInTestMode()) {
|
||||
logger.debug(
|
||||
"In test mode. Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
return [];
|
||||
}
|
||||
|
||||
const databaseIsValid = await checkOverlayBaseDatabase(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
"Abort uploading overlay-base database to cache",
|
||||
);
|
||||
if (!databaseIsValid) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Clean up the database using the overlay cleanup level.
|
||||
await withGroupAsync("Cleaning up databases", async () => {
|
||||
await codeql.databaseCleanupCluster(config, CleanupLevel.Overlay);
|
||||
});
|
||||
|
||||
const dbLocation = config.dbLocation;
|
||||
|
||||
const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
|
||||
if (databaseSizeBytes === undefined) {
|
||||
logger.warning(
|
||||
"Failed to determine database size. " +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (databaseSizeBytes > OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES) {
|
||||
const databaseSizeMB = Math.round(databaseSizeBytes / 1_000_000);
|
||||
logger.warning(
|
||||
`Database size (${databaseSizeMB} MB) ` +
|
||||
`exceeds maximum upload size (${OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB} MB). ` +
|
||||
"Skip uploading overlay-base database to cache.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const codeQlVersion = (await codeql.getVersion()).version;
|
||||
const checkoutPath = getRequiredInput("checkout_path");
|
||||
const cacheSaveKey = await getCacheSaveKey(
|
||||
config,
|
||||
codeQlVersion,
|
||||
checkoutPath,
|
||||
logger,
|
||||
);
|
||||
logger.info(
|
||||
`Uploading overlay-base database to Actions cache with key ${cacheSaveKey}`,
|
||||
);
|
||||
|
||||
let contents: string;
|
||||
try {
|
||||
const cacheId = await waitForResultWithTimeLimit(
|
||||
MAX_CACHE_OPERATION_MS,
|
||||
actionsCache.saveCache([dbLocation], cacheSaveKey),
|
||||
() => {},
|
||||
);
|
||||
if (cacheId === undefined) {
|
||||
logger.warning("Timed out while uploading overlay-base database");
|
||||
return false;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warning(
|
||||
"Failed to upload overlay-base database to cache: " +
|
||||
`${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
logger.info(`Successfully uploaded overlay-base database from ${dbLocation}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
export interface OverlayBaseDatabaseDownloadStats {
|
||||
databaseSizeBytes: number;
|
||||
databaseDownloadDurationMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads the overlay-base database from the GitHub Actions cache. If conditions
|
||||
* for downloading are not met, the function does nothing and returns false.
|
||||
*
|
||||
* @param codeql The CodeQL instance
|
||||
* @param config The configuration object
|
||||
* @param logger The logger instance
|
||||
* @returns A promise that resolves to download statistics if an overlay-base
|
||||
* database was successfully downloaded, or undefined if the download was
|
||||
* either not performed or failed.
|
||||
*/
|
||||
export async function downloadOverlayBaseDatabaseFromCache(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
logger: Logger,
|
||||
): Promise<OverlayBaseDatabaseDownloadStats | undefined> {
|
||||
const overlayDatabaseMode = config.overlayDatabaseMode;
|
||||
if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
|
||||
logger.debug(
|
||||
`Overlay database mode is ${overlayDatabaseMode}. ` +
|
||||
"Skip downloading overlay-base database from cache.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (!config.useOverlayDatabaseCaching) {
|
||||
logger.debug(
|
||||
"Overlay database caching is disabled. " +
|
||||
"Skip downloading overlay-base database from cache.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (isInTestMode()) {
|
||||
logger.debug(
|
||||
"In test mode. Skip downloading overlay-base database from cache.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const dbLocation = config.dbLocation;
|
||||
const codeQlVersion = (await codeql.getVersion()).version;
|
||||
const cacheRestoreKeyPrefix = await getCacheRestoreKeyPrefix(
|
||||
config,
|
||||
codeQlVersion,
|
||||
);
|
||||
|
||||
logger.info(
|
||||
"Looking in Actions cache for overlay-base database with " +
|
||||
`restore key ${cacheRestoreKeyPrefix}`,
|
||||
);
|
||||
|
||||
let databaseDownloadDurationMs = 0;
|
||||
try {
|
||||
const databaseDownloadStart = performance.now();
|
||||
const foundKey = await waitForResultWithTimeLimit(
|
||||
// This ten-minute limit for the cache restore operation is mainly to
|
||||
// guard against the possibility that the cache service is unresponsive
|
||||
// and hangs outside the data download.
|
||||
//
|
||||
// Data download (which is normally the most time-consuming part of the
|
||||
// restore operation) should not run long enough to hit this limit. Even
|
||||
// for an extremely large 10GB database, at a download speed of 40MB/s
|
||||
// (see below), the download should complete within five minutes. If we
|
||||
// do hit this limit, there are likely more serious problems other than
|
||||
// mere slow download speed.
|
||||
//
|
||||
// This is important because we don't want any ongoing file operations
|
||||
// on the database directory when we do hit this limit. Hitting this
|
||||
// time limit takes us to a fallback path where we re-initialize the
|
||||
// database from scratch at dbLocation, and having the cache restore
|
||||
// operation continue to write into dbLocation in the background would
|
||||
// really mess things up. We want to hit this limit only in the case
|
||||
// of a hung cache service, not just slow download speed.
|
||||
MAX_CACHE_OPERATION_MS,
|
||||
actionsCache.restoreCache(
|
||||
[dbLocation],
|
||||
cacheRestoreKeyPrefix,
|
||||
undefined,
|
||||
{
|
||||
// Azure SDK download (which is the default) uses 128MB segments; see
|
||||
// https://github.com/actions/toolkit/blob/main/packages/cache/README.md.
|
||||
// Setting segmentTimeoutInMs to 3000 translates to segment download
|
||||
// speed of about 40 MB/s, which should be achievable unless the
|
||||
// download is unreliable (in which case we do want to abort).
|
||||
segmentTimeoutInMs: 3000,
|
||||
},
|
||||
),
|
||||
() => {
|
||||
logger.info("Timed out downloading overlay-base database from cache");
|
||||
},
|
||||
);
|
||||
databaseDownloadDurationMs = Math.round(
|
||||
performance.now() - databaseDownloadStart,
|
||||
);
|
||||
|
||||
if (foundKey === undefined) {
|
||||
logger.info("No overlay-base database found in Actions cache");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Downloaded overlay-base database in cache with key ${foundKey}`,
|
||||
);
|
||||
} catch (error) {
|
||||
logger.warning(
|
||||
"Failed to download overlay-base database from cache: " +
|
||||
`${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const databaseIsValid = await checkOverlayBaseDatabase(
|
||||
codeql,
|
||||
config,
|
||||
logger,
|
||||
"Downloaded overlay-base database is invalid",
|
||||
);
|
||||
if (!databaseIsValid) {
|
||||
logger.warning("Downloaded overlay-base database failed validation");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger);
|
||||
if (databaseSizeBytes === undefined) {
|
||||
logger.info(
|
||||
"Filesystem error while accessing downloaded overlay-base database",
|
||||
);
|
||||
// The problem that warrants reporting download failure is not that we are
|
||||
// unable to determine the size of the database. Rather, it is that we
|
||||
// encountered a filesystem error while accessing the database, which
|
||||
// indicates that an overlay analysis will likely fail.
|
||||
return undefined;
|
||||
}
|
||||
|
||||
logger.info(`Successfully downloaded overlay-base database to ${dbLocation}`);
|
||||
return {
|
||||
databaseSizeBytes: Math.round(databaseSizeBytes),
|
||||
databaseDownloadDurationMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cache key for saving the overlay-base database to the GitHub
|
||||
* Actions cache.
|
||||
*
|
||||
* The key consists of the restore key prefix (which does not include the
|
||||
* commit SHA) and the commit SHA of the current checkout.
|
||||
*/
|
||||
export async function getCacheSaveKey(
|
||||
config: Config,
|
||||
codeQlVersion: string,
|
||||
checkoutPath: string,
|
||||
logger: Logger,
|
||||
): Promise<string> {
|
||||
let runId = 1;
|
||||
let attemptId = 1;
|
||||
try {
|
||||
runId = getWorkflowRunID();
|
||||
attemptId = getWorkflowRunAttempt();
|
||||
contents = await fs.promises.readFile(jsonFilePath, "utf8");
|
||||
} catch (e) {
|
||||
logger.warning(
|
||||
`Failed to get workflow run ID or attempt ID. Reason: ${getErrorMessage(e)}`,
|
||||
`Failed to read diff ranges JSON file at ${jsonFilePath}: ${e}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
const sha = await getCommitOid(checkoutPath);
|
||||
const restoreKeyPrefix = await getCacheRestoreKeyPrefix(
|
||||
config,
|
||||
codeQlVersion,
|
||||
|
||||
let diffRanges: Array<{ path: string }>;
|
||||
try {
|
||||
diffRanges = JSON.parse(contents) as Array<{ path: string }>;
|
||||
} catch (e) {
|
||||
logger.warning(
|
||||
`Failed to parse diff ranges JSON file at ${jsonFilePath}: ${e}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
logger.debug(
|
||||
`Read ${diffRanges.length} diff range(s) from ${jsonFilePath} for overlay changes.`,
|
||||
);
|
||||
return `${restoreKeyPrefix}${sha}-${runId}-${attemptId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the cache key prefix for restoring the overlay-base database from
|
||||
* the GitHub Actions cache.
|
||||
*
|
||||
* Actions cache supports using multiple restore keys to indicate preference,
|
||||
* and this function could in principle take advantage of that feature by
|
||||
* returning a list of restore key prefixes. However, since overlay-base
|
||||
* databases are built from the default branch and used in PR analysis, it is
|
||||
* exceedingly unlikely that the commit SHA will ever be the same.
|
||||
*
|
||||
* Therefore, this function returns only a single restore key prefix, which does
|
||||
* not include the commit SHA. This allows us to restore the most recent
|
||||
* compatible overlay-base database.
|
||||
*/
|
||||
export async function getCacheRestoreKeyPrefix(
|
||||
config: Config,
|
||||
codeQlVersion: string,
|
||||
): Promise<string> {
|
||||
const languages = [...config.languages].sort().join("_");
|
||||
|
||||
const cacheKeyComponents = {
|
||||
automationID: await getAutomationID(),
|
||||
// Add more components here as needed in the future
|
||||
};
|
||||
const componentsHash = createCacheKeyHash(cacheKeyComponents);
|
||||
|
||||
// For a cached overlay-base database to be considered compatible for overlay
|
||||
// analysis, all components in the cache restore key must match:
|
||||
//
|
||||
// CACHE_PREFIX: distinguishes overlay-base databases from other cache objects
|
||||
// CACHE_VERSION: cache format version
|
||||
// componentsHash: hash of additional components (see above for details)
|
||||
// languages: the languages included in the overlay-base database
|
||||
// codeQlVersion: CodeQL bundle version
|
||||
//
|
||||
// Technically we can also include languages and codeQlVersion in the
|
||||
// componentsHash, but including them explicitly in the cache key makes it
|
||||
// easier to debug and understand the cache key structure.
|
||||
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languages}-${codeQlVersion}-`;
|
||||
|
||||
// Diff-range paths are relative to the repo root (from the GitHub compare
|
||||
// API), but overlay changed files must be relative to sourceRoot (to match
|
||||
// getFileOidsUnderPath output). Convert and filter accordingly.
|
||||
const repoRoot = await getGitRoot(sourceRoot);
|
||||
if (repoRoot === undefined) {
|
||||
if (getOptionalInput("source-root")) {
|
||||
throw new Error(
|
||||
"Cannot determine git root to convert diff range paths relative to source-root. " +
|
||||
"Failing to avoid omitting files from the analysis.",
|
||||
);
|
||||
}
|
||||
logger.warning(
|
||||
"Cannot determine git root; returning diff range paths as-is.",
|
||||
);
|
||||
return [...new Set(diffRanges.map((r) => r.path))];
|
||||
}
|
||||
|
||||
const relativePaths = diffRanges
|
||||
.map((r) =>
|
||||
path
|
||||
.relative(sourceRoot, path.join(repoRoot, r.path))
|
||||
.replaceAll(path.sep, "/"),
|
||||
)
|
||||
.filter((rel) => !rel.startsWith(".."));
|
||||
return [...new Set(relativePaths)];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
export enum OverlayDatabaseMode {
|
||||
Overlay = "overlay",
|
||||
OverlayBase = "overlay-base",
|
||||
None = "none",
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import * as core from "@actions/core";
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { getGitHubVersion } from "./api-client";
|
||||
import { Feature, FeatureEnablement, initFeatures } from "./feature-flags";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage, parseBuiltInLanguage } from "./languages";
|
||||
import { getActionsLogger, Logger } from "./logging";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import {
|
||||
@@ -14,7 +14,6 @@ import {
|
||||
getCredentials,
|
||||
getProxyBinaryPath,
|
||||
getSafeErrorMessage,
|
||||
parseLanguage,
|
||||
ProxyInfo,
|
||||
sendFailedStatusReport,
|
||||
sendSuccessStatusReport,
|
||||
@@ -33,7 +32,7 @@ async function run(startedAt: Date) {
|
||||
|
||||
const logger = getActionsLogger();
|
||||
let features: FeatureEnablement | undefined;
|
||||
let language: KnownLanguage | undefined;
|
||||
let language: BuiltInLanguage | undefined;
|
||||
|
||||
try {
|
||||
// Make inputs accessible in the `post` step.
|
||||
@@ -56,7 +55,7 @@ async function run(startedAt: Date) {
|
||||
|
||||
// Get the language input.
|
||||
const languageInput = actionsUtil.getOptionalInput("language");
|
||||
language = languageInput ? parseLanguage(languageInput) : undefined;
|
||||
language = languageInput ? parseBuiltInLanguage(languageInput) : undefined;
|
||||
|
||||
// Query the FF for whether we should use the reduced registry mapping.
|
||||
const skipUnusedRegistries = await features.getValue(
|
||||
@@ -112,14 +111,14 @@ async function run(startedAt: Date) {
|
||||
logger,
|
||||
);
|
||||
|
||||
// Check that the private registries are reachable.
|
||||
// Perform best-effort checks that the private registries are reachable.
|
||||
await checkConnections(logger, proxyInfo);
|
||||
|
||||
// Report success if we have reached this point.
|
||||
await sendSuccessStatusReport(
|
||||
startedAt,
|
||||
{
|
||||
languages: language && [language],
|
||||
languages: language === undefined ? undefined : [language],
|
||||
},
|
||||
proxyConfig.all_credentials.map((c) => c.type),
|
||||
logger,
|
||||
@@ -199,6 +198,7 @@ async function startProxy(
|
||||
.map((credential) => ({
|
||||
type: credential.type,
|
||||
url: credential.url,
|
||||
"replaces-base": credential["replaces-base"],
|
||||
}));
|
||||
core.setOutput("proxy_urls", JSON.stringify(registry_urls));
|
||||
|
||||
|
||||
+315
-183
@@ -8,15 +8,16 @@ import sinon from "sinon";
|
||||
import * as apiClient from "./api-client";
|
||||
import * as defaults from "./defaults.json";
|
||||
import { setUpFeatureFlagTests } from "./feature-flags/testing-util";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { UnvalidatedObject, validateSchema } from "./json";
|
||||
import { makeFromSchema } from "./json/testing-util";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { getRunnerLogger, Logger } from "./logging";
|
||||
import * as startProxyExports from "./start-proxy";
|
||||
import { parseLanguage } from "./start-proxy";
|
||||
import * as statusReport from "./status-report";
|
||||
import {
|
||||
assertNotLogged,
|
||||
checkExpectedLogMessages,
|
||||
createFeatures,
|
||||
getRecordingLogger,
|
||||
makeTestToken,
|
||||
RecordingLogger,
|
||||
setupTests,
|
||||
@@ -232,7 +233,7 @@ test("getCredentials filters by language when specified", async (t) => {
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON(mixedCredentials),
|
||||
KnownLanguage.java,
|
||||
BuiltInLanguage.java,
|
||||
);
|
||||
t.is(credentials.length, 1);
|
||||
t.is(credentials[0].type, "maven_repository");
|
||||
@@ -243,7 +244,7 @@ test("getCredentials returns all for a language when specified", async (t) => {
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON(mixedCredentials),
|
||||
KnownLanguage.go,
|
||||
BuiltInLanguage.go,
|
||||
);
|
||||
t.is(credentials.length, 2);
|
||||
|
||||
@@ -252,6 +253,57 @@ test("getCredentials returns all for a language when specified", async (t) => {
|
||||
t.assert(credentialsTypes.includes("git_source"));
|
||||
});
|
||||
|
||||
test("getCredentials returns all goproxy_servers for Go when specified", async (t) => {
|
||||
const multipleGoproxyServers = [
|
||||
{ type: "goproxy_server", host: "goproxy1.example.com", token: "token1" },
|
||||
{ type: "goproxy_server", host: "goproxy2.example.com", token: "token2" },
|
||||
{ type: "git_source", host: "github.com/github", token: "mno" },
|
||||
];
|
||||
|
||||
const credentials = startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON(multipleGoproxyServers),
|
||||
BuiltInLanguage.go,
|
||||
);
|
||||
t.is(credentials.length, 3);
|
||||
|
||||
const goproxyServers = credentials.filter((c) => c.type === "goproxy_server");
|
||||
t.is(goproxyServers.length, 2);
|
||||
t.assert(goproxyServers.some((c) => c.host === "goproxy1.example.com"));
|
||||
t.assert(goproxyServers.some((c) => c.host === "goproxy2.example.com"));
|
||||
});
|
||||
|
||||
test("getCredentials returns all maven_repositories for Java when specified", async (t) => {
|
||||
const multipleMavenRepositories = [
|
||||
{
|
||||
type: "maven_repository",
|
||||
host: "maven1.pkg.github.com",
|
||||
token: "token1",
|
||||
},
|
||||
{
|
||||
type: "maven_repository",
|
||||
host: "maven2.pkg.github.com",
|
||||
token: "token2",
|
||||
},
|
||||
{ type: "git_source", host: "github.com/github", token: "mno" },
|
||||
];
|
||||
|
||||
const credentials = startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON(multipleMavenRepositories),
|
||||
BuiltInLanguage.java,
|
||||
);
|
||||
t.is(credentials.length, 2);
|
||||
|
||||
const mavenRepositories = credentials.filter(
|
||||
(c) => c.type === "maven_repository",
|
||||
);
|
||||
t.assert(mavenRepositories.some((c) => c.host === "maven1.pkg.github.com"));
|
||||
t.assert(mavenRepositories.some((c) => c.host === "maven2.pkg.github.com"));
|
||||
});
|
||||
|
||||
test("getCredentials returns all credentials when no language specified", async (t) => {
|
||||
const credentialsInput = toEncodedJSON(mixedCredentials);
|
||||
|
||||
@@ -299,180 +351,287 @@ test("getCredentials throws an error when non-printable characters are used", as
|
||||
}
|
||||
});
|
||||
|
||||
const validAzureCredential: startProxyExports.AzureConfig = {
|
||||
tenant_id: "12345678-1234-1234-1234-123456789012",
|
||||
client_id: "abcdef01-2345-6789-abcd-ef0123456789",
|
||||
};
|
||||
for (const oidcSchemaInfo of startProxyExports.oidcSchemas) {
|
||||
test(`getCredentials throws when non-printable characters are used (${oidcSchemaInfo.name} OIDC)`, (t) => {
|
||||
const validCredential = makeFromSchema(true, oidcSchemaInfo.schema);
|
||||
for (const key of Object.keys(validCredential)) {
|
||||
const invalidAuthConfig = {
|
||||
...validCredential,
|
||||
[key]: "123\x00",
|
||||
};
|
||||
const invalidCredential: startProxyExports.RawCredential = {
|
||||
type: "nuget_feed",
|
||||
host: `${key}.nuget.pkg.github.com`,
|
||||
...invalidAuthConfig,
|
||||
};
|
||||
const credentialsInput = toEncodedJSON([invalidCredential]);
|
||||
|
||||
const validAwsCredential: startProxyExports.AWSConfig = {
|
||||
aws_region: "us-east-1",
|
||||
account_id: "123456789012",
|
||||
role_name: "MY_ROLE",
|
||||
domain: "MY_DOMAIN",
|
||||
domain_owner: "987654321098",
|
||||
audience: "custom-audience",
|
||||
};
|
||||
|
||||
const validJFrogCredential: startProxyExports.JFrogConfig = {
|
||||
jfrog_oidc_provider_name: "MY_PROVIDER",
|
||||
audience: "jfrog-audience",
|
||||
identity_mapping_name: "my-mapping",
|
||||
};
|
||||
|
||||
test("getCredentials throws an error when non-printable characters are used for Azure OIDC", (t) => {
|
||||
for (const key of Object.keys(validAzureCredential)) {
|
||||
const invalidAzureCredential = {
|
||||
...validAzureCredential,
|
||||
[key]: "123\x00",
|
||||
};
|
||||
const invalidCredential: startProxyExports.RawCredential = {
|
||||
type: "nuget_feed",
|
||||
host: `${key}.nuget.pkg.github.com`,
|
||||
...invalidAzureCredential,
|
||||
};
|
||||
const credentialsInput = toEncodedJSON([invalidCredential]);
|
||||
|
||||
t.throws(
|
||||
() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
credentialsInput,
|
||||
undefined,
|
||||
),
|
||||
{
|
||||
message:
|
||||
"Invalid credentials - fields must contain only printable characters",
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("getCredentials throws an error when non-printable characters are used for AWS OIDC", (t) => {
|
||||
for (const key of Object.keys(validAwsCredential)) {
|
||||
const invalidAwsCredential = {
|
||||
...validAwsCredential,
|
||||
[key]: "123\x00",
|
||||
};
|
||||
const invalidCredential: startProxyExports.RawCredential = {
|
||||
type: "nuget_feed",
|
||||
host: `${key}.nuget.pkg.github.com`,
|
||||
...invalidAwsCredential,
|
||||
};
|
||||
const credentialsInput = toEncodedJSON([invalidCredential]);
|
||||
|
||||
t.throws(
|
||||
() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
credentialsInput,
|
||||
undefined,
|
||||
),
|
||||
{
|
||||
message:
|
||||
"Invalid credentials - fields must contain only printable characters",
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("getCredentials throws an error when non-printable characters are used for JFrog OIDC", (t) => {
|
||||
for (const key of Object.keys(validJFrogCredential)) {
|
||||
const invalidJFrogCredential = {
|
||||
...validJFrogCredential,
|
||||
[key]: "123\x00",
|
||||
};
|
||||
const invalidCredential: startProxyExports.RawCredential = {
|
||||
type: "nuget_feed",
|
||||
host: `${key}.nuget.pkg.github.com`,
|
||||
...invalidJFrogCredential,
|
||||
};
|
||||
const credentialsInput = toEncodedJSON([invalidCredential]);
|
||||
|
||||
t.throws(
|
||||
() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
credentialsInput,
|
||||
undefined,
|
||||
),
|
||||
{
|
||||
message:
|
||||
"Invalid credentials - fields must contain only printable characters",
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
t.throws(
|
||||
() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
credentialsInput,
|
||||
undefined,
|
||||
),
|
||||
{
|
||||
message:
|
||||
"Invalid credentials - fields must contain only printable characters",
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("getCredentials accepts OIDC configurations", (t) => {
|
||||
const oidcConfigurations = [
|
||||
{
|
||||
const oidcConfigurations = startProxyExports.oidcSchemas.map(
|
||||
(schemaInfo) => ({
|
||||
type: "nuget_feed",
|
||||
host: "azure.pkg.github.com",
|
||||
...validAzureCredential,
|
||||
},
|
||||
{
|
||||
type: "nuget_feed",
|
||||
host: "aws.pkg.github.com",
|
||||
...validAwsCredential,
|
||||
},
|
||||
{
|
||||
type: "nuget_feed",
|
||||
host: "jfrog.pkg.github.com",
|
||||
...validJFrogCredential,
|
||||
},
|
||||
];
|
||||
host: `${schemaInfo.name.toLowerCase()}.pkg.github.com`,
|
||||
...makeFromSchema(true, schemaInfo.schema),
|
||||
}),
|
||||
);
|
||||
|
||||
const credentials = startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON(oidcConfigurations),
|
||||
KnownLanguage.csharp,
|
||||
BuiltInLanguage.csharp,
|
||||
);
|
||||
t.is(credentials.length, 3);
|
||||
t.is(credentials.length, startProxyExports.oidcSchemas.length);
|
||||
|
||||
t.assert(credentials.every((c) => c.type === "nuget_feed"));
|
||||
t.assert(credentials.some((c) => startProxyExports.isAzureConfig(c)));
|
||||
t.assert(credentials.some((c) => startProxyExports.isAWSConfig(c)));
|
||||
t.assert(credentials.some((c) => startProxyExports.isJFrogConfig(c)));
|
||||
|
||||
for (const oidcSchemaInfo of startProxyExports.oidcSchemas) {
|
||||
t.assert(
|
||||
credentials.some((c) =>
|
||||
validateSchema(
|
||||
oidcSchemaInfo.schema,
|
||||
c as unknown as UnvalidatedObject<any>,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("getCredentials logs a warning when a PAT is used without a username", async (t) => {
|
||||
const loggedMessages = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
const likelyWrongCredentials = toEncodedJSON([
|
||||
const getCredentialsMacro = test.macro({
|
||||
exec: async (
|
||||
t: ExecutionContext<unknown>,
|
||||
credentials: startProxyExports.RawCredential[],
|
||||
checkAccepted: (
|
||||
t: ExecutionContext<unknown>,
|
||||
logger: RecordingLogger,
|
||||
results: startProxyExports.Credential[],
|
||||
) => void,
|
||||
) => {
|
||||
const logger = new RecordingLogger();
|
||||
const credentialsString = toEncodedJSON(credentials);
|
||||
|
||||
const results = startProxyExports.getCredentials(
|
||||
logger,
|
||||
undefined,
|
||||
credentialsString,
|
||||
undefined,
|
||||
);
|
||||
|
||||
checkAccepted(t, logger, results);
|
||||
},
|
||||
|
||||
title: (providedTitle = "") => `getCredentials - ${providedTitle}`,
|
||||
});
|
||||
|
||||
test(
|
||||
"warns for PAT-like password without a username",
|
||||
getCredentialsMacro,
|
||||
[
|
||||
{
|
||||
type: "git_server",
|
||||
host: "https://github.com/",
|
||||
password: `ghp_${makeTestToken()}`,
|
||||
},
|
||||
],
|
||||
(t, logger, results) => {
|
||||
// The configurations should be accepted, despite the likely problem.
|
||||
t.assert(results);
|
||||
t.is(results.length, 1);
|
||||
t.is(results[0].type, "git_server");
|
||||
t.is(results[0].host, "https://github.com/");
|
||||
|
||||
if (startProxyExports.hasUsernameAndPassword(results[0])) {
|
||||
t.assert(results[0].password?.startsWith("ghp_"));
|
||||
} else {
|
||||
t.fail("Expected a `UsernamePassword`-based credential.");
|
||||
}
|
||||
|
||||
// A warning should have been logged.
|
||||
checkExpectedLogMessages(t, logger.messages, [
|
||||
"using a GitHub Personal Access Token (PAT), but no username was provided",
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"no warning for PAT-like password with a username",
|
||||
getCredentialsMacro,
|
||||
[
|
||||
{
|
||||
type: "git_server",
|
||||
host: "https://github.com/",
|
||||
username: "someone",
|
||||
password: `ghp_${makeTestToken()}`,
|
||||
},
|
||||
],
|
||||
(t, logger, results) => {
|
||||
// The configurations should be accepted, despite the likely problem.
|
||||
t.assert(results);
|
||||
t.is(results.length, 1);
|
||||
t.is(results[0].type, "git_server");
|
||||
t.is(results[0].host, "https://github.com/");
|
||||
|
||||
if (startProxyExports.hasUsernameAndPassword(results[0])) {
|
||||
t.assert(results[0].password?.startsWith("ghp_"));
|
||||
} else {
|
||||
t.fail("Expected a `UsernamePassword`-based credential.");
|
||||
}
|
||||
|
||||
assertNotLogged(
|
||||
t,
|
||||
logger,
|
||||
"using a GitHub Personal Access Token (PAT), but no username was provided",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"warns for PAT-like token without a username",
|
||||
getCredentialsMacro,
|
||||
[
|
||||
{
|
||||
type: "git_server",
|
||||
host: "https://github.com/",
|
||||
token: `ghp_${makeTestToken()}`,
|
||||
},
|
||||
],
|
||||
(t, logger, results) => {
|
||||
// The configurations should be accepted, despite the likely problem.
|
||||
t.assert(results);
|
||||
t.is(results.length, 1);
|
||||
t.is(results[0].type, "git_server");
|
||||
t.is(results[0].host, "https://github.com/");
|
||||
|
||||
if (startProxyExports.isToken(results[0])) {
|
||||
t.assert(results[0].token?.startsWith("ghp_"));
|
||||
} else {
|
||||
t.fail("Expected a `Token`-based credential.");
|
||||
}
|
||||
|
||||
// A warning should have been logged.
|
||||
checkExpectedLogMessages(t, logger.messages, [
|
||||
"using a GitHub Personal Access Token (PAT), but no username was provided",
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"no warning for PAT-like token with a username",
|
||||
getCredentialsMacro,
|
||||
[
|
||||
{
|
||||
type: "git_server",
|
||||
host: "https://github.com/",
|
||||
username: "someone",
|
||||
token: `ghp_${makeTestToken()}`,
|
||||
},
|
||||
],
|
||||
(t, logger, results) => {
|
||||
// The configurations should be accepted, despite the likely problem.
|
||||
t.assert(results);
|
||||
t.is(results.length, 1);
|
||||
t.is(results[0].type, "git_server");
|
||||
t.is(results[0].host, "https://github.com/");
|
||||
|
||||
if (startProxyExports.isToken(results[0])) {
|
||||
t.assert(results[0].token?.startsWith("ghp_"));
|
||||
} else {
|
||||
t.fail("Expected a `Token`-based credential.");
|
||||
}
|
||||
|
||||
assertNotLogged(
|
||||
t,
|
||||
logger,
|
||||
"using a GitHub Personal Access Token (PAT), but no username was provided",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test("getCredentials validates 'replaces-base' correctly", async (t) => {
|
||||
// Valid cases.
|
||||
const credentialsInput = toEncodedJSON([
|
||||
{
|
||||
type: "maven_repository",
|
||||
host: "maven1.pkg.github.com",
|
||||
token: "abc",
|
||||
"replaces-base": false,
|
||||
},
|
||||
{
|
||||
type: "maven_repository",
|
||||
host: "maven2.pkg.github.com",
|
||||
token: "def",
|
||||
"replaces-base": true,
|
||||
},
|
||||
{
|
||||
type: "maven_repository",
|
||||
host: "maven3.pkg.github.com",
|
||||
token: "ghi",
|
||||
},
|
||||
]);
|
||||
|
||||
const results = startProxyExports.getCredentials(
|
||||
logger,
|
||||
undefined,
|
||||
likelyWrongCredentials,
|
||||
const credentials = startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
credentialsInput,
|
||||
BuiltInLanguage.java,
|
||||
false,
|
||||
);
|
||||
|
||||
// The configuration should be accepted, despite the likely problem.
|
||||
t.assert(results);
|
||||
t.is(results.length, 1);
|
||||
t.is(results[0].type, "git_server");
|
||||
t.is(results[0].host, "https://github.com/");
|
||||
t.is(credentials.length, 3);
|
||||
t.true(credentials.some((c) => c["replaces-base"] === true));
|
||||
t.true(credentials.some((c) => c["replaces-base"] === false));
|
||||
t.true(credentials.some((c) => c["replaces-base"] === undefined));
|
||||
|
||||
if (startProxyExports.isUsernamePassword(results[0])) {
|
||||
t.assert(results[0].password?.startsWith("ghp_"));
|
||||
} else {
|
||||
t.fail("Expected a `UsernamePassword`-based credential.");
|
||||
}
|
||||
|
||||
// A warning should have been logged.
|
||||
checkExpectedLogMessages(t, loggedMessages, [
|
||||
"using a GitHub Personal Access Token (PAT), but no username was provided",
|
||||
]);
|
||||
// Invalid cases.
|
||||
const baseInvalid = {
|
||||
type: "maven_repository",
|
||||
host: "maven4.pkg.github.com",
|
||||
token: "jkl",
|
||||
};
|
||||
t.throws(() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON([{ ...baseInvalid, "replaces-base": null }]),
|
||||
BuiltInLanguage.actions,
|
||||
false,
|
||||
),
|
||||
);
|
||||
t.throws(() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON([{ ...baseInvalid, "replaces-base": 123 }]),
|
||||
BuiltInLanguage.actions,
|
||||
false,
|
||||
),
|
||||
);
|
||||
t.throws(() =>
|
||||
startProxyExports.getCredentials(
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
toEncodedJSON([{ ...baseInvalid, "replaces-base": "true" }]),
|
||||
BuiltInLanguage.actions,
|
||||
false,
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test("getCredentials returns all credentials for Actions when using LANGUAGE_TO_REGISTRY_TYPE", async (t) => {
|
||||
@@ -482,7 +641,7 @@ test("getCredentials returns all credentials for Actions when using LANGUAGE_TO_
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
credentialsInput,
|
||||
KnownLanguage.actions,
|
||||
BuiltInLanguage.actions,
|
||||
false,
|
||||
);
|
||||
t.is(credentials.length, mixedCredentials.length);
|
||||
@@ -495,39 +654,12 @@ test("getCredentials returns no credentials for Actions when using NEW_LANGUAGE_
|
||||
getRunnerLogger(true),
|
||||
undefined,
|
||||
credentialsInput,
|
||||
KnownLanguage.actions,
|
||||
BuiltInLanguage.actions,
|
||||
true,
|
||||
);
|
||||
t.deepEqual(credentials, []);
|
||||
});
|
||||
|
||||
test("parseLanguage", async (t) => {
|
||||
// Exact matches
|
||||
t.deepEqual(parseLanguage("csharp"), KnownLanguage.csharp);
|
||||
t.deepEqual(parseLanguage("cpp"), KnownLanguage.cpp);
|
||||
t.deepEqual(parseLanguage("go"), KnownLanguage.go);
|
||||
t.deepEqual(parseLanguage("java"), KnownLanguage.java);
|
||||
t.deepEqual(parseLanguage("javascript"), KnownLanguage.javascript);
|
||||
t.deepEqual(parseLanguage("python"), KnownLanguage.python);
|
||||
t.deepEqual(parseLanguage("rust"), KnownLanguage.rust);
|
||||
|
||||
// Aliases
|
||||
t.deepEqual(parseLanguage("c"), KnownLanguage.cpp);
|
||||
t.deepEqual(parseLanguage("c++"), KnownLanguage.cpp);
|
||||
t.deepEqual(parseLanguage("c#"), KnownLanguage.csharp);
|
||||
t.deepEqual(parseLanguage("kotlin"), KnownLanguage.java);
|
||||
t.deepEqual(parseLanguage("typescript"), KnownLanguage.javascript);
|
||||
|
||||
// spaces and case-insensitivity
|
||||
t.deepEqual(parseLanguage(" \t\nCsHaRp\t\t"), KnownLanguage.csharp);
|
||||
t.deepEqual(parseLanguage(" \t\nkOtLin\t\t"), KnownLanguage.java);
|
||||
|
||||
// Not matches
|
||||
t.deepEqual(parseLanguage("foo"), undefined);
|
||||
t.deepEqual(parseLanguage(" "), undefined);
|
||||
t.deepEqual(parseLanguage(""), undefined);
|
||||
});
|
||||
|
||||
function mockGetApiClient(endpoints: any) {
|
||||
return (
|
||||
sinon
|
||||
|
||||
+38
-137
@@ -18,26 +18,18 @@ import {
|
||||
FeatureEnablement,
|
||||
} from "./feature-flags";
|
||||
import * as json from "./json";
|
||||
import { KnownLanguage } from "./languages";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import {
|
||||
Address,
|
||||
Registry,
|
||||
Credential,
|
||||
AuthConfig,
|
||||
isToken,
|
||||
isAzureConfig,
|
||||
Token,
|
||||
UsernamePassword,
|
||||
AzureConfig,
|
||||
isAWSConfig,
|
||||
AWSConfig,
|
||||
isJFrogConfig,
|
||||
JFrogConfig,
|
||||
isUsernamePassword,
|
||||
hasToken,
|
||||
hasUsernameAndPassword,
|
||||
hasUsername,
|
||||
RawCredential,
|
||||
} from "./start-proxy/types";
|
||||
import { getAuthConfig } from "./start-proxy/validation";
|
||||
import {
|
||||
ActionName,
|
||||
createStatusReportBase,
|
||||
@@ -156,7 +148,7 @@ export function getSafeErrorMessage(error: Error): string {
|
||||
export async function sendFailedStatusReport(
|
||||
logger: Logger,
|
||||
startedAt: Date,
|
||||
language: KnownLanguage | undefined,
|
||||
language: BuiltInLanguage | undefined,
|
||||
unwrappedError: unknown,
|
||||
) {
|
||||
const error = util.wrapError(unwrappedError);
|
||||
@@ -172,7 +164,7 @@ export async function sendFailedStatusReport(
|
||||
getActionsStatus(error),
|
||||
startedAt,
|
||||
{
|
||||
languages: language && [language],
|
||||
languages: language === undefined ? undefined : [language],
|
||||
},
|
||||
await util.checkDiskUsage(logger),
|
||||
logger,
|
||||
@@ -188,48 +180,6 @@ export const UPDATEJOB_PROXY_VERSION = "v2.0.20250624110901";
|
||||
const UPDATEJOB_PROXY_URL_PREFIX =
|
||||
"https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.22.0/";
|
||||
|
||||
/*
|
||||
* Language aliases supported by the start-proxy Action.
|
||||
*
|
||||
* In general, the CodeQL CLI is the source of truth for language aliases, and to
|
||||
* allow us to more easily support new languages, we want to avoid hardcoding these
|
||||
* aliases in the Action itself. However this is difficult to do in the start-proxy
|
||||
* Action since this Action does not use CodeQL, so we're accepting some hardcoding
|
||||
* for this Action.
|
||||
*/
|
||||
const LANGUAGE_ALIASES: { [lang: string]: KnownLanguage } = {
|
||||
c: KnownLanguage.cpp,
|
||||
"c++": KnownLanguage.cpp,
|
||||
"c#": KnownLanguage.csharp,
|
||||
kotlin: KnownLanguage.java,
|
||||
typescript: KnownLanguage.javascript,
|
||||
"javascript-typescript": KnownLanguage.javascript,
|
||||
"java-kotlin": KnownLanguage.java,
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse the start-proxy language input into its canonical CodeQL language name.
|
||||
*
|
||||
* Exported for testing. Do not use this outside of the start-proxy Action
|
||||
* to avoid complicating the process of adding new CodeQL languages.
|
||||
*/
|
||||
export function parseLanguage(language: string): KnownLanguage | undefined {
|
||||
// Normalize to lower case
|
||||
language = language.trim().toLowerCase();
|
||||
|
||||
// See if it's an exact match
|
||||
if (language in KnownLanguage) {
|
||||
return language as KnownLanguage;
|
||||
}
|
||||
|
||||
// Check language aliases
|
||||
if (language in LANGUAGE_ALIASES) {
|
||||
return LANGUAGE_ALIASES[language];
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isPAT(value: string) {
|
||||
return artifactScanner.isAuthToken(value, [
|
||||
artifactScanner.GITHUB_PAT_CLASSIC_PATTERN,
|
||||
@@ -237,7 +187,7 @@ function isPAT(value: string) {
|
||||
]);
|
||||
}
|
||||
|
||||
type RegistryMapping = Partial<Record<KnownLanguage, string[]>>;
|
||||
type RegistryMapping = Partial<Record<BuiltInLanguage, string[]>>;
|
||||
|
||||
const LANGUAGE_TO_REGISTRY_TYPE: RegistryMapping = {
|
||||
java: ["maven_repository"],
|
||||
@@ -293,75 +243,6 @@ function getRegistryAddress(
|
||||
}
|
||||
}
|
||||
|
||||
/** Extracts an `AuthConfig` value from `config`. */
|
||||
export function getAuthConfig(
|
||||
config: json.UnvalidatedObject<AuthConfig>,
|
||||
): AuthConfig {
|
||||
// Start by checking for the OIDC configurations, since they have required properties
|
||||
// which we can use to identify them.
|
||||
if (isAzureConfig(config)) {
|
||||
return {
|
||||
tenant_id: config.tenant_id,
|
||||
client_id: config.client_id,
|
||||
} satisfies AzureConfig;
|
||||
} else if (isAWSConfig(config)) {
|
||||
return {
|
||||
aws_region: config.aws_region,
|
||||
account_id: config.account_id,
|
||||
role_name: config.role_name,
|
||||
domain: config.domain,
|
||||
domain_owner: config.domain_owner,
|
||||
audience: config.audience,
|
||||
} satisfies AWSConfig;
|
||||
} else if (isJFrogConfig(config)) {
|
||||
return {
|
||||
jfrog_oidc_provider_name: config.jfrog_oidc_provider_name,
|
||||
identity_mapping_name: config.identity_mapping_name,
|
||||
audience: config.audience,
|
||||
} satisfies JFrogConfig;
|
||||
} else if (isToken(config)) {
|
||||
// There are three scenarios for non-OIDC authentication based on the registry type:
|
||||
//
|
||||
// 1. `username`+`token`
|
||||
// 2. A `token` that combines the username and actual token, separated by ':'.
|
||||
// 3. `username`+`password`
|
||||
//
|
||||
// In all three cases, all fields are optional. If the `token` field is present,
|
||||
// we accept the configuration as a `Token` typed configuration, with the `token`
|
||||
// value and an optional `username`. Otherwise, we accept the configuration
|
||||
// typed as `UsernamePassword` (in the `else` clause below) with optional
|
||||
// username and password. I.e. a private registry type that uses 1. or 2.,
|
||||
// but has no `token` configured, will get accepted as `UsernamePassword` here.
|
||||
|
||||
if (isDefined(config.token)) {
|
||||
// Mask token to reduce chance of accidental leakage in logs, if we have one.
|
||||
core.setSecret(config.token);
|
||||
}
|
||||
|
||||
return { username: config.username, token: config.token } satisfies Token;
|
||||
} else {
|
||||
let username: string | undefined = undefined;
|
||||
let password: string | undefined = undefined;
|
||||
|
||||
// Both "username" and "password" are optional. If we have reached this point, we need
|
||||
// to validate which of them are present and that they have the correct type if so.
|
||||
if ("password" in config && json.isString(config.password)) {
|
||||
// Mask password to reduce chance of accidental leakage in logs, if we have one.
|
||||
core.setSecret(config.password);
|
||||
password = config.password;
|
||||
}
|
||||
if ("username" in config && json.isString(config.username)) {
|
||||
username = config.username;
|
||||
}
|
||||
|
||||
// Return the `UsernamePassword` object. Both username and password may be undefined.
|
||||
return {
|
||||
username,
|
||||
password,
|
||||
} satisfies UsernamePassword;
|
||||
}
|
||||
}
|
||||
|
||||
// getCredentials returns registry credentials from action inputs.
|
||||
// It prefers `registries_credentials` over `registry_secrets`.
|
||||
// If neither is set, it returns an empty array.
|
||||
@@ -369,7 +250,7 @@ export function getCredentials(
|
||||
logger: Logger,
|
||||
registrySecrets: string | undefined,
|
||||
registriesCredentials: string | undefined,
|
||||
language: KnownLanguage | undefined,
|
||||
language: BuiltInLanguage | undefined,
|
||||
skipUnusedRegistries: boolean = false,
|
||||
): Credential[] {
|
||||
const registryMapping = skipUnusedRegistries
|
||||
@@ -447,15 +328,18 @@ export function getCredentials(
|
||||
}
|
||||
|
||||
// If the password or token looks like a GitHub PAT, warn if no username is configured.
|
||||
if (
|
||||
((!hasUsername(authConfig) || !isDefined(authConfig.username)) &&
|
||||
isUsernamePassword(authConfig) &&
|
||||
isDefined(authConfig.password) &&
|
||||
isPAT(authConfig.password)) ||
|
||||
(isToken(authConfig) &&
|
||||
isDefined(authConfig.token) &&
|
||||
isPAT(authConfig.token))
|
||||
) {
|
||||
const noUsername =
|
||||
!hasUsername(authConfig) || !isDefined(authConfig.username);
|
||||
const passwordIsPAT =
|
||||
hasUsernameAndPassword(authConfig) &&
|
||||
isDefined(authConfig.password) &&
|
||||
isPAT(authConfig.password);
|
||||
const tokenIsPAT =
|
||||
hasToken(authConfig) &&
|
||||
isDefined(authConfig.token) &&
|
||||
isPAT(authConfig.token);
|
||||
|
||||
if (noUsername && (passwordIsPAT || tokenIsPAT)) {
|
||||
logger.warning(
|
||||
`A ${e.type} private registry is configured for ${e.host || e.url} using a GitHub Personal Access Token (PAT), but no username was provided. ` +
|
||||
`This may not work correctly. When configuring a private registry using a PAT, select "Username and password" and enter the username of the user ` +
|
||||
@@ -463,8 +347,25 @@ export function getCredentials(
|
||||
);
|
||||
}
|
||||
|
||||
// Construct the base credential object.
|
||||
const baseCredential: Omit<Registry, keyof Address> = { type: e.type };
|
||||
|
||||
// If "replaces-base" is present, it must be a boolean.
|
||||
if ("replaces-base" in e) {
|
||||
if (
|
||||
isDefined(e["replaces-base"]) &&
|
||||
typeof e["replaces-base"] === "boolean"
|
||||
) {
|
||||
baseCredential["replaces-base"] = e["replaces-base"];
|
||||
} else {
|
||||
throw new ConfigurationError(
|
||||
"Invalid credentials - 'replaces-base' must be a boolean",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
out.push({
|
||||
type: e.type,
|
||||
...baseCredential,
|
||||
...authConfig,
|
||||
...address,
|
||||
});
|
||||
|
||||
@@ -7,7 +7,7 @@ import * as io from "@actions/io";
|
||||
import test, { ExecutionContext } from "ava";
|
||||
import sinon from "sinon";
|
||||
|
||||
import { JavaEnvVars, KnownLanguage } from "../languages";
|
||||
import { JavaEnvVars, BuiltInLanguage } from "../languages";
|
||||
import {
|
||||
checkExpectedLogMessages,
|
||||
getRecordingLogger,
|
||||
@@ -182,11 +182,11 @@ test.serial("checkProxyEnvVars - credentials are removed from URLs", (t) => {
|
||||
});
|
||||
|
||||
test.serial(
|
||||
"checkProxyEnvironment - includes base checks for all known languages",
|
||||
"checkProxyEnvironment - includes base checks for all built-in languages",
|
||||
async (t) => {
|
||||
stubToolrunner();
|
||||
|
||||
for (const language of Object.values(KnownLanguage)) {
|
||||
for (const language of Object.values(BuiltInLanguage)) {
|
||||
const messages: LoggedMessage[] = [];
|
||||
const logger = getRecordingLogger(messages);
|
||||
|
||||
@@ -204,7 +204,7 @@ test.serial(
|
||||
|
||||
stubToolrunner();
|
||||
|
||||
await checkProxyEnvironment(logger, KnownLanguage.java);
|
||||
await checkProxyEnvironment(logger, BuiltInLanguage.java);
|
||||
assertEnvVarLogMessages(t, Object.keys(ProxyEnvVars), messages, false);
|
||||
assertEnvVarLogMessages(t, JAVA_PROXY_ENV_VARS, messages, false);
|
||||
},
|
||||
|
||||
@@ -4,7 +4,7 @@ import * as path from "path";
|
||||
import * as toolrunner from "@actions/exec/lib/toolrunner";
|
||||
import * as io from "@actions/io";
|
||||
|
||||
import { JavaEnvVars, KnownLanguage, Language } from "../languages";
|
||||
import { JavaEnvVars, BuiltInLanguage, Language } from "../languages";
|
||||
import { Logger } from "../logging";
|
||||
import { getErrorMessage, isDefined } from "../util";
|
||||
|
||||
@@ -196,7 +196,7 @@ export async function checkProxyEnvironment(
|
||||
|
||||
// Check language-specific configurations. If we don't know the language,
|
||||
// then we perform all checks.
|
||||
if (language === undefined || language === KnownLanguage.java) {
|
||||
if (language === undefined || language === BuiltInLanguage.java) {
|
||||
checkJavaEnvVars(logger);
|
||||
|
||||
await showJavaSettings(logger);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user