switch to just triggering specific integration test jobs

.github/workflows/codeql.yml

@@ -1,6 +1,6 @@
 name: "CodeQL action"
 
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   build:

.github/workflows/integration-testing.yml

@@ -3,45 +3,490 @@ name: "Integration Testing"
 on: [push, pull_request]
 
 jobs:
-  multi-language-repo_test-autodetect-languages:
+  # multi-language-repo_test-autodetect-languages:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #   - name: Build code
+  #     shell: bash
+  #     run: ./build.sh
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+  #   - run: |
+  #       cd "$RUNNER_TEMP/codeql_databases"
+  #       # List all directories as there will be precisely one directory per database
+  #       # but there may be other files in this directory such as query suites.
+  #       if [ "$(ls -d */ | wc -l)" != 6 ] || \
+  #          [[ ! -d cpp ]] || \
+  #          [[ ! -d csharp ]] || \
+  #          [[ ! -d go ]] || \
+  #          [[ ! -d java ]] || \
+  #          [[ ! -d javascript ]] || \
+  #          [[ ! -d python ]]; then
+  #         echo "Did not find expected number of databases. Database dir contains: $(ls)"
+  #         exit 1
+  #       fi
+
+  # multi-language-repo_test-custom-queries-and-remote-config:
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       os: [ubuntu-latest, windows-latest, macos-latest]
+  #   runs-on: ${{ matrix.os }}
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #     with:
+  #       languages: cpp,csharp,java,javascript,python
+  #       config-file: github/codeql-action/tests/multi-language-repo/.github/codeql/custom-queries.yml@${{ github.sha }}
+  #   - name: Build code
+  #     shell: bash
+  #     run: ./build.sh
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+
+  # # Currently is not possible to analyze Go in conjunction with other languages in macos
+  # multi-language-repo_test-go-custom-queries:
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       os: [ubuntu-latest, windows-latest, macos-latest]
+  #   runs-on: ${{ matrix.os }}
+
+  #   steps:
+  #   - uses: actions/setup-go@v2
+  #     if: ${{ matrix.os ==  'macos-latest' }}
+  #     with:
+  #       go-version: '^1.13.1'
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #     with:
+  #       languages: go
+  #       config-file: ./.github/codeql/custom-queries.yml
+  #   - name: Build code
+  #     shell: bash
+  #     run: ./build.sh
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+
+  # multi-language-repo_rubocop:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - name: Set up Ruby
+  #     uses: ruby/setup-ruby@v1
+  #     with:
+  #       ruby-version: 2.6
+  #   - name: Install Code Scanning integration
+  #     run: bundle add code-scanning-rubocop --version 0.3.0 --skip-install
+  #   - name: Install dependencies
+  #     run: bundle install
+  #   - name: Rubocop run
+  #     run: |
+  #       bash -c "
+  #         bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
+  #         [[ $? -ne 2 ]]
+  #       "
+  #   - uses: ./../action/upload-sarif
+  #     with:
+  #       sarif_file: rubocop.sarif
+  #     env:
+  #       TEST_MODE: true
+
+  # test-proxy:
+  #   runs-on: ubuntu-latest
+  #   container:
+  #     image: ubuntu:18.04
+  #     options: --dns 127.0.0.1
+  #   services:
+  #     squid-proxy:
+  #       image: datadog/squid:latest
+  #       ports:
+  #         - 3128:3128
+  #   env:
+  #     https_proxy: http://squid-proxy:3128
+  #   steps:
+  #   - uses: actions/checkout@v2
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+  #   - uses: ./../action/init
+  #     with:
+  #       languages: javascript
+  #   - uses: ./../action/analyze
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-javascript-ubuntu:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       # Pass --config-file here, but not for other jobs in this workflow.
+  #       # This means we're testing the config file parsing in the runner
+  #       # but not slowing down all jobs unnecessarily as it doesn't add much
+  #       # testing the parsing on different operating systems and languages.
+  #       runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Run analyze
+  #     run: |
+  #       runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-javascript-windows:
+  #   runs-on: windows-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages javascript --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Run analyze
+  #     run: |
+  #       runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-javascript-macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Run analyze
+  #     run: |
+  #       runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-csharp-ubuntu:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     run: |
+  #       . ./codeql-runner/codeql-env.sh
+  #       dotnet build
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-csharp-windows:
+  #   runs-on: windows-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: powershell
+  #     run: |
+  #       cat ./codeql-runner/codeql-env.sh | Invoke-Expression
+  #       dotnet build
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  # runner-analyze-csharp-macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: bash
+  #     run: |
+  #       . ./codeql-runner/codeql-env.sh
+  #       dotnet build
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  
+  # runner-analyze-csharp-autobuild-ubuntu:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux autobuild
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+  
+  # runner-analyze-csharp-autobuild-windows:
+  #   runs-on: windows-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: powershell
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe autobuild
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  # runner-analyze-csharp-autobuild-macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Move codeql-action
+  #     shell: bash
+  #     run: |
+  #       mkdir ../action
+  #       mv * .github ../action/
+  #       mv ../action/tests/multi-language-repo/{*,.github} .
+
+  #   - name: Build runner
+  #     run: |
+  #       cd ../action/runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Run init
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  #   - name: Build code
+  #     shell: bash
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos autobuild
+
+  #   - name: Run analyze
+  #     run: |
+  #       ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+  #     env:
+  #       TEST_MODE: true
+
+  # runner-upload-sarif:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@v2
+
+  #   - name: Build runner
+  #     run: |
+  #       cd runner
+  #       npm install
+  #       npm run build-runner
+
+  #   - name: Upload with runner
+  #     run: |
+  #       # Deliberately don't use TEST_MODE here. This is specifically testing
+  #       # the compatibility with the API.
+  #       runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+# Test the behaviour when asked to scan a compiled language but no code is compiled
+  java:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - uses: ./../action/init
-    - name: Build code
-      shell: bash
-      run: ./build.sh
-    - uses: ./../action/analyze
-      env:
-        TEST_MODE: true
-    - run: |
-        cd "$RUNNER_TEMP/codeql_databases"
-        # List all directories as there will be precisely one directory per database
-        # but there may be other files in this directory such as query suites.
-        if [ "$(ls -d */ | wc -l)" != 6 ] || \
-           [[ ! -d cpp ]] || \
-           [[ ! -d csharp ]] || \
-           [[ ! -d go ]] || \
-           [[ ! -d java ]] || \
-           [[ ! -d javascript ]] || \
-           [[ ! -d python ]]; then
-          echo "Did not find expected number of databases. Database dir contains: $(ls)"
-          exit 1
-        fi
+      - uses: actions/checkout@v2
+      - name: Move codeql-action
+        shell: bash
+        run: |
+          mkdir ../action
+          shopt -s dotglob
+          mv * ../action/
+          # Do not copy anything back, so there's nothing left to scan
+      - uses: ./../action/init
+        with:
+          languages: java
+      - uses: ./../action/analyze
+        env:
+          TEST_MODE: true
+  
+  # Test the behaviour when asked to scan a non-compiled language but there is no code to scan
+  javascript:
+    runs-on: ubuntu-latest
 
-  multi-language-repo_test-custom-queries-and-remote-config:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Move codeql-action
+        shell: bash
+        run: |
+          mkdir ../action
+          shopt -s dotglob
+          mv * ../action/
+          # Do not copy anything back, so there's nothing left to scan
+      - uses: ./../action/init
+        with:
+          languages: javascript
+      - uses: ./../action/analyze
+        env:
+          TEST_MODE: true
+
+  matcher-catches-out-of-memory-error:
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2
@@ -59,388 +504,7 @@ jobs:
       shell: bash
       run: ./build.sh
     - uses: ./../action/analyze
-      env:
-        TEST_MODE: true
-
-  # Currently is not possible to analyze Go in conjunction with other languages in macos
-  multi-language-repo_test-go-custom-queries:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    - uses: actions/setup-go@v2
-      if: ${{ matrix.os ==  'macos-latest' }}
       with:
-        go-version: '^1.13.1'
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - uses: ./../action/init
-      with:
-        languages: go
-        config-file: ./.github/codeql/custom-queries.yml
-    - name: Build code
-      shell: bash
-      run: ./build.sh
-    - uses: ./../action/analyze
+        ram: 10
       env:
         TEST_MODE: true
-
-  multi-language-repo_rubocop:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: 2.6
-    - name: Install Code Scanning integration
-      run: bundle add code-scanning-rubocop --version 0.3.0 --skip-install
-    - name: Install dependencies
-      run: bundle install
-    - name: Rubocop run
-      run: |
-        bash -c "
-          bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
-          [[ $? -ne 2 ]]
-        "
-    - uses: ./../action/upload-sarif
-      with:
-        sarif_file: rubocop.sarif
-      env:
-        TEST_MODE: true
-
-  test-proxy:
-    runs-on: ubuntu-latest
-    container:
-      image: ubuntu:18.04
-      options: --dns 127.0.0.1
-    services:
-      squid-proxy:
-        image: datadog/squid:latest
-        ports:
-          - 3128:3128
-    env:
-      https_proxy: http://squid-proxy:3128
-    steps:
-    - uses: actions/checkout@v2
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-    - uses: ./../action/init
-      with:
-        languages: javascript
-    - uses: ./../action/analyze
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-javascript-ubuntu:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        # Pass --config-file here, but not for other jobs in this workflow.
-        # This means we're testing the config file parsing in the runner
-        # but not slowing down all jobs unnecessarily as it doesn't add much
-        # testing the parsing on different operating systems and languages.
-        runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Run analyze
-      run: |
-        runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-javascript-windows:
-    runs-on: windows-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages javascript --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Run analyze
-      run: |
-        runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-javascript-macos:
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages javascript --config-file ./.github/codeql/codeql-config.yml --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Run analyze
-      run: |
-        runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-csharp-ubuntu:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      run: |
-        . ./codeql-runner/codeql-env.sh
-        dotnet build
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-csharp-windows:
-    runs-on: windows-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: powershell
-      run: |
-        cat ./codeql-runner/codeql-env.sh | Invoke-Expression
-        dotnet build
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  runner-analyze-csharp-macos:
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: bash
-      run: |
-        . ./codeql-runner/codeql-env.sh
-        dotnet build
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  
-  runner-analyze-csharp-autobuild-ubuntu:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-linux init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      run: |
-        ../action/runner/dist/codeql-runner-linux autobuild
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-linux analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-  
-  runner-analyze-csharp-autobuild-windows:
-    runs-on: windows-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe init --repository $Env:GITHUB_REPOSITORY --languages csharp --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: powershell
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe autobuild
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-win.exe analyze --repository $Env:GITHUB_REPOSITORY --commit $Env:GITHUB_SHA --ref $Env:GITHUB_REF --github-url $Env:GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  runner-analyze-csharp-autobuild-macos:
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Move codeql-action
-      shell: bash
-      run: |
-        mkdir ../action
-        mv * .github ../action/
-        mv ../action/tests/multi-language-repo/{*,.github} .
-
-    - name: Build runner
-      run: |
-        cd ../action/runner
-        npm install
-        npm run build-runner
-
-    - name: Run init
-      run: |
-        ../action/runner/dist/codeql-runner-macos init --repository $GITHUB_REPOSITORY --languages csharp --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-
-    - name: Build code
-      shell: bash
-      run: |
-        ../action/runner/dist/codeql-runner-macos autobuild
-
-    - name: Run analyze
-      run: |
-        ../action/runner/dist/codeql-runner-macos analyze --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
-      env:
-        TEST_MODE: true
-
-  runner-upload-sarif:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build runner
-      run: |
-        cd runner
-        npm install
-        npm run build-runner
-
-    - name: Upload with runner
-      run: |
-        # Deliberately don't use TEST_MODE here. This is specifically testing
-        # the compatibility with the API.
-        runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}

.github/workflows/pr-checks.yml

@@ -1,6 +1,6 @@
 name: "PR checks"
 
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   lint-js:

.github/workflows/runner.yml

@@ -0,0 +1,26 @@
+name: "CodeQL runner"
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Build runner
+      run: |
+        cd runner
+        npm install
+        npm run build-runner
+
+    - name: Upload with runner
+      run: |
+        runner/dist/codeql-runner-linux upload \
+          --sarif-file src/testdata/empty-sarif.sarif \
+          --repository $GITHUB_REPOSITORY \
+          --commit $GITHUB_SHA \
+          --ref $GITHUB_REF \
+          --github-url $GITHUB_SERVER_URL \
+          --github-auth ${{ github.token }}