Merge remote-tracking branch 'origin/releases/v3' into backport-v2.23.2-b7bf0a3ed

src/analyze-action.ts

@@ -220,6 +220,8 @@ async function run() {
 
     const gitHubVersion = await getGitHubVersion();
 
+    util.checkActionVersion(actionsUtil.getActionVersion(), gitHubVersion);
+
     const features = new Features(
       gitHubVersion,
       repositoryNwo,

src/autobuild-action.ts

@@ -17,6 +17,7 @@ import {
   sendStatusReport,
 } from "./status-report";
 import {
+  checkActionVersion,
   checkDiskUsage,
   checkGitHubVersionInRange,
   initializeEnvironment,
@@ -77,6 +78,7 @@ async function run() {
 
     const gitHubVersion = await getGitHubVersion();
     checkGitHubVersionInRange(gitHubVersion, logger);
+    checkActionVersion(getActionVersion(), gitHubVersion);
 
     const config = await configUtils.getConfig(getTemporaryDirectory(), logger);
     if (config === undefined) {

src/defaults.json

@@ -1,6 +1,6 @@
 {
-  "bundleVersion": "codeql-bundle-v2.16.0",
-  "cliVersion": "2.16.0",
-  "priorBundleVersion": "codeql-bundle-v2.15.5",
-  "priorCliVersion": "2.15.5"
+  "bundleVersion": "codeql-bundle-v2.16.1",
+  "cliVersion": "2.16.1",
+  "priorBundleVersion": "codeql-bundle-v2.16.0",
+  "priorCliVersion": "2.16.0"
 }

src/environment.ts

@@ -1,3 +1,9 @@
+/**
+ * Environment variables used by the CodeQL Action.
+ *
+ * We recommend prefixing environment variables with `CODEQL_ACTION_`
+ * to reduce the risk that they are overwritten by other steps.
+ */
 export enum EnvVar {
   /** Whether the `analyze` Action completes successfully. */
   ANALYZE_DID_COMPLETE_SUCCESSFULLY = "CODEQL_ACTION_ANALYZE_DID_COMPLETE_SUCCESSFULLY",
@@ -35,6 +41,9 @@ export enum EnvVar {
   /** UUID representing the current job run. */
   JOB_RUN_UUID = "JOB_RUN_UUID",
 
+  /** Status for the entire job, submitted to the status report in `init-post` */
+  JOB_STATUS = "CODEQL_ACTION_JOB_STATUS",
+
   ODASA_TRACER_CONFIGURATION = "ODASA_TRACER_CONFIGURATION",
 
   /**

src/feature-flags.ts

@@ -44,8 +44,8 @@ export interface FeatureEnablement {
  * Each value of this enum should end with `_enabled`.
  */
 export enum Feature {
-  CodeqlJavaLombokEnabled = "codeql_java_lombok_enabled",
   CppDependencyInstallation = "cpp_dependency_installation_enabled",
+  CppTrapCachingEnabled = "cpp_trap_caching_enabled",
   DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
   DisablePythonDependencyInstallationEnabled = "disable_python_dependency_installation_enabled",
   PythonDefaultIsToSkipDependencyInstallationEnabled = "python_default_is_to_skip_dependency_installation_enabled",
@@ -58,16 +58,16 @@ export const featureConfig: Record<
   Feature,
   { envVar: string; minimumVersion: string | undefined; defaultValue: boolean }
 > = {
-  [Feature.CodeqlJavaLombokEnabled]: {
-    envVar: "CODEQL_JAVA_LOMBOK",
-    minimumVersion: "2.14.0",
-    defaultValue: false,
-  },
   [Feature.CppDependencyInstallation]: {
     envVar: "CODEQL_EXTRACTOR_CPP_AUTOINSTALL_DEPENDENCIES",
     minimumVersion: "2.15.0",
     defaultValue: false,
   },
+  [Feature.CppTrapCachingEnabled]: {
+    envVar: "CODEQL_CPP_TRAP_CACHING",
+    minimumVersion: "2.16.1",
+    defaultValue: false,
+  },
   [Feature.DisableKotlinAnalysisEnabled]: {
     envVar: "CODEQL_DISABLE_KOTLIN_ANALYSIS",
     minimumVersion: undefined,

src/init-action-post-helper.ts

@@ -1,3 +1,4 @@
+import * as core from "@actions/core";
 import * as github from "@actions/github";
 
 import * as actionsUtil from "./actions-util";
@@ -8,6 +9,7 @@ import { EnvVar } from "./environment";
 import { Feature, FeatureEnablement } from "./feature-flags";
 import { Logger } from "./logging";
 import { RepositoryNwo, parseRepositoryNwo } from "./repository";
+import { JobStatus } from "./status-report";
 import * as uploadLib from "./upload-lib";
 import {
   delay,
@@ -36,6 +38,10 @@ export interface UploadFailedSarifResult extends uploadLib.UploadStatusReport {
   sarifID?: string;
 }
 
+export interface JobStatusReport {
+  job_status: JobStatus;
+}
+
 function createFailedUploadFailedSarifResult(
   error: unknown,
 ): UploadFailedSarifResult {
@@ -121,6 +127,15 @@ export async function tryUploadSarifIfRunFailed(
   logger: Logger,
 ): Promise<UploadFailedSarifResult> {
   if (process.env[EnvVar.ANALYZE_DID_COMPLETE_SUCCESSFULLY] !== "true") {
+    // If analyze didn't complete successfully and the job status hasn't
+    // already been set to Failure/ConfigurationError previously, this
+    // means that something along the way failed in a step that is not
+    // owned by the Action, for example a manual build step. We
+    // consider this a configuration error.
+    core.exportVariable(
+      EnvVar.JOB_STATUS,
+      process.env[EnvVar.JOB_STATUS] ?? JobStatus.ConfigurationError,
+    );
     try {
       return await maybeUploadFailedSarif(
         config,
@@ -135,6 +150,10 @@ export async function tryUploadSarifIfRunFailed(
       return createFailedUploadFailedSarifResult(e);
     }
   } else {
+    core.exportVariable(
+      EnvVar.JOB_STATUS,
+      process.env[EnvVar.JOB_STATUS] ?? JobStatus.Success,
+    );
     return {
       upload_failed_run_skipped_because:
         "Analyze Action completed successfully",
@@ -282,3 +301,20 @@ async function removeUploadedSarif(
     );
   }
 }
+
+/**
+ * Returns the final job status sent in the `init-post` Action, based on the
+ * current value of the JOB_STATUS environment variable. If the variable is
+ * unset, or if its value is not one of the JobStatus enum values, returns
+ * Unknown. Otherwise it returns the status set in the environment variable.
+ */
+export function getFinalJobStatus(): JobStatus {
+  const jobStatusFromEnvironment = process.env[EnvVar.JOB_STATUS];
+  if (
+    !jobStatusFromEnvironment ||
+    !Object.values(JobStatus).includes(jobStatusFromEnvironment as JobStatus)
+  ) {
+    return JobStatus.Unknown;
+  }
+  return jobStatusFromEnvironment as JobStatus;
+}

src/init-action-post.ts

@@ -28,7 +28,8 @@ import {
 
 interface InitPostStatusReport
   extends StatusReportBase,
-    initActionPostHelper.UploadFailedSarifResult {}
+    initActionPostHelper.UploadFailedSarifResult,
+    initActionPostHelper.JobStatusReport {}
 
 async function runWrapper() {
   const startedAt = new Date();
@@ -83,6 +84,7 @@ async function runWrapper() {
   const statusReport: InitPostStatusReport = {
     ...statusReportBase,
     ...uploadFailedSarifResult,
+    job_status: initActionPostHelper.getFinalJobStatus(),
   };
   await sendStatusReport(statusReport);
 }

src/init-action.ts

@@ -54,6 +54,7 @@ import {
   isHostedRunner,
   UserError,
   wrapError,
+  checkActionVersion,
 } from "./util";
 import { validateWorkflow } from "./workflow";
 
@@ -212,6 +213,7 @@ async function run() {
 
   const gitHubVersion = await getGitHubVersion();
   checkGitHubVersionInRange(gitHubVersion, logger);
+  checkActionVersion(getActionVersion(), gitHubVersion);
 
   const repositoryNwo = parseRepositoryNwo(
     getRequiredEnvParam("GITHUB_REPOSITORY"),
@@ -423,19 +425,37 @@ async function run() {
       core.exportVariable(kotlinLimitVar, "1.9.20");
     }
 
-    if (config.languages.includes(Language.java)) {
+    if (
+      config.languages.includes(Language.java) &&
+      // Java Lombok support is enabled by default for >= 2.14.4
+      (await codeQlVersionAbove(codeql, "2.14.0")) &&
+      !(await codeQlVersionAbove(codeql, "2.14.4"))
+    ) {
       const envVar = "CODEQL_EXTRACTOR_JAVA_RUN_ANNOTATION_PROCESSORS";
       if (process.env[envVar]) {
         logger.info(
           `Environment variable ${envVar} already set. Not en/disabling CodeQL Java Lombok support`,
         );
-      } else if (
-        await features.getValue(Feature.CodeqlJavaLombokEnabled, codeql)
-      ) {
+      } else {
         logger.info("Enabling CodeQL Java Lombok support");
         core.exportVariable(envVar, "true");
+      }
+    }
+
+    if (config.languages.includes(Language.cpp)) {
+      const envVar = "CODEQL_EXTRACTOR_CPP_TRAP_CACHING";
+      if (process.env[envVar]) {
+        logger.info(
+          `Environment variable ${envVar} already set. Not en/disabling CodeQL C++ TRAP caching support`,
+        );
+      } else if (
+        getTrapCachingEnabled() &&
+        (await features.getValue(Feature.CppTrapCachingEnabled, codeql))
+      ) {
+        logger.info("Enabling CodeQL C++ TRAP caching support");
+        core.exportVariable(envVar, "true");
       } else {
-        logger.info("Disabling CodeQL Java Lombok support");
+        logger.info("Disabling CodeQL C++ TRAP caching support");
         core.exportVariable(envVar, "false");
       }
     }

src/resolve-environment-action.ts

@@ -1,6 +1,7 @@
 import * as core from "@actions/core";
 
 import {
+  getActionVersion,
   getOptionalInput,
   getRequiredInput,
   getTemporaryDirectory,
@@ -16,6 +17,7 @@ import {
   getActionsStatus,
 } from "./status-report";
 import {
+  checkActionVersion,
   checkDiskUsage,
   checkForTimeout,
   checkGitHubVersionInRange,
@@ -45,6 +47,7 @@ async function run() {
 
     const gitHubVersion = await getGitHubVersion();
     checkGitHubVersionInRange(gitHubVersion, logger);
+    checkActionVersion(getActionVersion(), gitHubVersion);
 
     const config = await configUtils.getConfig(getTemporaryDirectory(), logger);
     if (config === undefined) {

src/status-report.ts

@@ -32,12 +32,20 @@ export type ActionName =
   | "upload-sarif";
 
 export type ActionStatus =
-  | "aborted"
+  | "aborted" // Only used in the init Action, if init failed before initializing the tracer due to something other than a configuration error.
   | "failure"
   | "starting"
   | "success"
   | "user-error";
 
+/** Overall status of the entire job. String values match the Hydro schema. */
+export enum JobStatus {
+  Unknown = "JOB_STATUS_UNKNOWN",
+  Success = "JOB_STATUS_SUCCESS",
+  Failure = "JOB_STATUS_FAILURE",
+  ConfigurationError = "JOB_STATUS_CONFIGURATION_ERROR",
+}
+
 export interface StatusReportBase {
   /** Name of the action being executed. */
   action_name: ActionName;
@@ -133,6 +141,25 @@ export function getActionsStatus(
   }
 }
 
+/**
+ * Sets the overall job status environment variable to configuration error
+ * or failure, unless it's already been set to one of these values in a
+ * previous step.
+ */
+function setJobStatusIfUnsuccessful(actionStatus: ActionStatus) {
+  if (actionStatus === "user-error") {
+    core.exportVariable(
+      EnvVar.JOB_STATUS,
+      process.env[EnvVar.JOB_STATUS] ?? JobStatus.ConfigurationError,
+    );
+  } else if (actionStatus === "failure" || actionStatus === "aborted") {
+    core.exportVariable(
+      EnvVar.JOB_STATUS,
+      process.env[EnvVar.JOB_STATUS] ?? JobStatus.Failure,
+    );
+  }
+}
+
 // Any status report may include an array of EventReports associated with it.
 export interface EventReport {
   /** Time this event ended. */
@@ -273,6 +300,8 @@ const INCOMPATIBLE_MSG =
 export async function sendStatusReport<S extends StatusReportBase>(
   statusReport: S,
 ): Promise<boolean> {
+  setJobStatusIfUnsuccessful(statusReport.status);
+
   const statusReportJSON = JSON.stringify(statusReport);
   core.debug(`Sending status report: ${statusReportJSON}`);
   // If in test mode we don't want to upload the results

src/upload-sarif-action.ts

@@ -2,6 +2,7 @@ import * as core from "@actions/core";
 
 import * as actionsUtil from "./actions-util";
 import { getActionVersion } from "./actions-util";
+import { getGitHubVersion } from "./api-client";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import {
@@ -12,6 +13,7 @@ import {
 } from "./status-report";
 import * as upload_lib from "./upload-lib";
 import {
+  checkActionVersion,
   checkDiskUsage,
   getRequiredEnvParam,
   initializeEnvironment,
@@ -44,6 +46,10 @@ async function run() {
   const startedAt = new Date();
   const logger = getActionsLogger();
   initializeEnvironment(getActionVersion());
+
+  const gitHubVersion = await getGitHubVersion();
+  checkActionVersion(getActionVersion(), gitHubVersion);
+
   if (
     !(await sendStatusReport(
       await createStatusReportBase(

src/util.test.ts

@@ -2,8 +2,11 @@ import * as fs from "fs";
 import * as os from "os";
 import path from "path";
 
+import * as core from "@actions/core";
 import test from "ava";
+import * as sinon from "sinon";
 
+import * as api from "./api-client";
 import { EnvVar } from "./environment";
 import { getRunnerLogger } from "./logging";
 import { getRecordingLogger, LoggedMessage, setupTests } from "./testing-utils";
@@ -385,3 +388,64 @@ test("fixInvalidNotifications removes duplicate locations", (t) => {
     message: "Removed 1 duplicate locations from SARIF notification objects.",
   });
 });
+
+function formatGitHubVersion(version: util.GitHubVersion): string {
+  switch (version.type) {
+    case util.GitHubVariant.DOTCOM:
+      return "dotcom";
+    case util.GitHubVariant.GHE_DOTCOM:
+      return "GHE dotcom";
+    case util.GitHubVariant.GHES:
+      return `GHES ${version.version}`;
+    default:
+      util.assertNever(version);
+  }
+}
+
+const CHECK_ACTION_VERSION_TESTS: Array<[string, util.GitHubVersion, boolean]> =
+  [
+    ["2.2.1", { type: util.GitHubVariant.DOTCOM }, true],
+    ["2.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, true],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.10" }, false],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.11" }, true],
+    ["2.2.1", { type: util.GitHubVariant.GHES, version: "3.12" }, true],
+    ["3.2.1", { type: util.GitHubVariant.DOTCOM }, false],
+    ["3.2.1", { type: util.GitHubVariant.GHE_DOTCOM }, false],
+    ["3.2.1", { type: util.GitHubVariant.GHES, version: "3.10" }, false],
+    ["3.2.1", { type: util.GitHubVariant.GHES, version: "3.11" }, false],
+    ["3.2.1", { type: util.GitHubVariant.GHES, version: "3.12" }, false],
+  ];
+
+for (const [
+  version,
+  githubVersion,
+  shouldReportWarning,
+] of CHECK_ACTION_VERSION_TESTS) {
+  const reportWarningDescription = shouldReportWarning
+    ? "reports warning"
+    : "doesn't report warning";
+  const versionsDescription = `CodeQL Action version ${version} and GitHub version ${formatGitHubVersion(
+    githubVersion,
+  )}`;
+  test(`checkActionVersion ${reportWarningDescription} for ${versionsDescription}`, async (t) => {
+    const warningSpy = sinon.spy(core, "warning");
+    const versionStub = sinon
+      .stub(api, "getGitHubVersion")
+      .resolves(githubVersion);
+
+    // call checkActionVersion twice and assert below that warning is reported only once
+    util.checkActionVersion(version, await api.getGitHubVersion());
+    util.checkActionVersion(version, await api.getGitHubVersion());
+
+    if (shouldReportWarning) {
+      t.true(
+        warningSpy.calledOnceWithExactly(
+          sinon.match("CodeQL Action v2 will be deprecated"),
+        ),
+      );
+    } else {
+      t.false(warningSpy.called);
+    }
+    versionStub.restore();
+  });
+}

src/util.ts

@@ -354,7 +354,22 @@ export function getThreadsFlagValue(
   logger: Logger,
 ): number {
   let numThreads: number;
-  const maxThreads = os.cpus().length;
+  const maxThreadsCandidates = [os.cpus().length];
+  if (os.platform() === "linux") {
+    maxThreadsCandidates.push(
+      ...["/sys/fs/cgroup/cpuset.cpus.effective", "/sys/fs/cgroup/cpuset.cpus"]
+        .map((file) => getCgroupCpuCountFromCpus(file, logger))
+        .filter((count) => count !== undefined && count > 0)
+        .map((count) => count as number),
+    );
+    maxThreadsCandidates.push(
+      ...["/sys/fs/cgroup/cpu.max"]
+        .map((file) => getCgroupCpuCountFromCpuMax(file, logger))
+        .filter((count) => count !== undefined && count > 0)
+        .map((count) => count as number),
+    );
+  }
+  const maxThreads = Math.min(...maxThreadsCandidates);
   if (userInput) {
     numThreads = Number(userInput);
     if (Number.isNaN(numThreads)) {
@@ -380,6 +395,79 @@ export function getThreadsFlagValue(
   return numThreads;
 }
 
+/**
+ * Gets the number of available cores specified by the cgroup cpu.max file at the given path.
+ * Format of file: two values, the limit and the duration (period). If the limit is "max" then
+ * we return undefined and do not use this file to determine CPU limits.
+ */
+function getCgroupCpuCountFromCpuMax(
+  cpuMaxFile: string,
+  logger: Logger,
+): number | undefined {
+  if (!fs.existsSync(cpuMaxFile)) {
+    logger.debug(
+      `While resolving threads, did not find a cgroup CPU file at ${cpuMaxFile}.`,
+    );
+    return undefined;
+  }
+
+  const cpuMaxString = fs.readFileSync(cpuMaxFile, "utf-8");
+  const cpuMaxStringSplit = cpuMaxString.split(" ");
+  if (cpuMaxStringSplit.length !== 2) {
+    logger.debug(
+      `While resolving threads, did not use cgroup CPU file at ${cpuMaxFile} because it contained ${cpuMaxStringSplit.length} value(s) rather than the two expected.`,
+    );
+    return undefined;
+  }
+  const cpuLimit = cpuMaxStringSplit[0];
+  if (cpuLimit === "max") {
+    return undefined;
+  }
+  const duration = cpuMaxStringSplit[1];
+  const cpuCount = Math.floor(parseInt(cpuLimit) / parseInt(duration));
+
+  logger.info(
+    `While resolving threads, found a cgroup CPU file with ${cpuCount} CPUs in ${cpuMaxFile}.`,
+  );
+
+  return cpuCount;
+}
+
+/**
+ * Gets the number of available cores listed in the cgroup cpuset.cpus file at the given path.
+ */
+function getCgroupCpuCountFromCpus(
+  cpusFile: string,
+  logger: Logger,
+): number | undefined {
+  if (!fs.existsSync(cpusFile)) {
+    logger.debug(
+      `While resolving threads, did not find a cgroup CPUs file at ${cpusFile}.`,
+    );
+    return undefined;
+  }
+
+  let cpuCount = 0;
+  // Comma-separated numbers and ranges, for eg. 0-1,3
+  const cpusString = fs.readFileSync(cpusFile, "utf-8");
+  for (const token of cpusString.split(",")) {
+    if (!token.includes("-")) {
+      // Not a range
+      ++cpuCount;
+    } else {
+      const cpuStartIndex = parseInt(token.split("-")[0]);
+      const cpuEndIndex = parseInt(token.split("-")[1]);
+      cpuCount += cpuEndIndex - cpuStartIndex + 1;
+    }
+  }
+
+  logger.info(
+    `While resolving threads, found a cgroup CPUs file with ${cpuCount} CPUs in ${cpusFile}.`,
+  );
+
+  return cpuCount;
+}
+
 /**
  * Get the codeql `--threads` flag specified for the `threads` input.
  * If no value was specified, all available threads will be used.
@@ -944,3 +1032,44 @@ export async function checkDiskUsage(logger?: Logger): Promise<DiskUsage> {
     numTotalBytes: diskUsage.size,
   };
 }
+
+/**
+ * Prompt the customer to upgrade to CodeQL Action v3, if appropriate.
+ *
+ * Check whether a customer is running v2. If they are, and we can determine that the GitHub
+ * instance supports v3, then log a warning about v2's upcoming deprecation prompting the customer
+ * to upgrade to v3.
+ */
+export function checkActionVersion(
+  version: string,
+  githubVersion: GitHubVersion,
+) {
+  if (
+    !semver.satisfies(version, ">=3") && // do not warn if the customer is already running v3
+    !process.env.CODEQL_V2_DEPRECATION_WARNING // do not warn if we have already warned
+  ) {
+    // Only log a warning for versions of GHES that are compatible with CodeQL Action version 3.
+    //
+    // GHES 3.11 shipped without the v3 tag, but it also shipped without this warning message code.
+    // Therefore users who are seeing this warning message code have pulled in a new version of the
+    // Action, and with it the v3 tag.
+    if (
+      githubVersion.type === GitHubVariant.DOTCOM ||
+      githubVersion.type === GitHubVariant.GHE_DOTCOM ||
+      (githubVersion.type === GitHubVariant.GHES &&
+        semver.satisfies(
+          semver.coerce(githubVersion.version) ?? "0.0.0",
+          ">=3.11",
+        ))
+    ) {
+      core.warning(
+        "CodeQL Action v2 will be deprecated on December 5th, 2024. " +
+          "Please update all occurrences of the CodeQL Action in your workflow files to v3. " +
+          "For more information, see " +
+          "https://github.blog/changelog/2024-01-12-code-scanning-deprecation-of-codeql-action-v2/",
+      );
+      // set CODEQL_V2_DEPRECATION_WARNING env var to prevent the warning from being logged multiple times
+      core.exportVariable("CODEQL_V2_DEPRECATION_WARNING", "true");
+    }
+  }
+}