Only try zstd for specified version ranges of tar

src/codeql.ts

@@ -28,6 +28,7 @@ import {
 import { Language } from "./languages";
 import { Logger } from "./logging";
 import * as setupCodeql from "./setup-codeql";
+import { setupCodeQLBundle } from "./setup-codeql";
 import { ToolsFeature, isSupportedToolsFeature } from "./tools-features";
 import { shouldEnableIndirectTracing } from "./tracer-config";
 import * as util from "./util";
@@ -327,56 +328,6 @@ export const CODEQL_VERSION_SUBLANGUAGE_FILE_COVERAGE = "2.15.0";
  */
 const CODEQL_VERSION_INCLUDE_QUERY_HELP = "2.15.2";
 
-async function setupCodeQLBundlePreferringZstd(
-  toolsInput: string | undefined,
-  apiDetails: api.GitHubApiDetails,
-  tempDir: string,
-  variant: util.GitHubVariant,
-  defaultCliVersion: CodeQLDefaultVersionInfo,
-  features: FeatureEnablement,
-  logger: Logger,
-): Promise<setupCodeql.SetupCodeQLResult> {
-  let zstdError: unknown = undefined;
-
-  if (!toolsInput && (await features.getValue(Feature.ZstdBundle))) {
-    try {
-      return await setupCodeql.setupCodeQLBundle(
-        toolsInput,
-        apiDetails,
-        tempDir,
-        variant,
-        defaultCliVersion,
-        true,
-        logger,
-      );
-    } catch (e) {
-      logger.info(
-        "Failed to set up bundle compressed using zstd, falling back to bundle compressed using gzip.",
-      );
-      zstdError = e;
-    }
-  }
-
-  const result = await setupCodeql.setupCodeQLBundle(
-    toolsInput,
-    apiDetails,
-    tempDir,
-    variant,
-    defaultCliVersion,
-    false,
-    logger,
-  );
-
-  if (zstdError) {
-    result.toolsDownloadStatusReport = Object.assign(
-      {},
-      result.toolsDownloadStatusReport,
-      { zstdError: wrapError(zstdError).message },
-    );
-  }
-  return result;
-}
-
 /**
  * Set up CodeQL CLI access.
  *
@@ -411,7 +362,7 @@ export async function setupCodeQL(
       toolsDownloadStatusReport,
       toolsSource,
       toolsVersion,
-    } = await setupCodeQLBundlePreferringZstd(
+    } = await setupCodeQLBundle(
       toolsInput,
       apiDetails,
       tempDir,

src/setup-codeql.test.ts

@@ -11,6 +11,7 @@ import {
   LoggedMessage,
   SAMPLE_DEFAULT_CLI_VERSION,
   SAMPLE_DOTCOM_API_DETAILS,
+  createFeatures,
   getRecordingLogger,
   mockBundleDownloadApi,
   setupActionsVars,
@@ -172,7 +173,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to use
       "tmp/codeql_action_test/",
       GitHubVariant.DOTCOM,
       SAMPLE_DEFAULT_CLI_VERSION,
-      false,
+      createFeatures([]),
       logger,
     );
 
@@ -220,7 +221,7 @@ test("setupCodeQLBundle logs the CodeQL CLI version being used when asked to dow
       "tmp/codeql_action_test/",
       GitHubVariant.DOTCOM,
       SAMPLE_DEFAULT_CLI_VERSION,
-      false,
+      createFeatures([]),
       logger,
     );
 

src/setup-codeql.ts

@@ -15,10 +15,15 @@ import * as api from "./api-client";
 // creation scripts. Ensure that any changes to the format of this file are compatible with both of
 // these dependents.
 import * as defaults from "./defaults.json";
-import { CodeQLDefaultVersionInfo } from "./feature-flags";
+import {
+  CodeQLDefaultVersionInfo,
+  Feature,
+  FeatureEnablement,
+} from "./feature-flags";
 import { Logger } from "./logging";
+import * as tar from "./tar";
 import * as util from "./util";
-import { isGoodVersion } from "./util";
+import { isGoodVersion, wrapError } from "./util";
 
 export enum ToolsSource {
   Unknown = "UNKNOWN",
@@ -473,10 +478,8 @@ export async function tryGetFallbackToolcacheVersion(
   return fallbackVersion;
 }
 
-type CompressionMethod = "gzip" | "zstd";
-
 export interface ToolsDownloadStatusReport {
-  compressionMethod: CompressionMethod;
+  compressionMethod: tar.CompressionMethod;
   downloadDurationMs: number;
   extractionDurationMs: number;
 }
@@ -541,9 +544,8 @@ export const downloadCodeQL = async function (
 
   logger.debug("Extracting CodeQL bundle.");
   const extractionStart = performance.now();
-  const { compressionMethod, extractedBundlePath } = await extractBundle(
-    archivedBundlePath,
-  );
+  const { compressionMethod, outputPath: extractedBundlePath } =
+    await tar.extract(archivedBundlePath);
   const extractionDurationMs = Math.round(performance.now() - extractionStart);
   logger.debug(
     `Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionDurationMs} ms).`,
@@ -646,13 +648,12 @@ export interface SetupCodeQLResult {
 }
 
 /**
- * Obtains the CodeQL bundle, installs it in the toolcache if appropriate, and extracts it.
+ * Sets up a CodeQL bundle.
  *
- * @param checkVersion Whether to check that CodeQL CLI meets the minimum
- *        version requirement. Must be set to true outside tests.
- * @returns the path to the extracted bundle, and the version of the tools
+ * If `useZstdBundle` is true, and the requested CodeQL bundle needs to be downloaded,
+ * this function will attempt to download a zstd-compressed bundle.
  */
-export async function setupCodeQLBundle(
+async function setupCodeQLBundleWithZstdOption(
   toolsInput: string | undefined,
   apiDetails: api.GitHubApiDetails,
   tempDir: string,
@@ -705,6 +706,69 @@ export async function setupCodeQLBundle(
   return { codeqlFolder, toolsDownloadStatusReport, toolsSource, toolsVersion };
 }
 
+/**
+ * Obtains the CodeQL bundle, installs it in the toolcache if appropriate, and extracts it.
+ *
+ * @returns the path to the extracted bundle, and the version of the tools
+ */
+export async function setupCodeQLBundle(
+  toolsInput: string | undefined,
+  apiDetails: api.GitHubApiDetails,
+  tempDir: string,
+  variant: util.GitHubVariant,
+  defaultCliVersion: CodeQLDefaultVersionInfo,
+  features: FeatureEnablement,
+  logger: Logger,
+): Promise<SetupCodeQLResult> {
+  let zstdError: unknown = undefined;
+
+  if (!toolsInput && (await features.getValue(Feature.ZstdBundle))) {
+    try {
+      if (await tar.isZstdAvailable(logger)) {
+        return await setupCodeQLBundleWithZstdOption(
+          toolsInput,
+          apiDetails,
+          tempDir,
+          variant,
+          defaultCliVersion,
+          true,
+          logger,
+        );
+      } else {
+        logger.debug(
+          "Falling back to bundle compressed using gzip because the available version of tar was not " +
+            "recognized or is too old.",
+        );
+      }
+    } catch (e) {
+      logger.info(
+        "Failed to set up bundle compressed using zstd, falling back to bundle compressed using gzip.",
+      );
+      logger.debug(`Underlying error: ${e}`);
+      zstdError = e;
+    }
+  }
+
+  const result = await setupCodeQLBundleWithZstdOption(
+    toolsInput,
+    apiDetails,
+    tempDir,
+    variant,
+    defaultCliVersion,
+    false,
+    logger,
+  );
+
+  if (zstdError) {
+    result.toolsDownloadStatusReport = Object.assign(
+      {},
+      result.toolsDownloadStatusReport,
+      { zstdError: wrapError(zstdError).message },
+    );
+  }
+  return result;
+}
+
 async function cleanUpGlob(glob: string, name: string, logger: Logger) {
   logger.debug(`Cleaning up ${name}.`);
   try {
@@ -722,27 +786,3 @@ async function cleanUpGlob(glob: string, name: string, logger: Logger) {
     logger.warning(`Failed to clean up ${name}: ${e}.`);
   }
 }
-
-async function extractBundle(archivedBundlePath: string): Promise<{
-  compressionMethod: CompressionMethod;
-  extractedBundlePath: string;
-}> {
-  if (archivedBundlePath.endsWith(".tar.gz")) {
-    return {
-      compressionMethod: "gzip",
-      // While we could also ask tar to autodetect the compression method,
-      // we defensively keep the gzip call identical as requesting a gzipped
-      // bundle will soon be a fallback option.
-      extractedBundlePath: await toolcache.extractTar(archivedBundlePath),
-    };
-  }
-  return {
-    compressionMethod: "zstd",
-    // tar will autodetect the compression method
-    extractedBundlePath: await toolcache.extractTar(
-      archivedBundlePath,
-      undefined,
-      "x",
-    ),
-  };
-}

src/tar.ts

@@ -0,0 +1,91 @@
+import { ToolRunner } from "@actions/exec/lib/toolrunner";
+import * as toolcache from "@actions/tool-cache";
+import { safeWhich } from "@chrisgavin/safe-which";
+
+import { Logger } from "./logging";
+import { assertNever } from "./util";
+
+const MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
+const MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
+
+type TarVersion = {
+  type: "gnu" | "bsd";
+  version: string;
+};
+
+async function getTarVersion(): Promise<TarVersion> {
+  const tar = await safeWhich("tar");
+  let stdout = "";
+  const exitCode = await new ToolRunner(tar, ["--version"], {
+    listeners: {
+      stdout: (data: Buffer) => {
+        stdout += data.toString();
+      },
+    },
+  }).exec();
+  if (exitCode !== 0) {
+    throw new Error("Failed to call tar --version");
+  }
+  // Return whether this is GNU tar or BSD tar, and the version number
+  if (stdout.includes("GNU tar")) {
+    const match = stdout.match(/tar \(GNU tar\) ([0-9.]+)/);
+    if (!match || !match[1]) {
+      throw new Error("Failed to parse output of tar --version.");
+    }
+
+    return { type: "gnu", version: match[1] };
+  } else if (stdout.includes("bsdtar")) {
+    const match = stdout.match(/bsdtar ([0-9.]+)/);
+    if (!match || !match[1]) {
+      throw new Error("Failed to parse output of tar --version.");
+    }
+
+    return { type: "bsd", version: match[1] };
+  } else {
+    throw new Error("Unknown tar version");
+  }
+}
+
+export async function isZstdAvailable(logger: Logger): Promise<boolean> {
+  try {
+    const { type, version } = await getTarVersion();
+    logger.info(`Found ${type} tar version ${version}.`);
+    switch (type) {
+      case "gnu":
+        return version >= MIN_REQUIRED_GNU_TAR_VERSION;
+      case "bsd":
+        return version >= MIN_REQUIRED_BSD_TAR_VERSION;
+      default:
+        assertNever(type);
+    }
+  } catch (e) {
+    logger.error(
+      "Failed to determine tar version, therefore will assume zstd may not be available. " +
+        `The underlying error was: ${e}`,
+    );
+    return false;
+  }
+}
+
+export type CompressionMethod = "gzip" | "zstd";
+
+export async function extract(path: string): Promise<{
+  compressionMethod: CompressionMethod;
+  outputPath: string;
+}> {
+  if (path.endsWith(".tar.gz")) {
+    return {
+      compressionMethod: "gzip",
+      // While we could also ask tar to autodetect the compression method,
+      // we defensively keep the gzip call identical as requesting a gzipped
+      // bundle will soon be a fallback option.
+      outputPath: await toolcache.extractTar(path),
+    };
+  }
+  return {
+    compressionMethod: "zstd",
+    // By specifying only the "x" flag, we ask tar to autodetect the compression
+    // method.
+    outputPath: await toolcache.extractTar(path, undefined, "x"),
+  };
+}