Merge main into releases/v2 (#1287)

* Update changelog and version after v2.1.26

* Update checked-in dependencies

* Don't check for Go logs on failure (#1279)

* Update supported GitHub Enterprise Server versions. (#1275)

Co-authored-by: GitHub <noreply@github.com>

* TRAP Caching: Add timeouts to upload/download operations

* Add logging statements declaring state of the cli_config_file_enabled

It's possible to determine this otherwise, but this makes it easier to
spot.

* Avoid using single value as array

The user config parser in the CLI doesn't yet support it.

* Extract logging statements to separate function

* Correctly report CodeQL version when using cache (#1259)

* Correctly report CodeQL version when using cache

* Add JS generated files

* Add test for return value of `setupCodeQL`

* Fill in missing return value comment

* Convert "Invalid source root" errors to UserErrors

* Add changelog note for Go extraction reconciliation (#1286)

* Add changelog note for Go extraction reconciliation

* Update CHANGELOG.md

Co-authored-by: Henry Mercer <henrymercer@github.com>

* Update CHANGELOG.md

Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>

* Tweaks from PR review

Co-authored-by: Henry Mercer <henrymercer@github.com>
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>

* Update changelog for v2.1.27

Co-authored-by: github-actions[bot] <github-actions@github.com>
Co-authored-by: Chuan-kai Lin <cklin@github.com>
Co-authored-by: Angela P Wen <angelapwen@github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: GitHub <noreply@github.com>
Co-authored-by: Edoardo Pirovano <edoardo.pirovano@gmail.com>
Co-authored-by: Edoardo Pirovano <6748066+edoardopirovano@users.noreply.github.com>
Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>
Co-authored-by: Henry Mercer <henrymercer@github.com>

src/analyze.ts

@@ -239,6 +239,9 @@ export async function runQueries(
   }
 
   const codeql = await getCodeQL(config.codeQLCmd);
+
+  await util.logCodeScanningConfigInCli(codeql, featureFlags, logger);
+
   for (const language of config.languages) {
     const queries = config.queries[language];
     const queryFilters = validateQueryFilters(

src/api-compatibility.json

@@ -1 +1 @@
-{"maximumVersion": "3.7", "minimumVersion": "3.3"}
+{"maximumVersion": "3.7", "minimumVersion": "3.2"}

src/codeql.test.ts

@@ -81,7 +81,7 @@ async function mockApiAndSetupCodeQL({
   tmpDir: string;
   toolsInput?: { input?: string };
   version: string;
-}) {
+}): Promise<{ codeql: codeql.CodeQL; toolsVersion: string }> {
   const platform =
     process.platform === "win32"
       ? "win64"
@@ -104,7 +104,7 @@ async function mockApiAndSetupCodeQL({
       )
     );
 
-  await codeql.setupCodeQL(
+  return await codeql.setupCodeQL(
     toolsInput ? toolsInput.input : `${baseUrl}${relativeUrl}`,
     apiDetails ?? sampleApiDetails,
     tmpDir,
@@ -124,8 +124,9 @@ test("download codeql bundle cache", async (t) => {
     for (let i = 0; i < versions.length; i++) {
       const version = versions[i];
 
-      await mockApiAndSetupCodeQL({ version, tmpDir });
+      const codeQLConfig = await mockApiAndSetupCodeQL({ version, tmpDir });
       t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
+      t.deepEqual(codeQLConfig.toolsVersion, version);
     }
 
     t.is(toolcache.findAllVersions("CodeQL").length, 2);
@@ -136,15 +137,20 @@ test("download codeql bundle cache explicitly requested with pinned different ve
   await util.withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
 
-    await mockApiAndSetupCodeQL({
+    const pinnedCodeQLConfig = await mockApiAndSetupCodeQL({
       version: "20200601",
       isPinned: true,
       tmpDir,
     });
     t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
+    t.deepEqual(pinnedCodeQLConfig.toolsVersion, "20200601");
 
-    await mockApiAndSetupCodeQL({ version: "20200610", tmpDir });
+    const unpinnedCodeQLConfig = await mockApiAndSetupCodeQL({
+      version: "20200610",
+      tmpDir,
+    });
     t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
+    t.deepEqual(unpinnedCodeQLConfig.toolsVersion, "20200610");
   });
 });
 
@@ -152,15 +158,16 @@ test("don't download codeql bundle cache with pinned different version cached",
   await util.withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
 
-    await mockApiAndSetupCodeQL({
+    const pinnedCodeQLConfig = await mockApiAndSetupCodeQL({
       version: "20200601",
       isPinned: true,
       tmpDir,
     });
 
     t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
+    t.deepEqual(pinnedCodeQLConfig.toolsVersion, "20200601");
 
-    await codeql.setupCodeQL(
+    const codeQLConfig = await codeql.setupCodeQL(
       undefined,
       sampleApiDetails,
       tmpDir,
@@ -169,6 +176,7 @@ test("don't download codeql bundle cache with pinned different version cached",
       getRunnerLogger(true),
       false
     );
+    t.deepEqual(codeQLConfig.toolsVersion, "0.0.0-20200601");
 
     const cachedVersions = toolcache.findAllVersions("CodeQL");
 
@@ -180,16 +188,24 @@ test("download codeql bundle cache with different version cached (not pinned)",
   await util.withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
 
-    await mockApiAndSetupCodeQL({ version: "20200601", tmpDir });
+    const cachedCodeQLConfig = await mockApiAndSetupCodeQL({
+      version: "20200601",
+      tmpDir,
+    });
 
     t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
+    t.deepEqual(cachedCodeQLConfig.toolsVersion, "20200601");
 
-    await mockApiAndSetupCodeQL({
+    const codeQLConfig = await mockApiAndSetupCodeQL({
       version: defaults.bundleVersion,
       tmpDir,
       apiDetails: sampleApiDetails,
       toolsInput: { input: undefined },
     });
+    t.deepEqual(
+      codeQLConfig.toolsVersion,
+      defaults.bundleVersion.replace("codeql-bundle-", "")
+    );
 
     const cachedVersions = toolcache.findAllVersions("CodeQL");
 
@@ -201,20 +217,25 @@ test('download codeql bundle cache with pinned different version cached if "late
   await util.withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
 
-    await mockApiAndSetupCodeQL({
+    const pinnedCodeQLConfig = await mockApiAndSetupCodeQL({
       version: "20200601",
       isPinned: true,
       tmpDir,
     });
 
     t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
+    t.deepEqual(pinnedCodeQLConfig.toolsVersion, "20200601");
 
-    await mockApiAndSetupCodeQL({
+    const latestCodeQLConfig = await mockApiAndSetupCodeQL({
       version: defaults.bundleVersion,
       apiDetails: sampleApiDetails,
       toolsInput: { input: "latest" },
       tmpDir,
     });
+    t.deepEqual(
+      latestCodeQLConfig.toolsVersion,
+      defaults.bundleVersion.replace("codeql-bundle-", "")
+    );
 
     const cachedVersions = toolcache.findAllVersions("CodeQL");
 

src/codeql.ts

@@ -421,7 +421,7 @@ async function getCodeQLBundleDownloadURL(
  * @param logger
  * @param checkVersion Whether to check that CodeQL CLI meets the minimum
  *        version requirement. Must be set to true outside tests.
- * @returns
+ * @returns a { CodeQL, toolsVersion } object.
  */
 export async function setupCodeQL(
   codeqlURL: string | undefined,
@@ -479,6 +479,7 @@ export async function setupCodeQL(
               `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
             );
             codeqlFolder = tmpCodeqlFolder;
+            codeqlURLVersion = codeqlVersions[0];
           }
         }
       }

src/config-utils.ts

@@ -24,6 +24,7 @@ import {
   codeQlVersionAbove,
   getMlPoweredJsQueriesPack,
   GitHubVersion,
+  logCodeScanningConfigInCli,
   ML_POWERED_JS_QUERIES_PACK_NAME,
   useCodeScanningConfigInCli,
 } from "./util";
@@ -1704,6 +1705,8 @@ export async function initConfig(
   // When using the codescanning config in the CLI, pack downloads
   // happen in the CLI during the `database init` command, so no need
   // to download them here.
+  await logCodeScanningConfigInCli(codeQL, featureFlags, logger);
+
   if (!(await useCodeScanningConfigInCli(codeQL, featureFlags))) {
     const registries = parseRegistries(registriesInput);
     await downloadPacks(

src/init.ts

@@ -117,24 +117,7 @@ export async function runInit(
       }
     }
   } catch (e) {
-    // Handle the situation where init is called twice
-    // for the same database in the same job.
-    if (
-      e instanceof Error &&
-      e.message?.includes("Refusing to create databases") &&
-      e.message.includes("exists and is not an empty directory.")
-    ) {
-      throw new util.UserError(
-        `Is the "init" action called twice in the same job? ${e.message}`
-      );
-    } else if (
-      e instanceof Error &&
-      e.message?.includes("is not compatible with this CodeQL CLI")
-    ) {
-      throw new util.UserError(e.message);
-    } else {
-      throw e;
-    }
+    throw processError(e);
   }
   return await getCombinedTracerConfig(
     config,
@@ -144,6 +127,42 @@ export async function runInit(
   );
 }
 
+/**
+ * Possibly convert this error into a UserError in order to avoid
+ * counting this error towards our internal error budget.
+ *
+ * @param e The error to possibly convert to a UserError.
+ *
+ * @returns A UserError if the error is a known error that can be
+ *         attributed to the user, otherwise the original error.
+ */
+function processError(e: any): Error {
+  if (!(e instanceof Error)) {
+    return e;
+  }
+
+  if (
+    // Init action called twice
+    e.message?.includes("Refusing to create databases") &&
+    e.message?.includes("exists and is not an empty directory.")
+  ) {
+    return new util.UserError(
+      `Is the "init" action called twice in the same job? ${e.message}`
+    );
+  }
+
+  if (
+    // Version of CodeQL CLI is incompatible with this version of the CodeQL Action
+    e.message?.includes("is not compatible with this CodeQL CLI") ||
+    // Expected source location for database creation does not exist
+    e.message?.includes("Invalid source root")
+  ) {
+    return new util.UserError(e.message);
+  }
+
+  return e;
+}
+
 // Runs a powershell script to inject the tracer into a parent process
 // so it can tracer future processes, hopefully including the build process.
 // If processName is given then injects into the nearest parent process with

src/trap-caching.ts

@@ -8,7 +8,7 @@ import { CodeQL, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES } from "./codeql";
 import { Config } from "./config-utils";
 import { Language } from "./languages";
 import { Logger } from "./logging";
-import { codeQlVersionAbove, tryGetFolderBytes } from "./util";
+import { codeQlVersionAbove, tryGetFolderBytes, withTimeout } from "./util";
 
 // This constant should be bumped if we make a breaking change
 // to how the CodeQL Action stores or retrieves the TRAP cache,
@@ -24,6 +24,12 @@ const CACHE_SIZE_MB = 1024;
 // cache for us to consider it worth uploading.
 const MINIMUM_CACHE_MB_TO_UPLOAD = 10;
 
+// The maximum number of milliseconds to wait for TRAP cache
+// uploads or downloads to complete before continuing. Note
+// this timeout is per operation, so will be run as many
+// times as there are languages with TRAP caching enabled.
+const MAX_CACHE_OPERATION_MS = 120_000; // Two minutes
+
 export async function getTrapCachingExtractorConfigArgs(
   config: Config
 ): Promise<string[]> {
@@ -107,9 +113,17 @@ export async function downloadTrapCaches(
     logger.info(
       `Looking in Actions cache for TRAP cache with key ${preferredKey}`
     );
-    const found = await cache.restoreCache([cacheDir], preferredKey, [
-      await cachePrefix(codeql, language), // Fall back to any cache with the right key prefix
-    ]);
+    const found = await withTimeout(
+      MAX_CACHE_OPERATION_MS,
+      cache.restoreCache([cacheDir], preferredKey, [
+        await cachePrefix(codeql, language), // Fall back to any cache with the right key prefix
+      ]),
+      () => {
+        logger.info(
+          `Timed out waiting for TRAP cache download for ${language}, will continue without it`
+        );
+      }
+    );
     if (found === undefined) {
       // We didn't find a TRAP cache in the Actions cache, so the directory on disk is
       // still just an empty directory. There's no reason to tell the extractor to use it,
@@ -136,7 +150,6 @@ export async function uploadTrapCaches(
 ): Promise<boolean> {
   if (!(await actionsUtil.isAnalyzingDefaultBranch())) return false; // Only upload caches from the default branch
 
-  const toAwait: Array<Promise<number>> = [];
   for (const language of config.languages) {
     const cacheDir = config.trapCaches[language];
     if (cacheDir === undefined) continue;
@@ -159,9 +172,16 @@ export async function uploadTrapCaches(
       process.env.GITHUB_SHA || "unknown"
     );
     logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
-    toAwait.push(cache.saveCache([cacheDir], key));
+    await withTimeout(
+      MAX_CACHE_OPERATION_MS,
+      cache.saveCache([cacheDir], key),
+      () => {
+        logger.info(
+          `Timed out waiting for TRAP cache for ${language} to upload, will continue without uploading`
+        );
+      }
+    );
   }
-  await Promise.all(toAwait);
   return true;
 }
 

src/util.test.ts

@@ -601,3 +601,34 @@ function mockVersion(version) {
     },
   } as CodeQL;
 }
+
+const longTime = 999_999;
+const shortTime = 10;
+
+test("withTimeout on long task", async (t) => {
+  let longTaskTimedOut = false;
+  const longTask = new Promise((resolve) => {
+    setTimeout(() => {
+      resolve(42);
+    }, longTime);
+  });
+  const result = await util.withTimeout(shortTime, longTask, () => {
+    longTaskTimedOut = true;
+  });
+  t.deepEqual(longTaskTimedOut, true);
+  t.deepEqual(result, undefined);
+});
+
+test("withTimeout on short task", async (t) => {
+  let shortTaskTimedOut = false;
+  const shortTask = new Promise((resolve) => {
+    setTimeout(() => {
+      resolve(99);
+    }, shortTime);
+  });
+  const result = await util.withTimeout(longTime, shortTask, () => {
+    shortTaskTimedOut = true;
+  });
+  t.deepEqual(shortTaskTimedOut, false);
+  t.deepEqual(result, 99);
+});

src/util.ts

@@ -817,6 +817,22 @@ export async function useCodeScanningConfigInCli(
   return await codeQlVersionAbove(codeql, CODEQL_VERSION_CONFIG_FILES);
 }
 
+export async function logCodeScanningConfigInCli(
+  codeql: CodeQL,
+  featureFlags: FeatureFlags,
+  logger: Logger
+) {
+  if (await useCodeScanningConfigInCli(codeql, featureFlags)) {
+    logger.info(
+      "Code Scanning configuration file being processed in the codeql CLI."
+    );
+  } else {
+    logger.info(
+      "Code Scanning configuration file being processed in the codeql-action."
+    );
+  }
+}
+
 /*
  * Returns whether the path in the argument represents an existing directory.
  */
@@ -878,3 +894,27 @@ export async function tryGetFolderBytes(
     return undefined;
   }
 }
+
+/**
+ * Run a promise for a given amount of time, and if it doesn't resolve within
+ * that time, call the provided callback and then return undefined.
+ *
+ * @param timeoutMs The timeout in milliseconds.
+ * @param promise The promise to run.
+ * @param onTimeout A callback to call if the promise times out.
+ * @returns The result of the promise, or undefined if the promise times out.
+ */
+export async function withTimeout<T>(
+  timeoutMs: number,
+  promise: Promise<T>,
+  onTimeout: () => void
+): Promise<T | undefined> {
+  const timeout: Promise<undefined> = new Promise((resolve) => {
+    setTimeout(() => {
+      onTimeout();
+      resolve(undefined);
+    }, timeoutMs);
+  });
+
+  return await Promise.race([promise, timeout]);
+}