Merge branch 'main' into mbg/ignore-generated

src/codeql.ts

@@ -147,11 +147,20 @@ export interface CodeQL {
   ): Promise<void>;
   /**
    * Run 'codeql database bundle'.
+   *
+   * @param alsoIncludeRelativePaths Additional paths that should be included in the bundle if
+   * supported by the version of the CodeQL CLI.
+   *
+   * These paths are relative to the database root.
+   *
+   * Older versions of the CodeQL CLI do not support including additional paths in the bundle.
+   * In those cases, this parameter will be ignored.
    */
   databaseBundle(
     databasePath: string,
     outputFilePath: string,
     dbName: string,
+    alsoIncludeRelativePaths: string[],
   ): Promise<void>;
   /**
    * Run 'codeql database run-queries'. If no `queries` are specified, then the CLI
@@ -911,6 +920,7 @@ async function getCodeQLForCmd(
       databasePath: string,
       outputFilePath: string,
       databaseName: string,
+      alsoIncludeRelativePaths: string[],
     ): Promise<void> {
       const args = [
         "database",
@@ -920,6 +930,16 @@ async function getCodeQLForCmd(
         `--name=${databaseName}`,
         ...getExtraOptionsFromEnv(["database", "bundle"]),
       ];
+      if (
+        await this.supportsFeature(ToolsFeature.BundleSupportsIncludeOption)
+      ) {
+        args.push(
+          ...alsoIncludeRelativePaths.flatMap((relativePath) => [
+            "--include",
+            relativePath,
+          ]),
+        );
+      }
       await new toolrunner.ToolRunner(cmd, args).exec();
     },
     async databaseExportDiagnostics(

src/debug-artifacts.test.ts

@@ -24,6 +24,37 @@ test("sanitizeArtifactName", (t) => {
   );
 });
 
+test("getArtifactSuffix", (t) => {
+  // No suffix if there's no `matrix` input, it is invalid, or has no keys.
+  t.is(debugArtifacts.getArtifactSuffix(undefined), "");
+  t.is(debugArtifacts.getArtifactSuffix(""), "");
+  t.is(debugArtifacts.getArtifactSuffix("invalid json"), "");
+  t.is(debugArtifacts.getArtifactSuffix("{}"), "");
+  t.is(debugArtifacts.getArtifactSuffix("null"), "");
+  t.is(debugArtifacts.getArtifactSuffix("123"), "");
+  t.is(debugArtifacts.getArtifactSuffix('"string"'), "");
+
+  // Suffixes for non-empty, valid `matrix` inputs.
+  const testMatrices = [
+    { matrix: { language: "go" }, expected: "-go" },
+    {
+      matrix: { language: "javascript", "build-mode": "none" },
+      expected: "-none-javascript",
+    },
+    {
+      matrix: { "build-mode": "none", language: "javascript" },
+      expected: "-none-javascript",
+    },
+  ];
+
+  for (const testMatrix of testMatrices) {
+    const suffix = debugArtifacts.getArtifactSuffix(
+      JSON.stringify(testMatrix.matrix),
+    );
+    t.is(suffix, testMatrix.expected);
+  }
+});
+
 // These next tests check the correctness of the logic to determine whether or not
 // artifacts are uploaded in debug mode. Since it's not easy to mock the actual
 // call to upload an artifact, we just check that we get an "upload-failed" result,

src/debug-artifacts.ts

@@ -246,6 +246,42 @@ export async function tryUploadAllAvailableDebugArtifacts(
   }
 }
 
+/**
+ * When a build matrix is used, multiple different jobs arising from the matrix may attempt to upload
+ * workflow artifacts with the same base name. In that case, only one of the uploads will succeed and
+ * the others will fail. This function inspects the matrix object to compute a suffix for the artifact
+ * name that uniquely identifies the matrix values of the current job to avoid name clashes.
+ *
+ * @param matrix A stringified JSON value, usually the value of the `matrix` input.
+ * @returns A suffix that uniquely identifies the `matrix` value for the current job, or `""` if there
+ * is no matrix value.
+ */
+export function getArtifactSuffix(matrix: string | undefined): string {
+  let suffix = "";
+  if (matrix) {
+    try {
+      const matrixObject = JSON.parse(matrix);
+      if (matrixObject !== null && typeof matrixObject === "object") {
+        for (const matrixKey of Object.keys(matrixObject as object).sort())
+          suffix += `-${matrixObject[matrixKey]}`;
+      } else {
+        core.warning("User-specified `matrix` input is not an object.");
+      }
+    } catch {
+      core.warning(
+        "Could not parse user-specified `matrix` input into JSON. The debug artifact will not be named with the user's `matrix` input.",
+      );
+    }
+  }
+  return suffix;
+}
+
+// Enumerates different, possible outcomes for artifact uploads.
+export type UploadArtifactsResult =
+  | "no-artifacts-to-upload"
+  | "upload-successful"
+  | "upload-failed";
+
 export async function uploadDebugArtifacts(
   logger: Logger,
   toUpload: string[],
@@ -253,15 +289,7 @@ export async function uploadDebugArtifacts(
   artifactName: string,
   ghVariant: GitHubVariant,
   codeQlVersion: string | undefined,
-): Promise<
-  | "no-artifacts-to-upload"
-  | "upload-successful"
-  | "upload-failed"
-  | "upload-not-supported"
-> {
-  if (toUpload.length === 0) {
-    return "no-artifacts-to-upload";
-  }
+): Promise<UploadArtifactsResult | "upload-not-supported"> {
   const uploadSupported = isSafeArtifactUpload(codeQlVersion);
 
   if (!uploadSupported) {
@@ -271,6 +299,31 @@ export async function uploadDebugArtifacts(
     return "upload-not-supported";
   }
 
+  return uploadArtifacts(logger, toUpload, rootDir, artifactName, ghVariant);
+}
+
+/**
+ * Uploads the specified files as a single workflow artifact.
+ *
+ * @param logger The logger to use.
+ * @param toUpload The list of paths to include in the artifact.
+ * @param rootDir The root directory of the paths to include.
+ * @param artifactName The base name for the artifact.
+ * @param ghVariant The GitHub variant.
+ *
+ * @returns The outcome of the attempt to create and upload the artifact.
+ */
+export async function uploadArtifacts(
+  logger: Logger,
+  toUpload: string[],
+  rootDir: string,
+  artifactName: string,
+  ghVariant: GitHubVariant,
+): Promise<UploadArtifactsResult> {
+  if (toUpload.length === 0) {
+    return "no-artifacts-to-upload";
+  }
+
   // When running in test mode, perform a best effort scan of the debug artifacts. The artifact
   // scanner is basic and not reliable or fast enough for production use, but it can help catch
   // some issues early.
@@ -279,21 +332,7 @@ export async function uploadDebugArtifacts(
     core.exportVariable("CODEQL_ACTION_ARTIFACT_SCAN_FINISHED", "true");
   }
 
-  let suffix = "";
-  const matrix = getOptionalInput("matrix");
-  if (matrix) {
-    try {
-      for (const [, matrixVal] of Object.entries(
-        JSON.parse(matrix) as any[][],
-      ).sort())
-        suffix += `-${matrixVal}`;
-    } catch {
-      core.info(
-        "Could not parse user-specified `matrix` input into JSON. The debug artifact will not be named with the user's `matrix` input.",
-      );
-    }
-  }
-
+  const suffix = getArtifactSuffix(getOptionalInput("matrix"));
   const artifactUploader = await getArtifactUploaderClient(logger, ghVariant);
 
   try {

src/overlay-database-utils.ts

@@ -17,6 +17,7 @@ import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
 import { Logger, withGroupAsync } from "./logging";
 import {
   CleanupLevel,
+  getBaseDatabaseOidsFilePath,
   getCodeQLDatabasePath,
   getErrorMessage,
   isInTestMode,
@@ -98,10 +99,6 @@ async function readBaseDatabaseOidsFile(
   }
 }
 
-function getBaseDatabaseOidsFilePath(config: Config): string {
-  return path.join(config.dbLocation, "base-database-oids.json");
-}
-
 /**
  * Writes a JSON file containing the source-root-relative paths of files under
  * `sourceRoot` that have changed (added, removed, or modified) from the overlay

src/start-proxy-action-post.ts

@@ -8,7 +8,7 @@ import * as core from "@actions/core";
 import * as actionsUtil from "./actions-util";
 import { getGitHubVersion } from "./api-client";
 import * as configUtils from "./config-utils";
-import { getArtifactUploaderClient } from "./debug-artifacts";
+import { uploadArtifacts } from "./debug-artifacts";
 import { getActionsLogger } from "./logging";
 import { checkGitHubVersionInRange, getErrorMessage } from "./util";
 
@@ -44,19 +44,12 @@ async function runWrapper() {
       const gitHubVersion = await getGitHubVersion();
       checkGitHubVersionInRange(gitHubVersion, logger);
 
-      const artifactUploader = await getArtifactUploaderClient(
+      await uploadArtifacts(
         logger,
-        gitHubVersion.type,
-      );
-
-      await artifactUploader.uploadArtifact(
-        "proxy-log-file",
         [logFilePath],
         actionsUtil.getTemporaryDirectory(),
-        {
-          // ensure we don't keep the debug artifacts around for too long since they can be large.
-          retentionDays: 7,
-        },
+        "proxy-log-file",
+        gitHubVersion.type,
       );
     }
   } catch (error) {

src/tools-features.ts

@@ -4,6 +4,7 @@ import type { VersionInfo } from "./codeql";
 
 export enum ToolsFeature {
   BuiltinExtractorsSpecifyDefaultQueries = "builtinExtractorsSpecifyDefaultQueries",
+  BundleSupportsIncludeOption = "bundleSupportsIncludeOption",
   BundleSupportsOverlay = "bundleSupportsOverlay",
   DatabaseInterpretResultsSupportsSarifRunProperty = "databaseInterpretResultsSupportsSarifRunProperty",
   ForceOverwrite = "forceOverwrite",

src/util.ts

@@ -16,6 +16,12 @@ import { EnvVar } from "./environment";
 import { Language } from "./languages";
 import { Logger } from "./logging";
 
+/**
+ * The name of the file containing the base database OIDs, as stored in the
+ * root of the database location.
+ */
+const BASE_DATABASE_OIDS_FILE_NAME = "base-database-oids.json";
+
 /**
  * Specifies bundle versions that are known to be broken
  * and will not be used if found in the toolcache.
@@ -728,6 +734,10 @@ export async function codeQlVersionAtLeast(
   return semver.gte((await codeql.getVersion()).version, requiredVersion);
 }
 
+export function getBaseDatabaseOidsFilePath(config: Config): string {
+  return path.join(config.dbLocation, BASE_DATABASE_OIDS_FILE_NAME);
+}
+
 // Create a bundle for the given DB, if it doesn't already exist
 export async function bundleDb(
   config: Config,
@@ -745,7 +755,27 @@ export async function bundleDb(
   if (fs.existsSync(databaseBundlePath)) {
     await fs.promises.rm(databaseBundlePath, { force: true });
   }
-  await codeql.databaseBundle(databasePath, databaseBundlePath, dbName);
+  // When overlay is enabled, the base database OIDs file is included at the
+  // root of the database cluster. However when we bundle a database, we only
+  // include the per-language database. So, to ensure the base database OIDs
+  // file is included in the database bundle, we copy it from the cluster into
+  // the individual database location before bundling.
+  const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
+  const additionalFiles: string[] = [];
+  if (fs.existsSync(baseDatabaseOidsFilePath)) {
+    await fsPromises.copyFile(
+      baseDatabaseOidsFilePath,
+      path.join(databasePath, BASE_DATABASE_OIDS_FILE_NAME),
+    );
+    additionalFiles.push(BASE_DATABASE_OIDS_FILE_NAME);
+  }
+  // Create the bundle, including the base database OIDs file if it exists
+  await codeql.databaseBundle(
+    databasePath,
+    databaseBundlePath,
+    dbName,
+    additionalFiles,
+  );
   return databaseBundlePath;
 }