Clean up the database if it will be uploaded

src/analyze-action.ts

@@ -9,7 +9,6 @@ import {
   CodeQLAnalysisError,
   dbIsFinalized,
   QueriesStatusReport,
-  runCleanup,
   runFinalize,
   runQueries,
   setupDiffInformedQueryRun,
@@ -27,10 +26,7 @@ import { EnvVar } from "./environment";
 import { Features } from "./feature-flags";
 import { KnownLanguage } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
-import {
-  OverlayDatabaseMode,
-  uploadOverlayBaseDatabaseToCache,
-} from "./overlay-database-utils";
+import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils";
 import { getRepositoryNwo } from "./repository";
 import * as statusReport from "./status-report";
 import {
@@ -251,6 +247,12 @@ async function run() {
       );
     }
 
+    if (actionsUtil.getOptionalInput("cleanup-level") !== "") {
+      logger.info(
+        "The 'cleanup-level' input is ignored since the CodeQL Action no longer writes intermediate results to the database. This input can safely be removed from your workflow.",
+      );
+    }
+
     const apiDetails = getApiDetails();
     const outputDir = actionsUtil.getRequiredInput("output");
     core.exportVariable(EnvVar.SARIF_RESULTS_OUTPUT_DIR, outputDir);
@@ -298,31 +300,12 @@ async function run() {
       logger,
     );
 
-    if (actionsUtil.getOptionalInput("cleanup-level") !== "") {
-      logger.info(
-        "The 'cleanup-level' input is ignored since the CodeQL Action no longer writes intermediate results to the database. This input can safely be removed from your workflow.",
-      );
-    }
-
-    // An overlay-base database should always use the 'overlay' cleanup level
-    // to preserve the cached intermediate results.
-    //
-    // Otherwise, use cleanup level 'none'. We are already discarding
-    // intermediate results during evaluation with '--expect-discarded-cache',
-    // so there is nothing to clean up.
-    const cleanupLevel =
-      config.augmentationProperties.overlayDatabaseMode ===
-      OverlayDatabaseMode.OverlayBase
-        ? "overlay"
-        : "none";
-
     if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
       runStats = await runQueries(
         outputDir,
         memory,
         util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
         threads,
-        cleanupLevel,
         diffRangePackDir,
         actionsUtil.getOptionalInput("category"),
         config,
@@ -331,10 +314,6 @@ async function run() {
       );
     }
 
-    if (cleanupLevel !== "none") {
-      await runCleanup(config, cleanupLevel, logger);
-    }
-
     const dbLocations: { [lang: string]: string } = {};
     for (const language of config.languages) {
       dbLocations[language] = util.getCodeQLDatabasePath(config, language);
@@ -368,12 +347,14 @@ async function run() {
       logger.info("Not uploading results");
     }
 
-    // Possibly upload the database bundles for remote queries
-    await uploadDatabases(repositoryNwo, config, apiDetails, logger);
-
-    // Possibly upload the overlay-base database to actions cache
+    // Possibly upload the overlay-base database to actions cache.
+    // If databases are to be uploaded, they will first be cleaned up at the overlay level.
     await uploadOverlayBaseDatabaseToCache(codeql, config, logger);
 
+    // Possibly upload the database bundles for remote queries.
+    // If databases are to be uploaded, they will first be cleaned up at the clear level.
+    await uploadDatabases(repositoryNwo, codeql, config, apiDetails, logger);
+
     // Possibly upload the TRAP caches for later re-use
     const trapCacheUploadStartTime = performance.now();
     didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger);

src/analyze.test.ts

@@ -106,7 +106,6 @@ test("status report fields", async (t) => {
         memoryFlag,
         addSnippetsFlag,
         threadsFlag,
-        "brutal",
         undefined,
         undefined,
         config,

src/analyze.ts

@@ -612,7 +612,6 @@ export async function runQueries(
   memoryFlag: string,
   addSnippetsFlag: string,
   threadsFlag: string,
-  cleanupLevel: string,
   diffRangePackDir: string | undefined,
   automationDetailsId: string | undefined,
   config: configUtils.Config,
@@ -623,7 +622,11 @@ export async function runQueries(
   const queryFlags = [memoryFlag, threadsFlag];
   const incrementalMode: string[] = [];
 
-  if (cleanupLevel !== "overlay") {
+  // Preserve cached intermediate results for overlay-base databases.
+  if (
+    config.augmentationProperties.overlayDatabaseMode !==
+    OverlayDatabaseMode.OverlayBase
+  ) {
     queryFlags.push("--expect-discarded-cache");
   }
 
@@ -874,20 +877,6 @@ export async function warnIfGoInstalledAfterInit(
   }
 }
 
-export async function runCleanup(
-  config: configUtils.Config,
-  cleanupLevel: string,
-  logger: Logger,
-): Promise<void> {
-  logger.startGroup("Cleaning up databases");
-  for (const language of config.languages) {
-    const codeql = await getCodeQL(config.codeQLCmd);
-    const databasePath = util.getCodeQLDatabasePath(config, language);
-    await codeql.databaseCleanup(databasePath, cleanupLevel);
-  }
-  logger.endGroup();
-}
-
 export const exportedForTesting = {
   getDiffRanges,
 };

src/codeql.ts

@@ -155,6 +155,10 @@ export interface CodeQL {
    * Run 'codeql database cleanup'.
    */
   databaseCleanup(databasePath: string, cleanupLevel: string): Promise<void>;
+  /**
+   * Clean up all the databases within a database cluster.
+   */
+  databaseCleanupCluster(config: Config, cleanupLevel: string): Promise<void>;
   /**
    * Run 'codeql database bundle'.
    */
@@ -483,6 +487,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
     ),
     packDownload: resolveFunction(partialCodeql, "packDownload"),
     databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
+    databaseCleanupCluster: resolveFunction(
+      partialCodeql,
+      "databaseCleanupCluster",
+    ),
     databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
     databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
     databaseInterpretResults: resolveFunction(
@@ -997,6 +1005,15 @@ export async function getCodeQLForCmd(
       ];
       await runCli(cmd, codeqlArgs);
     },
+    async databaseCleanupCluster(
+      config: Config,
+      cleanupLevel: string,
+    ): Promise<void> {
+      for (const language of config.languages) {
+        const databasePath = util.getCodeQLDatabasePath(config, language);
+        await codeql.databaseCleanup(databasePath, cleanupLevel);
+      }
+    },
     async databaseBundle(
       databasePath: string,
       outputFilePath: string,

src/database-upload.test.ts

@@ -69,6 +69,17 @@ async function mockHttpRequests(databaseUploadStatusCode: number) {
   return databaseUploadSpy;
 }
 
+function getCodeQL() {
+  return setCodeQL({
+    async databaseBundle(_: string, outputFilePath: string) {
+      fs.writeFileSync(outputFilePath, "");
+    },
+    async databaseCleanupCluster() {
+      // Do nothing, as we are not testing cleanup here.
+    },
+  });
+}
+
 test("Abort database upload if 'upload-database' input set to false", async (t) => {
   await withTmpDir(async (tmpDir) => {
     setupActionsVars(tmpDir, tmpDir);
@@ -81,6 +92,7 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
     const loggedMessages = [];
     await uploadDatabases(
       testRepoName,
+      getCodeQL(),
       getTestConfig(tmpDir),
       testApiDetails,
       getRecordingLogger(loggedMessages),
@@ -111,6 +123,7 @@ test("Abort database upload if running against GHES", async (t) => {
     const loggedMessages = [];
     await uploadDatabases(
       testRepoName,
+      getCodeQL(),
       config,
       testApiDetails,
       getRecordingLogger(loggedMessages),
@@ -138,6 +151,7 @@ test("Abort database upload if not analyzing default branch", async (t) => {
     const loggedMessages = [];
     await uploadDatabases(
       testRepoName,
+      getCodeQL(),
       getTestConfig(tmpDir),
       testApiDetails,
       getRecordingLogger(loggedMessages),
@@ -163,15 +177,10 @@ test("Don't crash if uploading a database fails", async (t) => {
 
     await mockHttpRequests(500);
 
-    setCodeQL({
-      async databaseBundle(_: string, outputFilePath: string) {
-        fs.writeFileSync(outputFilePath, "");
-      },
-    });
-
     const loggedMessages = [] as LoggedMessage[];
     await uploadDatabases(
       testRepoName,
+      getCodeQL(),
       getTestConfig(tmpDir),
       testApiDetails,
       getRecordingLogger(loggedMessages),
@@ -199,15 +208,10 @@ test("Successfully uploading a database to github.com", async (t) => {
 
     await mockHttpRequests(201);
 
-    setCodeQL({
-      async databaseBundle(_: string, outputFilePath: string) {
-        fs.writeFileSync(outputFilePath, "");
-      },
-    });
-
     const loggedMessages = [] as LoggedMessage[];
     await uploadDatabases(
       testRepoName,
+      getCodeQL(),
       getTestConfig(tmpDir),
       testApiDetails,
       getRecordingLogger(loggedMessages),
@@ -233,15 +237,10 @@ test("Successfully uploading a database to GHEC-DR", async (t) => {
 
     const databaseUploadSpy = await mockHttpRequests(201);
 
-    setCodeQL({
-      async databaseBundle(_: string, outputFilePath: string) {
-        fs.writeFileSync(outputFilePath, "");
-      },
-    });
-
     const loggedMessages = [] as LoggedMessage[];
     await uploadDatabases(
       testRepoName,
+      getCodeQL(),
       getTestConfig(tmpDir),
       {
         auth: "1234",

src/database-upload.ts

@@ -2,16 +2,17 @@ import * as fs from "fs";
 
 import * as actionsUtil from "./actions-util";
 import { getApiClient, GitHubApiDetails } from "./api-client";
-import { getCodeQL } from "./codeql";
+import { type CodeQL } from "./codeql";
 import { Config } from "./config-utils";
 import * as gitUtils from "./git-utils";
-import { Logger } from "./logging";
+import { Logger, withGroupAsync } from "./logging";
 import { RepositoryNwo } from "./repository";
 import * as util from "./util";
 import { bundleDb, parseGitHubUrl } from "./util";
 
 export async function uploadDatabases(
   repositoryNwo: RepositoryNwo,
+  codeql: CodeQL,
   config: Config,
   apiDetails: GitHubApiDetails,
   logger: Logger,
@@ -41,8 +42,13 @@ export async function uploadDatabases(
     return;
   }
 
+  // Clean up the database, since intermediate results may still be written to the
+  // database if there is high RAM pressure.
+  await withGroupAsync("Cleaning up databases", async () => {
+    await codeql.databaseCleanupCluster(config, "clear");
+  });
+
   const client = getApiClient();
-  const codeql = await getCodeQL(config.codeQLCmd);
 
   const uploadsUrl = new URL(parseGitHubUrl(apiDetails.url));
   uploadsUrl.hostname = `uploads.${uploadsUrl.hostname}`;

src/overlay-database-utils.ts

@@ -7,7 +7,7 @@ import { getRequiredInput, getTemporaryDirectory } from "./actions-util";
 import { type CodeQL } from "./codeql";
 import { type Config } from "./config-utils";
 import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
-import { Logger } from "./logging";
+import { Logger, withGroupAsync } from "./logging";
 import { isInTestMode, tryGetFolderBytes, withTimeout } from "./util";
 
 export enum OverlayDatabaseMode {
@@ -206,6 +206,11 @@ export async function uploadOverlayBaseDatabaseToCache(
     return false;
   }
 
+  // Clean up the database using the overlay cleanup level.
+  await withGroupAsync("Cleaning up databases", async () => {
+    await codeql.databaseCleanupCluster(config, "overlay");
+  });
+
   const dbLocation = config.dbLocation;
   const codeQlVersion = (await codeql.getVersion()).version;
   const checkoutPath = getRequiredInput("checkout_path");