Merge pull request #3340 from github/kaspersv/check-for-overlayBaseSpecifier

Overlay: Check database metadata for overlayBaseSpecifier

lib/analyze-action-post.js

@@ -120735,6 +120735,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/analyze-action.js

@@ -88512,7 +88512,7 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
 var CACHE_VERSION = 1;
 var CACHE_PREFIX = "codeql-overlay-base-database";
 var MAX_CACHE_OPERATION_MS = 6e5;
-function checkOverlayBaseDatabase(config, logger, warningPrefix) {
+async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) {
   const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
   if (!fs3.existsSync(baseDatabaseOidsFilePath)) {
     logger.warning(
@@ -88520,6 +88520,23 @@ function checkOverlayBaseDatabase(config, logger, warningPrefix) {
     );
     return false;
   }
+  for (const language of config.languages) {
+    const dbPath = getCodeQLDatabasePath(config, language);
+    try {
+      const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
+      if (resolveDatabaseOutput === void 0 || !("overlayBaseSpecifier" in resolveDatabaseOutput)) {
+        logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
+        return false;
+      } else {
+        logger.debug(
+          `Overlay base specifier for ${language} overlay-base database found: ${resolveDatabaseOutput.overlayBaseSpecifier}`
+        );
+      }
+    } catch (e) {
+      logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
+      return false;
+    }
+  }
   return true;
 }
 async function cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger) {
@@ -88542,7 +88559,8 @@ async function cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger
     );
     return false;
   }
-  const databaseIsValid = checkOverlayBaseDatabase(
+  const databaseIsValid = await checkOverlayBaseDatabase(
+    codeql,
     config,
     logger,
     "Abort uploading overlay-base database to cache"
@@ -90887,6 +90905,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/autobuild-action.js

@@ -84973,6 +84973,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/init-action-post.js

@@ -125350,6 +125350,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/init-action.js

@@ -85914,7 +85914,7 @@ function computeChangedFiles(baseFileOids, overlayFileOids) {
 var CACHE_VERSION = 1;
 var CACHE_PREFIX = "codeql-overlay-base-database";
 var MAX_CACHE_OPERATION_MS = 6e5;
-function checkOverlayBaseDatabase(config, logger, warningPrefix) {
+async function checkOverlayBaseDatabase(codeql, config, logger, warningPrefix) {
   const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
   if (!fs3.existsSync(baseDatabaseOidsFilePath)) {
     logger.warning(
@@ -85922,6 +85922,23 @@ function checkOverlayBaseDatabase(config, logger, warningPrefix) {
     );
     return false;
   }
+  for (const language of config.languages) {
+    const dbPath = getCodeQLDatabasePath(config, language);
+    try {
+      const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
+      if (resolveDatabaseOutput === void 0 || !("overlayBaseSpecifier" in resolveDatabaseOutput)) {
+        logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
+        return false;
+      } else {
+        logger.debug(
+          `Overlay base specifier for ${language} overlay-base database found: ${resolveDatabaseOutput.overlayBaseSpecifier}`
+        );
+      }
+    } catch (e) {
+      logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
+      return false;
+    }
+  }
   return true;
 }
 async function downloadOverlayBaseDatabaseFromCache(codeql, config, logger) {
@@ -86009,7 +86026,8 @@ async function downloadOverlayBaseDatabaseFromCache(codeql, config, logger) {
     );
     return void 0;
   }
-  const databaseIsValid = checkOverlayBaseDatabase(
+  const databaseIsValid = await checkOverlayBaseDatabase(
+    codeql,
     config,
     logger,
     "Downloaded overlay-base database is invalid"
@@ -89147,6 +89165,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/resolve-environment-action.js

@@ -84672,6 +84672,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/setup-codeql-action.js

@@ -85975,6 +85975,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/upload-lib.js

@@ -88697,6 +88697,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

lib/upload-sarif-action.js

@@ -89215,6 +89215,23 @@ ${output}`
         );
       }
     },
+    async resolveDatabase(databasePath) {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"])
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+      try {
+        return JSON.parse(output);
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`
+        );
+      }
+    },
     async mergeResults(sarifFiles, outputFile, {
       mergeRunsFromEqualCategory = false
     }) {

src/codeql.ts

@@ -206,6 +206,7 @@ export interface CodeQL {
    * Run 'codeql resolve queries --format=startingpacks'.
    */
   resolveQueriesStartingPacks(queries: string[]): Promise<string[]>;
+  resolveDatabase(databasePath: string): Promise<ResolveDatabaseOutput>;
   /**
    * Run 'codeql github merge-results'.
    */
@@ -230,6 +231,10 @@ export interface VersionInfo {
   overlayVersion?: number;
 }
 
+export interface ResolveDatabaseOutput {
+  overlayBaseSpecifier?: string;
+}
+
 export interface ResolveLanguagesOutput {
   [language: string]: [string];
 }
@@ -493,6 +498,7 @@ export function createStubCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
       partialCodeql,
       "resolveQueriesStartingPacks",
     ),
+    resolveDatabase: resolveFunction(partialCodeql, "resolveDatabase"),
     mergeResults: resolveFunction(partialCodeql, "mergeResults"),
   };
 }
@@ -1003,6 +1009,26 @@ async function getCodeQLForCmd(
         );
       }
     },
+    async resolveDatabase(
+      databasePath: string,
+    ): Promise<ResolveDatabaseOutput> {
+      const codeqlArgs = [
+        "resolve",
+        "database",
+        databasePath,
+        "--format=json",
+        ...getExtraOptionsFromEnv(["resolve", "database"]),
+      ];
+      const output = await runCli(cmd, codeqlArgs, { noStreamStdout: true });
+
+      try {
+        return JSON.parse(output) as ResolveDatabaseOutput;
+      } catch (e) {
+        throw new Error(
+          `Unexpected output from codeql resolve database --format=json: ${e}`,
+        );
+      }
+    },
     async mergeResults(
       sarifFiles: string[],
       outputFile: string,

src/overlay-database-utils.test.ts

@@ -7,7 +7,9 @@ import * as sinon from "sinon";
 
 import * as actionsUtil from "./actions-util";
 import * as apiClient from "./api-client";
+import { ResolveDatabaseOutput } from "./codeql";
 import * as gitUtils from "./git-utils";
+import { KnownLanguage } from "./languages";
 import { getRunnerLogger } from "./logging";
 import {
   downloadOverlayBaseDatabaseFromCache,
@@ -95,6 +97,7 @@ interface DownloadOverlayBaseDatabaseTestCase {
   hasBaseDatabaseOidsFile: boolean;
   tryGetFolderBytesSucceeds: boolean;
   codeQLVersion: string;
+  resolveDatabaseOutput: ResolveDatabaseOutput | Error;
 }
 
 const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
@@ -105,6 +108,7 @@ const defaultDownloadTestCase: DownloadOverlayBaseDatabaseTestCase = {
   hasBaseDatabaseOidsFile: true,
   tryGetFolderBytesSucceeds: true,
   codeQLVersion: "2.20.5",
+  resolveDatabaseOutput: { overlayBaseSpecifier: "20250626:XXX" },
 };
 
 const testDownloadOverlayBaseDatabaseFromCache = test.macro({
@@ -119,9 +123,11 @@ const testDownloadOverlayBaseDatabaseFromCache = test.macro({
       await fs.promises.mkdir(dbLocation, { recursive: true });
 
       const logger = getRunnerLogger(true);
-      const config = createTestConfig({ dbLocation });
-
       const testCase = { ...defaultDownloadTestCase, ...partialTestCase };
+      const config = createTestConfig({
+        dbLocation,
+        languages: [KnownLanguage.java],
+      });
 
       config.overlayDatabaseMode = testCase.overlayDatabaseMode;
       config.useOverlayDatabaseCaching = testCase.useOverlayDatabaseCaching;
@@ -163,9 +169,23 @@ const testDownloadOverlayBaseDatabaseFromCache = test.macro({
         .resolves(testCase.tryGetFolderBytesSucceeds ? 1024 * 1024 : undefined);
       stubs.push(tryGetFolderBytesStub);
 
+      const codeql = mockCodeQLVersion(testCase.codeQLVersion);
+
+      if (testCase.resolveDatabaseOutput instanceof Error) {
+        const resolveDatabaseStub = sinon
+          .stub(codeql, "resolveDatabase")
+          .rejects(testCase.resolveDatabaseOutput);
+        stubs.push(resolveDatabaseStub);
+      } else {
+        const resolveDatabaseStub = sinon
+          .stub(codeql, "resolveDatabase")
+          .resolves(testCase.resolveDatabaseOutput);
+        stubs.push(resolveDatabaseStub);
+      }
+
       try {
         const result = await downloadOverlayBaseDatabaseFromCache(
-          mockCodeQLVersion(testCase.codeQLVersion),
+          codeql,
           config,
           logger,
         );
@@ -255,6 +275,24 @@ test(
   false,
 );
 
+test(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when downloaded database doesn't have an overlayBaseSpecifier",
+  {
+    resolveDatabaseOutput: {},
+  },
+  false,
+);
+
+test(
+  testDownloadOverlayBaseDatabaseFromCache,
+  "returns undefined when resolving database metadata fails",
+  {
+    resolveDatabaseOutput: new Error("Failed to resolve database metadata"),
+  },
+  false,
+);
+
 test(
   testDownloadOverlayBaseDatabaseFromCache,
   "returns undefined when filesystem error occurs",

src/overlay-database-utils.ts

@@ -17,6 +17,7 @@ import { getCommitOid, getFileOidsUnderPath } from "./git-utils";
 import { Logger, withGroupAsync } from "./logging";
 import {
   CleanupLevel,
+  getCodeQLDatabasePath,
   getErrorMessage,
   isInTestMode,
   tryGetFolderBytes,
@@ -176,11 +177,12 @@ const MAX_CACHE_OPERATION_MS = 600_000;
  * @param warningPrefix Prefix for the check failure warning message
  * @returns True if the verification succeeded, false otherwise
  */
-function checkOverlayBaseDatabase(
+async function checkOverlayBaseDatabase(
+  codeql: CodeQL,
   config: Config,
   logger: Logger,
   warningPrefix: string,
-): boolean {
+): Promise<boolean> {
   // An overlay-base database should contain the base database OIDs file.
   const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
   if (!fs.existsSync(baseDatabaseOidsFilePath)) {
@@ -189,6 +191,29 @@ function checkOverlayBaseDatabase(
     );
     return false;
   }
+
+  for (const language of config.languages) {
+    const dbPath = getCodeQLDatabasePath(config, language);
+    try {
+      const resolveDatabaseOutput = await codeql.resolveDatabase(dbPath);
+      if (
+        resolveDatabaseOutput === undefined ||
+        !("overlayBaseSpecifier" in resolveDatabaseOutput)
+      ) {
+        logger.info(`${warningPrefix}: no overlayBaseSpecifier defined`);
+        return false;
+      } else {
+        logger.debug(
+          `Overlay base specifier for ${language} overlay-base database found: ` +
+            `${resolveDatabaseOutput.overlayBaseSpecifier}`,
+        );
+      }
+    } catch (e) {
+      logger.warning(`${warningPrefix}: failed to resolve database: ${e}`);
+      return false;
+    }
+  }
+
   return true;
 }
 
@@ -232,7 +257,8 @@ export async function cleanupAndUploadOverlayBaseDatabaseToCache(
     return false;
   }
 
-  const databaseIsValid = checkOverlayBaseDatabase(
+  const databaseIsValid = await checkOverlayBaseDatabase(
+    codeql,
     config,
     logger,
     "Abort uploading overlay-base database to cache",
@@ -415,7 +441,8 @@ export async function downloadOverlayBaseDatabaseFromCache(
     return undefined;
   }
 
-  const databaseIsValid = checkOverlayBaseDatabase(
+  const databaseIsValid = await checkOverlayBaseDatabase(
+    codeql,
     config,
     logger,
     "Downloaded overlay-base database is invalid",