Merge pull request #3358 from github/henrymercer/database-upload-telemetry

Add status report for uploading databases to API

lib/analyze-action.js

@@ -91739,27 +91739,28 @@ var fs13 = __toESM(require("fs"));
 async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetails, features, logger) {
   if (getRequiredInput("upload-database") !== "true") {
     logger.debug("Database upload disabled in workflow. Skipping upload.");
-    return;
+    return [];
   }
   if (!config.analysisKinds.includes("code-scanning" /* CodeScanning */)) {
     logger.debug(
       `Not uploading database because 'analysis-kinds: ${"code-scanning" /* CodeScanning */}' is not enabled.`
     );
-    return;
+    return [];
   }
   if (isInTestMode()) {
     logger.debug("In test mode. Skipping database upload.");
-    return;
+    return [];
   }
   if (config.gitHubVersion.type !== "GitHub.com" /* DOTCOM */ && config.gitHubVersion.type !== "GitHub Enterprise Cloud with data residency" /* GHEC_DR */) {
     logger.debug("Not running against github.com or GHEC-DR. Skipping upload.");
-    return;
+    return [];
   }
   if (!await isAnalyzingDefaultBranch()) {
     logger.debug("Not analyzing default branch. Skipping upload.");
-    return;
+    return [];
   }
-  const cleanupLevel = config.overlayDatabaseMode === "overlay-base" /* OverlayBase */ && await features.getValue("upload_overlay_db_to_api" /* UploadOverlayDbToApi */) ? "overlay" /* Overlay */ : "clear" /* Clear */;
+  const shouldUploadOverlayBase = config.overlayDatabaseMode === "overlay-base" /* OverlayBase */ && await features.getValue("upload_overlay_db_to_api" /* UploadOverlayDbToApi */);
+  const cleanupLevel = shouldUploadOverlayBase ? "overlay" /* Overlay */ : "clear" /* Clear */;
   await withGroupAsync("Cleaning up databases", async () => {
     await codeql.databaseCleanupCluster(config, cleanupLevel);
   });
@@ -91770,6 +91771,7 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
   if (uploadsBaseUrl.endsWith("/")) {
     uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
   }
+  const reports = [];
   for (const language of config.languages) {
     try {
       const bundledDb = await bundleDb(config, language, codeql, language);
@@ -91779,6 +91781,7 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
         getRequiredInput("checkout_path")
       );
       try {
+        const startTime = performance.now();
         await client.request(
           `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
           {
@@ -91796,14 +91799,28 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
             }
           }
         );
+        const endTime = performance.now();
+        reports.push({
+          language,
+          zipped_upload_size_bytes: bundledDbSize,
+          is_overlay_base: shouldUploadOverlayBase,
+          upload_duration_ms: endTime - startTime
+        });
         logger.debug(`Successfully uploaded database for ${language}`);
       } finally {
         bundledDbReadStream.close();
       }
     } catch (e) {
-      logger.warning(`Failed to upload database for ${language}: ${e}`);
+      logger.warning(
+        `Failed to upload database for ${language}: ${getErrorMessage(e)}`
+      );
+      reports.push({
+        language,
+        error: getErrorMessage(e)
+      });
     }
   }
+  return reports;
 }
 
 // src/status-report.ts
@@ -93817,7 +93834,7 @@ async function postProcessAndUploadSarif(logger, features, uploadKind, checkoutP
 }
 
 // src/analyze-action.ts
-async function sendStatusReport2(startedAt, config, stats, error3, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, dependencyCacheResults, logger) {
+async function sendStatusReport2(startedAt, config, stats, error3, trapCacheUploadTime, dbCreationTimings, didUploadTrapCaches, trapCacheCleanup, dependencyCacheResults, databaseUploadResults, logger) {
   const status = getActionsStatus(error3, stats?.analyze_failure_language);
   const statusReportBase = await createStatusReportBase(
     "finish" /* Analyze */,
@@ -93835,7 +93852,8 @@ async function sendStatusReport2(startedAt, config, stats, error3, trapCacheUplo
       ...stats || {},
       ...dbCreationTimings || {},
       ...trapCacheCleanup || {},
-      dependency_caching_upload_results: dependencyCacheResults
+      dependency_caching_upload_results: dependencyCacheResults,
+      database_upload_results: databaseUploadResults
     };
     if (config && didUploadTrapCaches) {
       const trapCacheUploadStatusReport = {
@@ -93917,6 +93935,7 @@ async function run() {
   let dbCreationTimings = void 0;
   let didUploadTrapCaches = false;
   let dependencyCacheResults;
+  let databaseUploadResults = [];
   initializeEnvironment(getActionVersion());
   persistInputs();
   const logger = getActionsLogger();
@@ -94046,7 +94065,7 @@ async function run() {
       logger.info("Not uploading results");
     }
     await cleanupAndUploadOverlayBaseDatabaseToCache(codeql, config, logger);
-    await cleanupAndUploadDatabases(
+    databaseUploadResults = await cleanupAndUploadDatabases(
       repositoryNwo,
       codeql,
       config,
@@ -94100,6 +94119,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger
     );
     return;
@@ -94118,6 +94138,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger
     );
   } else if (runStats !== void 0) {
@@ -94131,6 +94152,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger
     );
   } else {
@@ -94144,6 +94166,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger
     );
   }

src/analyze-action.ts

@@ -20,7 +20,10 @@ import { runAutobuild } from "./autobuild";
 import { getTotalCacheSize, shouldStoreCache } from "./caching-utils";
 import { getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
-import { cleanupAndUploadDatabases } from "./database-upload";
+import {
+  cleanupAndUploadDatabases,
+  DatabaseUploadResult,
+} from "./database-upload";
 import {
   DependencyCacheUploadStatusReport,
   uploadDependencyCaches,
@@ -54,15 +57,13 @@ interface AnalysisStatusReport
   extends uploadLib.UploadStatusReport,
     QueriesStatusReport {}
 
-interface DependencyCachingUploadStatusReport {
-  dependency_caching_upload_results?: DependencyCacheUploadStatusReport;
-}
-
 interface FinishStatusReport
   extends StatusReportBase,
     DatabaseCreationTimings,
-    AnalysisStatusReport,
-    DependencyCachingUploadStatusReport {}
+    AnalysisStatusReport {
+  dependency_caching_upload_results?: DependencyCacheUploadStatusReport;
+  database_upload_results: DatabaseUploadResult[];
+}
 
 interface FinishWithTrapUploadStatusReport extends FinishStatusReport {
   /** Size of TRAP caches that we uploaded, in bytes. */
@@ -81,6 +82,7 @@ async function sendStatusReport(
   didUploadTrapCaches: boolean,
   trapCacheCleanup: TrapCacheCleanupStatusReport | undefined,
   dependencyCacheResults: DependencyCacheUploadStatusReport | undefined,
+  databaseUploadResults: DatabaseUploadResult[],
   logger: Logger,
 ) {
   const status = getActionsStatus(error, stats?.analyze_failure_language);
@@ -101,6 +103,7 @@ async function sendStatusReport(
       ...(dbCreationTimings || {}),
       ...(trapCacheCleanup || {}),
       dependency_caching_upload_results: dependencyCacheResults,
+      database_upload_results: databaseUploadResults,
     };
     if (config && didUploadTrapCaches) {
       const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = {
@@ -218,6 +221,7 @@ async function run() {
   let dbCreationTimings: DatabaseCreationTimings | undefined = undefined;
   let didUploadTrapCaches = false;
   let dependencyCacheResults: DependencyCacheUploadStatusReport | undefined;
+  let databaseUploadResults: DatabaseUploadResult[] = [];
   util.initializeEnvironment(actionsUtil.getActionVersion());
 
   // Make inputs accessible in the `post` step, details at
@@ -389,7 +393,7 @@ async function run() {
     // Possibly upload the database bundles for remote queries.
     // Note: Take care with the ordering of this call since databases may be cleaned up
     // at the `overlay` or `clear` level.
-    await cleanupAndUploadDatabases(
+    databaseUploadResults = await cleanupAndUploadDatabases(
       repositoryNwo,
       codeql,
       config,
@@ -461,6 +465,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger,
     );
     return;
@@ -483,6 +488,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger,
     );
   } else if (runStats !== undefined) {
@@ -496,6 +502,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger,
     );
   } else {
@@ -509,6 +516,7 @@ async function run() {
       didUploadTrapCaches,
       trapCacheCleanupTelemetry,
       dependencyCacheResults,
+      databaseUploadResults,
       logger,
     );
   }

src/database-upload.test.ts

@@ -231,7 +231,7 @@ test("Don't crash if uploading a database fails", async (t) => {
         (v) =>
           v.type === "warning" &&
           v.message ===
-            "Failed to upload database for javascript: Error: some error message",
+            "Failed to upload database for javascript: some error message",
       ) !== undefined,
     );
   });

src/database-upload.ts

@@ -13,6 +13,20 @@ import { RepositoryNwo } from "./repository";
 import * as util from "./util";
 import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
 
+/** Information about a database upload. */
+export interface DatabaseUploadResult {
+  /** Language of the database. */
+  language: string;
+  /** Size of the zipped database in bytes. */
+  zipped_upload_size_bytes?: number;
+  /** Whether the uploaded database is an overlay base. */
+  is_overlay_base?: boolean;
+  /** Time taken to upload database in milliseconds. */
+  upload_duration_ms?: number;
+  /** If there was an error during database upload, this is its message. */
+  error?: string;
+}
+
 export async function cleanupAndUploadDatabases(
   repositoryNwo: RepositoryNwo,
   codeql: CodeQL,
@@ -20,22 +34,22 @@ export async function cleanupAndUploadDatabases(
   apiDetails: GitHubApiDetails,
   features: FeatureEnablement,
   logger: Logger,
-): Promise<void> {
+): Promise<DatabaseUploadResult[]> {
   if (actionsUtil.getRequiredInput("upload-database") !== "true") {
     logger.debug("Database upload disabled in workflow. Skipping upload.");
-    return;
+    return [];
   }
 
   if (!config.analysisKinds.includes(AnalysisKind.CodeScanning)) {
     logger.debug(
       `Not uploading database because 'analysis-kinds: ${AnalysisKind.CodeScanning}' is not enabled.`,
     );
-    return;
+    return [];
   }
 
   if (util.isInTestMode()) {
     logger.debug("In test mode. Skipping database upload.");
-    return;
+    return [];
   }
 
   // Do nothing when not running against github.com
@@ -44,20 +58,22 @@ export async function cleanupAndUploadDatabases(
     config.gitHubVersion.type !== util.GitHubVariant.GHEC_DR
   ) {
     logger.debug("Not running against github.com or GHEC-DR. Skipping upload.");
-    return;
+    return [];
   }
 
   if (!(await gitUtils.isAnalyzingDefaultBranch())) {
     // We only want to upload a database if we are analyzing the default branch.
     logger.debug("Not analyzing default branch. Skipping upload.");
-    return;
+    return [];
   }
 
-  const cleanupLevel =
+  // If config.overlayDatabaseMode is OverlayBase, then we have overlay base databases for all languages.
+  const shouldUploadOverlayBase =
     config.overlayDatabaseMode === OverlayDatabaseMode.OverlayBase &&
-    (await features.getValue(Feature.UploadOverlayDbToApi))
-      ? CleanupLevel.Overlay
-      : CleanupLevel.Clear;
+    (await features.getValue(Feature.UploadOverlayDbToApi));
+  const cleanupLevel = shouldUploadOverlayBase
+    ? CleanupLevel.Overlay
+    : CleanupLevel.Clear;
 
   // Clean up the database, since intermediate results may still be written to the
   // database if there is high RAM pressure.
@@ -77,6 +93,7 @@ export async function cleanupAndUploadDatabases(
     uploadsBaseUrl = uploadsBaseUrl.slice(0, -1);
   }
 
+  const reports: DatabaseUploadResult[] = [];
   for (const language of config.languages) {
     try {
       // Upload the database bundle.
@@ -90,6 +107,7 @@ export async function cleanupAndUploadDatabases(
         actionsUtil.getRequiredInput("checkout_path"),
       );
       try {
+        const startTime = performance.now();
         await client.request(
           `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
           {
@@ -107,13 +125,27 @@ export async function cleanupAndUploadDatabases(
             },
           },
         );
+        const endTime = performance.now();
+        reports.push({
+          language,
+          zipped_upload_size_bytes: bundledDbSize,
+          is_overlay_base: shouldUploadOverlayBase,
+          upload_duration_ms: endTime - startTime,
+        });
         logger.debug(`Successfully uploaded database for ${language}`);
       } finally {
         bundledDbReadStream.close();
       }
     } catch (e) {
       // Log a warning but don't fail the workflow
-      logger.warning(`Failed to upload database for ${language}: ${e}`);
+      logger.warning(
+        `Failed to upload database for ${language}: ${util.getErrorMessage(e)}`,
+      );
+      reports.push({
+        language,
+        error: util.getErrorMessage(e),
+      });
     }
   }
+  return reports;
 }