Fix retries when uploading databases

lib/analyze-action-post.js

@@ -161404,6 +161404,7 @@ retry.VERSION = VERSION7;
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -161418,10 +161419,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/analyze-action.js

@@ -106782,6 +106782,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -106796,10 +106797,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );
@@ -110969,40 +110967,59 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai
         includeDiagnostics: false
       });
       bundledDbSize = fs13.statSync(bundledDb).size;
-      const bundledDbReadStream = fs13.createReadStream(bundledDb);
       const commitOid = await getCommitOid(
         getRequiredInput("checkout_path")
       );
-      try {
-        const startTime = performance.now();
-        await client.request(
-          `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
-          {
-            baseUrl: uploadsBaseUrl,
-            owner: repositoryNwo.owner,
-            repo: repositoryNwo.repo,
-            language,
-            name: `${language}-database`,
-            commit_oid: commitOid,
-            data: bundledDbReadStream,
-            headers: {
-              authorization: `token ${apiDetails.auth}`,
-              "Content-Type": "application/zip",
-              "Content-Length": bundledDbSize
+      const maxAttempts = 4;
+      let uploadDurationMs;
+      for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        const bundledDbReadStream = fs13.createReadStream(bundledDb);
+        try {
+          const attemptStartTime = performance.now();
+          await client.request(
+            `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
+            {
+              baseUrl: uploadsBaseUrl,
+              owner: repositoryNwo.owner,
+              repo: repositoryNwo.repo,
+              language,
+              name: `${language}-database`,
+              commit_oid: commitOid,
+              data: bundledDbReadStream,
+              headers: {
+                authorization: `token ${apiDetails.auth}`,
+                "Content-Type": "application/zip",
+                "Content-Length": bundledDbSize
+              },
+              request: {
+                retries: 0
+              }
             }
+          );
+          uploadDurationMs = performance.now() - attemptStartTime;
+          break;
+        } catch (e) {
+          const httpError = asHTTPError(e);
+          const isRetryable = !httpError || !DO_NOT_RETRY_STATUSES.includes(httpError.status);
+          if (!isRetryable || attempt === maxAttempts) {
+            throw e;
           }
-        );
-        const endTime = performance.now();
-        reports.push({
-          language,
-          zipped_upload_size_bytes: bundledDbSize,
-          is_overlay_base: shouldUploadOverlayBase,
-          upload_duration_ms: endTime - startTime
-        });
-        logger.debug(`Successfully uploaded database for ${language}`);
-      } finally {
-        bundledDbReadStream.close();
+          const backoffMs = 15e3 * Math.pow(2, attempt - 1);
+          logger.debug(
+            `Database upload attempt ${attempt} of ${maxAttempts} failed for ${language}: ${getErrorMessage(e)}. Retrying in ${backoffMs / 1e3}s...`
+          );
+          await new Promise((resolve8) => setTimeout(resolve8, backoffMs));
+        } finally {
+          bundledDbReadStream.close();
+        }
       }
+      reports.push({
+        language,
+        zipped_upload_size_bytes: bundledDbSize,
+        is_overlay_base: shouldUploadOverlayBase,
+        upload_duration_ms: uploadDurationMs
+      });
+      logger.debug(`Successfully uploaded database for ${language}`);
     } catch (e) {
       logger.warning(
         `Failed to upload database for ${language}: ${getErrorMessage(e)}`

lib/autobuild-action.js

@@ -103423,6 +103423,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -103437,10 +103438,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/init-action-post.js

@@ -164624,6 +164624,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -164638,10 +164639,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/init-action.js

@@ -104131,6 +104131,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -104145,10 +104146,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/resolve-environment-action.js

@@ -103431,6 +103431,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -103445,10 +103446,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/setup-codeql-action.js

@@ -103540,6 +103540,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -103554,10 +103555,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/start-proxy-action-post.js

@@ -161287,6 +161287,7 @@ retry.VERSION = VERSION7;
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -161301,10 +161302,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/start-proxy-action.js

@@ -120510,6 +120510,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -120524,10 +120525,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/upload-lib.js

@@ -106416,6 +106416,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -106430,10 +106431,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/upload-sarif-action-post.js

@@ -161287,6 +161287,7 @@ retry.VERSION = VERSION7;
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -161301,10 +161302,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

lib/upload-sarif-action.js

@@ -106465,6 +106465,7 @@ function parseRepositoryNwo(input) {
 
 // src/api-client.ts
 var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
+var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
 function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) {
   const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth;
   const retryingOctokit = githubUtils.GitHub.plugin(retry);
@@ -106479,10 +106480,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {})
         error: core5.error
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451]
+        doNotRetry: DO_NOT_RETRY_STATUSES
       }
     })
   );

src/api-client.test.ts

@@ -5,6 +5,7 @@ import * as sinon from "sinon";
 
 import * as actionsUtil from "./actions-util";
 import * as api from "./api-client";
+import { DO_NOT_RETRY_STATUSES } from "./api-client";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";
 
@@ -37,7 +38,7 @@ test.serial("getApiClient", async (t) => {
       log: sinon.match.any,
       userAgent: `CodeQL-Action/${actionsUtil.getActionVersion()}`,
       retry: {
-        doNotRetry: [400, 410, 422, 451],
+        doNotRetry: DO_NOT_RETRY_STATUSES,
       },
     }),
   );

src/api-client.ts

@@ -19,6 +19,15 @@ import {
 
 const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version";
 
+/**
+ * HTTP status codes that should not be retried.
+ *
+ * The default Octokit list is 400, 401, 403, 404, 410, 422, and 451. We have
+ * observed transient errors with authentication, so we remove 401, 403, and 404
+ * from the default list to ensure that these errors are retried.
+ */
+export const DO_NOT_RETRY_STATUSES = [400, 410, 422, 451];
+
 export type GitHubApiCombinedDetails = GitHubApiDetails &
   GitHubApiExternalRepoDetails;
 
@@ -52,10 +61,7 @@ function createApiClientWithDetails(
         error: core.error,
       },
       retry: {
-        // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors
-        // with authentication, so we remove 401, 403, and 404 from the default list to ensure that
-        // these errors are retried.
-        doNotRetry: [400, 410, 422, 451],
+        doNotRetry: DO_NOT_RETRY_STATUSES,
       },
     }),
   );

src/database-upload.ts

@@ -2,7 +2,11 @@ import * as fs from "fs";
 
 import * as actionsUtil from "./actions-util";
 import { AnalysisKind } from "./analyses";
-import { getApiClient, GitHubApiDetails } from "./api-client";
+import {
+  DO_NOT_RETRY_STATUSES,
+  getApiClient,
+  GitHubApiDetails,
+} from "./api-client";
 import { type CodeQL } from "./codeql";
 import { Config } from "./config-utils";
 import { Feature, FeatureEnablement } from "./feature-flags";
@@ -11,7 +15,7 @@ import { Logger, withGroupAsync } from "./logging";
 import { OverlayDatabaseMode } from "./overlay";
 import { RepositoryNwo } from "./repository";
 import * as util from "./util";
-import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
+import { asHTTPError, bundleDb, CleanupLevel, parseGitHubUrl } from "./util";
 
 /** Information about a database upload. */
 export interface DatabaseUploadResult {
@@ -105,40 +109,63 @@ export async function cleanupAndUploadDatabases(
         includeDiagnostics: false,
       });
       bundledDbSize = fs.statSync(bundledDb).size;
-      const bundledDbReadStream = fs.createReadStream(bundledDb);
       const commitOid = await gitUtils.getCommitOid(
         actionsUtil.getRequiredInput("checkout_path"),
       );
-      try {
-        const startTime = performance.now();
-        await client.request(
-          `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
-          {
-            baseUrl: uploadsBaseUrl,
-            owner: repositoryNwo.owner,
-            repo: repositoryNwo.repo,
-            language,
-            name: `${language}-database`,
-            commit_oid: commitOid,
-            data: bundledDbReadStream,
-            headers: {
-              authorization: `token ${apiDetails.auth}`,
-              "Content-Type": "application/zip",
-              "Content-Length": bundledDbSize,
+      // Upload with manual retry logic. We disable Octokit's built-in retries
+      // because the request body is a ReadStream, which can only be consumed
+      // once.
+      const maxAttempts = 4; // 1 initial attempt + 3 retries, identical to the default retry behavior of Octokit
+      let uploadDurationMs: number | undefined;
+      for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        const bundledDbReadStream = fs.createReadStream(bundledDb);
+        try {
+          const attemptStartTime = performance.now();
+          await client.request(
+            `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`,
+            {
+              baseUrl: uploadsBaseUrl,
+              owner: repositoryNwo.owner,
+              repo: repositoryNwo.repo,
+              language,
+              name: `${language}-database`,
+              commit_oid: commitOid,
+              data: bundledDbReadStream,
+              headers: {
+                authorization: `token ${apiDetails.auth}`,
+                "Content-Type": "application/zip",
+                "Content-Length": bundledDbSize,
+              },
+              request: {
+                retries: 0,
+              },
             },
-          },
-        );
-        const endTime = performance.now();
-        reports.push({
-          language,
-          zipped_upload_size_bytes: bundledDbSize,
-          is_overlay_base: shouldUploadOverlayBase,
-          upload_duration_ms: endTime - startTime,
-        });
-        logger.debug(`Successfully uploaded database for ${language}`);
-      } finally {
-        bundledDbReadStream.close();
+          );
+          uploadDurationMs = performance.now() - attemptStartTime;
+          break;
+        } catch (e) {
+          const httpError = asHTTPError(e);
+          const isRetryable =
+            !httpError || !DO_NOT_RETRY_STATUSES.includes(httpError.status);
+          if (!isRetryable || attempt === maxAttempts) {
+            throw e;
+          }
+          const backoffMs = 15_000 * Math.pow(2, attempt - 1); // 15s, 30s, 60s
+          logger.debug(
+            `Database upload attempt ${attempt} of ${maxAttempts} failed for ${language}: ${util.getErrorMessage(e)}. Retrying in ${backoffMs / 1000}s...`,
+          );
+          await new Promise((resolve) => setTimeout(resolve, backoffMs));
+        } finally {
+          bundledDbReadStream.close();
+        }
       }
+      reports.push({
+        language,
+        zipped_upload_size_bytes: bundledDbSize,
+        is_overlay_base: shouldUploadOverlayBase,
+        upload_duration_ms: uploadDurationMs,
+      });
+      logger.debug(`Successfully uploaded database for ${language}`);
     } catch (e) {
       // Log a warning but don't fail the workflow
       logger.warning(