diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index 454c2d9fb..fb54b6631 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -161404,6 +161404,7 @@ retry.VERSION = VERSION7; // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -161418,10 +161419,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/analyze-action.js b/lib/analyze-action.js index a65d7175c..cb0bf128f 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -106782,6 +106782,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -106796,10 +106797,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); @@ -110969,40 +110967,59 @@ async function cleanupAndUploadDatabases(repositoryNwo, codeql, config, apiDetai includeDiagnostics: false }); bundledDbSize = fs13.statSync(bundledDb).size; - const bundledDbReadStream = fs13.createReadStream(bundledDb); const commitOid = await getCommitOid( getRequiredInput("checkout_path") ); - try { - const startTime = performance.now(); - await client.request( - `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`, - { - baseUrl: uploadsBaseUrl, - owner: repositoryNwo.owner, - repo: repositoryNwo.repo, - language, - name: `${language}-database`, - commit_oid: commitOid, - data: bundledDbReadStream, - headers: { - authorization: `token ${apiDetails.auth}`, - "Content-Type": "application/zip", - "Content-Length": bundledDbSize + const maxAttempts = 4; + let uploadDurationMs; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const bundledDbReadStream = fs13.createReadStream(bundledDb); + try { + const attemptStartTime = performance.now(); + await client.request( + `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`, + { + baseUrl: uploadsBaseUrl, + owner: repositoryNwo.owner, + repo: repositoryNwo.repo, + language, + name: `${language}-database`, + commit_oid: commitOid, + data: bundledDbReadStream, + headers: { + authorization: `token ${apiDetails.auth}`, + "Content-Type": "application/zip", + "Content-Length": bundledDbSize + }, + request: { + retries: 0 + } } + ); + uploadDurationMs = performance.now() - attemptStartTime; + break; + } catch (e) { + const httpError = asHTTPError(e); + const isRetryable = !httpError || !DO_NOT_RETRY_STATUSES.includes(httpError.status); + if (!isRetryable || attempt === maxAttempts) { + throw e; } - ); - const endTime = performance.now(); - reports.push({ - language, - zipped_upload_size_bytes: bundledDbSize, - is_overlay_base: shouldUploadOverlayBase, - upload_duration_ms: endTime - startTime - }); - logger.debug(`Successfully uploaded database for ${language}`); - } finally { - bundledDbReadStream.close(); + const backoffMs = 15e3 * Math.pow(2, attempt - 1); + logger.debug( + `Database upload attempt ${attempt} of ${maxAttempts} failed for ${language}: ${getErrorMessage(e)}. Retrying in ${backoffMs / 1e3}s...` + ); + await new Promise((resolve8) => setTimeout(resolve8, backoffMs)); + } finally { + bundledDbReadStream.close(); + } } + reports.push({ + language, + zipped_upload_size_bytes: bundledDbSize, + is_overlay_base: shouldUploadOverlayBase, + upload_duration_ms: uploadDurationMs + }); + logger.debug(`Successfully uploaded database for ${language}`); } catch (e) { logger.warning( `Failed to upload database for ${language}: ${getErrorMessage(e)}` diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index acd1b250e..234da35b3 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -103423,6 +103423,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -103437,10 +103438,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/init-action-post.js b/lib/init-action-post.js index 479129b24..67e8f2689 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -164624,6 +164624,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -164638,10 +164639,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/init-action.js b/lib/init-action.js index 5d5a6fa59..e305ac6f1 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -104131,6 +104131,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -104145,10 +104146,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index aa3673bd3..11fa8c0c0 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -103431,6 +103431,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -103445,10 +103446,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/setup-codeql-action.js b/lib/setup-codeql-action.js index a9eb08eb5..e35c84838 100644 --- a/lib/setup-codeql-action.js +++ b/lib/setup-codeql-action.js @@ -103540,6 +103540,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -103554,10 +103555,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/start-proxy-action-post.js b/lib/start-proxy-action-post.js index 6fdfe2d8b..610a35a72 100644 --- a/lib/start-proxy-action-post.js +++ b/lib/start-proxy-action-post.js @@ -161287,6 +161287,7 @@ retry.VERSION = VERSION7; // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -161301,10 +161302,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/start-proxy-action.js b/lib/start-proxy-action.js index 84519a068..0b3ed7b0c 100644 --- a/lib/start-proxy-action.js +++ b/lib/start-proxy-action.js @@ -120510,6 +120510,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -120524,10 +120525,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/upload-lib.js b/lib/upload-lib.js index 2f5585bd1..38765bcc6 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -106416,6 +106416,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -106430,10 +106431,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/upload-sarif-action-post.js b/lib/upload-sarif-action-post.js index dab78eb86..f6a2421c2 100644 --- a/lib/upload-sarif-action-post.js +++ b/lib/upload-sarif-action-post.js @@ -161287,6 +161287,7 @@ retry.VERSION = VERSION7; // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -161301,10 +161302,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index 53f385653..fd3672bde 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -106465,6 +106465,7 @@ function parseRepositoryNwo(input) { // src/api-client.ts var GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +var DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) { const auth2 = allowExternal && apiDetails.externalRepoAuth || apiDetails.auth; const retryingOctokit = githubUtils.GitHub.plugin(retry); @@ -106479,10 +106480,7 @@ function createApiClientWithDetails(apiDetails, { allowExternal = false } = {}) error: core5.error }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451] + doNotRetry: DO_NOT_RETRY_STATUSES } }) ); diff --git a/src/api-client.test.ts b/src/api-client.test.ts index d0311d0dc..f8846e768 100644 --- a/src/api-client.test.ts +++ b/src/api-client.test.ts @@ -5,6 +5,7 @@ import * as sinon from "sinon"; import * as actionsUtil from "./actions-util"; import * as api from "./api-client"; +import { DO_NOT_RETRY_STATUSES } from "./api-client"; import { setupTests } from "./testing-utils"; import * as util from "./util"; @@ -37,7 +38,7 @@ test.serial("getApiClient", async (t) => { log: sinon.match.any, userAgent: `CodeQL-Action/${actionsUtil.getActionVersion()}`, retry: { - doNotRetry: [400, 410, 422, 451], + doNotRetry: DO_NOT_RETRY_STATUSES, }, }), ); diff --git a/src/api-client.ts b/src/api-client.ts index 13babcd38..4b8cb7b34 100644 --- a/src/api-client.ts +++ b/src/api-client.ts @@ -19,6 +19,15 @@ import { const GITHUB_ENTERPRISE_VERSION_HEADER = "x-github-enterprise-version"; +/** + * HTTP status codes that should not be retried. + * + * The default Octokit list is 400, 401, 403, 404, 410, 422, and 451. We have + * observed transient errors with authentication, so we remove 401, 403, and 404 + * from the default list to ensure that these errors are retried. + */ +export const DO_NOT_RETRY_STATUSES = [400, 410, 422, 451]; + export type GitHubApiCombinedDetails = GitHubApiDetails & GitHubApiExternalRepoDetails; @@ -52,10 +61,7 @@ function createApiClientWithDetails( error: core.error, }, retry: { - // The default is 400, 401, 403, 404, 410, 422, and 451. We have observed transient errors - // with authentication, so we remove 401, 403, and 404 from the default list to ensure that - // these errors are retried. - doNotRetry: [400, 410, 422, 451], + doNotRetry: DO_NOT_RETRY_STATUSES, }, }), ); diff --git a/src/database-upload.ts b/src/database-upload.ts index 41546697f..b9e2f5b06 100644 --- a/src/database-upload.ts +++ b/src/database-upload.ts @@ -2,7 +2,11 @@ import * as fs from "fs"; import * as actionsUtil from "./actions-util"; import { AnalysisKind } from "./analyses"; -import { getApiClient, GitHubApiDetails } from "./api-client"; +import { + DO_NOT_RETRY_STATUSES, + getApiClient, + GitHubApiDetails, +} from "./api-client"; import { type CodeQL } from "./codeql"; import { Config } from "./config-utils"; import { Feature, FeatureEnablement } from "./feature-flags"; @@ -11,7 +15,7 @@ import { Logger, withGroupAsync } from "./logging"; import { OverlayDatabaseMode } from "./overlay"; import { RepositoryNwo } from "./repository"; import * as util from "./util"; -import { bundleDb, CleanupLevel, parseGitHubUrl } from "./util"; +import { asHTTPError, bundleDb, CleanupLevel, parseGitHubUrl } from "./util"; /** Information about a database upload. */ export interface DatabaseUploadResult { @@ -105,40 +109,63 @@ export async function cleanupAndUploadDatabases( includeDiagnostics: false, }); bundledDbSize = fs.statSync(bundledDb).size; - const bundledDbReadStream = fs.createReadStream(bundledDb); const commitOid = await gitUtils.getCommitOid( actionsUtil.getRequiredInput("checkout_path"), ); - try { - const startTime = performance.now(); - await client.request( - `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`, - { - baseUrl: uploadsBaseUrl, - owner: repositoryNwo.owner, - repo: repositoryNwo.repo, - language, - name: `${language}-database`, - commit_oid: commitOid, - data: bundledDbReadStream, - headers: { - authorization: `token ${apiDetails.auth}`, - "Content-Type": "application/zip", - "Content-Length": bundledDbSize, + // Upload with manual retry logic. We disable Octokit's built-in retries + // because the request body is a ReadStream, which can only be consumed + // once. + const maxAttempts = 4; // 1 initial attempt + 3 retries, identical to the default retry behavior of Octokit + let uploadDurationMs: number | undefined; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const bundledDbReadStream = fs.createReadStream(bundledDb); + try { + const attemptStartTime = performance.now(); + await client.request( + `POST /repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name&commit_oid=:commit_oid`, + { + baseUrl: uploadsBaseUrl, + owner: repositoryNwo.owner, + repo: repositoryNwo.repo, + language, + name: `${language}-database`, + commit_oid: commitOid, + data: bundledDbReadStream, + headers: { + authorization: `token ${apiDetails.auth}`, + "Content-Type": "application/zip", + "Content-Length": bundledDbSize, + }, + request: { + retries: 0, + }, }, - }, - ); - const endTime = performance.now(); - reports.push({ - language, - zipped_upload_size_bytes: bundledDbSize, - is_overlay_base: shouldUploadOverlayBase, - upload_duration_ms: endTime - startTime, - }); - logger.debug(`Successfully uploaded database for ${language}`); - } finally { - bundledDbReadStream.close(); + ); + uploadDurationMs = performance.now() - attemptStartTime; + break; + } catch (e) { + const httpError = asHTTPError(e); + const isRetryable = + !httpError || !DO_NOT_RETRY_STATUSES.includes(httpError.status); + if (!isRetryable || attempt === maxAttempts) { + throw e; + } + const backoffMs = 15_000 * Math.pow(2, attempt - 1); // 15s, 30s, 60s + logger.debug( + `Database upload attempt ${attempt} of ${maxAttempts} failed for ${language}: ${util.getErrorMessage(e)}. Retrying in ${backoffMs / 1000}s...`, + ); + await new Promise((resolve) => setTimeout(resolve, backoffMs)); + } finally { + bundledDbReadStream.close(); + } } + reports.push({ + language, + zipped_upload_size_bytes: bundledDbSize, + is_overlay_base: shouldUploadOverlayBase, + upload_duration_ms: uploadDurationMs, + }); + logger.debug(`Successfully uploaded database for ${language}`); } catch (e) { // Log a warning but don't fail the workflow logger.warning(