diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f647efdda..b5a745e7b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ This is the log of notable changes to EAS CLI and related packages. - [build-tools] Prevent user npm minimum release age settings from applying to internal EAS CLI commands. ([#4296](https://github.com/expo/eas-cli/pull/4296) by [@sjchmiela](https://github.com/sjchmiela)) - [build-tools] Reduce serve-sim preview resolution from 1280 px to 960 px to lower streaming bandwidth. ([#4297](https://github.com/expo/eas-cli/pull/4297) by [@szdziedzic](https://github.com/szdziedzic)) +- [eas-cli] Add dynamic retries to deploy command that scale up more generously, depending on deployment size ([#4299](https://github.com/expo/eas-cli/pull/4299) by [@kitten](https://github.com/kitten)) ### 🧹 Chores diff --git a/packages/eas-cli/src/worker/__tests__/upload.test.ts b/packages/eas-cli/src/worker/__tests__/upload.test.ts new file mode 100644 index 0000000000..325dd8d5d6 --- /dev/null +++ b/packages/eas-cli/src/worker/__tests__/upload.test.ts @@ -0,0 +1,443 @@ +import cliProgress from 'cli-progress'; +import fetch, { Headers, Response } from 'node-fetch'; + +import Log from '../../log'; +import { AssetFileEntry } from '../assets'; +import { + UploadPayload, + batchUploadAsync, + callUploadApiAsync, + createProgressBar, + uploadAsync, +} from '../upload'; + +jest.mock('node-fetch', () => ({ + ...jest.requireActual('node-fetch'), + __esModule: true, + default: jest.fn(), +})); + +const mockedFetch = jest.mocked(fetch); + +const asset: AssetFileEntry = { + normalizedPath: 'index.html', + path: __filename, + size: 123, + sha512: 'a'.repeat(128), + type: 'text/html', +}; + +const response = (status = 200, body: unknown = { success: true }): Response => + new Response(JSON.stringify(body), { + status, + statusText: status === 200 ? 'OK' : 'Request failed', + headers: { 'content-type': 'application/json' }, + }); + +async function runTimersAndExpectRejection( + promise: Promise, + expected: string | RegExp +): Promise { + const expectation = expect(promise).rejects.toThrow(expected); + await jest.runAllTimersAsync(); + await expectation; +} + +describe(uploadAsync, () => { + beforeEach(() => { + jest.useFakeTimers({ now: 0 }); + jest.spyOn(Math, 'random').mockReturnValue(0); + jest.spyOn(Log, 'warn').mockImplementation(() => {}); + mockedFetch.mockReset(); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('uploads an individual asset with its hash and metadata', async () => { + mockedFetch.mockResolvedValue(response()); + + const result = await uploadAsync({ baseURL: 'https://eas.expo.app/?token=secret' }, { asset }); + + expect(result.payload).toEqual({ asset }); + expect(mockedFetch).toHaveBeenCalledTimes(1); + const [url, init] = mockedFetch.mock.calls[0]; + expect(String(url)).toBe(`https://eas.expo.app/asset/${asset.sha512}?token=secret`); + expect(init?.method).toBe('POST'); + expect((init?.headers as Headers).get('content-type')).toBe('text/html'); + expect((init?.headers as Headers).get('content-length')).toBe('123'); + expect(init?.body).toBeDefined(); + }); + + it('uploads a worker deployment file without changing the URL', async () => { + mockedFetch.mockResolvedValue(response()); + + await uploadAsync({ baseURL: 'https://eas.expo.app/deploy' }, { filePath: __filename }); + + const [url, init] = mockedFetch.mock.calls[0]; + expect(String(url)).toBe('https://eas.expo.app/deploy'); + expect(init?.method).toBe('POST'); + expect(init?.body).toBeDefined(); + }); + + it('uploads multipart assets to the batch endpoint and reports progress', async () => { + mockedFetch.mockResolvedValue(response()); + const onProgress = jest.fn(); + + await uploadAsync( + { baseURL: 'https://eas.expo.app/asset/?token=secret' }, + { multipart: [asset, { ...asset, sha512: 'b'.repeat(128) }] }, + onProgress + ); + + const [url, init] = mockedFetch.mock.calls[0]; + expect(String(url)).toBe('https://eas.expo.app/asset/batch?token=secret'); + expect(init?.method).toBe('PATCH'); + expect((init?.headers as Headers).get('content-type')).toMatch(/^multipart\/form-data;/); + expect(onProgress).toHaveBeenNthCalledWith(1, 0); + expect(onProgress).toHaveBeenLastCalledWith(1); + }); + + it.each([408, 409, 429, 500, 503])('retries HTTP %s responses', async status => { + mockedFetch.mockImplementation(async () => response(status, { error: `status ${status}` })); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }); + await runTimersAndExpectRejection(promise, `status ${status}`); + + expect(mockedFetch).toHaveBeenCalledTimes(5); + }); + + it('does not retry a terminal HTTP response', async () => { + mockedFetch.mockResolvedValue(response(400, { error: 'invalid upload' })); + + await expect(uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset })).rejects.toThrow( + 'invalid upload' + ); + expect(mockedFetch).toHaveBeenCalledTimes(1); + }); + + it('reports payload-too-large responses without retrying', async () => { + mockedFetch.mockResolvedValue(response(413)); + + await expect(uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset })).rejects.toThrow( + 'File size exceeded the upload limit' + ); + expect(mockedFetch).toHaveBeenCalledTimes(1); + }); + + it('reports CDN HTML errors with the request ID', async () => { + mockedFetch.mockResolvedValue( + new Response('', { + status: 403, + statusText: 'Forbidden', + headers: { 'content-type': 'text/html', 'cf-ray': 'ray-id' }, + }) + ); + + await expect(uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset })).rejects.toThrow( + 'Request ID ray-id' + ); + }); + + it('switches to network mode and preserves the original network error', async () => { + const networkError = Object.assign(new Error('getaddrinfo ENOTFOUND eas.expo.app'), { + code: 'ENOTFOUND', + }); + mockedFetch.mockRejectedValue(networkError); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }); + await runTimersAndExpectRejection(promise, networkError.message); + + expect(mockedFetch).toHaveBeenCalledTimes(8); + expect(Log.warn).toHaveBeenCalledTimes(1); + expect(Log.warn).toHaveBeenCalledWith( + `The upload encountered an error but is still retrying: ${networkError.message}` + ); + }); + + it('smoothly increases retries based on the total request count', async () => { + mockedFetch.mockImplementation(async () => response(503, { error: 'overloaded' })); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }, undefined, { + totalRequests: 32, + }); + await runTimersAndExpectRejection(promise, 'overloaded'); + + // log10(32) / 3 results in six retries, plus the initial request. + expect(mockedFetch).toHaveBeenCalledTimes(7); + }); + + it('caps request-count scaling at 1,000 requests', async () => { + mockedFetch.mockImplementation(async () => response(503, { error: 'overloaded' })); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }, undefined, { + totalRequests: 1_000_000, + }); + await runTimersAndExpectRejection(promise, 'overloaded'); + + // The scaled 40s ceiling cuts the last attempt the backoff schedule would have allowed. + expect(mockedFetch).toHaveBeenCalledTimes(8); + }); + + it('keeps the retry state when it switches to network mode', async () => { + let calls = 0; + mockedFetch.mockImplementation(async () => { + if (++calls <= 7) { + return response(503, { error: 'overloaded' }); + } + throw new Error('socket disconnected'); + }); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }, undefined, { + totalRequests: 1_000, + }); + await runTimersAndExpectRejection(promise, 'socket disconnected'); + + // The warning must not repeat once the network-mode attempts take over. + expect(Log.warn).toHaveBeenCalledTimes(1); + expect(Log.warn).toHaveBeenCalledWith( + 'The upload encountered an error but is still retrying: overloaded' + ); + }); + + it('warns before a short retry budget is exhausted', async () => { + mockedFetch.mockImplementation(async () => response(503, { error: 'overloaded' })); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }); + await runTimersAndExpectRejection(promise, 'overloaded'); + + expect(Log.warn).toHaveBeenCalledTimes(1); + }); + + it('stops retrying at the scaled upload ceiling', async () => { + mockedFetch.mockImplementation(async () => response(503, { error: 'overloaded' })); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }, undefined, { + totalRequests: 1_000, + }); + await runTimersAndExpectRejection(promise, 'overloaded'); + + // The ceiling stops the next retry from being scheduled, so the last attempt already in + // flight can overshoot it by up to one maxTimeout (10s at this scale). + expect(Date.now()).toBeLessThanOrEqual(40_000 + 10_000); + }); + + it('clears the network flag after a success', async () => { + const state = { shared: { hasSeenNetworkError: false, hasWarned: false } }; + mockedFetch.mockResolvedValueOnce(response(503)).mockResolvedValueOnce(response()); + + const promise = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }, undefined, { + state, + }); + await jest.runAllTimersAsync(); + await promise; + + expect(state.shared.hasSeenNetworkError).toBe(false); + }); + + it('keeps retry warning clocks independent when retry state is shared', async () => { + const shared = { hasSeenNetworkError: false, hasWarned: false }; + mockedFetch + .mockResolvedValueOnce(response(503, { error: 'overloaded' })) + .mockResolvedValueOnce(response()) + .mockImplementation(async () => response(503, { error: 'overloaded' })); + + const retryingUpload = uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }, undefined, { + state: { shared }, + }); + await uploadAsync({ baseURL: 'https://eas.expo.app' }, { asset }, undefined, { + state: { shared }, + }); + await runTimersAndExpectRejection(retryingUpload, 'overloaded'); + + expect(Log.warn).toHaveBeenCalledTimes(1); + }); + + it('does not retry or enter network mode after cancellation', async () => { + const controller = new AbortController(); + controller.abort(); + const abortError = Object.assign(new Error('The operation was aborted'), { + name: 'AbortError', + }); + mockedFetch.mockRejectedValue(abortError); + + await expect( + uploadAsync({ baseURL: 'https://eas.expo.app', signal: controller.signal }, { asset }) + ).rejects.toBe(abortError); + expect(mockedFetch).toHaveBeenCalledTimes(1); + expect(Log.warn).not.toHaveBeenCalled(); + }); +}); + +describe(callUploadApiAsync, () => { + beforeEach(() => { + jest.useFakeTimers({ now: 0 }); + jest.spyOn(Math, 'random').mockReturnValue(0); + jest.spyOn(Log, 'warn').mockImplementation(() => {}); + mockedFetch.mockReset(); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('returns a successful JSON response', async () => { + mockedFetch.mockResolvedValue(response(200, { result: 'complete' })); + + await expect(callUploadApiAsync('https://eas.expo.app/finalize')).resolves.toEqual({ + result: 'complete', + }); + }); + + it('retries server errors and passes through the final error', async () => { + mockedFetch.mockResolvedValue(response(503)); + + const promise = callUploadApiAsync('https://eas.expo.app/finalize'); + await runTimersAndExpectRejection(promise, 'Deployment failed: Request failed'); + // Deploy API calls get a somewhat larger retry budget than a single asset upload. + expect(mockedFetch).toHaveBeenCalledTimes(7); + }); + + it('retries invalid JSON responses', async () => { + mockedFetch + .mockResolvedValueOnce(new Response('invalid JSON')) + .mockResolvedValueOnce(response(200, { success: true })); + + const promise = callUploadApiAsync('https://eas.expo.app/finalize'); + await jest.runAllTimersAsync(); + await expect(promise).resolves.toEqual({ success: true }); + expect(mockedFetch).toHaveBeenCalledTimes(2); + }); + + it('uses network retries for fetch errors', async () => { + const networkError = new Error('socket disconnected'); + mockedFetch.mockRejectedValue(networkError); + + const promise = callUploadApiAsync('https://eas.expo.app/finalize'); + await runTimersAndExpectRejection(promise, networkError.message); + // Network mode adds attempts but shares the 30s deadline, so it cannot double the wait. + expect(mockedFetch).toHaveBeenCalledTimes(10); + }); + + it('stops retrying at the 30s ceiling', async () => { + mockedFetch.mockResolvedValue(response(503)); + + const promise = callUploadApiAsync('https://eas.expo.app/finalize'); + await runTimersAndExpectRejection(promise, 'Deployment failed: Request failed'); + + expect(Date.now()).toBeLessThanOrEqual(30_000); + }); + + it('shares the ceiling with the network-mode attempts', async () => { + mockedFetch.mockRejectedValue(new Error('socket disconnected')); + + const promise = callUploadApiAsync('https://eas.expo.app/finalize'); + await runTimersAndExpectRejection(promise, 'socket disconnected'); + + expect(Date.now()).toBeLessThanOrEqual(30_000 + 5_000); + }); + + it('does not retry an aborted request', async () => { + const controller = new AbortController(); + controller.abort(); + const abortError = Object.assign(new Error('The operation was aborted'), { + name: 'AbortError', + }); + mockedFetch.mockRejectedValue(abortError); + + await expect( + callUploadApiAsync('https://eas.expo.app/finalize', { signal: controller.signal as any }) + ).rejects.toBe(abortError); + expect(mockedFetch).toHaveBeenCalledTimes(1); + }); +}); + +describe(batchUploadAsync, () => { + beforeEach(() => { + mockedFetch.mockReset(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('uploads every payload and reports aggregate progress', async () => { + mockedFetch.mockResolvedValue(response()); + const payloads: UploadPayload[] = [ + { asset }, + { asset: { ...asset, sha512: 'b'.repeat(128) } }, + { asset: { ...asset, sha512: 'c'.repeat(128) } }, + ]; + const progress = jest.fn(); + + const results = []; + for await (const result of batchUploadAsync( + { baseURL: 'https://eas.expo.app' }, + payloads, + progress + )) { + results.push(result); + } + + expect(mockedFetch).toHaveBeenCalledTimes(3); + expect(results.at(-1)?.progress).toBe(1); + expect(progress).toHaveBeenLastCalledWith(1); + }); + + it('accepts an empty upload queue', async () => { + const results = []; + for await (const result of batchUploadAsync({ baseURL: 'https://eas.expo.app' }, [])) { + results.push(result); + } + + expect(results).toEqual([]); + expect(mockedFetch).not.toHaveBeenCalled(); + }); + + it('aborts sibling requests after the first upload failure', async () => { + mockedFetch.mockImplementation(async (url, init) => { + if (String(url).includes(asset.sha512)) { + return response(400, { error: 'terminal failure' }); + } + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + reject(new Error('aborted')); + }); + }); + }); + + const iterator = batchUploadAsync({ baseURL: 'https://eas.expo.app' }, [ + { asset }, + { asset: { ...asset, sha512: 'b'.repeat(128) } }, + ]); + + await iterator.next(); + await iterator.next(); + await expect(iterator.next()).rejects.toThrow('terminal failure'); + }); +}); + +describe(createProgressBar, () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('starts, updates, and stops the upload progress bar', () => { + const start = jest.spyOn(cliProgress.SingleBar.prototype, 'start').mockImplementation(() => {}); + const update = jest + .spyOn(cliProgress.SingleBar.prototype, 'update') + .mockImplementation(() => {}); + const stop = jest.spyOn(cliProgress.SingleBar.prototype, 'stop').mockImplementation(() => {}); + + const progressBar = createProgressBar('Uploading 3 assets'); + progressBar.update(0.5); + progressBar.stop(); + + expect(start).toHaveBeenCalledWith(1, 0); + expect(update).toHaveBeenCalledWith(0.5); + expect(stop).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/eas-cli/src/worker/upload.ts b/packages/eas-cli/src/worker/upload.ts index bc85d93806..9f8261e4a5 100644 --- a/packages/eas-cli/src/worker/upload.ts +++ b/packages/eas-cli/src/worker/upload.ts @@ -7,6 +7,7 @@ import os from 'node:os'; import { Readable } from 'node:stream'; import promiseRetry from 'promise-retry'; +import Log from '../log'; import { AssetFileEntry } from './assets'; import { createMultipartBodyFromFilesAsync, @@ -15,7 +16,86 @@ import { } from './utils/multipart'; const MAX_CONCURRENCY = Math.min(10, Math.max(os.availableParallelism() * 2, 20)); -const MAX_RETRIES = 4; +const MAX_RETRY_WARNING_DELAY_MS = 30_000; + +const UPLOAD_RETRY_LIMITS: RetryLimits = { retries: 4, maxTimeout: 5_000, maxRetryTime: 20_000 }; +const API_RETRY_LIMITS: RetryLimits = { retries: 6, maxTimeout: 5_000, maxRetryTime: 30_000 }; + +interface SharedRetryState { + hasSeenNetworkError: boolean; + hasWarned: boolean; +} + +interface RetryState { + shared: SharedRetryState; + firstRetryAt?: number; +} + +interface UploadRetryOptions { + totalRequests?: number; + state?: RetryState; + deadlineAt?: number; +} + +interface RetryLimits { + retries: number; + maxTimeout: number; + maxRetryTime: number; +} + +interface RetryOptions { + retries: number; + factor: number; + minTimeout: number; + maxTimeout: number; + maxRetryTime: number; + randomize: boolean; +} + +interface RetryWarningContext { + state: RetryState; + warningDelayMs: number; + subject: string; +} + +const getRetryScale = (totalRequests: number): number => + Math.min(1, Math.log10(Math.max(1, totalRequests)) / 3); + +const getErrorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +const getRetryOptions = ( + limits: RetryLimits, + totalRequests: number, + networkMode: boolean +): RetryOptions => { + const scale = getRetryScale(totalRequests); + return { + retries: Math.round(limits.retries * (1 + scale)) + (networkMode ? 2 : 0), + factor: 2, + minTimeout: 1_000, + maxTimeout: Math.round(limits.maxTimeout * (1 + scale)), + maxRetryTime: Math.round(limits.maxRetryTime * (1 + scale)), + randomize: true, + }; +}; + +const getRetryWarningDelay = (maxRetryTime: number): number => + Math.min(MAX_RETRY_WARNING_DELAY_MS, Math.round(maxRetryTime / 2)); + +const retryWithWarning = ( + retry: (error: unknown) => never, + error: unknown, + attempt: number, + { state, warningDelayMs, subject }: RetryWarningContext +): never => { + state.firstRetryAt ??= Date.now(); + if (!state.shared.hasWarned && attempt > 1 && Date.now() - state.firstRetryAt >= warningDelayMs) { + state.shared.hasWarned = true; + Log.warn(`${subject} encountered an error but is still retrying: ${getErrorMessage(error)}`); + } + return retry(error); +}; export type UploadPayload = | { filePath: string } @@ -56,110 +136,156 @@ type OnProgressUpdateCallback = (progress: number) => void; export async function uploadAsync( init: UploadRequestInit, payload: UploadPayload, - onProgressUpdate?: OnProgressUpdateCallback + onProgressUpdate?: OnProgressUpdateCallback, + retryOptions: UploadRetryOptions = {}, + networkMode = retryOptions.state?.shared.hasSeenNetworkError ?? false ): Promise { - return await promiseRetry( - async retry => { - if (onProgressUpdate) { - onProgressUpdate(0); - } + const state = retryOptions.state ?? { + shared: { hasSeenNetworkError: false, hasWarned: false }, + }; + const retryOptionsForAttempts = getRetryOptions( + UPLOAD_RETRY_LIMITS, + retryOptions.totalRequests ?? 1, + networkMode + ); + const warningDelayMs = getRetryWarningDelay(retryOptionsForAttempts.maxRetryTime); + const deadlineAt = retryOptions.deadlineAt ?? Date.now() + retryOptionsForAttempts.maxRetryTime; + retryOptionsForAttempts.maxRetryTime = Math.max(0, deadlineAt - Date.now()); + const warningContext: RetryWarningContext = { + state, + warningDelayMs, + subject: 'The upload', + }; + return await promiseRetry(async (retry, attempt) => { + if (onProgressUpdate) { + onProgressUpdate(0); + } - const headers = new Headers(init.headers); - - const url = new URL(`${init.baseURL}`); - let errorPrefix: string; - let body: BodyInit | undefined; - let method = init.method || 'POST'; - if ('asset' in payload) { - const { asset } = payload; - errorPrefix = `Upload of "${asset.normalizedPath}" failed`; - if (asset.type) { - headers.set('content-type', asset.type); - } - if (asset.size) { - headers.set('content-length', `${asset.size}`); - } - method = 'POST'; - url.pathname = `/asset/${asset.sha512}`; - body = Readable.from(createReadStreamAsync(asset), { objectMode: false }); - } else if ('filePath' in payload) { - const { filePath } = payload; - errorPrefix = 'Worker deployment failed'; - body = fs.createReadStream(filePath); - } else if ('multipart' in payload) { - const { multipart } = payload; - errorPrefix = `Upload of ${multipart.length} assets failed`; - headers.set('content-type', multipartContentType); - method = 'PATCH'; - url.pathname = '/asset/batch'; - body = Readable.from(createMultipartBodyFromFilesAsync(multipart, onProgressUpdate), { - objectMode: false, - }); + const headers = new Headers(init.headers); + + const url = new URL(`${init.baseURL}`); + let errorPrefix: string; + let body: BodyInit | undefined; + let method = init.method || 'POST'; + if ('asset' in payload) { + const { asset } = payload; + errorPrefix = `Upload of "${asset.normalizedPath}" failed`; + if (asset.type) { + headers.set('content-type', asset.type); } + if (asset.size) { + headers.set('content-length', `${asset.size}`); + } + method = 'POST'; + url.pathname = `/asset/${asset.sha512}`; + body = Readable.from(createReadStreamAsync(asset), { objectMode: false }); + } else if ('filePath' in payload) { + const { filePath } = payload; + errorPrefix = 'Worker deployment failed'; + body = fs.createReadStream(filePath); + } else if ('multipart' in payload) { + const { multipart } = payload; + errorPrefix = `Upload of ${multipart.length} assets failed`; + headers.set('content-type', multipartContentType); + method = 'PATCH'; + url.pathname = '/asset/batch'; + body = Readable.from(createMultipartBodyFromFilesAsync(multipart, onProgressUpdate), { + objectMode: false, + }); + } - let response: Response; - try { - response = await fetch(url, { - method, - body, - headers, - agent: getAgent(), - signal: init.signal as any, - }); - } catch (error) { - return retry(error); + let response: Response; + try { + response = await fetch(url, { + method, + body, + headers, + agent: getAgent(), + signal: init.signal as any, + }); + } catch (error) { + if (init.signal?.aborted) { + throw error; } + if (!networkMode) { + state.shared.hasSeenNetworkError = true; + return await uploadAsync( + init, + payload, + onProgressUpdate, + { ...retryOptions, state, deadlineAt }, + true + ); + } + return retryWithWarning(retry, error, attempt, warningContext); + } - const getErrorMessageAsync = async (): Promise => { - const rayId = response.headers.get('cf-ray'); - const contentType = response.headers.get('Content-Type'); - if (contentType?.startsWith('text/html')) { - // NOTE(@kitten): We've received a CDN error most likely. There's not much we can do - // except for quoting the Request ID, so a user can send it to us. We can check - // why a request was blocked by looking up a WAF event via the "Ray ID" here: - // https://dash.cloudflare.com/e6f39f67f543faa6038768e8f37e4234/expo.app/security/events - let message = `CDN firewall has aborted the upload with ${response.statusText}.`; - if (rayId) { - message += `\nReport this error quoting Request ID ${rayId}`; - } - return `${errorPrefix}: ${message}`; - } else { - const json = await response.json().catch(() => null); - return json?.error ?? `${errorPrefix}: ${response.statusText}`; + const getErrorMessageAsync = async (): Promise => { + const rayId = response.headers.get('cf-ray'); + const contentType = response.headers.get('Content-Type'); + if (contentType?.startsWith('text/html')) { + // NOTE(@kitten): We've received a CDN error most likely. There's not much we can do + // except for quoting the Request ID, so a user can send it to us. We can check + // why a request was blocked by looking up a WAF event via the "Ray ID" here: + // https://dash.cloudflare.com/e6f39f67f543faa6038768e8f37e4234/expo.app/security/events + let message = `CDN firewall has aborted the upload with ${response.statusText}.`; + if (rayId) { + message += `\nReport this error quoting Request ID ${rayId}`; } - }; - - if ( - response.status === 408 || - response.status === 409 || - response.status === 429 || - (response.status >= 500 && response.status <= 599) - ) { - return retry(new Error(await getErrorMessageAsync())); - } else if (response.status === 413) { - const message = `${errorPrefix!}: File size exceeded the upload limit`; - throw new Error(message); - } else if (!response.ok) { - throw new Error(await getErrorMessageAsync()); - } else if (onProgressUpdate) { - onProgressUpdate(1); + return `${errorPrefix}: ${message}`; + } else { + const json = await response.json().catch(() => null); + return json?.error ?? `${errorPrefix}: ${response.statusText}`; } + }; - return { - payload, - response, - }; - }, - { - retries: MAX_RETRIES, - minTimeout: 50, - randomize: false, + if ( + response.status === 408 || + response.status === 409 || + response.status === 429 || + (response.status >= 500 && response.status <= 599) + ) { + return retryWithWarning( + retry, + new Error(await getErrorMessageAsync()), + attempt, + warningContext + ); + } else if (response.status === 413) { + const message = `${errorPrefix!}: File size exceeded the upload limit`; + throw new Error(message); + } else if (!response.ok) { + throw new Error(await getErrorMessageAsync()); + } else if (onProgressUpdate) { + onProgressUpdate(1); } - ); + + state.shared.hasSeenNetworkError = false; + + return { + payload, + response, + }; + }, retryOptionsForAttempts); } -export async function callUploadApiAsync(url: string | URL, init?: RequestInit): Promise { - return await promiseRetry(async retry => { +async function callUploadApiWithRetryAsync( + url: string | URL, + init: RequestInit | undefined, + networkMode = false, + state: RetryState = { shared: { hasSeenNetworkError: false, hasWarned: false } }, + deadlineAt?: number +): Promise { + const retryOptions = getRetryOptions(API_RETRY_LIMITS, 1, networkMode); + const warningDelayMs = getRetryWarningDelay(retryOptions.maxRetryTime); + const retryDeadlineAt = deadlineAt ?? Date.now() + retryOptions.maxRetryTime; + retryOptions.maxRetryTime = Math.max(0, retryDeadlineAt - Date.now()); + const warningContext: RetryWarningContext = { + state, + warningDelayMs, + subject: 'The deployment', + }; + return await promiseRetry(async (retry, attempt) => { let response: Response; try { response = await fetch(url, { @@ -167,17 +293,33 @@ export async function callUploadApiAsync(url: string | URL, init?: RequestInit): agent: getAgent(), }); } catch (error) { - return retry(error); + if (init?.signal?.aborted) { + throw error; + } + if (!networkMode) { + state.shared.hasSeenNetworkError = true; + return await callUploadApiWithRetryAsync(url, init, true, state, retryDeadlineAt); + } + return retryWithWarning(retry, error, attempt, warningContext); } if (response.status >= 500 && response.status <= 599) { - retry(new Error(`Deployment failed: ${response.statusText}`)); + retryWithWarning( + retry, + new Error(`Deployment failed: ${response.statusText}`), + attempt, + warningContext + ); } try { return await response.json(); } catch (error) { - retry(error); + return retryWithWarning(retry, error, attempt, warningContext); } - }); + }, retryOptions); +} + +export async function callUploadApiAsync(url: string | URL, init?: RequestInit): Promise { + return await callUploadApiWithRetryAsync(url, init); } export interface UploadPending { @@ -192,6 +334,7 @@ export async function* batchUploadAsync( ): AsyncGenerator { const progressTracker = new Array(payloads.length).fill(0); const controller = new AbortController(); + const sharedRetryState: SharedRetryState = { hasSeenNetworkError: false, hasWarned: false }; const queue = new Set>(); const initWithSignal = { ...init, signal: controller.signal }; const getProgressValue = (): number => { @@ -216,7 +359,10 @@ export async function* batchUploadAsync( progressTracker[currentIndex] = progress; sendProgressUpdate(); }); - const uploadPromise = uploadAsync(initWithSignal, payload, onChildProgressUpdate).then( + const uploadPromise = uploadAsync(initWithSignal, payload, onChildProgressUpdate, { + totalRequests: payloads.length, + state: { shared: sharedRetryState }, + }).then( result => { queue.delete(uploadPromise); progressTracker[currentIndex] = 1;