diff --git a/packages/core/src/utils/retry.test.ts b/packages/core/src/utils/retry.test.ts index 8ab6500259..43f038cfaa 100644 --- a/packages/core/src/utils/retry.test.ts +++ b/packages/core/src/utils/retry.test.ts @@ -101,33 +101,33 @@ describe('retryWithBackoff', () => { expect(mockFn).toHaveBeenCalledTimes(3); }); - it('should default to 3 maxAttempts if no options are provided', async () => { - // This function will fail more than 3 times to ensure all retries are used. - const mockFn = createFailingFunction(5); + it('should default to 10 maxAttempts if no options are provided', async () => { + // This function will fail more than 10 times to ensure all retries are used. + const mockFn = createFailingFunction(15); const promise = retryWithBackoff(mockFn); await Promise.all([ - expect(promise).rejects.toThrow('Simulated error attempt 3'), + expect(promise).rejects.toThrow('Simulated error attempt 10'), vi.runAllTimersAsync(), ]); - expect(mockFn).toHaveBeenCalledTimes(3); + expect(mockFn).toHaveBeenCalledTimes(10); }); - it('should default to 3 maxAttempts if options.maxAttempts is undefined', async () => { - // This function will fail more than 3 times to ensure all retries are used. - const mockFn = createFailingFunction(5); + it('should default to 10 maxAttempts if options.maxAttempts is undefined', async () => { + // This function will fail more than 10 times to ensure all retries are used. + const mockFn = createFailingFunction(15); const promise = retryWithBackoff(mockFn, { maxAttempts: undefined }); - // Expect it to fail with the error from the 3rd attempt. + // Expect it to fail with the error from the 10th attempt. await Promise.all([ - expect(promise).rejects.toThrow('Simulated error attempt 3'), + expect(promise).rejects.toThrow('Simulated error attempt 10'), vi.runAllTimersAsync(), ]); - expect(mockFn).toHaveBeenCalledTimes(3); + expect(mockFn).toHaveBeenCalledTimes(10); }); it('should not retry if shouldRetry returns false', async () => { @@ -541,7 +541,13 @@ describe('retryWithBackoff', () => { await vi.runAllTimersAsync(); await assertionPromise; - expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 12345); + expect(setTimeoutSpy).toHaveBeenCalledWith( + expect.any(Function), + expect.any(Number), + ); + const calledDelayMs = setTimeoutSpy.mock.calls[0][1]; + expect(calledDelayMs).toBeGreaterThanOrEqual(12345); + expect(calledDelayMs).toBeLessThanOrEqual(12345 * 1.2); }); it.each([[AuthType.USE_GEMINI], [AuthType.USE_VERTEX_AI], [undefined]])( diff --git a/packages/core/src/utils/retry.ts b/packages/core/src/utils/retry.ts index 8b3fb1f200..17c4a656ed 100644 --- a/packages/core/src/utils/retry.ts +++ b/packages/core/src/utils/retry.ts @@ -18,7 +18,7 @@ import { getErrorStatus, ModelNotFoundError } from './httpErrors.js'; import type { RetryAvailabilityContext } from '../availability/modelPolicy.js'; export type { RetryAvailabilityContext }; -export const DEFAULT_MAX_ATTEMPTS = 3; +export const DEFAULT_MAX_ATTEMPTS = 10; export interface RetryOptions { maxAttempts: number; @@ -302,13 +302,18 @@ export async function retryWithBackoff( classifiedError instanceof RetryableQuotaError && classifiedError.retryDelayMs !== undefined ) { + currentDelay = Math.max(currentDelay, classifiedError.retryDelayMs); + // Positive jitter up to +20% while respecting server minimum delay + const jitter = currentDelay * 0.2 * Math.random(); + const delayWithJitter = currentDelay + jitter; debugLogger.warn( - `Attempt ${attempt} failed: ${classifiedError.message}. Retrying after ${classifiedError.retryDelayMs}ms...`, + `Attempt ${attempt} failed: ${classifiedError.message}. Retrying after ${Math.round(delayWithJitter)}ms...`, ); if (onRetry) { - onRetry(attempt, error, classifiedError.retryDelayMs); + onRetry(attempt, error, delayWithJitter); } - await delay(classifiedError.retryDelayMs, signal); + await delay(delayWithJitter, signal); + currentDelay = Math.min(maxDelayMs, currentDelay * 2); continue; } else { const errorStatus = getErrorStatus(error);