From 7c1a90244a3a5b8f4f283c28177f1393eb38d19d Mon Sep 17 00:00:00 2001 From: Sandy Tao Date: Tue, 14 Oct 2025 09:17:31 -0700 Subject: [PATCH] fix(core): add retry logic for specific fetch errors (#11066) --- packages/cli/src/config/config.ts | 1 + packages/cli/src/config/settings.ts | 1 + packages/cli/src/config/settingsSchema.ts | 10 ++++ packages/core/src/config/config.ts | 7 +++ packages/core/src/core/geminiChat.test.ts | 65 +++++++++++++++++++++++ packages/core/src/core/geminiChat.ts | 1 + packages/core/src/utils/retry.test.ts | 35 ++++++++++++ packages/core/src/utils/retry.ts | 26 +++++++-- 8 files changed, 143 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 6a807c9b0c..5d44dabffa 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -760,6 +760,7 @@ export async function loadCliConfig( settings.tools?.enableMessageBusIntegration ?? false, codebaseInvestigatorSettings: settings.experimental?.codebaseInvestigatorSettings, + retryFetchErrors: settings.general?.retryFetchErrors ?? false, }); } diff --git a/packages/cli/src/config/settings.ts b/packages/cli/src/config/settings.ts index b5bc4be045..59ffb42ce6 100644 --- a/packages/cli/src/config/settings.ts +++ b/packages/cli/src/config/settings.ts @@ -106,6 +106,7 @@ const MIGRATION_MAP: Record = { memoryDiscoveryMaxDirs: 'context.discoveryMaxDirs', model: 'model.name', preferredEditor: 'general.preferredEditor', + retryFetchErrors: 'general.retryFetchErrors', sandbox: 'tools.sandbox', selectedAuthType: 'security.auth.selectedType', enableInteractiveShell: 'tools.shell.enableInteractiveShell', diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 887dedb205..27d2e64c50 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -184,6 +184,16 @@ const SETTINGS_SCHEMA = { 'Enable AI-powered prompt completion suggestions while typing.', showInDialog: true, }, + retryFetchErrors: { + type: 'boolean', + label: 'Retry Fetch Errors', + category: 'General', + requiresRestart: false, + default: false, + description: + 'Retry on "exception TypeError: fetch failed sending request" errors.', + showInDialog: false, + }, debugKeystrokeLogging: { type: 'boolean', label: 'Debug Keystroke Logging', diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 795008dc8b..617f70f964 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -278,6 +278,7 @@ export interface ConfigParameters { enableMessageBusIntegration?: boolean; codebaseInvestigatorSettings?: CodebaseInvestigatorSettings; continueOnFailedApiCall?: boolean; + retryFetchErrors?: boolean; enableShellOutputEfficiency?: boolean; } @@ -372,6 +373,7 @@ export class Config { private readonly enableMessageBusIntegration: boolean; private readonly codebaseInvestigatorSettings?: CodebaseInvestigatorSettings; private readonly continueOnFailedApiCall: boolean; + private readonly retryFetchErrors: boolean; private readonly enableShellOutputEfficiency: boolean; constructor(params: ConfigParameters) { @@ -479,6 +481,7 @@ export class Config { this.outputSettings = { format: params.output?.format ?? OutputFormat.TEXT, }; + this.retryFetchErrors = params.retryFetchErrors ?? false; if (params.contextFileName) { setGeminiMdFilename(params.contextFileName); @@ -970,6 +973,10 @@ export class Config { return this.continueOnFailedApiCall; } + getRetryFetchErrors(): boolean { + return this.retryFetchErrors; + } + getEnableShellOutputEfficiency(): boolean { return this.enableShellOutputEfficiency; } diff --git a/packages/core/src/core/geminiChat.test.ts b/packages/core/src/core/geminiChat.test.ts index 1e7f455955..e5a017e711 100644 --- a/packages/core/src/core/geminiChat.test.ts +++ b/packages/core/src/core/geminiChat.test.ts @@ -127,6 +127,7 @@ describe('GeminiChat', () => { getTool: vi.fn(), }), getContentGenerator: vi.fn().mockReturnValue(mockContentGenerator), + getRetryFetchErrors: vi.fn().mockReturnValue(false), } as unknown as Config; // Disable 429 simulation for tests @@ -1113,6 +1114,70 @@ describe('GeminiChat', () => { ).toHaveBeenCalledTimes(2); }); + it('should retry on specific fetch errors when configured', async () => { + vi.mocked(mockConfig.getRetryFetchErrors).mockReturnValue(true); + + const fetchError = new Error( + 'exception TypeError: fetch failed sending request', + ); + + vi.mocked(mockContentGenerator.generateContentStream) + .mockRejectedValueOnce(fetchError) + .mockResolvedValueOnce( + (async function* () { + yield { + candidates: [ + { + content: { parts: [{ text: 'Success after fetch error' }] }, + finishReason: 'STOP', + }, + ], + } as unknown as GenerateContentResponse; + })(), + ); + + mockRetryWithBackoff.mockImplementation(async (apiCall, options) => { + try { + return await apiCall(); + } catch (error) { + if ( + options?.retryFetchErrors && + error instanceof Error && + error.message.includes( + 'exception TypeError: fetch failed sending request', + ) + ) { + return await apiCall(); + } + throw error; + } + }); + + const stream = await chat.sendMessageStream( + 'test-model', + { message: 'test' }, + 'prompt-id-fetch-error-retry', + ); + + const events: StreamEvent[] = []; + for await (const event of stream) { + events.push(event); + } + + expect( + mockContentGenerator.generateContentStream, + ).toHaveBeenCalledTimes(2); + + expect( + events.some( + (e) => + e.type === StreamEventType.CHUNK && + e.value.candidates?.[0]?.content?.parts?.[0]?.text === + 'Success after fetch error', + ), + ).toBe(true); + }); + afterEach(() => { // Reset to default behavior mockRetryWithBackoff.mockImplementation(async (apiCall) => apiCall()); diff --git a/packages/core/src/core/geminiChat.ts b/packages/core/src/core/geminiChat.ts index 45b37c9aeb..a090728200 100644 --- a/packages/core/src/core/geminiChat.ts +++ b/packages/core/src/core/geminiChat.ts @@ -382,6 +382,7 @@ export class GeminiChat { const streamResponse = await retryWithBackoff(apiCall, { onPersistent429: onPersistent429Callback, authType: this.config.getContentGeneratorConfig()?.authType, + retryFetchErrors: this.config.getRetryFetchErrors(), }); return this.processStreamResponse(model, streamResponse); diff --git a/packages/core/src/utils/retry.test.ts b/packages/core/src/utils/retry.test.ts index 60861d1de6..6b2d4e4312 100644 --- a/packages/core/src/utils/retry.test.ts +++ b/packages/core/src/utils/retry.test.ts @@ -304,6 +304,41 @@ describe('retryWithBackoff', () => { }); }); + describe('Fetch error retries', () => { + const fetchErrorMsg = 'exception TypeError: fetch failed sending request'; + + it('should retry on specific fetch error when retryFetchErrors is true', async () => { + const mockFn = vi.fn(); + mockFn.mockRejectedValueOnce(new Error(fetchErrorMsg)); + mockFn.mockResolvedValueOnce('success'); + + const promise = retryWithBackoff(mockFn, { + retryFetchErrors: true, + initialDelayMs: 10, + }); + + await vi.runAllTimersAsync(); + + const result = await promise; + expect(result).toBe('success'); + expect(mockFn).toHaveBeenCalledTimes(2); + }); + + it.each([false, undefined])( + 'should not retry on specific fetch error when retryFetchErrors is %s', + async (retryFetchErrors) => { + const mockFn = vi.fn().mockRejectedValue(new Error(fetchErrorMsg)); + + const promise = retryWithBackoff(mockFn, { + retryFetchErrors, + }); + + await expect(promise).rejects.toThrow(fetchErrorMsg); + expect(mockFn).toHaveBeenCalledTimes(1); + }, + ); + }); + describe('Flash model fallback for OAuth users', () => { it('should trigger fallback for OAuth personal users on TerminalQuotaError', async () => { const fallbackCallback = vi.fn().mockResolvedValue('gemini-2.5-flash'); diff --git a/packages/core/src/utils/retry.ts b/packages/core/src/utils/retry.ts index 93a2fc5035..7de10eb8d1 100644 --- a/packages/core/src/utils/retry.ts +++ b/packages/core/src/utils/retry.ts @@ -13,6 +13,9 @@ import { TerminalQuotaError, } from './googleQuotaErrors.js'; +const FETCH_FAILED_MESSAGE = + 'exception TypeError: fetch failed sending request'; + export interface HttpError extends Error { status?: number; } @@ -21,13 +24,14 @@ export interface RetryOptions { maxAttempts: number; initialDelayMs: number; maxDelayMs: number; - shouldRetryOnError: (error: Error) => boolean; + shouldRetryOnError: (error: Error, retryFetchErrors?: boolean) => boolean; shouldRetryOnContent?: (content: GenerateContentResponse) => boolean; onPersistent429?: ( authType?: string, error?: unknown, ) => Promise; authType?: string; + retryFetchErrors?: boolean; } const DEFAULT_RETRY_OPTIONS: RetryOptions = { @@ -41,9 +45,21 @@ const DEFAULT_RETRY_OPTIONS: RetryOptions = { * Default predicate function to determine if a retry should be attempted. * Retries on 429 (Too Many Requests) and 5xx server errors. * @param error The error object. + * @param retryFetchErrors Whether to retry on specific fetch errors. * @returns True if the error is a transient error, false otherwise. */ -function defaultShouldRetry(error: Error | unknown): boolean { +function defaultShouldRetry( + error: Error | unknown, + retryFetchErrors?: boolean, +): boolean { + if ( + retryFetchErrors && + error instanceof Error && + error.message.includes(FETCH_FAILED_MESSAGE) + ) { + return true; + } + // Priority check for ApiError if (error instanceof ApiError) { // Explicitly do not retry 400 (Bad Request) @@ -96,6 +112,7 @@ export async function retryWithBackoff( authType, shouldRetryOnError, shouldRetryOnContent, + retryFetchErrors, } = { ...DEFAULT_RETRY_OPTIONS, ...cleanOptions, @@ -155,7 +172,10 @@ export async function retryWithBackoff( } // Generic retry logic for other errors - if (attempt >= maxAttempts || !shouldRetryOnError(error as Error)) { + if ( + attempt >= maxAttempts || + !shouldRetryOnError(error as Error, retryFetchErrors) + ) { throw error; }