fix(core): add retry logic for specific fetch errors (#11066)

This commit is contained in:
Sandy Tao
2025-10-14 09:17:31 -07:00
committed by GitHub
parent c86ee4cc83
commit 7c1a90244a
8 changed files with 143 additions and 3 deletions

View File

@@ -760,6 +760,7 @@ export async function loadCliConfig(
settings.tools?.enableMessageBusIntegration ?? false,
codebaseInvestigatorSettings:
settings.experimental?.codebaseInvestigatorSettings,
retryFetchErrors: settings.general?.retryFetchErrors ?? false,
});
}

View File

@@ -106,6 +106,7 @@ const MIGRATION_MAP: Record<string, string> = {
memoryDiscoveryMaxDirs: 'context.discoveryMaxDirs',
model: 'model.name',
preferredEditor: 'general.preferredEditor',
retryFetchErrors: 'general.retryFetchErrors',
sandbox: 'tools.sandbox',
selectedAuthType: 'security.auth.selectedType',
enableInteractiveShell: 'tools.shell.enableInteractiveShell',

View File

@@ -184,6 +184,16 @@ const SETTINGS_SCHEMA = {
'Enable AI-powered prompt completion suggestions while typing.',
showInDialog: true,
},
retryFetchErrors: {
type: 'boolean',
label: 'Retry Fetch Errors',
category: 'General',
requiresRestart: false,
default: false,
description:
'Retry on "exception TypeError: fetch failed sending request" errors.',
showInDialog: false,
},
debugKeystrokeLogging: {
type: 'boolean',
label: 'Debug Keystroke Logging',

View File

@@ -278,6 +278,7 @@ export interface ConfigParameters {
enableMessageBusIntegration?: boolean;
codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
continueOnFailedApiCall?: boolean;
retryFetchErrors?: boolean;
enableShellOutputEfficiency?: boolean;
}
@@ -372,6 +373,7 @@ export class Config {
private readonly enableMessageBusIntegration: boolean;
private readonly codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
private readonly continueOnFailedApiCall: boolean;
private readonly retryFetchErrors: boolean;
private readonly enableShellOutputEfficiency: boolean;
constructor(params: ConfigParameters) {
@@ -479,6 +481,7 @@ export class Config {
this.outputSettings = {
format: params.output?.format ?? OutputFormat.TEXT,
};
this.retryFetchErrors = params.retryFetchErrors ?? false;
if (params.contextFileName) {
setGeminiMdFilename(params.contextFileName);
@@ -970,6 +973,10 @@ export class Config {
return this.continueOnFailedApiCall;
}
getRetryFetchErrors(): boolean {
return this.retryFetchErrors;
}
getEnableShellOutputEfficiency(): boolean {
return this.enableShellOutputEfficiency;
}

View File

@@ -127,6 +127,7 @@ describe('GeminiChat', () => {
getTool: vi.fn(),
}),
getContentGenerator: vi.fn().mockReturnValue(mockContentGenerator),
getRetryFetchErrors: vi.fn().mockReturnValue(false),
} as unknown as Config;
// Disable 429 simulation for tests
@@ -1113,6 +1114,70 @@ describe('GeminiChat', () => {
).toHaveBeenCalledTimes(2);
});
it('should retry on specific fetch errors when configured', async () => {
vi.mocked(mockConfig.getRetryFetchErrors).mockReturnValue(true);
const fetchError = new Error(
'exception TypeError: fetch failed sending request',
);
vi.mocked(mockContentGenerator.generateContentStream)
.mockRejectedValueOnce(fetchError)
.mockResolvedValueOnce(
(async function* () {
yield {
candidates: [
{
content: { parts: [{ text: 'Success after fetch error' }] },
finishReason: 'STOP',
},
],
} as unknown as GenerateContentResponse;
})(),
);
mockRetryWithBackoff.mockImplementation(async (apiCall, options) => {
try {
return await apiCall();
} catch (error) {
if (
options?.retryFetchErrors &&
error instanceof Error &&
error.message.includes(
'exception TypeError: fetch failed sending request',
)
) {
return await apiCall();
}
throw error;
}
});
const stream = await chat.sendMessageStream(
'test-model',
{ message: 'test' },
'prompt-id-fetch-error-retry',
);
const events: StreamEvent[] = [];
for await (const event of stream) {
events.push(event);
}
expect(
mockContentGenerator.generateContentStream,
).toHaveBeenCalledTimes(2);
expect(
events.some(
(e) =>
e.type === StreamEventType.CHUNK &&
e.value.candidates?.[0]?.content?.parts?.[0]?.text ===
'Success after fetch error',
),
).toBe(true);
});
afterEach(() => {
// Reset to default behavior
mockRetryWithBackoff.mockImplementation(async (apiCall) => apiCall());

View File

@@ -382,6 +382,7 @@ export class GeminiChat {
const streamResponse = await retryWithBackoff(apiCall, {
onPersistent429: onPersistent429Callback,
authType: this.config.getContentGeneratorConfig()?.authType,
retryFetchErrors: this.config.getRetryFetchErrors(),
});
return this.processStreamResponse(model, streamResponse);

View File

@@ -304,6 +304,41 @@ describe('retryWithBackoff', () => {
});
});
describe('Fetch error retries', () => {
const fetchErrorMsg = 'exception TypeError: fetch failed sending request';
it('should retry on specific fetch error when retryFetchErrors is true', async () => {
const mockFn = vi.fn();
mockFn.mockRejectedValueOnce(new Error(fetchErrorMsg));
mockFn.mockResolvedValueOnce('success');
const promise = retryWithBackoff(mockFn, {
retryFetchErrors: true,
initialDelayMs: 10,
});
await vi.runAllTimersAsync();
const result = await promise;
expect(result).toBe('success');
expect(mockFn).toHaveBeenCalledTimes(2);
});
it.each([false, undefined])(
'should not retry on specific fetch error when retryFetchErrors is %s',
async (retryFetchErrors) => {
const mockFn = vi.fn().mockRejectedValue(new Error(fetchErrorMsg));
const promise = retryWithBackoff(mockFn, {
retryFetchErrors,
});
await expect(promise).rejects.toThrow(fetchErrorMsg);
expect(mockFn).toHaveBeenCalledTimes(1);
},
);
});
describe('Flash model fallback for OAuth users', () => {
it('should trigger fallback for OAuth personal users on TerminalQuotaError', async () => {
const fallbackCallback = vi.fn().mockResolvedValue('gemini-2.5-flash');

View File

@@ -13,6 +13,9 @@ import {
TerminalQuotaError,
} from './googleQuotaErrors.js';
const FETCH_FAILED_MESSAGE =
'exception TypeError: fetch failed sending request';
export interface HttpError extends Error {
status?: number;
}
@@ -21,13 +24,14 @@ export interface RetryOptions {
maxAttempts: number;
initialDelayMs: number;
maxDelayMs: number;
shouldRetryOnError: (error: Error) => boolean;
shouldRetryOnError: (error: Error, retryFetchErrors?: boolean) => boolean;
shouldRetryOnContent?: (content: GenerateContentResponse) => boolean;
onPersistent429?: (
authType?: string,
error?: unknown,
) => Promise<string | boolean | null>;
authType?: string;
retryFetchErrors?: boolean;
}
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
@@ -41,9 +45,21 @@ const DEFAULT_RETRY_OPTIONS: RetryOptions = {
* Default predicate function to determine if a retry should be attempted.
* Retries on 429 (Too Many Requests) and 5xx server errors.
* @param error The error object.
* @param retryFetchErrors Whether to retry on specific fetch errors.
* @returns True if the error is a transient error, false otherwise.
*/
function defaultShouldRetry(error: Error | unknown): boolean {
function defaultShouldRetry(
error: Error | unknown,
retryFetchErrors?: boolean,
): boolean {
if (
retryFetchErrors &&
error instanceof Error &&
error.message.includes(FETCH_FAILED_MESSAGE)
) {
return true;
}
// Priority check for ApiError
if (error instanceof ApiError) {
// Explicitly do not retry 400 (Bad Request)
@@ -96,6 +112,7 @@ export async function retryWithBackoff<T>(
authType,
shouldRetryOnError,
shouldRetryOnContent,
retryFetchErrors,
} = {
...DEFAULT_RETRY_OPTIONS,
...cleanOptions,
@@ -155,7 +172,10 @@ export async function retryWithBackoff<T>(
}
// Generic retry logic for other errors
if (attempt >= maxAttempts || !shouldRetryOnError(error as Error)) {
if (
attempt >= maxAttempts ||
!shouldRetryOnError(error as Error, retryFetchErrors)
) {
throw error;
}