mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-03-10 14:10:37 -07:00
fix(core): add retry logic for specific fetch errors (#11066)
This commit is contained in:
@@ -760,6 +760,7 @@ export async function loadCliConfig(
|
||||
settings.tools?.enableMessageBusIntegration ?? false,
|
||||
codebaseInvestigatorSettings:
|
||||
settings.experimental?.codebaseInvestigatorSettings,
|
||||
retryFetchErrors: settings.general?.retryFetchErrors ?? false,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -106,6 +106,7 @@ const MIGRATION_MAP: Record<string, string> = {
|
||||
memoryDiscoveryMaxDirs: 'context.discoveryMaxDirs',
|
||||
model: 'model.name',
|
||||
preferredEditor: 'general.preferredEditor',
|
||||
retryFetchErrors: 'general.retryFetchErrors',
|
||||
sandbox: 'tools.sandbox',
|
||||
selectedAuthType: 'security.auth.selectedType',
|
||||
enableInteractiveShell: 'tools.shell.enableInteractiveShell',
|
||||
|
||||
@@ -184,6 +184,16 @@ const SETTINGS_SCHEMA = {
|
||||
'Enable AI-powered prompt completion suggestions while typing.',
|
||||
showInDialog: true,
|
||||
},
|
||||
retryFetchErrors: {
|
||||
type: 'boolean',
|
||||
label: 'Retry Fetch Errors',
|
||||
category: 'General',
|
||||
requiresRestart: false,
|
||||
default: false,
|
||||
description:
|
||||
'Retry on "exception TypeError: fetch failed sending request" errors.',
|
||||
showInDialog: false,
|
||||
},
|
||||
debugKeystrokeLogging: {
|
||||
type: 'boolean',
|
||||
label: 'Debug Keystroke Logging',
|
||||
|
||||
@@ -278,6 +278,7 @@ export interface ConfigParameters {
|
||||
enableMessageBusIntegration?: boolean;
|
||||
codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
|
||||
continueOnFailedApiCall?: boolean;
|
||||
retryFetchErrors?: boolean;
|
||||
enableShellOutputEfficiency?: boolean;
|
||||
}
|
||||
|
||||
@@ -372,6 +373,7 @@ export class Config {
|
||||
private readonly enableMessageBusIntegration: boolean;
|
||||
private readonly codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
|
||||
private readonly continueOnFailedApiCall: boolean;
|
||||
private readonly retryFetchErrors: boolean;
|
||||
private readonly enableShellOutputEfficiency: boolean;
|
||||
|
||||
constructor(params: ConfigParameters) {
|
||||
@@ -479,6 +481,7 @@ export class Config {
|
||||
this.outputSettings = {
|
||||
format: params.output?.format ?? OutputFormat.TEXT,
|
||||
};
|
||||
this.retryFetchErrors = params.retryFetchErrors ?? false;
|
||||
|
||||
if (params.contextFileName) {
|
||||
setGeminiMdFilename(params.contextFileName);
|
||||
@@ -970,6 +973,10 @@ export class Config {
|
||||
return this.continueOnFailedApiCall;
|
||||
}
|
||||
|
||||
getRetryFetchErrors(): boolean {
|
||||
return this.retryFetchErrors;
|
||||
}
|
||||
|
||||
getEnableShellOutputEfficiency(): boolean {
|
||||
return this.enableShellOutputEfficiency;
|
||||
}
|
||||
|
||||
@@ -127,6 +127,7 @@ describe('GeminiChat', () => {
|
||||
getTool: vi.fn(),
|
||||
}),
|
||||
getContentGenerator: vi.fn().mockReturnValue(mockContentGenerator),
|
||||
getRetryFetchErrors: vi.fn().mockReturnValue(false),
|
||||
} as unknown as Config;
|
||||
|
||||
// Disable 429 simulation for tests
|
||||
@@ -1113,6 +1114,70 @@ describe('GeminiChat', () => {
|
||||
).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('should retry on specific fetch errors when configured', async () => {
|
||||
vi.mocked(mockConfig.getRetryFetchErrors).mockReturnValue(true);
|
||||
|
||||
const fetchError = new Error(
|
||||
'exception TypeError: fetch failed sending request',
|
||||
);
|
||||
|
||||
vi.mocked(mockContentGenerator.generateContentStream)
|
||||
.mockRejectedValueOnce(fetchError)
|
||||
.mockResolvedValueOnce(
|
||||
(async function* () {
|
||||
yield {
|
||||
candidates: [
|
||||
{
|
||||
content: { parts: [{ text: 'Success after fetch error' }] },
|
||||
finishReason: 'STOP',
|
||||
},
|
||||
],
|
||||
} as unknown as GenerateContentResponse;
|
||||
})(),
|
||||
);
|
||||
|
||||
mockRetryWithBackoff.mockImplementation(async (apiCall, options) => {
|
||||
try {
|
||||
return await apiCall();
|
||||
} catch (error) {
|
||||
if (
|
||||
options?.retryFetchErrors &&
|
||||
error instanceof Error &&
|
||||
error.message.includes(
|
||||
'exception TypeError: fetch failed sending request',
|
||||
)
|
||||
) {
|
||||
return await apiCall();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
const stream = await chat.sendMessageStream(
|
||||
'test-model',
|
||||
{ message: 'test' },
|
||||
'prompt-id-fetch-error-retry',
|
||||
);
|
||||
|
||||
const events: StreamEvent[] = [];
|
||||
for await (const event of stream) {
|
||||
events.push(event);
|
||||
}
|
||||
|
||||
expect(
|
||||
mockContentGenerator.generateContentStream,
|
||||
).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(
|
||||
events.some(
|
||||
(e) =>
|
||||
e.type === StreamEventType.CHUNK &&
|
||||
e.value.candidates?.[0]?.content?.parts?.[0]?.text ===
|
||||
'Success after fetch error',
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
// Reset to default behavior
|
||||
mockRetryWithBackoff.mockImplementation(async (apiCall) => apiCall());
|
||||
|
||||
@@ -382,6 +382,7 @@ export class GeminiChat {
|
||||
const streamResponse = await retryWithBackoff(apiCall, {
|
||||
onPersistent429: onPersistent429Callback,
|
||||
authType: this.config.getContentGeneratorConfig()?.authType,
|
||||
retryFetchErrors: this.config.getRetryFetchErrors(),
|
||||
});
|
||||
|
||||
return this.processStreamResponse(model, streamResponse);
|
||||
|
||||
@@ -304,6 +304,41 @@ describe('retryWithBackoff', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Fetch error retries', () => {
|
||||
const fetchErrorMsg = 'exception TypeError: fetch failed sending request';
|
||||
|
||||
it('should retry on specific fetch error when retryFetchErrors is true', async () => {
|
||||
const mockFn = vi.fn();
|
||||
mockFn.mockRejectedValueOnce(new Error(fetchErrorMsg));
|
||||
mockFn.mockResolvedValueOnce('success');
|
||||
|
||||
const promise = retryWithBackoff(mockFn, {
|
||||
retryFetchErrors: true,
|
||||
initialDelayMs: 10,
|
||||
});
|
||||
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
const result = await promise;
|
||||
expect(result).toBe('success');
|
||||
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each([false, undefined])(
|
||||
'should not retry on specific fetch error when retryFetchErrors is %s',
|
||||
async (retryFetchErrors) => {
|
||||
const mockFn = vi.fn().mockRejectedValue(new Error(fetchErrorMsg));
|
||||
|
||||
const promise = retryWithBackoff(mockFn, {
|
||||
retryFetchErrors,
|
||||
});
|
||||
|
||||
await expect(promise).rejects.toThrow(fetchErrorMsg);
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('Flash model fallback for OAuth users', () => {
|
||||
it('should trigger fallback for OAuth personal users on TerminalQuotaError', async () => {
|
||||
const fallbackCallback = vi.fn().mockResolvedValue('gemini-2.5-flash');
|
||||
|
||||
@@ -13,6 +13,9 @@ import {
|
||||
TerminalQuotaError,
|
||||
} from './googleQuotaErrors.js';
|
||||
|
||||
const FETCH_FAILED_MESSAGE =
|
||||
'exception TypeError: fetch failed sending request';
|
||||
|
||||
export interface HttpError extends Error {
|
||||
status?: number;
|
||||
}
|
||||
@@ -21,13 +24,14 @@ export interface RetryOptions {
|
||||
maxAttempts: number;
|
||||
initialDelayMs: number;
|
||||
maxDelayMs: number;
|
||||
shouldRetryOnError: (error: Error) => boolean;
|
||||
shouldRetryOnError: (error: Error, retryFetchErrors?: boolean) => boolean;
|
||||
shouldRetryOnContent?: (content: GenerateContentResponse) => boolean;
|
||||
onPersistent429?: (
|
||||
authType?: string,
|
||||
error?: unknown,
|
||||
) => Promise<string | boolean | null>;
|
||||
authType?: string;
|
||||
retryFetchErrors?: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
||||
@@ -41,9 +45,21 @@ const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
||||
* Default predicate function to determine if a retry should be attempted.
|
||||
* Retries on 429 (Too Many Requests) and 5xx server errors.
|
||||
* @param error The error object.
|
||||
* @param retryFetchErrors Whether to retry on specific fetch errors.
|
||||
* @returns True if the error is a transient error, false otherwise.
|
||||
*/
|
||||
function defaultShouldRetry(error: Error | unknown): boolean {
|
||||
function defaultShouldRetry(
|
||||
error: Error | unknown,
|
||||
retryFetchErrors?: boolean,
|
||||
): boolean {
|
||||
if (
|
||||
retryFetchErrors &&
|
||||
error instanceof Error &&
|
||||
error.message.includes(FETCH_FAILED_MESSAGE)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Priority check for ApiError
|
||||
if (error instanceof ApiError) {
|
||||
// Explicitly do not retry 400 (Bad Request)
|
||||
@@ -96,6 +112,7 @@ export async function retryWithBackoff<T>(
|
||||
authType,
|
||||
shouldRetryOnError,
|
||||
shouldRetryOnContent,
|
||||
retryFetchErrors,
|
||||
} = {
|
||||
...DEFAULT_RETRY_OPTIONS,
|
||||
...cleanOptions,
|
||||
@@ -155,7 +172,10 @@ export async function retryWithBackoff<T>(
|
||||
}
|
||||
|
||||
// Generic retry logic for other errors
|
||||
if (attempt >= maxAttempts || !shouldRetryOnError(error as Error)) {
|
||||
if (
|
||||
attempt >= maxAttempts ||
|
||||
!shouldRetryOnError(error as Error, retryFetchErrors)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user