mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-05-12 12:54:07 -07:00
fix(core): add retry logic for specific fetch errors (#11066)
This commit is contained in:
@@ -760,6 +760,7 @@ export async function loadCliConfig(
|
|||||||
settings.tools?.enableMessageBusIntegration ?? false,
|
settings.tools?.enableMessageBusIntegration ?? false,
|
||||||
codebaseInvestigatorSettings:
|
codebaseInvestigatorSettings:
|
||||||
settings.experimental?.codebaseInvestigatorSettings,
|
settings.experimental?.codebaseInvestigatorSettings,
|
||||||
|
retryFetchErrors: settings.general?.retryFetchErrors ?? false,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -106,6 +106,7 @@ const MIGRATION_MAP: Record<string, string> = {
|
|||||||
memoryDiscoveryMaxDirs: 'context.discoveryMaxDirs',
|
memoryDiscoveryMaxDirs: 'context.discoveryMaxDirs',
|
||||||
model: 'model.name',
|
model: 'model.name',
|
||||||
preferredEditor: 'general.preferredEditor',
|
preferredEditor: 'general.preferredEditor',
|
||||||
|
retryFetchErrors: 'general.retryFetchErrors',
|
||||||
sandbox: 'tools.sandbox',
|
sandbox: 'tools.sandbox',
|
||||||
selectedAuthType: 'security.auth.selectedType',
|
selectedAuthType: 'security.auth.selectedType',
|
||||||
enableInteractiveShell: 'tools.shell.enableInteractiveShell',
|
enableInteractiveShell: 'tools.shell.enableInteractiveShell',
|
||||||
|
|||||||
@@ -184,6 +184,16 @@ const SETTINGS_SCHEMA = {
|
|||||||
'Enable AI-powered prompt completion suggestions while typing.',
|
'Enable AI-powered prompt completion suggestions while typing.',
|
||||||
showInDialog: true,
|
showInDialog: true,
|
||||||
},
|
},
|
||||||
|
retryFetchErrors: {
|
||||||
|
type: 'boolean',
|
||||||
|
label: 'Retry Fetch Errors',
|
||||||
|
category: 'General',
|
||||||
|
requiresRestart: false,
|
||||||
|
default: false,
|
||||||
|
description:
|
||||||
|
'Retry on "exception TypeError: fetch failed sending request" errors.',
|
||||||
|
showInDialog: false,
|
||||||
|
},
|
||||||
debugKeystrokeLogging: {
|
debugKeystrokeLogging: {
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
label: 'Debug Keystroke Logging',
|
label: 'Debug Keystroke Logging',
|
||||||
|
|||||||
@@ -278,6 +278,7 @@ export interface ConfigParameters {
|
|||||||
enableMessageBusIntegration?: boolean;
|
enableMessageBusIntegration?: boolean;
|
||||||
codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
|
codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
|
||||||
continueOnFailedApiCall?: boolean;
|
continueOnFailedApiCall?: boolean;
|
||||||
|
retryFetchErrors?: boolean;
|
||||||
enableShellOutputEfficiency?: boolean;
|
enableShellOutputEfficiency?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -372,6 +373,7 @@ export class Config {
|
|||||||
private readonly enableMessageBusIntegration: boolean;
|
private readonly enableMessageBusIntegration: boolean;
|
||||||
private readonly codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
|
private readonly codebaseInvestigatorSettings?: CodebaseInvestigatorSettings;
|
||||||
private readonly continueOnFailedApiCall: boolean;
|
private readonly continueOnFailedApiCall: boolean;
|
||||||
|
private readonly retryFetchErrors: boolean;
|
||||||
private readonly enableShellOutputEfficiency: boolean;
|
private readonly enableShellOutputEfficiency: boolean;
|
||||||
|
|
||||||
constructor(params: ConfigParameters) {
|
constructor(params: ConfigParameters) {
|
||||||
@@ -479,6 +481,7 @@ export class Config {
|
|||||||
this.outputSettings = {
|
this.outputSettings = {
|
||||||
format: params.output?.format ?? OutputFormat.TEXT,
|
format: params.output?.format ?? OutputFormat.TEXT,
|
||||||
};
|
};
|
||||||
|
this.retryFetchErrors = params.retryFetchErrors ?? false;
|
||||||
|
|
||||||
if (params.contextFileName) {
|
if (params.contextFileName) {
|
||||||
setGeminiMdFilename(params.contextFileName);
|
setGeminiMdFilename(params.contextFileName);
|
||||||
@@ -970,6 +973,10 @@ export class Config {
|
|||||||
return this.continueOnFailedApiCall;
|
return this.continueOnFailedApiCall;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getRetryFetchErrors(): boolean {
|
||||||
|
return this.retryFetchErrors;
|
||||||
|
}
|
||||||
|
|
||||||
getEnableShellOutputEfficiency(): boolean {
|
getEnableShellOutputEfficiency(): boolean {
|
||||||
return this.enableShellOutputEfficiency;
|
return this.enableShellOutputEfficiency;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -127,6 +127,7 @@ describe('GeminiChat', () => {
|
|||||||
getTool: vi.fn(),
|
getTool: vi.fn(),
|
||||||
}),
|
}),
|
||||||
getContentGenerator: vi.fn().mockReturnValue(mockContentGenerator),
|
getContentGenerator: vi.fn().mockReturnValue(mockContentGenerator),
|
||||||
|
getRetryFetchErrors: vi.fn().mockReturnValue(false),
|
||||||
} as unknown as Config;
|
} as unknown as Config;
|
||||||
|
|
||||||
// Disable 429 simulation for tests
|
// Disable 429 simulation for tests
|
||||||
@@ -1113,6 +1114,70 @@ describe('GeminiChat', () => {
|
|||||||
).toHaveBeenCalledTimes(2);
|
).toHaveBeenCalledTimes(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should retry on specific fetch errors when configured', async () => {
|
||||||
|
vi.mocked(mockConfig.getRetryFetchErrors).mockReturnValue(true);
|
||||||
|
|
||||||
|
const fetchError = new Error(
|
||||||
|
'exception TypeError: fetch failed sending request',
|
||||||
|
);
|
||||||
|
|
||||||
|
vi.mocked(mockContentGenerator.generateContentStream)
|
||||||
|
.mockRejectedValueOnce(fetchError)
|
||||||
|
.mockResolvedValueOnce(
|
||||||
|
(async function* () {
|
||||||
|
yield {
|
||||||
|
candidates: [
|
||||||
|
{
|
||||||
|
content: { parts: [{ text: 'Success after fetch error' }] },
|
||||||
|
finishReason: 'STOP',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown as GenerateContentResponse;
|
||||||
|
})(),
|
||||||
|
);
|
||||||
|
|
||||||
|
mockRetryWithBackoff.mockImplementation(async (apiCall, options) => {
|
||||||
|
try {
|
||||||
|
return await apiCall();
|
||||||
|
} catch (error) {
|
||||||
|
if (
|
||||||
|
options?.retryFetchErrors &&
|
||||||
|
error instanceof Error &&
|
||||||
|
error.message.includes(
|
||||||
|
'exception TypeError: fetch failed sending request',
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return await apiCall();
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const stream = await chat.sendMessageStream(
|
||||||
|
'test-model',
|
||||||
|
{ message: 'test' },
|
||||||
|
'prompt-id-fetch-error-retry',
|
||||||
|
);
|
||||||
|
|
||||||
|
const events: StreamEvent[] = [];
|
||||||
|
for await (const event of stream) {
|
||||||
|
events.push(event);
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(
|
||||||
|
mockContentGenerator.generateContentStream,
|
||||||
|
).toHaveBeenCalledTimes(2);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
events.some(
|
||||||
|
(e) =>
|
||||||
|
e.type === StreamEventType.CHUNK &&
|
||||||
|
e.value.candidates?.[0]?.content?.parts?.[0]?.text ===
|
||||||
|
'Success after fetch error',
|
||||||
|
),
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
// Reset to default behavior
|
// Reset to default behavior
|
||||||
mockRetryWithBackoff.mockImplementation(async (apiCall) => apiCall());
|
mockRetryWithBackoff.mockImplementation(async (apiCall) => apiCall());
|
||||||
|
|||||||
@@ -382,6 +382,7 @@ export class GeminiChat {
|
|||||||
const streamResponse = await retryWithBackoff(apiCall, {
|
const streamResponse = await retryWithBackoff(apiCall, {
|
||||||
onPersistent429: onPersistent429Callback,
|
onPersistent429: onPersistent429Callback,
|
||||||
authType: this.config.getContentGeneratorConfig()?.authType,
|
authType: this.config.getContentGeneratorConfig()?.authType,
|
||||||
|
retryFetchErrors: this.config.getRetryFetchErrors(),
|
||||||
});
|
});
|
||||||
|
|
||||||
return this.processStreamResponse(model, streamResponse);
|
return this.processStreamResponse(model, streamResponse);
|
||||||
|
|||||||
@@ -304,6 +304,41 @@ describe('retryWithBackoff', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Fetch error retries', () => {
|
||||||
|
const fetchErrorMsg = 'exception TypeError: fetch failed sending request';
|
||||||
|
|
||||||
|
it('should retry on specific fetch error when retryFetchErrors is true', async () => {
|
||||||
|
const mockFn = vi.fn();
|
||||||
|
mockFn.mockRejectedValueOnce(new Error(fetchErrorMsg));
|
||||||
|
mockFn.mockResolvedValueOnce('success');
|
||||||
|
|
||||||
|
const promise = retryWithBackoff(mockFn, {
|
||||||
|
retryFetchErrors: true,
|
||||||
|
initialDelayMs: 10,
|
||||||
|
});
|
||||||
|
|
||||||
|
await vi.runAllTimersAsync();
|
||||||
|
|
||||||
|
const result = await promise;
|
||||||
|
expect(result).toBe('success');
|
||||||
|
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([false, undefined])(
|
||||||
|
'should not retry on specific fetch error when retryFetchErrors is %s',
|
||||||
|
async (retryFetchErrors) => {
|
||||||
|
const mockFn = vi.fn().mockRejectedValue(new Error(fetchErrorMsg));
|
||||||
|
|
||||||
|
const promise = retryWithBackoff(mockFn, {
|
||||||
|
retryFetchErrors,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(promise).rejects.toThrow(fetchErrorMsg);
|
||||||
|
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
describe('Flash model fallback for OAuth users', () => {
|
describe('Flash model fallback for OAuth users', () => {
|
||||||
it('should trigger fallback for OAuth personal users on TerminalQuotaError', async () => {
|
it('should trigger fallback for OAuth personal users on TerminalQuotaError', async () => {
|
||||||
const fallbackCallback = vi.fn().mockResolvedValue('gemini-2.5-flash');
|
const fallbackCallback = vi.fn().mockResolvedValue('gemini-2.5-flash');
|
||||||
|
|||||||
@@ -13,6 +13,9 @@ import {
|
|||||||
TerminalQuotaError,
|
TerminalQuotaError,
|
||||||
} from './googleQuotaErrors.js';
|
} from './googleQuotaErrors.js';
|
||||||
|
|
||||||
|
const FETCH_FAILED_MESSAGE =
|
||||||
|
'exception TypeError: fetch failed sending request';
|
||||||
|
|
||||||
export interface HttpError extends Error {
|
export interface HttpError extends Error {
|
||||||
status?: number;
|
status?: number;
|
||||||
}
|
}
|
||||||
@@ -21,13 +24,14 @@ export interface RetryOptions {
|
|||||||
maxAttempts: number;
|
maxAttempts: number;
|
||||||
initialDelayMs: number;
|
initialDelayMs: number;
|
||||||
maxDelayMs: number;
|
maxDelayMs: number;
|
||||||
shouldRetryOnError: (error: Error) => boolean;
|
shouldRetryOnError: (error: Error, retryFetchErrors?: boolean) => boolean;
|
||||||
shouldRetryOnContent?: (content: GenerateContentResponse) => boolean;
|
shouldRetryOnContent?: (content: GenerateContentResponse) => boolean;
|
||||||
onPersistent429?: (
|
onPersistent429?: (
|
||||||
authType?: string,
|
authType?: string,
|
||||||
error?: unknown,
|
error?: unknown,
|
||||||
) => Promise<string | boolean | null>;
|
) => Promise<string | boolean | null>;
|
||||||
authType?: string;
|
authType?: string;
|
||||||
|
retryFetchErrors?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
||||||
@@ -41,9 +45,21 @@ const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
|||||||
* Default predicate function to determine if a retry should be attempted.
|
* Default predicate function to determine if a retry should be attempted.
|
||||||
* Retries on 429 (Too Many Requests) and 5xx server errors.
|
* Retries on 429 (Too Many Requests) and 5xx server errors.
|
||||||
* @param error The error object.
|
* @param error The error object.
|
||||||
|
* @param retryFetchErrors Whether to retry on specific fetch errors.
|
||||||
* @returns True if the error is a transient error, false otherwise.
|
* @returns True if the error is a transient error, false otherwise.
|
||||||
*/
|
*/
|
||||||
function defaultShouldRetry(error: Error | unknown): boolean {
|
function defaultShouldRetry(
|
||||||
|
error: Error | unknown,
|
||||||
|
retryFetchErrors?: boolean,
|
||||||
|
): boolean {
|
||||||
|
if (
|
||||||
|
retryFetchErrors &&
|
||||||
|
error instanceof Error &&
|
||||||
|
error.message.includes(FETCH_FAILED_MESSAGE)
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
// Priority check for ApiError
|
// Priority check for ApiError
|
||||||
if (error instanceof ApiError) {
|
if (error instanceof ApiError) {
|
||||||
// Explicitly do not retry 400 (Bad Request)
|
// Explicitly do not retry 400 (Bad Request)
|
||||||
@@ -96,6 +112,7 @@ export async function retryWithBackoff<T>(
|
|||||||
authType,
|
authType,
|
||||||
shouldRetryOnError,
|
shouldRetryOnError,
|
||||||
shouldRetryOnContent,
|
shouldRetryOnContent,
|
||||||
|
retryFetchErrors,
|
||||||
} = {
|
} = {
|
||||||
...DEFAULT_RETRY_OPTIONS,
|
...DEFAULT_RETRY_OPTIONS,
|
||||||
...cleanOptions,
|
...cleanOptions,
|
||||||
@@ -155,7 +172,10 @@ export async function retryWithBackoff<T>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Generic retry logic for other errors
|
// Generic retry logic for other errors
|
||||||
if (attempt >= maxAttempts || !shouldRetryOnError(error as Error)) {
|
if (
|
||||||
|
attempt >= maxAttempts ||
|
||||||
|
!shouldRetryOnError(error as Error, retryFetchErrors)
|
||||||
|
) {
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user