fix(core): treat retryable errors with >5 min delay as terminal quota errors (#21881)

This commit is contained in:
Gaurav
2026-03-10 07:53:51 -07:00
committed by GitHub
parent 0486a1675a
commit 94ab449e65
2 changed files with 99 additions and 14 deletions
@@ -134,21 +134,21 @@ describe('classifyGoogleError', () => {
expect((result as TerminalQuotaError).cause).toBe(apiError); expect((result as TerminalQuotaError).cause).toBe(apiError);
}); });
it('should return RetryableQuotaError for long retry delays', () => { it('should return TerminalQuotaError for retry delays over 5 minutes', () => {
const apiError: GoogleApiError = { const apiError: GoogleApiError = {
code: 429, code: 429,
message: 'Too many requests', message: 'Too many requests',
details: [ details: [
{ {
'@type': 'type.googleapis.com/google.rpc.RetryInfo', '@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '301s', // Any delay is now retryable retryDelay: '301s', // Over 5 min threshold => terminal
}, },
], ],
}; };
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError); vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error()); const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(RetryableQuotaError); expect(result).toBeInstanceOf(TerminalQuotaError);
expect((result as RetryableQuotaError).retryDelayMs).toBe(301000); expect((result as TerminalQuotaError).retryDelayMs).toBe(301000);
}); });
it('should return RetryableQuotaError for short retry delays', () => { it('should return RetryableQuotaError for short retry delays', () => {
@@ -285,6 +285,34 @@ describe('classifyGoogleError', () => {
); );
}); });
it('should return TerminalQuotaError for Cloud Code RATE_LIMIT_EXCEEDED with retry delay over 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message:
'You have exhausted your capacity on this model. Your quota will reset after 10m.',
details: [
{
'@type': 'type.googleapis.com/google.rpc.ErrorInfo',
reason: 'RATE_LIMIT_EXCEEDED',
domain: 'cloudcode-pa.googleapis.com',
metadata: {
uiMessage: 'true',
model: 'gemini-2.5-pro',
},
},
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '600s',
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(TerminalQuotaError);
expect((result as TerminalQuotaError).retryDelayMs).toBe(600000);
expect((result as TerminalQuotaError).reason).toBe('RATE_LIMIT_EXCEEDED');
});
it('should return TerminalQuotaError for Cloud Code QUOTA_EXHAUSTED', () => { it('should return TerminalQuotaError for Cloud Code QUOTA_EXHAUSTED', () => {
const apiError: GoogleApiError = { const apiError: GoogleApiError = {
code: 429, code: 429,
@@ -427,6 +455,40 @@ describe('classifyGoogleError', () => {
} }
}); });
it('should return TerminalQuotaError when fallback "Please retry in" delay exceeds 5 minutes', () => {
const errorWithEmptyDetails = {
error: {
code: 429,
message: 'Resource exhausted. Please retry in 400s',
details: [],
},
};
const result = classifyGoogleError(errorWithEmptyDetails);
expect(result).toBeInstanceOf(TerminalQuotaError);
if (result instanceof TerminalQuotaError) {
expect(result.retryDelayMs).toBe(400000);
}
});
it('should return RetryableQuotaError when retry delay is exactly 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message: 'Too many requests',
details: [
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '300s',
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(RetryableQuotaError);
expect((result as RetryableQuotaError).retryDelayMs).toBe(300000);
});
it('should return RetryableQuotaError without delay time for generic 429 without specific message', () => { it('should return RetryableQuotaError without delay time for generic 429 without specific message', () => {
const generic429 = { const generic429 = {
status: 429, status: 429,
+33 -10
View File
@@ -100,6 +100,13 @@ function parseDurationInSeconds(duration: string): number | null {
return null; return null;
} }
/**
* Maximum retry delay (in seconds) before a retryable error is treated as terminal.
* If the server suggests waiting longer than this, the user is effectively locked out,
* so we trigger the fallback/credits flow instead of silently waiting.
*/
const MAX_RETRYABLE_DELAY_SECONDS = 300; // 5 minutes
/** /**
* Valid Cloud Code API domains for VALIDATION_REQUIRED errors. * Valid Cloud Code API domains for VALIDATION_REQUIRED errors.
*/ */
@@ -248,15 +255,15 @@ export function classifyGoogleError(error: unknown): unknown {
if (match?.[1]) { if (match?.[1]) {
const retryDelaySeconds = parseDurationInSeconds(match[1]); const retryDelaySeconds = parseDurationInSeconds(match[1]);
if (retryDelaySeconds !== null) { if (retryDelaySeconds !== null) {
return new RetryableQuotaError( const cause = googleApiError ?? {
errorMessage, code: status ?? 429,
googleApiError ?? { message: errorMessage,
code: status ?? 429, details: [],
message: errorMessage, };
details: [], if (retryDelaySeconds > MAX_RETRYABLE_DELAY_SECONDS) {
}, return new TerminalQuotaError(errorMessage, cause, retryDelaySeconds);
retryDelaySeconds, }
); return new RetryableQuotaError(errorMessage, cause, retryDelaySeconds);
} }
} else if (status === 429 || status === 499) { } else if (status === 429 || status === 499) {
// Fallback: If it is a 429 or 499 but doesn't have a specific "retry in" message, // Fallback: If it is a 429 or 499 but doesn't have a specific "retry in" message,
@@ -325,10 +332,19 @@ export function classifyGoogleError(error: unknown): unknown {
if (errorInfo.domain) { if (errorInfo.domain) {
if (isCloudCodeDomain(errorInfo.domain)) { if (isCloudCodeDomain(errorInfo.domain)) {
if (errorInfo.reason === 'RATE_LIMIT_EXCEEDED') { if (errorInfo.reason === 'RATE_LIMIT_EXCEEDED') {
const effectiveDelay = delaySeconds ?? 10;
if (effectiveDelay > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(
`${googleApiError.message}`,
googleApiError,
effectiveDelay,
errorInfo.reason,
);
}
return new RetryableQuotaError( return new RetryableQuotaError(
`${googleApiError.message}`, `${googleApiError.message}`,
googleApiError, googleApiError,
delaySeconds ?? 10, effectiveDelay,
); );
} }
if (errorInfo.reason === 'QUOTA_EXHAUSTED') { if (errorInfo.reason === 'QUOTA_EXHAUSTED') {
@@ -345,6 +361,13 @@ export function classifyGoogleError(error: unknown): unknown {
// 2. Check for delays in RetryInfo // 2. Check for delays in RetryInfo
if (retryInfo?.retryDelay && delaySeconds) { if (retryInfo?.retryDelay && delaySeconds) {
if (delaySeconds > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(
`${googleApiError.message}\nSuggested retry after ${retryInfo.retryDelay}.`,
googleApiError,
delaySeconds,
);
}
return new RetryableQuotaError( return new RetryableQuotaError(
`${googleApiError.message}\nSuggested retry after ${retryInfo.retryDelay}.`, `${googleApiError.message}\nSuggested retry after ${retryInfo.retryDelay}.`,
googleApiError, googleApiError,