fix(core): treat retryable errors with >5 min delay as terminal quota errors (#21881)

This commit is contained in:
Gaurav
2026-03-10 07:53:51 -07:00
committed by GitHub
parent 0486a1675a
commit 94ab449e65
2 changed files with 99 additions and 14 deletions

View File

@@ -134,21 +134,21 @@ describe('classifyGoogleError', () => {
expect((result as TerminalQuotaError).cause).toBe(apiError);
});
it('should return RetryableQuotaError for long retry delays', () => {
it('should return TerminalQuotaError for retry delays over 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message: 'Too many requests',
details: [
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '301s', // Any delay is now retryable
retryDelay: '301s', // Over 5 min threshold => terminal
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(RetryableQuotaError);
expect((result as RetryableQuotaError).retryDelayMs).toBe(301000);
expect(result).toBeInstanceOf(TerminalQuotaError);
expect((result as TerminalQuotaError).retryDelayMs).toBe(301000);
});
it('should return RetryableQuotaError for short retry delays', () => {
@@ -285,6 +285,34 @@ describe('classifyGoogleError', () => {
);
});
it('should return TerminalQuotaError for Cloud Code RATE_LIMIT_EXCEEDED with retry delay over 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message:
'You have exhausted your capacity on this model. Your quota will reset after 10m.',
details: [
{
'@type': 'type.googleapis.com/google.rpc.ErrorInfo',
reason: 'RATE_LIMIT_EXCEEDED',
domain: 'cloudcode-pa.googleapis.com',
metadata: {
uiMessage: 'true',
model: 'gemini-2.5-pro',
},
},
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '600s',
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(TerminalQuotaError);
expect((result as TerminalQuotaError).retryDelayMs).toBe(600000);
expect((result as TerminalQuotaError).reason).toBe('RATE_LIMIT_EXCEEDED');
});
it('should return TerminalQuotaError for Cloud Code QUOTA_EXHAUSTED', () => {
const apiError: GoogleApiError = {
code: 429,
@@ -427,6 +455,40 @@ describe('classifyGoogleError', () => {
}
});
it('should return TerminalQuotaError when fallback "Please retry in" delay exceeds 5 minutes', () => {
const errorWithEmptyDetails = {
error: {
code: 429,
message: 'Resource exhausted. Please retry in 400s',
details: [],
},
};
const result = classifyGoogleError(errorWithEmptyDetails);
expect(result).toBeInstanceOf(TerminalQuotaError);
if (result instanceof TerminalQuotaError) {
expect(result.retryDelayMs).toBe(400000);
}
});
it('should return RetryableQuotaError when retry delay is exactly 5 minutes', () => {
const apiError: GoogleApiError = {
code: 429,
message: 'Too many requests',
details: [
{
'@type': 'type.googleapis.com/google.rpc.RetryInfo',
retryDelay: '300s',
},
],
};
vi.spyOn(errorParser, 'parseGoogleApiError').mockReturnValue(apiError);
const result = classifyGoogleError(new Error());
expect(result).toBeInstanceOf(RetryableQuotaError);
expect((result as RetryableQuotaError).retryDelayMs).toBe(300000);
});
it('should return RetryableQuotaError without delay time for generic 429 without specific message', () => {
const generic429 = {
status: 429,

View File

@@ -100,6 +100,13 @@ function parseDurationInSeconds(duration: string): number | null {
return null;
}
/**
* Maximum retry delay (in seconds) before a retryable error is treated as terminal.
* If the server suggests waiting longer than this, the user is effectively locked out,
* so we trigger the fallback/credits flow instead of silently waiting.
*/
const MAX_RETRYABLE_DELAY_SECONDS = 300; // 5 minutes
/**
* Valid Cloud Code API domains for VALIDATION_REQUIRED errors.
*/
@@ -248,15 +255,15 @@ export function classifyGoogleError(error: unknown): unknown {
if (match?.[1]) {
const retryDelaySeconds = parseDurationInSeconds(match[1]);
if (retryDelaySeconds !== null) {
return new RetryableQuotaError(
errorMessage,
googleApiError ?? {
code: status ?? 429,
message: errorMessage,
details: [],
},
retryDelaySeconds,
);
const cause = googleApiError ?? {
code: status ?? 429,
message: errorMessage,
details: [],
};
if (retryDelaySeconds > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(errorMessage, cause, retryDelaySeconds);
}
return new RetryableQuotaError(errorMessage, cause, retryDelaySeconds);
}
} else if (status === 429 || status === 499) {
// Fallback: If it is a 429 or 499 but doesn't have a specific "retry in" message,
@@ -325,10 +332,19 @@ export function classifyGoogleError(error: unknown): unknown {
if (errorInfo.domain) {
if (isCloudCodeDomain(errorInfo.domain)) {
if (errorInfo.reason === 'RATE_LIMIT_EXCEEDED') {
const effectiveDelay = delaySeconds ?? 10;
if (effectiveDelay > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(
`${googleApiError.message}`,
googleApiError,
effectiveDelay,
errorInfo.reason,
);
}
return new RetryableQuotaError(
`${googleApiError.message}`,
googleApiError,
delaySeconds ?? 10,
effectiveDelay,
);
}
if (errorInfo.reason === 'QUOTA_EXHAUSTED') {
@@ -345,6 +361,13 @@ export function classifyGoogleError(error: unknown): unknown {
// 2. Check for delays in RetryInfo
if (retryInfo?.retryDelay && delaySeconds) {
if (delaySeconds > MAX_RETRYABLE_DELAY_SECONDS) {
return new TerminalQuotaError(
`${googleApiError.message}\nSuggested retry after ${retryInfo.retryDelay}.`,
googleApiError,
delaySeconds,
);
}
return new RetryableQuotaError(
`${googleApiError.message}\nSuggested retry after ${retryInfo.retryDelay}.`,
googleApiError,