From 761f604c1675333dbe51f84fe223435b337986d7 Mon Sep 17 00:00:00 2001 From: luisfelipe-alt Date: Wed, 5 Aug 2026 15:38:37 -0700 Subject: [PATCH] fix(core): unwrap and parse nested gaxios streaming errors from cause message (#28689) --- packages/a2a-server/src/agent/task.ts | 6 +- packages/core/src/utils/errorParsing.test.ts | 10 ++ packages/core/src/utils/googleErrors.test.ts | 109 ++++++++++++++++++ packages/core/src/utils/googleErrors.ts | 83 ++++++++++++- packages/core/src/utils/googleQuotaErrors.ts | 22 +++- .../core/src/utils/quotaErrorDetection.ts | 6 +- 6 files changed, 229 insertions(+), 7 deletions(-) diff --git a/packages/a2a-server/src/agent/task.ts b/packages/a2a-server/src/agent/task.ts index ee4bfbb5bb..2d13542187 100644 --- a/packages/a2a-server/src/agent/task.ts +++ b/packages/a2a-server/src/agent/task.ts @@ -131,9 +131,9 @@ export class Task { this.autoExecute = autoExecute; this.config.setFallbackModelHandler( // For a2a-server, we want to automatically switch to the fallback model - // for future requests without retrying the current one. The 'stop' - // intent achieves this. - async () => 'stop', + // and retry the current request seamlessly. The 'retry_always' intent + // achieves this, ensuring a smooth fallback experience for the user. + async () => 'retry_always', ); } diff --git a/packages/core/src/utils/errorParsing.test.ts b/packages/core/src/utils/errorParsing.test.ts index 291145d2e8..d3c3a70c61 100644 --- a/packages/core/src/utils/errorParsing.test.ts +++ b/packages/core/src/utils/errorParsing.test.ts @@ -109,6 +109,16 @@ describe('parseAndFormatApiError', () => { expect(result).toContain(vertexMessage); }); + it('should format a StructuredError with status: undefined', () => { + const error: StructuredError = { + message: 'Rate limit exceeded (simulated 429 error, limit: 0)', + status: undefined, + }; + const expected = + '[API Error: Rate limit exceeded (simulated 429 error, limit: 0)]'; + expect(parseAndFormatApiError(error)).toBe(expected); + }); + it('should handle an unknown error type', () => { const error = 12345; const expected = '[API Error: An unknown error occurred.]'; diff --git a/packages/core/src/utils/googleErrors.test.ts b/packages/core/src/utils/googleErrors.test.ts index 5a21e72801..109012fccf 100644 --- a/packages/core/src/utils/googleErrors.test.ts +++ b/packages/core/src/utils/googleErrors.test.ts @@ -445,4 +445,113 @@ describe('parseGoogleApiError', () => { expect(parsed?.code).toBe(429); expect(parsed?.message).toBe('Quota exceeded'); }); + + it('should parse an error wrapped inside cause.message by gaxios', () => { + const mockError = { + code: 429, + status: 429, + cause: { + message: JSON.stringify([ + { + error: { + code: 429, + message: + 'No capacity available for model gemini-3.1-pro-preview on the server', + details: [ + { + '@type': 'type.googleapis.com/google.rpc.ErrorInfo', + reason: 'MODEL_CAPACITY_EXHAUSTED', + domain: 'cloudcode-pa.googleapis.com', + metadata: { model: 'gemini-3.1-pro-preview' }, + }, + ], + }, + }, + ]), + code: 429, + status: 'Too Many Requests', + }, + }; + + const parsed = parseGoogleApiError(mockError); + expect(parsed).not.toBeNull(); + expect(parsed?.code).toBe(429); + expect(parsed?.message).toBe( + 'No capacity available for model gemini-3.1-pro-preview on the server', + ); + expect(parsed?.details).toHaveLength(1); + expect(parsed?.details[0]['@type']).toBe( + 'type.googleapis.com/google.rpc.ErrorInfo', + ); + }); + + it('should parse an error where cause is a plain ErrorShape and propagate outer code', () => { + const mockError = { + code: 429, + cause: { + message: 'Quota exceeded on the server', + }, + }; + + const parsed = parseGoogleApiError(mockError); + expect(parsed).not.toBeNull(); + expect(parsed?.code).toBe(429); + expect(parsed?.message).toBe('Quota exceeded on the server'); + }); + + it('should parse an error where cause is a standard Error object and propagate outer status', () => { + const mockError = { + status: 503, + cause: new Error('Service Unavailable'), + }; + + const parsed = parseGoogleApiError(mockError); + expect(parsed).not.toBeNull(); + expect(parsed?.code).toBe(503); + expect(parsed?.message).toBe('Service Unavailable'); + }); + + it('should defensively parse numeric string status codes from outer error', () => { + const mockError = { + status: '503', + cause: new Error('Service Unavailable'), + }; + + const parsed = parseGoogleApiError(mockError); + expect(parsed).not.toBeNull(); + expect(parsed?.code).toBe(503); + expect(parsed?.message).toBe('Service Unavailable'); + }); + + it('should return null for non-numeric string status codes from outer error', () => { + const mockError = { + status: 'Too Many Requests', + cause: new Error('Quota exceeded'), + }; + + const parsed = parseGoogleApiError(mockError); + expect(parsed).toBeNull(); + }); + + it('should return null for empty or whitespace-only string status codes from outer error', () => { + const mockError = { + status: ' ', + cause: new Error('Quota exceeded'), + }; + + const parsed = parseGoogleApiError(mockError); + expect(parsed).toBeNull(); + }); + + it('should parse an error where cause is a plain string and propagate outer status', () => { + const mockError = { + status: 429, + cause: 'Quota exceeded on the server', + }; + + const parsed = parseGoogleApiError(mockError); + expect(parsed).not.toBeNull(); + expect(parsed?.code).toBe(429); + expect(parsed?.message).toBe('Quota exceeded on the server'); + }); }); diff --git a/packages/core/src/utils/googleErrors.ts b/packages/core/src/utils/googleErrors.ts index bcb57425b3..f17a711bb7 100644 --- a/packages/core/src/utils/googleErrors.ts +++ b/packages/core/src/utils/googleErrors.ts @@ -153,6 +153,18 @@ export function parseGoogleApiError(error: unknown): GoogleApiError | null { return null; } + // Skip parsing if the error is already a classified quota error + if ( + typeof error === 'object' && + error !== null && + 'name' in error && + (error.name === 'TerminalQuotaError' || + error.name === 'RetryableQuotaError' || + error.name === 'ValidationRequiredError') + ) { + return null; + } + let errorObj: unknown = error; // If error is a string, try to parse it. @@ -174,7 +186,9 @@ export function parseGoogleApiError(error: unknown): GoogleApiError | null { } let currentError: ErrorShape | undefined = - fromGaxiosError(errorObj) ?? fromApiError(errorObj); + fromGaxiosError(errorObj) ?? + fromApiError(errorObj) ?? + fromCauseError(errorObj); let depth = 0; const maxDepth = 10; @@ -371,3 +385,70 @@ function fromApiError(errorObj: object): ErrorShape | undefined { } return outerError; } + +function fromCauseError(errorObj: object): ErrorShape | undefined { + const err = errorObj as { + code?: unknown; + status?: unknown; + cause?: unknown; + }; + if (!err.cause) return undefined; + + const rawCode = err.code ?? err.status; + const fallbackCode = + typeof rawCode === 'number' + ? rawCode + : typeof rawCode === 'string' && + rawCode.trim() !== '' && + !isNaN(Number(rawCode)) + ? Number(rawCode) + : undefined; + + const resolveError = ( + resolved: ErrorShape | undefined, + ): ErrorShape | undefined => { + if (!resolved) return undefined; + const message = resolved.message; + const details = resolved.details; + const code = resolved.code ?? fallbackCode; + return { + ...(message !== undefined ? { message } : {}), + ...(details !== undefined ? { details } : {}), + ...(code !== undefined ? { code } : {}), + }; + }; + + if (typeof err.cause === 'object' && err.cause !== null) { + if ( + 'error' in err.cause && + err.cause.error && + isErrorShape(err.cause.error) + ) { + return resolveError(err.cause.error); + } + if ('message' in err.cause && err.cause.message) { + if (typeof err.cause.message === 'string') { + const parsed = fromApiError({ message: err.cause.message }); + if (parsed) return resolveError(parsed); + } else if ( + typeof err.cause.message === 'object' && + err.cause.message !== null + ) { + const msgObj = err.cause.message as { error?: unknown }; + if (msgObj.error && isErrorShape(msgObj.error)) { + return resolveError(msgObj.error); + } + } + } + if (isErrorShape(err.cause)) { + return resolveError(err.cause); + } + } + if (typeof err.cause === 'string' && err.cause.trim() !== '') { + const parsed = fromApiError({ message: err.cause }) ?? { + message: err.cause, + }; + return resolveError(parsed); + } + return undefined; +} diff --git a/packages/core/src/utils/googleQuotaErrors.ts b/packages/core/src/utils/googleQuotaErrors.ts index 9d656964fe..e6bd6387ce 100644 --- a/packages/core/src/utils/googleQuotaErrors.ts +++ b/packages/core/src/utils/googleQuotaErrors.ts @@ -28,15 +28,17 @@ enum GoogleApiType { export class TerminalQuotaError extends Error { retryDelayMs?: number; reason?: string; + status?: number; constructor( message: string, - override readonly cause: GoogleApiError, + override readonly cause?: GoogleApiError, retryDelaySeconds?: number, reason?: string, ) { super(message); this.name = 'TerminalQuotaError'; + this.status = cause?.code; this.retryDelayMs = retryDelaySeconds ? retryDelaySeconds * 1000 : undefined; @@ -53,14 +55,16 @@ export class TerminalQuotaError extends Error { */ export class RetryableQuotaError extends Error { retryDelayMs?: number; + status?: number; constructor( message: string, - override readonly cause: GoogleApiError, + override readonly cause?: GoogleApiError, retryDelaySeconds?: number, ) { super(message); this.name = 'RetryableQuotaError'; + this.status = cause?.code; this.retryDelayMs = retryDelaySeconds ? retryDelaySeconds * 1000 : undefined; @@ -217,6 +221,20 @@ function classifyValidationRequiredError( * @returns A classified error or the original `unknown` error. */ export function classifyGoogleError(error: unknown): unknown { + if ( + error instanceof TerminalQuotaError || + error instanceof RetryableQuotaError || + error instanceof ValidationRequiredError || + (typeof error === 'object' && + error !== null && + 'name' in error && + (error.name === 'TerminalQuotaError' || + error.name === 'RetryableQuotaError' || + error.name === 'ValidationRequiredError')) + ) { + return error; + } + const googleApiError = parseGoogleApiError(error); const status = googleApiError?.code ?? getErrorStatus(error); const errorMessage = googleApiError?.message || extractErrorMessage(error); diff --git a/packages/core/src/utils/quotaErrorDetection.ts b/packages/core/src/utils/quotaErrorDetection.ts index 73947049ae..ed04b23d94 100644 --- a/packages/core/src/utils/quotaErrorDetection.ts +++ b/packages/core/src/utils/quotaErrorDetection.ts @@ -41,7 +41,11 @@ export function isStructuredError(error: unknown): error is StructuredError { if (typeof error.message !== 'string') { return false; } - if ('status' in error && typeof error.status !== 'number') { + if ( + 'status' in error && + error.status !== undefined && + typeof error.status !== 'number' + ) { return false; } return true;