diff --git a/packages/core/src/core/geminiChat_network_retry.test.ts b/packages/core/src/core/geminiChat_network_retry.test.ts index 4683e29261..83d5848e75 100644 --- a/packages/core/src/core/geminiChat_network_retry.test.ts +++ b/packages/core/src/core/geminiChat_network_retry.test.ts @@ -519,4 +519,70 @@ describe('GeminiChat Network Retries', () => { }), ); }); + + it('should retry on OpenSSL 3.x SSL error during stream iteration (ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC)', async () => { + // OpenSSL 3.x produces a different error code format than OpenSSL 1.x + const sslError = new Error( + 'request to https://cloudcode-pa.googleapis.com/v1internal:streamGenerateContent failed', + ) as NodeJS.ErrnoException & { type?: string }; + sslError.type = 'system'; + sslError.errno = + 'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC' as unknown as number; + sslError.code = 'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC'; + + vi.mocked(mockContentGenerator.generateContentStream) + .mockImplementationOnce(async () => + (async function* () { + yield { + candidates: [ + { content: { parts: [{ text: 'Partial response...' }] } }, + ], + } as unknown as GenerateContentResponse; + throw sslError; + })(), + ) + .mockImplementationOnce(async () => + (async function* () { + yield { + candidates: [ + { + content: { parts: [{ text: 'Complete response after retry' }] }, + finishReason: 'STOP', + }, + ], + } as unknown as GenerateContentResponse; + })(), + ); + + const stream = await chat.sendMessageStream( + { model: 'test-model' }, + 'test message', + 'prompt-id-ssl3-mid-stream', + new AbortController().signal, + LlmRole.MAIN, + ); + + const events: StreamEvent[] = []; + for await (const event of stream) { + events.push(event); + } + + const retryEvent = events.find((e) => e.type === StreamEventType.RETRY); + expect(retryEvent).toBeDefined(); + + const successChunk = events.find( + (e) => + e.type === StreamEventType.CHUNK && + e.value.candidates?.[0]?.content?.parts?.[0]?.text === + 'Complete response after retry', + ); + expect(successChunk).toBeDefined(); + + expect(mockLogNetworkRetryAttempt).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + error_type: 'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC', + }), + ); + }); }); diff --git a/packages/core/src/utils/retry.test.ts b/packages/core/src/utils/retry.test.ts index a5b5a8b657..29758e6e92 100644 --- a/packages/core/src/utils/retry.test.ts +++ b/packages/core/src/utils/retry.test.ts @@ -511,6 +511,40 @@ describe('retryWithBackoff', () => { expect(mockFn).toHaveBeenCalledTimes(2); }); + it('should retry on OpenSSL 3.x SSL error code (ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC)', async () => { + const error = new Error('SSL error'); + (error as any).code = 'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC'; + const mockFn = vi + .fn() + .mockRejectedValueOnce(error) + .mockResolvedValue('success'); + + const promise = retryWithBackoff(mockFn, { + initialDelayMs: 1, + maxDelayMs: 1, + }); + await vi.runAllTimersAsync(); + await expect(promise).resolves.toBe('success'); + expect(mockFn).toHaveBeenCalledTimes(2); + }); + + it('should retry on unknown SSL BAD_RECORD_MAC variant via substring fallback', async () => { + const error = new Error('SSL error'); + (error as any).code = 'ERR_SSL_SOME_FUTURE_BAD_RECORD_MAC'; + const mockFn = vi + .fn() + .mockRejectedValueOnce(error) + .mockResolvedValue('success'); + + const promise = retryWithBackoff(mockFn, { + initialDelayMs: 1, + maxDelayMs: 1, + }); + await vi.runAllTimersAsync(); + await expect(promise).resolves.toBe('success'); + expect(mockFn).toHaveBeenCalledTimes(2); + }); + it('should retry on gaxios-style SSL error with code property', async () => { // This matches the exact structure from issue #17318 const error = new Error( diff --git a/packages/core/src/utils/retry.ts b/packages/core/src/utils/retry.ts index 46765216b9..5b3ac4f113 100644 --- a/packages/core/src/utils/retry.ts +++ b/packages/core/src/utils/retry.ts @@ -53,14 +53,30 @@ const RETRYABLE_NETWORK_CODES = [ 'ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', - // SSL/TLS transient errors - 'ERR_SSL_SSLV3_ALERT_BAD_RECORD_MAC', 'ERR_SSL_WRONG_VERSION_NUMBER', - 'ERR_SSL_DECRYPTION_FAILED_OR_BAD_RECORD_MAC', - 'ERR_SSL_BAD_RECORD_MAC', 'EPROTO', // Generic protocol error (often SSL-related) ]; +// Node.js builds SSL error codes by prepending ERR_SSL_ to the uppercased +// OpenSSL reason string with spaces replaced by underscores (see +// TLSWrap::ClearOut in node/src/crypto/crypto_tls.cc). The reason string +// format varies by OpenSSL version (e.g. ERR_SSL_SSLV3_ALERT_BAD_RECORD_MAC +// on OpenSSL 1.x, ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC on OpenSSL 3.x), so +// match the stable suffix instead of enumerating every variant. +const RETRYABLE_SSL_ERROR_PATTERN = /^ERR_SSL_.*BAD_RECORD_MAC/i; + +/** + * Returns true if the error code should be retried: either an exact match + * against RETRYABLE_NETWORK_CODES, or an SSL BAD_RECORD_MAC variant (the + * OpenSSL reason-string portion of the code varies across OpenSSL versions). + */ +function isRetryableSslErrorCode(code: string): boolean { + return ( + RETRYABLE_NETWORK_CODES.includes(code) || + RETRYABLE_SSL_ERROR_PATTERN.test(code) + ); +} + function getNetworkErrorCode(error: unknown): string | undefined { const getCode = (obj: unknown): string | undefined => { if (typeof obj !== 'object' || obj === null) { @@ -112,7 +128,7 @@ export function getRetryErrorType(error: unknown): string { } const errorCode = getNetworkErrorCode(error); - if (errorCode && RETRYABLE_NETWORK_CODES.includes(errorCode)) { + if (errorCode && isRetryableSslErrorCode(errorCode)) { return errorCode; } @@ -153,7 +169,7 @@ export function isRetryableError( ): boolean { // Check for common network error codes const errorCode = getNetworkErrorCode(error); - if (errorCode && RETRYABLE_NETWORK_CODES.includes(errorCode)) { + if (errorCode && isRetryableSslErrorCode(errorCode)) { return true; }