Fix(core): retry additional OpenSSL 3.x SSL errors during streaming (#16075) (#25187)

This commit is contained in:
Rob Clevenger
2026-04-14 19:50:22 -07:00
committed by GitHub
parent 8d05bdbe49
commit 06e7621b26
3 changed files with 122 additions and 6 deletions

View File

@@ -519,4 +519,70 @@ describe('GeminiChat Network Retries', () => {
}),
);
});
it('should retry on OpenSSL 3.x SSL error during stream iteration (ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC)', async () => {
// OpenSSL 3.x produces a different error code format than OpenSSL 1.x
const sslError = new Error(
'request to https://cloudcode-pa.googleapis.com/v1internal:streamGenerateContent failed',
) as NodeJS.ErrnoException & { type?: string };
sslError.type = 'system';
sslError.errno =
'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC' as unknown as number;
sslError.code = 'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC';
vi.mocked(mockContentGenerator.generateContentStream)
.mockImplementationOnce(async () =>
(async function* () {
yield {
candidates: [
{ content: { parts: [{ text: 'Partial response...' }] } },
],
} as unknown as GenerateContentResponse;
throw sslError;
})(),
)
.mockImplementationOnce(async () =>
(async function* () {
yield {
candidates: [
{
content: { parts: [{ text: 'Complete response after retry' }] },
finishReason: 'STOP',
},
],
} as unknown as GenerateContentResponse;
})(),
);
const stream = await chat.sendMessageStream(
{ model: 'test-model' },
'test message',
'prompt-id-ssl3-mid-stream',
new AbortController().signal,
LlmRole.MAIN,
);
const events: StreamEvent[] = [];
for await (const event of stream) {
events.push(event);
}
const retryEvent = events.find((e) => e.type === StreamEventType.RETRY);
expect(retryEvent).toBeDefined();
const successChunk = events.find(
(e) =>
e.type === StreamEventType.CHUNK &&
e.value.candidates?.[0]?.content?.parts?.[0]?.text ===
'Complete response after retry',
);
expect(successChunk).toBeDefined();
expect(mockLogNetworkRetryAttempt).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
error_type: 'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC',
}),
);
});
});

View File

@@ -511,6 +511,40 @@ describe('retryWithBackoff', () => {
expect(mockFn).toHaveBeenCalledTimes(2);
});
it('should retry on OpenSSL 3.x SSL error code (ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC)', async () => {
const error = new Error('SSL error');
(error as any).code = 'ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC';
const mockFn = vi
.fn()
.mockRejectedValueOnce(error)
.mockResolvedValue('success');
const promise = retryWithBackoff(mockFn, {
initialDelayMs: 1,
maxDelayMs: 1,
});
await vi.runAllTimersAsync();
await expect(promise).resolves.toBe('success');
expect(mockFn).toHaveBeenCalledTimes(2);
});
it('should retry on unknown SSL BAD_RECORD_MAC variant via substring fallback', async () => {
const error = new Error('SSL error');
(error as any).code = 'ERR_SSL_SOME_FUTURE_BAD_RECORD_MAC';
const mockFn = vi
.fn()
.mockRejectedValueOnce(error)
.mockResolvedValue('success');
const promise = retryWithBackoff(mockFn, {
initialDelayMs: 1,
maxDelayMs: 1,
});
await vi.runAllTimersAsync();
await expect(promise).resolves.toBe('success');
expect(mockFn).toHaveBeenCalledTimes(2);
});
it('should retry on gaxios-style SSL error with code property', async () => {
// This matches the exact structure from issue #17318
const error = new Error(

View File

@@ -53,14 +53,30 @@ const RETRYABLE_NETWORK_CODES = [
'ENOTFOUND',
'EAI_AGAIN',
'ECONNREFUSED',
// SSL/TLS transient errors
'ERR_SSL_SSLV3_ALERT_BAD_RECORD_MAC',
'ERR_SSL_WRONG_VERSION_NUMBER',
'ERR_SSL_DECRYPTION_FAILED_OR_BAD_RECORD_MAC',
'ERR_SSL_BAD_RECORD_MAC',
'EPROTO', // Generic protocol error (often SSL-related)
];
// Node.js builds SSL error codes by prepending ERR_SSL_ to the uppercased
// OpenSSL reason string with spaces replaced by underscores (see
// TLSWrap::ClearOut in node/src/crypto/crypto_tls.cc). The reason string
// format varies by OpenSSL version (e.g. ERR_SSL_SSLV3_ALERT_BAD_RECORD_MAC
// on OpenSSL 1.x, ERR_SSL_SSL/TLS_ALERT_BAD_RECORD_MAC on OpenSSL 3.x), so
// match the stable suffix instead of enumerating every variant.
const RETRYABLE_SSL_ERROR_PATTERN = /^ERR_SSL_.*BAD_RECORD_MAC/i;
/**
* Returns true if the error code should be retried: either an exact match
* against RETRYABLE_NETWORK_CODES, or an SSL BAD_RECORD_MAC variant (the
* OpenSSL reason-string portion of the code varies across OpenSSL versions).
*/
function isRetryableSslErrorCode(code: string): boolean {
return (
RETRYABLE_NETWORK_CODES.includes(code) ||
RETRYABLE_SSL_ERROR_PATTERN.test(code)
);
}
function getNetworkErrorCode(error: unknown): string | undefined {
const getCode = (obj: unknown): string | undefined => {
if (typeof obj !== 'object' || obj === null) {
@@ -112,7 +128,7 @@ export function getRetryErrorType(error: unknown): string {
}
const errorCode = getNetworkErrorCode(error);
if (errorCode && RETRYABLE_NETWORK_CODES.includes(errorCode)) {
if (errorCode && isRetryableSslErrorCode(errorCode)) {
return errorCode;
}
@@ -153,7 +169,7 @@ export function isRetryableError(
): boolean {
// Check for common network error codes
const errorCode = getNetworkErrorCode(error);
if (errorCode && RETRYABLE_NETWORK_CODES.includes(errorCode)) {
if (errorCode && isRetryableSslErrorCode(errorCode)) {
return true;
}