/** * @license * Copyright 2025 Google LLC * SPDX-License-Identifier: Apache-2.0 */ import type { GenerateContentResponse } from '@google/genai'; import { ApiError } from '@google/genai'; import { AuthType } from '../core/contentGenerator.js'; import { classifyGoogleError, RetryableQuotaError, TerminalQuotaError, } from './googleQuotaErrors.js'; const FETCH_FAILED_MESSAGE = 'exception TypeError: fetch failed sending request'; export interface HttpError extends Error { status?: number; } export interface RetryOptions { maxAttempts: number; initialDelayMs: number; maxDelayMs: number; shouldRetryOnError: (error: Error, retryFetchErrors?: boolean) => boolean; shouldRetryOnContent?: (content: GenerateContentResponse) => boolean; onPersistent429?: ( authType?: string, error?: unknown, ) => Promise; authType?: string; retryFetchErrors?: boolean; } const DEFAULT_RETRY_OPTIONS: RetryOptions = { maxAttempts: 3, initialDelayMs: 5000, maxDelayMs: 30000, // 30 seconds shouldRetryOnError: defaultShouldRetry, }; /** * Default predicate function to determine if a retry should be attempted. * Retries on 429 (Too Many Requests) and 5xx server errors. * @param error The error object. * @param retryFetchErrors Whether to retry on specific fetch errors. * @returns True if the error is a transient error, false otherwise. */ function defaultShouldRetry( error: Error | unknown, retryFetchErrors?: boolean, ): boolean { if ( retryFetchErrors && error instanceof Error && error.message.includes(FETCH_FAILED_MESSAGE) ) { return true; } // Priority check for ApiError if (error instanceof ApiError) { // Explicitly do not retry 400 (Bad Request) if (error.status === 400) return false; return error.status === 429 || (error.status >= 500 && error.status < 600); } // Check for status using helper (handles other error shapes) const status = getErrorStatus(error); if (status !== undefined) { return status === 429 || (status >= 500 && status < 600); } return false; } /** * Delays execution for a specified number of milliseconds. * @param ms The number of milliseconds to delay. * @returns A promise that resolves after the delay. */ function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } /** * Retries a function with exponential backoff and jitter. * @param fn The asynchronous function to retry. * @param options Optional retry configuration. * @returns A promise that resolves with the result of the function if successful. * @throws The last error encountered if all attempts fail. */ export async function retryWithBackoff( fn: () => Promise, options?: Partial, ): Promise { if (options?.maxAttempts !== undefined && options.maxAttempts <= 0) { throw new Error('maxAttempts must be a positive number.'); } const cleanOptions = options ? Object.fromEntries(Object.entries(options).filter(([_, v]) => v != null)) : {}; const { maxAttempts, initialDelayMs, maxDelayMs, onPersistent429, authType, shouldRetryOnError, shouldRetryOnContent, retryFetchErrors, } = { ...DEFAULT_RETRY_OPTIONS, ...cleanOptions, }; let attempt = 0; let currentDelay = initialDelayMs; while (attempt < maxAttempts) { attempt++; try { const result = await fn(); if ( shouldRetryOnContent && shouldRetryOnContent(result as GenerateContentResponse) ) { const jitter = currentDelay * 0.3 * (Math.random() * 2 - 1); const delayWithJitter = Math.max(0, currentDelay + jitter); await delay(delayWithJitter); currentDelay = Math.min(maxDelayMs, currentDelay * 2); continue; } return result; } catch (error) { const classifiedError = classifyGoogleError(error); if (classifiedError instanceof TerminalQuotaError) { if (onPersistent429 && authType === AuthType.LOGIN_WITH_GOOGLE) { try { const fallbackModel = await onPersistent429( authType, classifiedError, ); if (fallbackModel) { attempt = 0; // Reset attempts and retry with the new model. currentDelay = initialDelayMs; continue; } } catch (fallbackError) { console.warn('Model fallback failed:', fallbackError); } } throw classifiedError; // Throw if no fallback or fallback failed. } if (classifiedError instanceof RetryableQuotaError) { if (attempt >= maxAttempts) { throw classifiedError; } console.warn( `Attempt ${attempt} failed: ${classifiedError.message}. Retrying after ${classifiedError.retryDelayMs}ms...`, ); await delay(classifiedError.retryDelayMs); continue; } // Generic retry logic for other errors if ( attempt >= maxAttempts || !shouldRetryOnError(error as Error, retryFetchErrors) ) { throw error; } const errorStatus = getErrorStatus(error); logRetryAttempt(attempt, error, errorStatus); // Exponential backoff with jitter for non-quota errors const jitter = currentDelay * 0.3 * (Math.random() * 2 - 1); const delayWithJitter = Math.max(0, currentDelay + jitter); await delay(delayWithJitter); currentDelay = Math.min(maxDelayMs, currentDelay * 2); } } throw new Error('Retry attempts exhausted'); } /** * Extracts the HTTP status code from an error object. * @param error The error object. * @returns The HTTP status code, or undefined if not found. */ export function getErrorStatus(error: unknown): number | undefined { if (typeof error === 'object' && error !== null) { if ('status' in error && typeof error.status === 'number') { return error.status; } // Check for error.response.status (common in axios errors) if ( 'response' in error && typeof (error as { response?: unknown }).response === 'object' && (error as { response?: unknown }).response !== null ) { const response = ( error as { response: { status?: unknown; headers?: unknown } } ).response; if ('status' in response && typeof response.status === 'number') { return response.status; } } } return undefined; } /** * Logs a message for a retry attempt when using exponential backoff. * @param attempt The current attempt number. * @param error The error that caused the retry. * @param errorStatus The HTTP status code of the error, if available. */ function logRetryAttempt( attempt: number, error: unknown, errorStatus?: number, ): void { let message = `Attempt ${attempt} failed. Retrying with backoff...`; if (errorStatus) { message = `Attempt ${attempt} failed with status ${errorStatus}. Retrying with backoff...`; } if (errorStatus === 429) { console.warn(message, error); } else if (errorStatus && errorStatus >= 500 && errorStatus < 600) { console.error(message, error); } else if (error instanceof Error) { // Fallback for errors that might not have a status but have a message if (error.message.includes('429')) { console.warn( `Attempt ${attempt} failed with 429 error (no Retry-After header). Retrying with backoff...`, error, ); } else if (error.message.match(/5\d{2}/)) { console.error( `Attempt ${attempt} failed with 5xx error. Retrying with backoff...`, error, ); } else { console.warn(message, error); // Default to warn for other errors } } else { console.warn(message, error); // Default to warn if error type is unknown } }