diff --git a/packages/core/src/utils/retry.ts b/packages/core/src/utils/retry.ts index a45ba0c0b0..07b552834d 100644 --- a/packages/core/src/utils/retry.ts +++ b/packages/core/src/utils/retry.ts @@ -208,6 +208,46 @@ export function isRetryableError( return false; } +/** + * Enriches quota-related errors with helpful hints if using a shared Google project + * without a dedicated user project set in their environment. + */ +function enrichQuotaError(error: Error, authType?: string): Error { + const isQuotaError = + error instanceof TerminalQuotaError || + error instanceof RetryableQuotaError || + error.name === 'TerminalQuotaError' || + error.name === 'RetryableQuotaError'; + + if ( + isQuotaError && + (authType === 'oauth-personal' || + authType === 'compute-default-credentials' || + authType === 'LOGIN_WITH_GOOGLE' || + authType === 'COMPUTE_ADC') + ) { + const hasUserProject = !!( + process.env['GOOGLE_CLOUD_PROJECT'] || + process.env['GOOGLE_CLOUD_PROJECT_ID'] + ); + if (!hasUserProject) { + const enrichment = + '\n\nšŸ’” Tip: The shared Google Cloud project is experiencing high traffic and has hit its quota limits. ' + + 'To get dedicated, uninterrupted quota, please set your own Google Cloud project by running:\n' + + ' gcloud config set project [PROJECT_ID]\n' + + 'or by setting the GOOGLE_CLOUD_PROJECT environment variable.'; + if (!error.message.includes('šŸ’” Tip:')) { + Object.defineProperty(error, 'message', { + value: error.message + enrichment, + writable: true, + configurable: true, + }); + } + } + } + return error; +} + /** * Retries a function with exponential backoff and jitter. * @param fn The asynchronous function to retry. @@ -321,7 +361,9 @@ export async function retryWithBackoff( } } // Terminal/not_found already recorded; nothing else to mark here. - throw classifiedError; // Throw if no fallback or fallback failed. + throw classifiedError instanceof Error + ? enrichQuotaError(classifiedError, authType) + : classifiedError; // Throw if no fallback or fallback failed. } // Handle ValidationRequiredError - user needs to verify before proceeding @@ -370,7 +412,7 @@ export async function retryWithBackoff( } } throw classifiedError instanceof RetryableQuotaError - ? classifiedError + ? enrichQuotaError(classifiedError, authType) : error; } diff --git a/packages/core/src/utils/sharedProjectThrottling.test.ts b/packages/core/src/utils/sharedProjectThrottling.test.ts new file mode 100644 index 0000000000..7ad9b60615 --- /dev/null +++ b/packages/core/src/utils/sharedProjectThrottling.test.ts @@ -0,0 +1,169 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; +import fs from 'node:fs'; +import { retryWithBackoff } from './retry.js'; +import { AuthType } from '../core/contentGenerator.js'; +import { TerminalQuotaError } from './googleQuotaErrors.js'; +import type { GoogleApiError } from './googleErrors.js'; + +vi.mock('node:fs'); + +describe('Shared Project Throttling Integration', () => { + let mockGoogleApiError: GoogleApiError; + + beforeEach(() => { + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.statSync).mockReturnValue({ + isDirectory: () => true, + } as fs.Stats); + mockGoogleApiError = { + code: 429, + message: + 'Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_requests', + details: [ + { + '@type': 'type.googleapis.com/google.rpc.QuotaFailure', + violations: [ + { + quotaMetric: + 'generativelanguage.googleapis.com/generate_content_requests', + quotaId: + 'GenerateRequestsPerMinutePerProjectPerModel-SharedProject', + quotaDimensions: { + location: 'global', + model: 'gemini-2.5-pro', + }, + quotaValue: '0', + }, + ], + }, + ], + }; + }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('fails completely when both Pro and Flash fallback models hit shared project quota limits', async () => { + let currentModel = 'gemini-2.5-pro'; + const modelsAttempted: string[] = []; + + // Simulate API calls that fail on both models + const mockApiCall = vi.fn().mockImplementation(async () => { + modelsAttempted.push(currentModel); + throw new TerminalQuotaError( + `Quota exhausted for model ${currentModel} on shared project`, + mockGoogleApiError, + ); + }); + + // Fallback handler changes the active model to Flash on persistent 429 + const mockPersistent429Callback = vi.fn( + async (_authType?: string, _error?: unknown) => { + if (currentModel === 'gemini-2.5-pro') { + currentModel = 'gemini-2.5-flash'; + return 'gemini-2.5-flash'; + } + return null; // No further fallback models + }, + ); + + const promise = retryWithBackoff(mockApiCall, { + maxAttempts: 1, + initialDelayMs: 1, + maxDelayMs: 5, + onPersistent429: mockPersistent429Callback, + authType: AuthType.LOGIN_WITH_GOOGLE, + }); + + await expect(promise).rejects.toThrow( + 'Quota exhausted for model gemini-2.5-flash on shared project', + ); + + // Check that both models were tried and both failed due to the shared project limits + expect(modelsAttempted).toEqual(['gemini-2.5-pro', 'gemini-2.5-flash']); + expect(mockPersistent429Callback).toHaveBeenCalledTimes(2); + }); + + it('appends helpful troubleshooting hint when no user project is configured and auth is LOGIN_WITH_GOOGLE', async () => { + vi.stubEnv('GOOGLE_CLOUD_PROJECT', ''); + vi.stubEnv('GOOGLE_CLOUD_PROJECT_ID', ''); + + const mockApiCall = vi + .fn() + .mockRejectedValue( + new TerminalQuotaError('Daily limit reached', mockGoogleApiError), + ); + + const promise = retryWithBackoff(mockApiCall, { + maxAttempts: 1, + initialDelayMs: 1, + maxDelayMs: 5, + authType: AuthType.LOGIN_WITH_GOOGLE, + }); + + let caughtError: Error | undefined; + try { + await promise; + } catch (e) { + caughtError = e instanceof Error ? e : new Error(String(e)); + } + + expect(caughtError).toBeDefined(); + expect(caughtError?.message).toContain( + 'šŸ’” Tip: The shared Google Cloud project is experiencing high traffic', + ); + expect(caughtError?.message).toContain( + 'gcloud config set project [PROJECT_ID]', + ); + }); + + it('does not append troubleshooting hint if a dedicated user project is set in environment', async () => { + vi.stubEnv('GOOGLE_CLOUD_PROJECT', 'my-dedicated-project-123'); + + const mockApiCall = vi + .fn() + .mockRejectedValue( + new TerminalQuotaError('Daily limit reached', mockGoogleApiError), + ); + + const promise = retryWithBackoff(mockApiCall, { + maxAttempts: 1, + initialDelayMs: 1, + maxDelayMs: 5, + authType: AuthType.LOGIN_WITH_GOOGLE, + }); + + const caughtError = await promise.catch((e) => e); + const errorMsg = + caughtError instanceof Error ? caughtError.message : String(caughtError); + expect(errorMsg).not.toContain('šŸ’” Tip:'); + }); + + it('does not append troubleshooting hint for non-Google/ADC auth types', async () => { + vi.stubEnv('GOOGLE_CLOUD_PROJECT', ''); + + const mockApiCall = vi + .fn() + .mockRejectedValue( + new TerminalQuotaError('Daily limit reached', mockGoogleApiError), + ); + + const promise = retryWithBackoff(mockApiCall, { + maxAttempts: 1, + initialDelayMs: 1, + maxDelayMs: 5, + authType: AuthType.USE_GEMINI, // API Key auth type + }); + + const caughtError = await promise.catch((e) => e); + const errorMsg = + caughtError instanceof Error ? caughtError.message : String(caughtError); + expect(errorMsg).not.toContain('šŸ’” Tip:'); + }); +});