mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-07-13 19:40:28 -07:00
fix(core): enrich shared project quota limit errors with setup hint (#28391)
This commit is contained in:
@@ -208,6 +208,46 @@ export function isRetryableError(
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enriches quota-related errors with helpful hints if using a shared Google project
|
||||
* without a dedicated user project set in their environment.
|
||||
*/
|
||||
function enrichQuotaError(error: Error, authType?: string): Error {
|
||||
const isQuotaError =
|
||||
error instanceof TerminalQuotaError ||
|
||||
error instanceof RetryableQuotaError ||
|
||||
error.name === 'TerminalQuotaError' ||
|
||||
error.name === 'RetryableQuotaError';
|
||||
|
||||
if (
|
||||
isQuotaError &&
|
||||
(authType === 'oauth-personal' ||
|
||||
authType === 'compute-default-credentials' ||
|
||||
authType === 'LOGIN_WITH_GOOGLE' ||
|
||||
authType === 'COMPUTE_ADC')
|
||||
) {
|
||||
const hasUserProject = !!(
|
||||
process.env['GOOGLE_CLOUD_PROJECT'] ||
|
||||
process.env['GOOGLE_CLOUD_PROJECT_ID']
|
||||
);
|
||||
if (!hasUserProject) {
|
||||
const enrichment =
|
||||
'\n\n💡 Tip: The shared Google Cloud project is experiencing high traffic and has hit its quota limits. ' +
|
||||
'To get dedicated, uninterrupted quota, please set your own Google Cloud project by running:\n' +
|
||||
' gcloud config set project [PROJECT_ID]\n' +
|
||||
'or by setting the GOOGLE_CLOUD_PROJECT environment variable.';
|
||||
if (!error.message.includes('💡 Tip:')) {
|
||||
Object.defineProperty(error, 'message', {
|
||||
value: error.message + enrichment,
|
||||
writable: true,
|
||||
configurable: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retries a function with exponential backoff and jitter.
|
||||
* @param fn The asynchronous function to retry.
|
||||
@@ -321,7 +361,9 @@ export async function retryWithBackoff<T>(
|
||||
}
|
||||
}
|
||||
// Terminal/not_found already recorded; nothing else to mark here.
|
||||
throw classifiedError; // Throw if no fallback or fallback failed.
|
||||
throw classifiedError instanceof Error
|
||||
? enrichQuotaError(classifiedError, authType)
|
||||
: classifiedError; // Throw if no fallback or fallback failed.
|
||||
}
|
||||
|
||||
// Handle ValidationRequiredError - user needs to verify before proceeding
|
||||
@@ -370,7 +412,7 @@ export async function retryWithBackoff<T>(
|
||||
}
|
||||
}
|
||||
throw classifiedError instanceof RetryableQuotaError
|
||||
? classifiedError
|
||||
? enrichQuotaError(classifiedError, authType)
|
||||
: error;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import { retryWithBackoff } from './retry.js';
|
||||
import { AuthType } from '../core/contentGenerator.js';
|
||||
import { TerminalQuotaError } from './googleQuotaErrors.js';
|
||||
import type { GoogleApiError } from './googleErrors.js';
|
||||
|
||||
vi.mock('node:fs');
|
||||
|
||||
describe('Shared Project Throttling Integration', () => {
|
||||
let mockGoogleApiError: GoogleApiError;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(fs.existsSync).mockReturnValue(true);
|
||||
vi.mocked(fs.statSync).mockReturnValue({
|
||||
isDirectory: () => true,
|
||||
} as fs.Stats);
|
||||
mockGoogleApiError = {
|
||||
code: 429,
|
||||
message:
|
||||
'Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_requests',
|
||||
details: [
|
||||
{
|
||||
'@type': 'type.googleapis.com/google.rpc.QuotaFailure',
|
||||
violations: [
|
||||
{
|
||||
quotaMetric:
|
||||
'generativelanguage.googleapis.com/generate_content_requests',
|
||||
quotaId:
|
||||
'GenerateRequestsPerMinutePerProjectPerModel-SharedProject',
|
||||
quotaDimensions: {
|
||||
location: 'global',
|
||||
model: 'gemini-2.5-pro',
|
||||
},
|
||||
quotaValue: '0',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it('fails completely when both Pro and Flash fallback models hit shared project quota limits', async () => {
|
||||
let currentModel = 'gemini-2.5-pro';
|
||||
const modelsAttempted: string[] = [];
|
||||
|
||||
// Simulate API calls that fail on both models
|
||||
const mockApiCall = vi.fn().mockImplementation(async () => {
|
||||
modelsAttempted.push(currentModel);
|
||||
throw new TerminalQuotaError(
|
||||
`Quota exhausted for model ${currentModel} on shared project`,
|
||||
mockGoogleApiError,
|
||||
);
|
||||
});
|
||||
|
||||
// Fallback handler changes the active model to Flash on persistent 429
|
||||
const mockPersistent429Callback = vi.fn(
|
||||
async (_authType?: string, _error?: unknown) => {
|
||||
if (currentModel === 'gemini-2.5-pro') {
|
||||
currentModel = 'gemini-2.5-flash';
|
||||
return 'gemini-2.5-flash';
|
||||
}
|
||||
return null; // No further fallback models
|
||||
},
|
||||
);
|
||||
|
||||
const promise = retryWithBackoff(mockApiCall, {
|
||||
maxAttempts: 1,
|
||||
initialDelayMs: 1,
|
||||
maxDelayMs: 5,
|
||||
onPersistent429: mockPersistent429Callback,
|
||||
authType: AuthType.LOGIN_WITH_GOOGLE,
|
||||
});
|
||||
|
||||
await expect(promise).rejects.toThrow(
|
||||
'Quota exhausted for model gemini-2.5-flash on shared project',
|
||||
);
|
||||
|
||||
// Check that both models were tried and both failed due to the shared project limits
|
||||
expect(modelsAttempted).toEqual(['gemini-2.5-pro', 'gemini-2.5-flash']);
|
||||
expect(mockPersistent429Callback).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('appends helpful troubleshooting hint when no user project is configured and auth is LOGIN_WITH_GOOGLE', async () => {
|
||||
vi.stubEnv('GOOGLE_CLOUD_PROJECT', '');
|
||||
vi.stubEnv('GOOGLE_CLOUD_PROJECT_ID', '');
|
||||
|
||||
const mockApiCall = vi
|
||||
.fn()
|
||||
.mockRejectedValue(
|
||||
new TerminalQuotaError('Daily limit reached', mockGoogleApiError),
|
||||
);
|
||||
|
||||
const promise = retryWithBackoff(mockApiCall, {
|
||||
maxAttempts: 1,
|
||||
initialDelayMs: 1,
|
||||
maxDelayMs: 5,
|
||||
authType: AuthType.LOGIN_WITH_GOOGLE,
|
||||
});
|
||||
|
||||
let caughtError: Error | undefined;
|
||||
try {
|
||||
await promise;
|
||||
} catch (e) {
|
||||
caughtError = e instanceof Error ? e : new Error(String(e));
|
||||
}
|
||||
|
||||
expect(caughtError).toBeDefined();
|
||||
expect(caughtError?.message).toContain(
|
||||
'💡 Tip: The shared Google Cloud project is experiencing high traffic',
|
||||
);
|
||||
expect(caughtError?.message).toContain(
|
||||
'gcloud config set project [PROJECT_ID]',
|
||||
);
|
||||
});
|
||||
|
||||
it('does not append troubleshooting hint if a dedicated user project is set in environment', async () => {
|
||||
vi.stubEnv('GOOGLE_CLOUD_PROJECT', 'my-dedicated-project-123');
|
||||
|
||||
const mockApiCall = vi
|
||||
.fn()
|
||||
.mockRejectedValue(
|
||||
new TerminalQuotaError('Daily limit reached', mockGoogleApiError),
|
||||
);
|
||||
|
||||
const promise = retryWithBackoff(mockApiCall, {
|
||||
maxAttempts: 1,
|
||||
initialDelayMs: 1,
|
||||
maxDelayMs: 5,
|
||||
authType: AuthType.LOGIN_WITH_GOOGLE,
|
||||
});
|
||||
|
||||
const caughtError = await promise.catch((e) => e);
|
||||
const errorMsg =
|
||||
caughtError instanceof Error ? caughtError.message : String(caughtError);
|
||||
expect(errorMsg).not.toContain('💡 Tip:');
|
||||
});
|
||||
|
||||
it('does not append troubleshooting hint for non-Google/ADC auth types', async () => {
|
||||
vi.stubEnv('GOOGLE_CLOUD_PROJECT', '');
|
||||
|
||||
const mockApiCall = vi
|
||||
.fn()
|
||||
.mockRejectedValue(
|
||||
new TerminalQuotaError('Daily limit reached', mockGoogleApiError),
|
||||
);
|
||||
|
||||
const promise = retryWithBackoff(mockApiCall, {
|
||||
maxAttempts: 1,
|
||||
initialDelayMs: 1,
|
||||
maxDelayMs: 5,
|
||||
authType: AuthType.USE_GEMINI, // API Key auth type
|
||||
});
|
||||
|
||||
const caughtError = await promise.catch((e) => e);
|
||||
const errorMsg =
|
||||
caughtError instanceof Error ? caughtError.message : String(caughtError);
|
||||
expect(errorMsg).not.toContain('💡 Tip:');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user