fix(core): enrich shared project quota limit errors with setup hint (#28391)

This commit is contained in:
amelidev
2026-07-13 11:22:16 -06:00
committed by GitHub
parent f354eebaf4
commit 42ee2b74c7
2 changed files with 213 additions and 2 deletions
+44 -2
View File
@@ -208,6 +208,46 @@ export function isRetryableError(
return false;
}
/**
* Enriches quota-related errors with helpful hints if using a shared Google project
* without a dedicated user project set in their environment.
*/
function enrichQuotaError(error: Error, authType?: string): Error {
const isQuotaError =
error instanceof TerminalQuotaError ||
error instanceof RetryableQuotaError ||
error.name === 'TerminalQuotaError' ||
error.name === 'RetryableQuotaError';
if (
isQuotaError &&
(authType === 'oauth-personal' ||
authType === 'compute-default-credentials' ||
authType === 'LOGIN_WITH_GOOGLE' ||
authType === 'COMPUTE_ADC')
) {
const hasUserProject = !!(
process.env['GOOGLE_CLOUD_PROJECT'] ||
process.env['GOOGLE_CLOUD_PROJECT_ID']
);
if (!hasUserProject) {
const enrichment =
'\n\n💡 Tip: The shared Google Cloud project is experiencing high traffic and has hit its quota limits. ' +
'To get dedicated, uninterrupted quota, please set your own Google Cloud project by running:\n' +
' gcloud config set project [PROJECT_ID]\n' +
'or by setting the GOOGLE_CLOUD_PROJECT environment variable.';
if (!error.message.includes('💡 Tip:')) {
Object.defineProperty(error, 'message', {
value: error.message + enrichment,
writable: true,
configurable: true,
});
}
}
}
return error;
}
/**
* Retries a function with exponential backoff and jitter.
* @param fn The asynchronous function to retry.
@@ -321,7 +361,9 @@ export async function retryWithBackoff<T>(
}
}
// Terminal/not_found already recorded; nothing else to mark here.
throw classifiedError; // Throw if no fallback or fallback failed.
throw classifiedError instanceof Error
? enrichQuotaError(classifiedError, authType)
: classifiedError; // Throw if no fallback or fallback failed.
}
// Handle ValidationRequiredError - user needs to verify before proceeding
@@ -370,7 +412,7 @@ export async function retryWithBackoff<T>(
}
}
throw classifiedError instanceof RetryableQuotaError
? classifiedError
? enrichQuotaError(classifiedError, authType)
: error;
}
@@ -0,0 +1,169 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
import fs from 'node:fs';
import { retryWithBackoff } from './retry.js';
import { AuthType } from '../core/contentGenerator.js';
import { TerminalQuotaError } from './googleQuotaErrors.js';
import type { GoogleApiError } from './googleErrors.js';
vi.mock('node:fs');
describe('Shared Project Throttling Integration', () => {
let mockGoogleApiError: GoogleApiError;
beforeEach(() => {
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.statSync).mockReturnValue({
isDirectory: () => true,
} as fs.Stats);
mockGoogleApiError = {
code: 429,
message:
'Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_requests',
details: [
{
'@type': 'type.googleapis.com/google.rpc.QuotaFailure',
violations: [
{
quotaMetric:
'generativelanguage.googleapis.com/generate_content_requests',
quotaId:
'GenerateRequestsPerMinutePerProjectPerModel-SharedProject',
quotaDimensions: {
location: 'global',
model: 'gemini-2.5-pro',
},
quotaValue: '0',
},
],
},
],
};
});
afterEach(() => {
vi.unstubAllEnvs();
});
it('fails completely when both Pro and Flash fallback models hit shared project quota limits', async () => {
let currentModel = 'gemini-2.5-pro';
const modelsAttempted: string[] = [];
// Simulate API calls that fail on both models
const mockApiCall = vi.fn().mockImplementation(async () => {
modelsAttempted.push(currentModel);
throw new TerminalQuotaError(
`Quota exhausted for model ${currentModel} on shared project`,
mockGoogleApiError,
);
});
// Fallback handler changes the active model to Flash on persistent 429
const mockPersistent429Callback = vi.fn(
async (_authType?: string, _error?: unknown) => {
if (currentModel === 'gemini-2.5-pro') {
currentModel = 'gemini-2.5-flash';
return 'gemini-2.5-flash';
}
return null; // No further fallback models
},
);
const promise = retryWithBackoff(mockApiCall, {
maxAttempts: 1,
initialDelayMs: 1,
maxDelayMs: 5,
onPersistent429: mockPersistent429Callback,
authType: AuthType.LOGIN_WITH_GOOGLE,
});
await expect(promise).rejects.toThrow(
'Quota exhausted for model gemini-2.5-flash on shared project',
);
// Check that both models were tried and both failed due to the shared project limits
expect(modelsAttempted).toEqual(['gemini-2.5-pro', 'gemini-2.5-flash']);
expect(mockPersistent429Callback).toHaveBeenCalledTimes(2);
});
it('appends helpful troubleshooting hint when no user project is configured and auth is LOGIN_WITH_GOOGLE', async () => {
vi.stubEnv('GOOGLE_CLOUD_PROJECT', '');
vi.stubEnv('GOOGLE_CLOUD_PROJECT_ID', '');
const mockApiCall = vi
.fn()
.mockRejectedValue(
new TerminalQuotaError('Daily limit reached', mockGoogleApiError),
);
const promise = retryWithBackoff(mockApiCall, {
maxAttempts: 1,
initialDelayMs: 1,
maxDelayMs: 5,
authType: AuthType.LOGIN_WITH_GOOGLE,
});
let caughtError: Error | undefined;
try {
await promise;
} catch (e) {
caughtError = e instanceof Error ? e : new Error(String(e));
}
expect(caughtError).toBeDefined();
expect(caughtError?.message).toContain(
'💡 Tip: The shared Google Cloud project is experiencing high traffic',
);
expect(caughtError?.message).toContain(
'gcloud config set project [PROJECT_ID]',
);
});
it('does not append troubleshooting hint if a dedicated user project is set in environment', async () => {
vi.stubEnv('GOOGLE_CLOUD_PROJECT', 'my-dedicated-project-123');
const mockApiCall = vi
.fn()
.mockRejectedValue(
new TerminalQuotaError('Daily limit reached', mockGoogleApiError),
);
const promise = retryWithBackoff(mockApiCall, {
maxAttempts: 1,
initialDelayMs: 1,
maxDelayMs: 5,
authType: AuthType.LOGIN_WITH_GOOGLE,
});
const caughtError = await promise.catch((e) => e);
const errorMsg =
caughtError instanceof Error ? caughtError.message : String(caughtError);
expect(errorMsg).not.toContain('💡 Tip:');
});
it('does not append troubleshooting hint for non-Google/ADC auth types', async () => {
vi.stubEnv('GOOGLE_CLOUD_PROJECT', '');
const mockApiCall = vi
.fn()
.mockRejectedValue(
new TerminalQuotaError('Daily limit reached', mockGoogleApiError),
);
const promise = retryWithBackoff(mockApiCall, {
maxAttempts: 1,
initialDelayMs: 1,
maxDelayMs: 5,
authType: AuthType.USE_GEMINI, // API Key auth type
});
const caughtError = await promise.catch((e) => e);
const errorMsg =
caughtError instanceof Error ? caughtError.message : String(caughtError);
expect(errorMsg).not.toContain('💡 Tip:');
});
});