Compare commits

..

2 Commits

Author SHA1 Message Date
jacob314 68e33a1103 fix(test): regenerate settings documentation and schema 2026-04-23 14:34:43 -07:00
jacob314 19601955fd feat(routing): availability-aware auto-routing with best-effort pro
Adds settings and logic to detect slow/hanging Pro model requests, marking them as temporarily unavailable and automatically triggering a fallback to Flash. Introduces a proTimeoutMinutes and bestEffortPro strategy configuration.
2026-04-23 13:50:24 -07:00
27 changed files with 367 additions and 356 deletions
+10 -7
View File
@@ -99,13 +99,16 @@ they appear in the UI.
### Model
| UI Label | Setting | Description | Default |
| ----------------------------- | ---------------------------- | -------------------------------------------------------------------------------------- | ----------- |
| Model | `model.name` | The Gemini model to use for conversations. | `undefined` |
| Max Session Turns | `model.maxSessionTurns` | Maximum number of user/model/tool turns to keep in a session. -1 means unlimited. | `-1` |
| Context Compression Threshold | `model.compressionThreshold` | The fraction of context usage at which to trigger context compression (e.g. 0.2, 0.3). | `0.5` |
| Disable Loop Detection | `model.disableLoopDetection` | Disable automatic detection and prevention of infinite loops. | `false` |
| Skip Next Speaker Check | `model.skipNextSpeakerCheck` | Skip the next speaker check. | `true` |
| UI Label | Setting | Description | Default |
| --------------------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ----------- |
| Model | `model.name` | The Gemini model to use for conversations. | `undefined` |
| Max Session Turns | `model.maxSessionTurns` | Maximum number of user/model/tool turns to keep in a session. -1 means unlimited. | `-1` |
| Context Compression Threshold | `model.compressionThreshold` | The fraction of context usage at which to trigger context compression (e.g. 0.2, 0.3). | `0.5` |
| Disable Loop Detection | `model.disableLoopDetection` | Disable automatic detection and prevention of infinite loops. | `false` |
| Skip Next Speaker Check | `model.skipNextSpeakerCheck` | Skip the next speaker check. | `true` |
| Best Effort Pro | `model.autoRouting.bestEffortPro` | Always prefer the Pro model unless it is unavailable (e.g., due to timeouts or quota), ignoring other routing hints. | `false` |
| Pro Timeout (Minutes) | `model.autoRouting.proTimeoutMinutes` | If a Pro request takes longer than this many minutes, it will be marked as temporarily unavailable and fallback to Flash. | `5` |
| Pro Timeout Fallback Duration (Minutes) | `model.autoRouting.proTimeoutFallbackDurationMinutes` | How long to route to Flash after Pro times out. | `60` |
### Agents
+14
View File
@@ -483,6 +483,20 @@ their corresponding top-level category object in your `settings.json` file.
- **Description:** Skip the next speaker check.
- **Default:** `true`
- **`model.autoRouting.bestEffortPro`** (boolean):
- **Description:** Always prefer the Pro model unless it is unavailable (e.g.,
due to timeouts or quota), ignoring other routing hints.
- **Default:** `false`
- **`model.autoRouting.proTimeoutMinutes`** (number):
- **Description:** If a Pro request takes longer than this many minutes, it
will be marked as temporarily unavailable and fallback to Flash.
- **Default:** `5`
- **`model.autoRouting.proTimeoutFallbackDurationMinutes`** (number):
- **Description:** How long to route to Flash after Pro times out.
- **Default:** `60`
#### `modelConfigs`
- **`modelConfigs.aliases`** (object):
@@ -58,7 +58,7 @@ describe.skipIf(skipOnDarwin)('Interactive Mode', () => {
);
await run.expectText('Chat history compressed', 5000);
}, 60000);
});
// TODO: Context compression is broken and doesn't include the system
// instructions or tool counts, so it thinks compression is beneficial when
@@ -102,9 +102,6 @@ describe.skipIf(skipOnDarwin)('Interactive Mode', () => {
});
const run = await rig.runInteractive();
await run.expectText('tips for getting started:', 5000);
// Wait for the async command loaders to finish so /compress is recognized
await new Promise((r) => setTimeout(r, 2000));
await run.type('/compress');
await run.type('\r');
@@ -119,5 +116,5 @@ describe.skipIf(skipOnDarwin)('Interactive Mode', () => {
foundEvent,
'chat_compression telemetry event should not be found for NOOP',
).toBe(false);
}, 60000);
});
});
@@ -33,8 +33,6 @@ describe('Interactive file system', () => {
rig.createFile(fileName, '1.0.0');
const run = await rig.runInteractive();
await run.expectText('tips for getting started:', 5000);
await new Promise((r) => setTimeout(r, 1000));
// Step 1: Read the file
const readPrompt = `Read the version from ${fileName}`;
@@ -58,5 +56,5 @@ describe('Interactive file system', () => {
// Wait for telemetry to flush and file system to sync, especially in sandboxed environments
await rig.waitForTelemetryReady();
}, 60000);
});
});
+1
View File
@@ -1044,6 +1044,7 @@ export async function loadCliConfig(
format: (argv.outputFormat ?? settings.output?.format) as OutputFormat,
},
gemmaModelRouter: settings.experimental?.gemmaModelRouter,
autoRouting: settings.model?.autoRouting,
adk: settings.experimental?.adk,
fakeResponses: argv.fakeResponses,
recordResponses: argv.recordResponses,
+40
View File
@@ -1112,6 +1112,46 @@ const SETTINGS_SCHEMA = {
description: 'Skip the next speaker check.',
showInDialog: true,
},
autoRouting: {
type: 'object',
label: 'Auto Routing',
category: 'Model',
requiresRestart: false,
default: {},
description: 'Settings for automatic model routing.',
showInDialog: false,
properties: {
bestEffortPro: {
type: 'boolean',
label: 'Best Effort Pro',
category: 'Model',
requiresRestart: false,
default: false,
description:
'Always prefer the Pro model unless it is unavailable (e.g., due to timeouts or quota), ignoring other routing hints.',
showInDialog: true,
},
proTimeoutMinutes: {
type: 'number',
label: 'Pro Timeout (Minutes)',
category: 'Model',
requiresRestart: false,
default: 5,
description:
'If a Pro request takes longer than this many minutes, it will be marked as temporarily unavailable and fallback to Flash.',
showInDialog: true,
},
proTimeoutFallbackDurationMinutes: {
type: 'number',
label: 'Pro Timeout Fallback Duration (Minutes)',
category: 'Model',
requiresRestart: false,
default: 60,
description: 'How long to route to Flash after Pro times out.',
showInDialog: true,
},
},
},
},
},
@@ -124,6 +124,9 @@ export const createMockConfig = (overrides: Partial<Config> = {}): Config =>
getCompressionThreshold: vi.fn().mockResolvedValue(undefined),
getUserCaching: vi.fn().mockResolvedValue(false),
getNumericalRoutingEnabled: vi.fn().mockResolvedValue(false),
getBestEffortProEnabled: vi.fn().mockResolvedValue(false),
getProTimeoutMinutes: vi.fn().mockResolvedValue(5),
getProTimeoutFallbackDurationMinutes: vi.fn().mockResolvedValue(60),
getClassifierThreshold: vi.fn().mockResolvedValue(undefined),
getBannerTextNoCapacityIssues: vi.fn().mockResolvedValue(''),
getBannerTextCapacityIssues: vi.fn().mockResolvedValue(''),
@@ -8,13 +8,15 @@ export type ModelId = string;
type TerminalUnavailabilityReason = 'quota' | 'capacity';
export type TurnUnavailabilityReason = 'retry_once_per_turn';
export type TemporaryUnavailabilityReason = 'timeout';
export type UnavailabilityReason =
| TerminalUnavailabilityReason
| TurnUnavailabilityReason
| TemporaryUnavailabilityReason
| 'unknown';
export type ModelHealthStatus = 'terminal' | 'sticky_retry';
export type ModelHealthStatus = 'terminal' | 'sticky_retry' | 'temporary';
type HealthState =
| { status: 'terminal'; reason: TerminalUnavailabilityReason }
@@ -22,6 +24,11 @@ type HealthState =
status: 'sticky_retry';
reason: TurnUnavailabilityReason;
consumed: boolean;
}
| {
status: 'temporary';
reason: TemporaryUnavailabilityReason;
untilMs: number;
};
export interface ModelAvailabilitySnapshot {
@@ -48,6 +55,18 @@ export class ModelAvailabilityService {
});
}
markTemporarilyUnavailable(
model: ModelId,
reason: TemporaryUnavailabilityReason,
durationMs: number,
) {
this.setState(model, {
status: 'temporary',
reason,
untilMs: Date.now() + durationMs,
});
}
markHealthy(model: ModelId) {
this.clearState(model);
}
@@ -95,6 +114,15 @@ export class ModelAvailabilityService {
return { available: false, reason: state.reason };
}
if (state.status === 'temporary') {
if (Date.now() < state.untilMs) {
return { available: false, reason: state.reason };
} else {
this.clearState(model);
return { available: true };
}
}
return { available: true };
}
+25
View File
@@ -679,6 +679,11 @@ export interface ConfigParameters {
policyUpdateConfirmationRequest?: PolicyUpdateConfirmationRequest;
output?: OutputSettings;
gemmaModelRouter?: GemmaModelRouterSettings;
autoRouting?: {
bestEffortPro?: boolean;
proTimeoutMinutes?: number;
proTimeoutFallbackDurationMinutes?: number;
};
adk?: ADKSettings;
disableModelRouterForAuth?: AuthType[];
continueOnFailedApiCall?: boolean;
@@ -963,6 +968,9 @@ export class Config implements McpContext, AgentLoopContext {
private readonly planEnabled: boolean;
private readonly trackerEnabled: boolean;
private readonly planModeRoutingEnabled: boolean;
private readonly autoRoutingBestEffortPro: boolean;
private readonly autoRoutingProTimeoutMinutes: number;
private readonly autoRoutingProTimeoutFallbackDurationMinutes: number;
private readonly modelSteering: boolean;
private memoryContextManager?: MemoryContextManager;
private readonly contextManagement: ContextManagementConfig;
@@ -1117,6 +1125,11 @@ export class Config implements McpContext, AgentLoopContext {
this.planEnabled = params.plan ?? true;
this.trackerEnabled = params.tracker ?? false;
this.planModeRoutingEnabled = params.planSettings?.modelRouting ?? true;
this.autoRoutingBestEffortPro = params.autoRouting?.bestEffortPro ?? false;
this.autoRoutingProTimeoutMinutes =
params.autoRouting?.proTimeoutMinutes ?? 5;
this.autoRoutingProTimeoutFallbackDurationMinutes =
params.autoRouting?.proTimeoutFallbackDurationMinutes ?? 60;
this.enableEventDrivenScheduler = params.enableEventDrivenScheduler ?? true;
this.skillsSupport = params.skillsSupport ?? true;
this.disabledSkills = params.disabledSkills ?? [];
@@ -3144,6 +3157,18 @@ export class Config implements McpContext, AgentLoopContext {
return flag?.boolValue ?? true;
}
async getBestEffortProEnabled(): Promise<boolean> {
return this.autoRoutingBestEffortPro;
}
async getProTimeoutMinutes(): Promise<number> {
return this.autoRoutingProTimeoutMinutes;
}
async getProTimeoutFallbackDurationMinutes(): Promise<number> {
return this.autoRoutingProTimeoutFallbackDurationMinutes;
}
/**
* Returns the resolved complexity threshold for routing.
* If a remote threshold is provided and within range (0-100), it is returned.
@@ -109,6 +109,8 @@ describe('BaseLlmClient', () => {
.mockReturnValue({ authType: AuthType.USE_GEMINI }),
getEmbeddingModel: vi.fn().mockReturnValue('test-embedding-model'),
isInteractive: vi.fn().mockReturnValue(false),
getProTimeoutMinutes: vi.fn().mockResolvedValue(5),
getProTimeoutFallbackDurationMinutes: vi.fn().mockResolvedValue(60),
modelConfigService: {
getResolvedConfig: vi
.fn()
+8
View File
@@ -325,11 +325,19 @@ export class BaseLlmClient {
);
};
const proTimeoutMinutes = await this.config.getProTimeoutMinutes();
const proTimeoutFallbackDurationMinutes =
await this.config.getProTimeoutFallbackDurationMinutes();
return await retryWithBackoff(apiCall, {
shouldRetryOnContent,
maxAttempts:
availabilityMaxAttempts ?? maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
getAvailabilityContext,
timeoutFallback: {
timeoutMs: proTimeoutMinutes * 60 * 1000,
fallbackDurationMs: proTimeoutFallbackDurationMinutes * 60 * 1000,
},
onPersistent429: this.config.isInteractive()
? (authType, error) =>
handleFallback(this.config, currentModel, authType, error)
@@ -160,6 +160,8 @@ describe('GeminiChat', () => {
authType: 'oauth-personal',
model: currentModel,
})),
getProTimeoutMinutes: vi.fn().mockResolvedValue(5),
getProTimeoutFallbackDurationMinutes: vi.fn().mockResolvedValue(60),
getModel: vi.fn().mockImplementation(() => currentModel),
setModel: vi.fn().mockImplementation((m: string) => {
currentModel = m;
+8
View File
@@ -687,6 +687,10 @@ export class GeminiChat {
);
};
const proTimeoutMinutes = await this.context.config.getProTimeoutMinutes();
const proTimeoutFallbackDurationMinutes =
await this.context.config.getProTimeoutFallbackDurationMinutes();
const streamResponse = await retryWithBackoff(apiCall, {
onPersistent429: onPersistent429Callback,
onValidationRequired: onValidationRequiredCallback,
@@ -696,6 +700,10 @@ export class GeminiChat {
maxAttempts:
availabilityMaxAttempts ?? this.context.config.getMaxAttempts(),
getAvailabilityContext,
timeoutFallback: {
timeoutMs: proTimeoutMinutes * 60 * 1000,
fallbackDurationMs: proTimeoutFallbackDurationMinutes * 60 * 1000,
},
onRetry: (attempt, error, delayMs) => {
coreEvents.emitRetryAttempt({
attempt,
@@ -103,6 +103,8 @@ describe('GeminiChat Network Retries', () => {
authType: 'oauth-personal',
model: 'test-model',
}),
getProTimeoutMinutes: vi.fn().mockResolvedValue(5),
getProTimeoutFallbackDurationMinutes: vi.fn().mockResolvedValue(60),
getModel: vi.fn().mockReturnValue('gemini-pro'),
getActiveModel: vi.fn().mockReturnValue('gemini-pro'),
setActiveModel: vi.fn(),
@@ -32,6 +32,9 @@ vi.mock('./strategies/overrideStrategy.js');
vi.mock('./strategies/approvalModeStrategy.js');
vi.mock('./strategies/classifierStrategy.js');
vi.mock('./strategies/numericalClassifierStrategy.js');
import { BestEffortProStrategy } from './strategies/bestEffortProStrategy.js';
vi.mock('./strategies/bestEffortProStrategy.js');
vi.mock('./strategies/gemmaClassifierStrategy.js');
vi.mock('../telemetry/loggers.js');
vi.mock('../telemetry/types.js');
@@ -74,6 +77,7 @@ describe('ModelRouterService', () => {
[
new FallbackStrategy(),
new OverrideStrategy(),
new BestEffortProStrategy(),
new ApprovalModeStrategy(),
new ClassifierStrategy(),
new NumericalClassifierStrategy(),
@@ -104,13 +108,14 @@ describe('ModelRouterService', () => {
const compositeStrategyArgs = vi.mocked(CompositeStrategy).mock.calls[0];
const childStrategies = compositeStrategyArgs[0];
expect(childStrategies.length).toBe(6);
expect(childStrategies.length).toBe(7);
expect(childStrategies[0]).toBeInstanceOf(FallbackStrategy);
expect(childStrategies[1]).toBeInstanceOf(OverrideStrategy);
expect(childStrategies[2]).toBeInstanceOf(ApprovalModeStrategy);
expect(childStrategies[3]).toBeInstanceOf(ClassifierStrategy);
expect(childStrategies[4]).toBeInstanceOf(NumericalClassifierStrategy);
expect(childStrategies[5]).toBeInstanceOf(DefaultStrategy);
expect(childStrategies[2]).toBeInstanceOf(BestEffortProStrategy);
expect(childStrategies[3]).toBeInstanceOf(ApprovalModeStrategy);
expect(childStrategies[4]).toBeInstanceOf(ClassifierStrategy);
expect(childStrategies[5]).toBeInstanceOf(NumericalClassifierStrategy);
expect(childStrategies[6]).toBeInstanceOf(DefaultStrategy);
expect(compositeStrategyArgs[1]).toBe('agent-router');
});
@@ -133,14 +138,15 @@ describe('ModelRouterService', () => {
const compositeStrategyArgs = vi.mocked(CompositeStrategy).mock.calls[0];
const childStrategies = compositeStrategyArgs[0];
expect(childStrategies.length).toBe(7);
expect(childStrategies.length).toBe(8);
expect(childStrategies[0]).toBeInstanceOf(FallbackStrategy);
expect(childStrategies[1]).toBeInstanceOf(OverrideStrategy);
expect(childStrategies[2]).toBeInstanceOf(ApprovalModeStrategy);
expect(childStrategies[3]).toBeInstanceOf(GemmaClassifierStrategy);
expect(childStrategies[4]).toBeInstanceOf(ClassifierStrategy);
expect(childStrategies[5]).toBeInstanceOf(NumericalClassifierStrategy);
expect(childStrategies[6]).toBeInstanceOf(DefaultStrategy);
expect(childStrategies[2]).toBeInstanceOf(BestEffortProStrategy);
expect(childStrategies[3]).toBeInstanceOf(ApprovalModeStrategy);
expect(childStrategies[4]).toBeInstanceOf(GemmaClassifierStrategy);
expect(childStrategies[5]).toBeInstanceOf(ClassifierStrategy);
expect(childStrategies[6]).toBeInstanceOf(NumericalClassifierStrategy);
expect(childStrategies[7]).toBeInstanceOf(DefaultStrategy);
expect(compositeStrategyArgs[1]).toBe('agent-router');
});
@@ -18,6 +18,7 @@ import { NumericalClassifierStrategy } from './strategies/numericalClassifierStr
import { CompositeStrategy } from './strategies/compositeStrategy.js';
import { FallbackStrategy } from './strategies/fallbackStrategy.js';
import { OverrideStrategy } from './strategies/overrideStrategy.js';
import { BestEffortProStrategy } from './strategies/bestEffortProStrategy.js';
import { ApprovalModeStrategy } from './strategies/approvalModeStrategy.js';
import { logModelRouting } from '../telemetry/loggers.js';
@@ -43,6 +44,9 @@ export class ModelRouterService {
strategies.push(new FallbackStrategy());
strategies.push(new OverrideStrategy());
// Best Effort Pro is next.
strategies.push(new BestEffortProStrategy());
// Approval mode is next.
strategies.push(new ApprovalModeStrategy());
@@ -0,0 +1,83 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../../config/config.js';
import { isAutoModel, resolveModel } from '../../config/models.js';
import type {
RoutingStrategy,
RoutingDecision,
RoutingContext,
} from '../routingStrategy.js';
/**
* A routing strategy that respects the "Best Effort Pro" setting.
* If the setting is enabled and the Pro model is available, it routes to Pro
* regardless of complexity. If Pro is unavailable, it routes to Flash.
*/
export class BestEffortProStrategy implements RoutingStrategy {
name = 'best-effort-pro';
async route(
context: RoutingContext,
config: Config,
): Promise<RoutingDecision | null> {
const requestedModel = config.getModel();
if (!isAutoModel(requestedModel)) {
return null;
}
const isBestEffortProEnabled = await config.getBestEffortProEnabled();
if (!isBestEffortProEnabled) {
return null;
}
const useGemini3_1 = (await config.getGemini31Launched?.()) ?? false;
const useGemini3_1FlashLite =
(await config.getGemini31FlashLiteLaunched?.()) ?? false;
const hasAccessToPreview = config.getHasAccessToPreviewModel?.() ?? true;
const availabilityService = config.getModelAvailabilityService();
const proModel = resolveModel(
'gemini-3.1-pro',
useGemini3_1,
useGemini3_1FlashLite,
false,
hasAccessToPreview,
config,
);
const flashModel = resolveModel(
'gemini-3.1-flash',
useGemini3_1,
useGemini3_1FlashLite,
false,
hasAccessToPreview,
config,
);
const proSnapshot = availabilityService.snapshot(proModel);
if (proSnapshot.available) {
return {
model: proModel,
metadata: {
source: this.name,
latencyMs: 0,
reasoning:
'Best Effort Pro is enabled and the Pro model is available.',
},
};
} else {
return {
model: flashModel,
metadata: {
source: this.name,
latencyMs: 0,
reasoning: `Best Effort Pro is enabled, but Pro is unavailable (${proSnapshot.reason}). Falling back to Flash.`,
},
};
}
}
}
+2 -4
View File
@@ -121,10 +121,8 @@ describe('Telemetry Metrics', () => {
return actualApi;
});
const { getCommonMetricAttributes } = await import(
'./telemetryAttributes.js'
);
(getCommonMetricAttributes as Mock).mockReturnValue({
const { getCommonAttributes } = await import('./telemetryAttributes.js');
(getCommonAttributes as Mock).mockReturnValue({
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
+2 -2
View File
@@ -24,7 +24,7 @@ import type {
TokenStorageInitializationEvent,
} from './types.js';
import { AuthType } from '../core/contentGenerator.js';
import { getCommonMetricAttributes } from './telemetryAttributes.js';
import { getCommonAttributes } from './telemetryAttributes.js';
import { sanitizeHookName } from './sanitize.js';
const EVENT_CHAT_COMPRESSION = 'gemini_cli.chat_compression';
@@ -104,7 +104,7 @@ const EXIT_FAIL_COUNT = 'gemini_cli.exit.fail.count';
const PLAN_EXECUTION_COUNT = 'gemini_cli.plan.execution.count';
const baseMetricDefinition = {
getCommonAttributes: getCommonMetricAttributes,
getCommonAttributes,
};
const COUNTER_DEFINITIONS = {
@@ -1,115 +0,0 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest';
import {
getCommonAttributes,
getCommonMetricAttributes,
} from './telemetryAttributes.js';
import type { Config } from '../config/config.js';
import { UserAccountManager } from '../utils/userAccountManager.js';
import { InstallationManager } from '../utils/installationManager.js';
vi.mock('../utils/userAccountManager.js');
vi.mock('../utils/installationManager.js');
describe('telemetryAttributes', () => {
let mockConfig: Partial<Config>;
beforeEach(() => {
vi.resetAllMocks();
mockConfig = {
getSessionId: vi.fn().mockReturnValue('mock-session-id'),
isInteractive: vi.fn().mockReturnValue(true),
getExperiments: vi.fn().mockReturnValue(undefined),
getContentGeneratorConfig: vi.fn().mockReturnValue(undefined),
};
(
UserAccountManager.prototype.getCachedGoogleAccount as Mock
).mockReturnValue(undefined);
(InstallationManager.prototype.getInstallationId as Mock).mockReturnValue(
'mock-install-id',
);
});
describe('getCommonMetricAttributes', () => {
it('should return interactive and auth_type when defined', () => {
mockConfig.getContentGeneratorConfig = vi
.fn()
.mockReturnValue({ authType: 'oauth-personal' });
const attributes = getCommonMetricAttributes(mockConfig as Config);
expect(attributes).toEqual({
interactive: true,
auth_type: 'oauth-personal',
});
});
it('should return only interactive when auth_type is not defined', () => {
const attributes = getCommonMetricAttributes(mockConfig as Config);
expect(attributes).toEqual({
interactive: true,
});
});
});
describe('getCommonAttributes', () => {
it('should include all common attributes', () => {
(
UserAccountManager.prototype.getCachedGoogleAccount as Mock
).mockReturnValue('test@google.com');
mockConfig.getExperiments = vi
.fn()
.mockReturnValue({ experimentIds: [123, 456] });
mockConfig.getContentGeneratorConfig = vi
.fn()
.mockReturnValue({ authType: 'adc' });
const attributes = getCommonAttributes(mockConfig as Config);
expect(attributes).toEqual({
'session.id': 'mock-session-id',
'installation.id': 'mock-install-id',
interactive: true,
'user.email': 'test@google.com',
auth_type: 'adc',
'experiments.ids': '123,456',
});
});
it('should safely truncate experiments string to not exceed 1000 characters and not cut mid-ID', () => {
// Generate a list of experiment IDs that will produce a string > 1000 chars
const expIds = [];
for (let i = 0; i < 200; i++) {
// e.g., 100000000 -> 9 chars + 1 comma = 10 chars per ID
expIds.push(100000000 + i);
}
mockConfig.getExperiments = vi
.fn()
.mockReturnValue({ experimentIds: expIds });
const attributes = getCommonAttributes(mockConfig as Config);
const expString = attributes['experiments.ids'] as string;
expect(expString.length).toBeLessThanOrEqual(1000);
// Verify the last ID is complete (not cut off) by checking if it's one of our expected IDs
const ids = expString.split(',');
const lastIdStr = ids[ids.length - 1];
const lastIdNumber = parseInt(lastIdStr, 10);
expect(lastIdNumber).toBeGreaterThanOrEqual(100000000);
expect(lastIdNumber).toBeLessThan(100000200);
// Also ensure no trailing comma
expect(expString.endsWith(',')).toBe(false);
});
});
});
@@ -12,36 +12,19 @@ import { UserAccountManager } from '../utils/userAccountManager.js';
const userAccountManager = new UserAccountManager();
const installationManager = new InstallationManager();
export function getCommonMetricAttributes(config: Config): Attributes {
const authType = config.getContentGeneratorConfig()?.authType;
return {
interactive: config.isInteractive(),
...(authType && { auth_type: authType }),
};
}
export function getCommonAttributes(config: Config): Attributes {
const email = userAccountManager.getCachedGoogleAccount();
const experiments = config.getExperiments();
let experimentsIdsStr = '';
if (experiments && experiments.experimentIds.length > 0) {
experimentsIdsStr = experiments.experimentIds.join(',');
if (experimentsIdsStr.length > 1000) {
experimentsIdsStr = experimentsIdsStr.substring(0, 1000);
const lastCommaIndex = experimentsIdsStr.lastIndexOf(',');
if (lastCommaIndex > 0) {
experimentsIdsStr = experimentsIdsStr.substring(0, lastCommaIndex);
}
}
}
const authType = config.getContentGeneratorConfig()?.authType;
return {
...getCommonMetricAttributes(config),
'session.id': config.getSessionId(),
'installation.id': installationManager.getInstallationId(),
interactive: config.isInteractive(),
...(email && { 'user.email': email }),
...(experimentsIdsStr && { 'experiments.ids': experimentsIdsStr }),
...(authType && { auth_type: authType }),
...(experiments &&
experiments.experimentIds.length > 0 && {
'experiments.ids': experiments.experimentIds,
}),
};
}
+9 -78
View File
@@ -52,95 +52,26 @@ describe('truncateForTelemetry', () => {
});
it('should correctly truncate strings with multi-byte unicode characters (emojis)', () => {
// 5 emojis, each is a surrogate pair (2 code units)
// 5 emojis, each is multiple bytes in UTF-16
const emojis = '👋🌍🚀🔥🎉';
// Truncating to 2 code units
const result = truncateForTelemetry(emojis, 2);
// Truncating to length 5 (which is 2.5 emojis in UTF-16 length terms)
// truncateString will stop after the full grapheme clusters that fit within 5
const result = truncateForTelemetry(emojis, 5);
expect(result).toBe('👋...[TRUNCATED: original length 10]');
expect(result).toBe('👋🌍...[TRUNCATED: original length 10]');
});
it('should stringify and structurally truncate objects if exceeding limits', () => {
it('should stringify and truncate objects if exceeding maxLength', () => {
const obj = { message: 'hello world', nested: { a: 1 } };
const stringified = JSON.stringify(obj);
const result = truncateForTelemetry(obj, 10);
expect(result).toBe(
JSON.stringify({
message: 'hello worl...[TRUNCATED: original length 11]',
nested: { a: 1 },
}),
stringified.substring(0, 10) +
`...[TRUNCATED: original length ${stringified.length}]`,
);
});
it('should structurally truncate arrays and depth', () => {
const obj = {
arr: [1, 2, 3],
deep: { level1: { level2: { level3: { a: 1 } } } },
};
const result = truncateForTelemetry(obj, 100, 2, 3);
expect(result).toBe(
JSON.stringify({
arr: [1, 2, '[TRUNCATED: Array of length 3]'],
deep: { level1: { level2: '[TRUNCATED: Max Depth Reached]' } },
}),
);
});
it('should handle objects with a toJSON method', () => {
const date = new Date('2026-04-13T00:00:00.000Z');
const result = truncateForTelemetry(date);
expect(result).toBe('2026-04-13T00:00:00.000Z');
});
it('should handle getters via direct property access', () => {
const obj = {
get myGetter() {
return 'getter value';
},
get errorGetter() {
throw new Error('getter error');
},
};
const result = truncateForTelemetry(obj);
expect(result).toBe(
JSON.stringify({
myGetter: 'getter value',
errorGetter: '[ERROR: Failed to read property]',
}),
);
});
it('should truncate extremely long keys', () => {
const longKey = 'a'.repeat(150);
const obj = {
[longKey]: 'value',
};
const result = truncateForTelemetry(obj);
const expectedKey = 'a'.repeat(100) + '...[TRUNCATED_KEY]';
expect(result).toBe(
JSON.stringify({
[expectedKey]: 'value',
}),
);
});
it('should enforce a global payload string limit without breaking JSON', () => {
const obj = {
a: 'x'.repeat(100),
b: 'y'.repeat(100),
};
// Capping global string length to 50
const result = truncateForTelemetry(obj, 100, 100, 4, 100, 50) as string;
// It should replace the entire object with a valid JSON string indicating truncation
expect(result).toBe(
'"[TRUNCATED: Payload exceeded global limit of 50 characters. Original length: 215]"',
);
// Prove it remains perfectly parseable JSON
expect(() => JSON.parse(result)).not.toThrow();
});
it('should stringify objects unchanged if within maxLength', () => {
const obj = { a: 1 };
expect(truncateForTelemetry(obj, 100)).toBe(JSON.stringify(obj));
+18 -99
View File
@@ -14,6 +14,7 @@ import {
import { debugLogger } from '../utils/debugLogger.js';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
import { truncateString } from '../utils/textUtils.js';
import {
GEN_AI_AGENT_DESCRIPTION,
GEN_AI_AGENT_NAME,
@@ -44,17 +45,6 @@ export const spanRegistry = new FinalizationRegistry((endSpan: () => void) => {
}
});
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null;
}
function hasToJSON(value: unknown): value is { toJSON: () => unknown } {
if (!isRecord(value)) return false;
if (!('toJSON' in value)) return false;
const toJSONFn = value['toJSON'];
return typeof toJSONFn === 'function';
}
/**
* Truncates a value for inclusion in telemetry attributes.
*
@@ -64,98 +54,27 @@ function hasToJSON(value: unknown): value is { toJSON: () => unknown } {
*/
export function truncateForTelemetry(
value: unknown,
maxStringLength = 10000,
maxArrayLength = 100,
maxDepth = 4,
maxObjectKeys = 100,
maxGlobalStringLength = 50000,
maxLength = 10000,
): AttributeValue | undefined {
const truncateObj = (v: unknown, depth: number): unknown => {
if (typeof v === 'string') {
if (v.length > maxStringLength) {
let truncatedStr = v.slice(0, maxStringLength);
if (truncatedStr.length > 0 && /[\uD800-\uDBFF]$/.test(truncatedStr)) {
truncatedStr = truncatedStr.slice(0, -1);
}
return truncatedStr + `...[TRUNCATED: original length ${v.length}]`;
}
return v;
}
if (
typeof v === 'number' ||
typeof v === 'boolean' ||
v === null ||
v === undefined
) {
return v;
}
if (hasToJSON(v)) {
try {
return truncateObj(v.toJSON(), depth);
} catch {
// Ignore and fall back to manual structural iteration
}
}
if (typeof v === 'object') {
if (depth >= maxDepth) {
return `[TRUNCATED: Max Depth Reached]`;
}
if (Array.isArray(v)) {
if (v.length > maxArrayLength) {
const truncatedArray = v
.slice(0, maxArrayLength)
.map((item) => truncateObj(item, depth + 1));
truncatedArray.push(`[TRUNCATED: Array of length ${v.length}]`);
return truncatedArray;
}
return v.map((item) => truncateObj(item, depth + 1));
}
const newObj: Record<string, unknown> = {};
let numKeys = 0;
const recordV = isRecord(v) ? v : {};
for (const key in recordV) {
if (!Object.prototype.hasOwnProperty.call(recordV, key)) continue;
if (numKeys >= maxObjectKeys) {
newObj['__truncated'] =
`[TRUNCATED: Object with >${maxObjectKeys} keys]`;
break;
}
const truncatedKey =
key.length > 100 ? key.slice(0, 100) + '...[TRUNCATED_KEY]' : key;
try {
const val = recordV[key];
newObj[truncatedKey] = truncateObj(val, depth + 1);
} catch {
newObj[truncatedKey] = '[ERROR: Failed to read property]';
}
numKeys++;
}
return newObj;
}
return undefined;
};
const truncated = truncateObj(value, 0);
if (
typeof truncated === 'string' ||
typeof truncated === 'number' ||
typeof truncated === 'boolean'
) {
return truncated as AttributeValue;
if (typeof value === 'string') {
return truncateString(
value,
maxLength,
`...[TRUNCATED: original length ${value.length}]`,
) as AttributeValue;
}
if (truncated === null || truncated === undefined) {
return undefined;
if (typeof value === 'object' && value !== null) {
const stringified = safeJsonStringify(value);
return truncateString(
stringified,
maxLength,
`...[TRUNCATED: original length ${stringified.length}]`,
) as AttributeValue;
}
const stringified = safeJsonStringify(truncated);
if (stringified.length > maxGlobalStringLength) {
return `"[TRUNCATED: Payload exceeded global limit of ${maxGlobalStringLength} characters. Original length: ${stringified.length}]"`;
if (typeof value === 'number' || typeof value === 'boolean') {
return value as AttributeValue;
}
return stringified as AttributeValue;
return undefined;
}
function isAsyncIterable<T>(value: T): value is T & AsyncIterable<unknown> {
+5 -6
View File
@@ -31,7 +31,6 @@ import type { AgentTerminateMode } from '../agents/types.js';
import { getCommonAttributes } from './telemetryAttributes.js';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
import { truncateForTelemetry } from './trace.js';
import {
toInputMessages,
toOutputMessages,
@@ -452,7 +451,7 @@ export class ApiRequestEvent implements BaseTelemetryEvent {
}
if (shouldIncludePayloads(config) && this.prompt.contents) {
attributes['gen_ai.input.messages'] = truncateForTelemetry(
attributes['gen_ai.input.messages'] = JSON.stringify(
toInputMessages(this.prompt.contents),
);
}
@@ -549,7 +548,7 @@ export class ApiErrorEvent implements BaseTelemetryEvent {
}
if (shouldIncludePayloads(config) && this.prompt.contents) {
attributes['gen_ai.input.messages'] = truncateForTelemetry(
attributes['gen_ai.input.messages'] = JSON.stringify(
toInputMessages(this.prompt.contents),
);
}
@@ -619,7 +618,7 @@ function toGenerateContentConfigAttributes(
'gen_ai.request.max_tokens': config.maxOutputTokens,
'gen_ai.output.type': toOutputType(config.responseMimeType),
'gen_ai.request.stop_sequences': config.stopSequences,
'gen_ai.system_instructions': truncateForTelemetry(
'gen_ai.system_instructions': JSON.stringify(
toSystemInstruction(config.systemInstruction),
),
};
@@ -717,7 +716,7 @@ export class ApiResponseEvent implements BaseTelemetryEvent {
'gen_ai.response.finish_reasons': this.finish_reasons,
...(shouldIncludePayloads(config)
? {
'gen_ai.output.messages': truncateForTelemetry(
'gen_ai.output.messages': JSON.stringify(
toOutputMessages(this.response.candidates),
),
}
@@ -732,7 +731,7 @@ export class ApiResponseEvent implements BaseTelemetryEvent {
}
if (shouldIncludePayloads(config) && this.prompt.contents) {
attributes['gen_ai.input.messages'] = truncateForTelemetry(
attributes['gen_ai.input.messages'] = JSON.stringify(
toInputMessages(this.prompt.contents),
);
}
+42
View File
@@ -37,6 +37,10 @@ export interface RetryOptions {
signal?: AbortSignal;
getAvailabilityContext?: () => RetryAvailabilityContext | undefined;
onRetry?: (attempt: number, error: unknown, delayMs: number) => void;
timeoutFallback?: {
timeoutMs: number;
fallbackDurationMs: number;
};
}
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
@@ -240,6 +244,7 @@ export async function retryWithBackoff<T>(
signal,
getAvailabilityContext,
onRetry,
timeoutFallback,
} = {
...DEFAULT_RETRY_OPTIONS,
shouldRetryOnError: isRetryableError,
@@ -248,6 +253,7 @@ export async function retryWithBackoff<T>(
let attempt = 0;
let currentDelay = initialDelayMs;
let startTime = Date.now();
const throwIfAborted = () => {
if (signal?.aborted) {
throw createAbortError();
@@ -294,6 +300,42 @@ export async function retryWithBackoff<T>(
const errorCode = getErrorStatus(error);
const isTimeout =
(error instanceof Error &&
error.message.toLowerCase().includes('timeout')) ||
getRetryErrorType(error) === 'ETIMEDOUT' ||
getRetryErrorType(error) === 'FETCH_FAILED';
if (isTimeout && timeoutFallback) {
if (Date.now() - startTime >= timeoutFallback.timeoutMs) {
const successContext = getAvailabilityContext?.();
if (successContext) {
successContext.service.markTemporarilyUnavailable(
successContext.policy.model,
'timeout',
timeoutFallback.fallbackDurationMs,
);
}
if (onPersistent429) {
try {
const fallbackModel = await onPersistent429(
authType,
new Error('Request timed out'),
);
if (fallbackModel) {
attempt = 0;
currentDelay = initialDelayMs;
startTime = Date.now();
continue;
}
} catch (fallbackError) {
debugLogger.warn('Model fallback failed:', fallbackError);
}
}
throw error;
}
}
if (
classifiedError instanceof TerminalQuotaError ||
classifiedError instanceof ModelNotFoundError
-1
View File
@@ -457,7 +457,6 @@ export class TestRig {
// Nightly releases sometimes becomes out of sync with local code and
// triggers auto-update, which causes tests to fail.
enableAutoUpdate: false,
enableAutoUpdateNotification: false,
},
telemetry: {
enabled: true,
+31
View File
@@ -702,6 +702,37 @@
"markdownDescription": "Skip the next speaker check.\n\n- Category: `Model`\n- Requires restart: `no`\n- Default: `true`",
"default": true,
"type": "boolean"
},
"autoRouting": {
"title": "Auto Routing",
"description": "Settings for automatic model routing.",
"markdownDescription": "Settings for automatic model routing.\n\n- Category: `Model`\n- Requires restart: `no`\n- Default: `{}`",
"default": {},
"type": "object",
"properties": {
"bestEffortPro": {
"title": "Best Effort Pro",
"description": "Always prefer the Pro model unless it is unavailable (e.g., due to timeouts or quota), ignoring other routing hints.",
"markdownDescription": "Always prefer the Pro model unless it is unavailable (e.g., due to timeouts or quota), ignoring other routing hints.\n\n- Category: `Model`\n- Requires restart: `no`\n- Default: `false`",
"default": false,
"type": "boolean"
},
"proTimeoutMinutes": {
"title": "Pro Timeout (Minutes)",
"description": "If a Pro request takes longer than this many minutes, it will be marked as temporarily unavailable and fallback to Flash.",
"markdownDescription": "If a Pro request takes longer than this many minutes, it will be marked as temporarily unavailable and fallback to Flash.\n\n- Category: `Model`\n- Requires restart: `no`\n- Default: `5`",
"default": 5,
"type": "number"
},
"proTimeoutFallbackDurationMinutes": {
"title": "Pro Timeout Fallback Duration (Minutes)",
"description": "How long to route to Flash after Pro times out.",
"markdownDescription": "How long to route to Flash after Pro times out.\n\n- Category: `Model`\n- Requires restart: `no`\n- Default: `60`",
"default": 60,
"type": "number"
}
},
"additionalProperties": false
}
},
"additionalProperties": false