refactor(config): stabilize synchronous experiment access and add real-time updates

- Revert getExperimentValue to a synchronous interface to maintain compatibility with existing callers.
- Add Config.updateExperimentalSettings() to support immediate application of /experiment set/unset changes.
- Implement CLI argument normalization to handle kebab-case, camelCase, and snake_case consistently.
- Enhance getExperimentFlagIdFromName to be more resilient to different naming conventions.
- Rename getNumericalRoutingEnabled to isNumericalRoutingEnabled for better idiomatic consistency.
- Update documentation in experimentation skill to recommend strongly-typed wrappers in Config.ts.
- Add regression tests for CLI overrides and update all relevant routing and command tests.
- Fix lint errors by removing unnecessary await/Promise.all for synchronous config methods.
This commit is contained in:
mkorwel
2026-02-19 20:42:00 -06:00
committed by Matt Korwel
parent 8d041e2acd
commit 6dd2d219d9
14 changed files with 148 additions and 65 deletions
@@ -102,9 +102,13 @@ export function getExperimentFlagName(flagId: number): string | undefined {
}
/**
* Gets the ID of an experiment flag from its kebab-case name.
* Gets the ID of an experiment flag from its name (supports kebab-case or camelCase).
*/
export function getExperimentFlagIdFromName(name: string): number | undefined {
const constantName = name.toUpperCase().replace(/-/g, '_');
// Convert enableNumericalRouting or enable-numerical-routing to ENABLE_NUMERICAL_ROUTING
const constantName = name
.replace(/([a-z])([A-Z])/g, '$1_$2') // camelCase to snake_case
.toUpperCase()
.replace(/-/g, '_'); // kebab-case to snake_case
return (ExperimentFlags as Record<string, number>)[constantName];
}
@@ -0,0 +1,49 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { Config } from './config.js';
import { ExperimentFlags } from '../code_assist/experiments/flagNames.js';
describe('Config CLI Override', () => {
const sessionId = 'test-session';
const targetDir = process.cwd();
const cwd = process.cwd();
const model = 'gemini-pro';
it('should prioritize CLI argument over local setting', () => {
const config = new Config({
sessionId,
targetDir,
cwd,
model,
debugMode: false,
experimentalCliArgs: { 'enable-numerical-routing': true },
experimentalSettings: { 'enable-numerical-routing': false },
});
expect(config.isNumericalRoutingEnabled()).toBe(true);
});
it('should prioritize CLI argument over remote experiment', () => {
const config = new Config({
sessionId,
targetDir,
cwd,
model,
debugMode: false,
experimentalCliArgs: { 'enable-numerical-routing': true },
experiments: {
flags: {
[ExperimentFlags.ENABLE_NUMERICAL_ROUTING]: { boolValue: false },
},
experimentIds: [],
},
});
expect(config.isNumericalRoutingEnabled()).toBe(true);
});
});
+21 -9
View File
@@ -942,7 +942,7 @@ export class Config implements McpContext, AgentLoopContext {
private readonly enableAgents: boolean;
private agents: AgentSettings;
private readonly experimentalSettings: Record<string, unknown>;
private experimentalSettings: Record<string, unknown>;
private readonly experimentalCliArgs: Record<string, unknown>;
private readonly enableEventDrivenScheduler: boolean;
private readonly skillsSupport: boolean;
@@ -3020,15 +3020,15 @@ export class Config implements McpContext, AgentLoopContext {
return remoteThreshold;
}
async getUserCaching(): Promise<boolean | undefined> {
getUserCaching(): boolean | undefined {
return this.getExperimentValue<boolean>(ExperimentFlags.USER_CACHING);
}
async getPlanModeRoutingEnabled(): Promise<boolean> {
getPlanModeRoutingEnabled(): boolean {
return this.planModeRoutingEnabled;
}
async getNumericalRoutingEnabled(): Promise<boolean> {
isNumericalRoutingEnabled(): boolean {
return (
this.getExperimentValue<boolean>(
ExperimentFlags.ENABLE_NUMERICAL_ROUTING,
@@ -3041,8 +3041,8 @@ export class Config implements McpContext, AgentLoopContext {
* If a remote threshold is provided and within range (0-100), it is returned.
* Otherwise, the default threshold (90) is returned.
*/
async getResolvedClassifierThreshold(): Promise<number> {
const remoteValue = await this.getClassifierThreshold();
getResolvedClassifierThreshold(): number {
const remoteValue = this.getClassifierThreshold();
const defaultValue = 90;
if (
@@ -3057,13 +3057,13 @@ export class Config implements McpContext, AgentLoopContext {
return defaultValue;
}
async getClassifierThreshold(): Promise<number | undefined> {
getClassifierThreshold(): number | undefined {
return this.getExperimentValue<number>(
ExperimentFlags.CLASSIFIER_THRESHOLD,
);
}
async getBannerTextNoCapacityIssues(): Promise<string> {
getBannerTextNoCapacityIssues(): string {
return (
this.getExperimentValue<string>(
ExperimentFlags.BANNER_TEXT_NO_CAPACITY_ISSUES,
@@ -3071,7 +3071,7 @@ export class Config implements McpContext, AgentLoopContext {
);
}
async getBannerTextCapacityIssues(): Promise<string> {
getBannerTextCapacityIssues(): string {
return (
this.getExperimentValue<string>(
ExperimentFlags.BANNER_TEXT_CAPACITY_ISSUES,
@@ -3738,6 +3738,18 @@ export class Config implements McpContext, AgentLoopContext {
return ExperimentMetadata[flagId]?.defaultValue as unknown as T;
}
/**
* Updates experimental settings.
*/
updateExperimentalSettings(settings: Record<string, unknown>): void {
// Only update if settings have actually changed to avoid unnecessary re-initialization logic
// if we add any in the future.
this.experimentalSettings = {
...this.experimentalSettings,
...settings,
};
}
/**
* Set experiments configuration
*/
@@ -51,15 +51,12 @@ describe('ModelRouterService', () => {
mockBaseLlmClient = {} as BaseLlmClient;
mockLocalLiteRtLmClient = {} as LocalLiteRtLmClient;
vi.spyOn(mockConfig, 'getBaseLlmClient').mockReturnValue(mockBaseLlmClient);
vi.spyOn(mockConfig, 'getLocalLiteRtLmClient').mockReturnValue(
mockLocalLiteRtLmClient,
);
vi.spyOn(mockConfig, 'getNumericalRoutingEnabled').mockResolvedValue(true);
vi.spyOn(mockConfig, 'getResolvedClassifierThreshold').mockResolvedValue(
90,
);
vi.spyOn(mockConfig, 'getClassifierThreshold').mockResolvedValue(undefined);
vi.spyOn(mockConfig, 'getGemmaModelRouterSettings').mockReturnValue({
vi.spyOn(mockConfig, 'getLocalLiteRtLmClient').mockReturnValue(
mockLocalLiteRtLmClient,
);
vi.spyOn(mockConfig, 'isNumericalRoutingEnabled').mockReturnValue(true);
vi.spyOn(mockConfig, 'getResolvedClassifierThreshold').mockReturnValue(90);
vi.spyOn(mockConfig, 'getClassifierThreshold').mockReturnValue(undefined); vi.spyOn(mockConfig, 'getGemmaModelRouterSettings').mockReturnValue({
enabled: false,
classifier: {
host: 'http://localhost:1234',
@@ -76,10 +76,8 @@ export class ModelRouterService {
const startTime = Date.now();
let decision: RoutingDecision;
const [enableNumericalRouting, thresholdValue] = await Promise.all([
this.config.getNumericalRoutingEnabled(),
this.config.getResolvedClassifierThreshold(),
]);
const enableNumericalRouting = this.config.isNumericalRoutingEnabled();
const thresholdValue = this.config.getResolvedClassifierThreshold();
const classifierThreshold = String(thresholdValue);
let failed = false;
@@ -57,7 +57,7 @@ describe('ClassifierStrategy', () => {
getResolvedConfig: vi.fn().mockReturnValue(mockResolvedConfig),
},
getModel: vi.fn().mockReturnValue(DEFAULT_GEMINI_MODEL_AUTO),
getNumericalRoutingEnabled: vi.fn().mockResolvedValue(false),
isNumericalRoutingEnabled: vi.fn().mockReturnValue(false),
getGemini31Launched: vi.fn().mockResolvedValue(false),
getGemini31FlashLiteLaunched: vi.fn().mockResolvedValue(false),
getUseCustomToolModel: vi.fn().mockImplementation(async () => {
@@ -78,7 +78,7 @@ describe('ClassifierStrategy', () => {
});
it('should return null if numerical routing is enabled and model is Gemini 3', async () => {
vi.mocked(mockConfig.getNumericalRoutingEnabled).mockResolvedValue(true);
vi.mocked(mockConfig.isNumericalRoutingEnabled).mockReturnValue(true);
vi.mocked(mockConfig.getModel).mockReturnValue(PREVIEW_GEMINI_MODEL_AUTO);
const decision = await strategy.route(
@@ -93,7 +93,7 @@ describe('ClassifierStrategy', () => {
});
it('should NOT return null if numerical routing is enabled but model is NOT Gemini 3', async () => {
vi.mocked(mockConfig.getNumericalRoutingEnabled).mockResolvedValue(true);
vi.mocked(mockConfig.isNumericalRoutingEnabled).mockReturnValue(true);
vi.mocked(mockConfig.getModel).mockReturnValue(DEFAULT_GEMINI_MODEL_AUTO);
vi.mocked(mockBaseLlmClient.generateJson).mockResolvedValue({
reasoning: 'test',
@@ -137,10 +137,7 @@ export class ClassifierStrategy implements RoutingStrategy {
const startTime = Date.now();
try {
const model = context.requestedModel ?? config.getModel();
if (
(await config.getNumericalRoutingEnabled()) &&
isGemini3Model(model, config)
) {
if (config.isNumericalRoutingEnabled() && isGemini3Model(model, config)) {
return null;
}
@@ -55,9 +55,9 @@ describe('NumericalClassifierStrategy', () => {
},
getModel: vi.fn().mockReturnValue(PREVIEW_GEMINI_MODEL_AUTO),
getSessionId: vi.fn().mockReturnValue('control-group-id'), // Default to Control Group (Hash 71 >= 50)
getNumericalRoutingEnabled: vi.fn().mockResolvedValue(true),
getResolvedClassifierThreshold: vi.fn().mockResolvedValue(90),
getClassifierThreshold: vi.fn().mockResolvedValue(undefined),
isNumericalRoutingEnabled: vi.fn().mockReturnValue(true),
getResolvedClassifierThreshold: vi.fn().mockReturnValue(90),
getClassifierThreshold: vi.fn().mockReturnValue(undefined),
getGemini31Launched: vi.fn().mockResolvedValue(false),
getGemini31FlashLiteLaunched: vi.fn().mockResolvedValue(false),
getUseCustomToolModel: vi.fn().mockImplementation(async () => {
@@ -82,7 +82,7 @@ describe('NumericalClassifierStrategy', () => {
});
it('should return null if numerical routing is disabled', async () => {
vi.mocked(mockConfig.getNumericalRoutingEnabled).mockResolvedValue(false);
vi.mocked(mockConfig.isNumericalRoutingEnabled).mockReturnValue(false);
const decision = await strategy.route(
mockContext,
@@ -105,7 +105,7 @@ export class NumericalClassifierStrategy implements RoutingStrategy {
const startTime = Date.now();
try {
const model = context.requestedModel ?? config.getModel();
if (!(await config.getNumericalRoutingEnabled())) {
if (!config.isNumericalRoutingEnabled()) {
return null;
}
@@ -187,8 +187,8 @@ export class NumericalClassifierStrategy implements RoutingStrategy {
groupLabel: string;
modelAlias: typeof FLASH_MODEL | typeof PRO_MODEL;
}> {
const threshold = await config.getResolvedClassifierThreshold();
const remoteThresholdValue = await config.getClassifierThreshold();
const threshold = config.getResolvedClassifierThreshold();
const remoteThresholdValue = config.getClassifierThreshold();
let groupLabel: string;
if (threshold === remoteThresholdValue) {