feat(routing): Initialize model routing architecture (#8153)

This commit is contained in:
Abhi
2025-09-11 13:38:50 -04:00
committed by GitHub
parent 5504f933e1
commit 538e6cd19a
28 changed files with 1263 additions and 67 deletions

View File

@@ -0,0 +1,96 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { ModelRouterService } from './modelRouterService.js';
import { Config } from '../config/config.js';
import type { BaseLlmClient } from '../core/baseLlmClient.js';
import type { RoutingContext, RoutingDecision } from './routingStrategy.js';
import { DefaultStrategy } from './strategies/defaultStrategy.js';
import { CompositeStrategy } from './strategies/compositeStrategy.js';
import { FallbackStrategy } from './strategies/fallbackStrategy.js';
import { OverrideStrategy } from './strategies/overrideStrategy.js';
vi.mock('../config/config.js');
vi.mock('../core/baseLlmClient.js');
vi.mock('./strategies/defaultStrategy.js');
vi.mock('./strategies/compositeStrategy.js');
vi.mock('./strategies/fallbackStrategy.js');
vi.mock('./strategies/overrideStrategy.js');
describe('ModelRouterService', () => {
let service: ModelRouterService;
let mockConfig: Config;
let mockBaseLlmClient: BaseLlmClient;
let mockContext: RoutingContext;
let mockCompositeStrategy: CompositeStrategy;
beforeEach(() => {
vi.clearAllMocks();
mockConfig = new Config({} as never);
mockBaseLlmClient = {} as BaseLlmClient;
vi.spyOn(mockConfig, 'getBaseLlmClient').mockReturnValue(mockBaseLlmClient);
mockCompositeStrategy = new CompositeStrategy(
[new FallbackStrategy(), new OverrideStrategy(), new DefaultStrategy()],
'agent-router',
);
vi.mocked(CompositeStrategy).mockImplementation(
() => mockCompositeStrategy,
);
service = new ModelRouterService(mockConfig);
mockContext = {
history: [],
request: [{ text: 'test prompt' }],
signal: new AbortController().signal,
};
});
it('should initialize with a CompositeStrategy', () => {
expect(CompositeStrategy).toHaveBeenCalled();
expect(service['strategy']).toBeInstanceOf(CompositeStrategy);
});
it('should initialize the CompositeStrategy with the correct child strategies in order', () => {
// This test relies on the mock implementation detail of the constructor
const compositeStrategyArgs = vi.mocked(CompositeStrategy).mock.calls[0];
const childStrategies = compositeStrategyArgs[0];
expect(childStrategies.length).toBe(3);
expect(childStrategies[0]).toBeInstanceOf(FallbackStrategy);
expect(childStrategies[1]).toBeInstanceOf(OverrideStrategy);
expect(childStrategies[2]).toBeInstanceOf(DefaultStrategy);
expect(compositeStrategyArgs[1]).toBe('agent-router');
});
describe('route()', () => {
it('should delegate routing to the composite strategy', async () => {
const strategyDecision: RoutingDecision = {
model: 'strategy-chosen-model',
metadata: {
source: 'test-router/fallback',
latencyMs: 10,
reasoning: 'Strategy reasoning',
},
};
const strategySpy = vi
.spyOn(mockCompositeStrategy, 'route')
.mockResolvedValue(strategyDecision);
const decision = await service.route(mockContext);
expect(strategySpy).toHaveBeenCalledWith(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(decision).toEqual(strategyDecision);
});
});
});

View File

@@ -0,0 +1,54 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../config/config.js';
import type {
RoutingContext,
RoutingDecision,
TerminalStrategy,
} from './routingStrategy.js';
import { DefaultStrategy } from './strategies/defaultStrategy.js';
import { CompositeStrategy } from './strategies/compositeStrategy.js';
import { FallbackStrategy } from './strategies/fallbackStrategy.js';
import { OverrideStrategy } from './strategies/overrideStrategy.js';
/**
* A centralized service for making model routing decisions.
*/
export class ModelRouterService {
private config: Config;
private strategy: TerminalStrategy;
constructor(config: Config) {
this.config = config;
this.strategy = this.initializeDefaultStrategy();
}
private initializeDefaultStrategy(): TerminalStrategy {
// Initialize the composite strategy with the desired priority order.
// The strategies are ordered in order of highest priority.
return new CompositeStrategy(
[new FallbackStrategy(), new OverrideStrategy(), new DefaultStrategy()],
'agent-router',
);
}
/**
* Determines which model to use for a given request context.
*
* @param context The full context of the request.
* @returns A promise that resolves to a RoutingDecision.
*/
async route(context: RoutingContext): Promise<RoutingDecision> {
const decision = await this.strategy.route(
context,
this.config,
this.config.getBaseLlmClient(),
);
return decision;
}
}

View File

@@ -0,0 +1,76 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Content, PartListUnion } from '@google/genai';
import type { BaseLlmClient } from '../core/baseLlmClient.js';
import type { Config } from '../config/config.js';
/**
* The output of a routing decision. It specifies which model to use and why.
*/
export interface RoutingDecision {
/** The model identifier string to use for the next API call (e.g., 'gemini-2.5-pro'). */
model: string;
/**
* Metadata about the routing decision for logging purposes.
*/
metadata: {
source: string;
latencyMs: number;
reasoning: string;
error?: string;
};
}
/**
* The context provided to the router for making a decision.
*/
export interface RoutingContext {
/** The full history of the conversation. */
history: Content[];
/** The immediate request parts to be processed. */
request: PartListUnion;
/** An abort signal to cancel an LLM call during routing. */
signal: AbortSignal;
}
/**
* The core interface that all routing strategies must implement.
* Strategies implementing this interface may decline a request by returning null.
*/
export interface RoutingStrategy {
/** The name of the strategy (e.g., 'fallback', 'override', 'composite'). */
readonly name: string;
/**
* Determines which model to use for a given request context.
* @param context The full context of the request.
* @param config The current configuration.
* @param client A reference to the GeminiClient, allowing the strategy to make its own API calls if needed.
* @returns A promise that resolves to a RoutingDecision, or null if the strategy is not applicable.
*/
route(
context: RoutingContext,
config: Config,
baseLlmClient: BaseLlmClient,
): Promise<RoutingDecision | null>;
}
/**
* A strategy that is guaranteed to return a decision. It must not return null.
* This is used to ensure that a composite chain always terminates.
*/
export interface TerminalStrategy extends RoutingStrategy {
/**
* Determines which model to use for a given request context.
* @returns A promise that resolves to a RoutingDecision.
*/
route(
context: RoutingContext,
config: Config,
baseLlmClient: BaseLlmClient,
): Promise<RoutingDecision>;
}

View File

@@ -0,0 +1,215 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { CompositeStrategy } from './compositeStrategy.js';
import type {
RoutingContext,
RoutingDecision,
RoutingStrategy,
TerminalStrategy,
} from '../routingStrategy.js';
import type { Config } from '../../config/config.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
describe('CompositeStrategy', () => {
let mockContext: RoutingContext;
let mockConfig: Config;
let mockBaseLlmClient: BaseLlmClient;
let mockStrategy1: RoutingStrategy;
let mockStrategy2: RoutingStrategy;
let mockTerminalStrategy: TerminalStrategy;
beforeEach(() => {
vi.clearAllMocks();
mockContext = {} as RoutingContext;
mockConfig = {} as Config;
mockBaseLlmClient = {} as BaseLlmClient;
mockStrategy1 = {
name: 'strategy1',
route: vi.fn().mockResolvedValue(null),
};
mockStrategy2 = {
name: 'strategy2',
route: vi.fn().mockResolvedValue(null),
};
mockTerminalStrategy = {
name: 'terminal',
route: vi.fn().mockResolvedValue({
model: 'terminal-model',
metadata: {
source: 'terminal',
latencyMs: 10,
reasoning: 'Terminal decision',
},
}),
};
});
it('should try strategies in order and return the first successful decision', async () => {
const decision: RoutingDecision = {
model: 'strategy2-model',
metadata: {
source: 'strategy2',
latencyMs: 20,
reasoning: 'Strategy 2 decided',
},
};
vi.spyOn(mockStrategy2, 'route').mockResolvedValue(decision);
const composite = new CompositeStrategy(
[mockStrategy1, mockStrategy2, mockTerminalStrategy],
'test-router',
);
const result = await composite.route(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(mockStrategy1.route).toHaveBeenCalledWith(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(mockStrategy2.route).toHaveBeenCalledWith(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(mockTerminalStrategy.route).not.toHaveBeenCalled();
expect(result.model).toBe('strategy2-model');
expect(result.metadata.source).toBe('test-router/strategy2');
});
it('should fall back to the terminal strategy if no other strategy provides a decision', async () => {
const composite = new CompositeStrategy(
[mockStrategy1, mockStrategy2, mockTerminalStrategy],
'test-router',
);
const result = await composite.route(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(mockStrategy1.route).toHaveBeenCalledTimes(1);
expect(mockStrategy2.route).toHaveBeenCalledTimes(1);
expect(mockTerminalStrategy.route).toHaveBeenCalledTimes(1);
expect(result.model).toBe('terminal-model');
expect(result.metadata.source).toBe('test-router/terminal');
});
it('should handle errors in non-terminal strategies and continue', async () => {
const consoleErrorSpy = vi
.spyOn(console, 'error')
.mockImplementation(() => {});
vi.spyOn(mockStrategy1, 'route').mockRejectedValue(
new Error('Strategy 1 failed'),
);
const composite = new CompositeStrategy(
[mockStrategy1, mockTerminalStrategy],
'test-router',
);
const result = await composite.route(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(consoleErrorSpy).toHaveBeenCalledWith(
"[Routing] Strategy 'strategy1' failed. Continuing to next strategy. Error:",
expect.any(Error),
);
expect(result.model).toBe('terminal-model');
consoleErrorSpy.mockRestore();
});
it('should re-throw an error from the terminal strategy', async () => {
const consoleErrorSpy = vi
.spyOn(console, 'error')
.mockImplementation(() => {});
const terminalError = new Error('Terminal strategy failed');
vi.spyOn(mockTerminalStrategy, 'route').mockRejectedValue(terminalError);
const composite = new CompositeStrategy([mockTerminalStrategy]);
await expect(
composite.route(mockContext, mockConfig, mockBaseLlmClient),
).rejects.toThrow(terminalError);
expect(consoleErrorSpy).toHaveBeenCalledWith(
"[Routing] Critical Error: Terminal strategy 'terminal' failed. Routing cannot proceed. Error:",
terminalError,
);
consoleErrorSpy.mockRestore();
});
it('should correctly finalize the decision metadata', async () => {
const decision: RoutingDecision = {
model: 'some-model',
metadata: {
source: 'child-source',
latencyMs: 50,
reasoning: 'Child reasoning',
},
};
vi.spyOn(mockStrategy1, 'route').mockResolvedValue(decision);
const composite = new CompositeStrategy(
[mockStrategy1, mockTerminalStrategy],
'my-composite',
);
const result = await composite.route(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(result.model).toBe('some-model');
expect(result.metadata.source).toBe('my-composite/child-source');
expect(result.metadata.reasoning).toBe('Child reasoning');
// It should keep the child's latency
expect(result.metadata.latencyMs).toBe(50);
});
it('should calculate total latency if child latency is not provided', async () => {
const decision: RoutingDecision = {
model: 'some-model',
metadata: {
source: 'child-source',
// No latencyMs here
latencyMs: 0,
reasoning: 'Child reasoning',
},
};
vi.spyOn(mockStrategy1, 'route').mockResolvedValue(decision);
const composite = new CompositeStrategy(
[mockStrategy1, mockTerminalStrategy],
'my-composite',
);
const result = await composite.route(
mockContext,
mockConfig,
mockBaseLlmClient,
);
expect(result.metadata.latencyMs).toBeGreaterThanOrEqual(0);
});
});

View File

@@ -0,0 +1,109 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../../config/config.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import type {
RoutingContext,
RoutingDecision,
RoutingStrategy,
TerminalStrategy,
} from '../routingStrategy.js';
/**
* A strategy that attempts a list of child strategies in order (Chain of Responsibility).
*/
export class CompositeStrategy implements TerminalStrategy {
readonly name: string;
private strategies: [...RoutingStrategy[], TerminalStrategy];
/**
* Initializes the CompositeStrategy.
* @param strategies The strategies to try, in order of priority. The last strategy must be terminal.
* @param name The name of this composite configuration (e.g., 'router' or 'composite').
*/
constructor(
strategies: [...RoutingStrategy[], TerminalStrategy],
name: string = 'composite',
) {
this.strategies = strategies;
this.name = name;
}
async route(
context: RoutingContext,
config: Config,
baseLlmClient: BaseLlmClient,
): Promise<RoutingDecision> {
const startTime = performance.now();
// Separate non-terminal strategies from the terminal one.
// This separation allows TypeScript to understand the control flow guarantees.
const nonTerminalStrategies = this.strategies.slice(
0,
-1,
) as RoutingStrategy[];
const terminalStrategy = this.strategies[
this.strategies.length - 1
] as TerminalStrategy;
// Try non-terminal strategies, allowing them to fail gracefully.
for (const strategy of nonTerminalStrategies) {
try {
const decision = await strategy.route(context, config, baseLlmClient);
if (decision) {
return this.finalizeDecision(decision, startTime);
}
} catch (error) {
console.error(
`[Routing] Strategy '${strategy.name}' failed. Continuing to next strategy. Error:`,
error,
);
}
}
// If no other strategy matched, execute the terminal strategy.
try {
const decision = await terminalStrategy.route(
context,
config,
baseLlmClient,
);
return this.finalizeDecision(decision, startTime);
} catch (error) {
console.error(
`[Routing] Critical Error: Terminal strategy '${terminalStrategy.name}' failed. Routing cannot proceed. Error:`,
error,
);
throw error;
}
}
/**
* Helper function to enhance the decision metadata with composite information.
*/
private finalizeDecision(
decision: RoutingDecision,
startTime: number,
): RoutingDecision {
const endTime = performance.now();
const totalLatency = endTime - startTime;
// Combine the source paths: composite_name/child_source (e.g. 'router/default')
const compositeSource = `${this.name}/${decision.metadata.source}`;
return {
...decision,
metadata: {
...decision.metadata,
source: compositeSource,
latencyMs: decision.metadata.latencyMs || totalLatency,
},
};
}
}

View File

@@ -0,0 +1,32 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { DefaultStrategy } from './defaultStrategy.js';
import type { RoutingContext } from '../routingStrategy.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import { DEFAULT_GEMINI_MODEL } from '../../config/models.js';
import type { Config } from '../../config/config.js';
describe('DefaultStrategy', () => {
it('should always route to the default Gemini model', async () => {
const strategy = new DefaultStrategy();
const mockContext = {} as RoutingContext;
const mockConfig = {} as Config;
const mockClient = {} as BaseLlmClient;
const decision = await strategy.route(mockContext, mockConfig, mockClient);
expect(decision).toEqual({
model: DEFAULT_GEMINI_MODEL,
metadata: {
source: 'default',
latencyMs: 0,
reasoning: `Routing to default model: ${DEFAULT_GEMINI_MODEL}`,
},
});
});
});

View File

@@ -0,0 +1,33 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../../config/config.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import type {
RoutingContext,
RoutingDecision,
TerminalStrategy,
} from '../routingStrategy.js';
import { DEFAULT_GEMINI_MODEL } from '../../config/models.js';
export class DefaultStrategy implements TerminalStrategy {
readonly name = 'default';
async route(
_context: RoutingContext,
_config: Config,
_baseLlmClient: BaseLlmClient,
): Promise<RoutingDecision> {
return {
model: DEFAULT_GEMINI_MODEL,
metadata: {
source: this.name,
latencyMs: 0,
reasoning: `Routing to default model: ${DEFAULT_GEMINI_MODEL}`,
},
};
}
}

View File

@@ -0,0 +1,86 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { FallbackStrategy } from './fallbackStrategy.js';
import type { RoutingContext } from '../routingStrategy.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import type { Config } from '../../config/config.js';
import {
DEFAULT_GEMINI_MODEL,
DEFAULT_GEMINI_FLASH_MODEL,
DEFAULT_GEMINI_FLASH_LITE_MODEL,
} from '../../config/models.js';
describe('FallbackStrategy', () => {
const strategy = new FallbackStrategy();
const mockContext = {} as RoutingContext;
const mockClient = {} as BaseLlmClient;
it('should return null when not in fallback mode', async () => {
const mockConfig = {
isInFallbackMode: () => false,
getModel: () => DEFAULT_GEMINI_MODEL,
} as Config;
const decision = await strategy.route(mockContext, mockConfig, mockClient);
expect(decision).toBeNull();
});
describe('when in fallback mode', () => {
it('should downgrade a pro model to the flash model', async () => {
const mockConfig = {
isInFallbackMode: () => true,
getModel: () => DEFAULT_GEMINI_MODEL,
} as Config;
const decision = await strategy.route(
mockContext,
mockConfig,
mockClient,
);
expect(decision).not.toBeNull();
expect(decision?.model).toBe(DEFAULT_GEMINI_FLASH_MODEL);
expect(decision?.metadata.source).toBe('fallback');
expect(decision?.metadata.reasoning).toContain('In fallback mode');
});
it('should honor a lite model request', async () => {
const mockConfig = {
isInFallbackMode: () => true,
getModel: () => DEFAULT_GEMINI_FLASH_LITE_MODEL,
} as Config;
const decision = await strategy.route(
mockContext,
mockConfig,
mockClient,
);
expect(decision).not.toBeNull();
expect(decision?.model).toBe(DEFAULT_GEMINI_FLASH_LITE_MODEL);
expect(decision?.metadata.source).toBe('fallback');
});
it('should use the flash model if flash is requested', async () => {
const mockConfig = {
isInFallbackMode: () => true,
getModel: () => DEFAULT_GEMINI_FLASH_MODEL,
} as Config;
const decision = await strategy.route(
mockContext,
mockConfig,
mockClient,
);
expect(decision).not.toBeNull();
expect(decision?.model).toBe(DEFAULT_GEMINI_FLASH_MODEL);
expect(decision?.metadata.source).toBe('fallback');
});
});
});

View File

@@ -0,0 +1,43 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../../config/config.js';
import { getEffectiveModel } from '../../config/models.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import type {
RoutingContext,
RoutingDecision,
RoutingStrategy,
} from '../routingStrategy.js';
export class FallbackStrategy implements RoutingStrategy {
readonly name = 'fallback';
async route(
_context: RoutingContext,
config: Config,
_baseLlmClient: BaseLlmClient,
): Promise<RoutingDecision | null> {
const isInFallbackMode: boolean = config.isInFallbackMode();
if (!isInFallbackMode) {
return null;
}
const effectiveModel = getEffectiveModel(
isInFallbackMode,
config.getModel(),
);
return {
model: effectiveModel,
metadata: {
source: this.name,
latencyMs: 0,
reasoning: `In fallback mode. Using: ${effectiveModel}`,
},
};
}
}

View File

@@ -0,0 +1,55 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { OverrideStrategy } from './overrideStrategy.js';
import type { RoutingContext } from '../routingStrategy.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import type { Config } from '../../config/config.js';
describe('OverrideStrategy', () => {
const strategy = new OverrideStrategy();
const mockContext = {} as RoutingContext;
const mockClient = {} as BaseLlmClient;
it('should return null when no override model is specified', async () => {
const mockConfig = {
getModel: () => '', // Simulate no model override
} as Config;
const decision = await strategy.route(mockContext, mockConfig, mockClient);
expect(decision).toBeNull();
});
it('should return a decision with the override model when one is specified', async () => {
const overrideModel = 'gemini-2.5-pro-custom';
const mockConfig = {
getModel: () => overrideModel,
} as Config;
const decision = await strategy.route(mockContext, mockConfig, mockClient);
expect(decision).not.toBeNull();
expect(decision?.model).toBe(overrideModel);
expect(decision?.metadata.source).toBe('override');
expect(decision?.metadata.reasoning).toContain(
'Routing bypassed by forced model directive',
);
expect(decision?.metadata.reasoning).toContain(overrideModel);
});
it('should handle different override model names', async () => {
const overrideModel = 'gemini-2.5-flash-experimental';
const mockConfig = {
getModel: () => overrideModel,
} as Config;
const decision = await strategy.route(mockContext, mockConfig, mockClient);
expect(decision).not.toBeNull();
expect(decision?.model).toBe(overrideModel);
});
});

View File

@@ -0,0 +1,40 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../../config/config.js';
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import type {
RoutingContext,
RoutingDecision,
RoutingStrategy,
} from '../routingStrategy.js';
/**
* Handles cases where the user explicitly specifies a model (override).
*/
export class OverrideStrategy implements RoutingStrategy {
readonly name = 'override';
async route(
_context: RoutingContext,
config: Config,
_baseLlmClient: BaseLlmClient,
): Promise<RoutingDecision | null> {
const overrideModel = config.getModel();
if (overrideModel) {
return {
model: overrideModel,
metadata: {
source: this.name,
latencyMs: 0,
reasoning: `Routing bypassed by forced model directive. Using: ${overrideModel}`,
},
};
}
// No override specified, pass to the next strategy.
return null;
}
}