mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-04-14 23:31:13 -07:00
feat(routing): Initialize model routing architecture (#8153)
This commit is contained in:
96
packages/core/src/routing/modelRouterService.test.ts
Normal file
96
packages/core/src/routing/modelRouterService.test.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { ModelRouterService } from './modelRouterService.js';
|
||||
import { Config } from '../config/config.js';
|
||||
import type { BaseLlmClient } from '../core/baseLlmClient.js';
|
||||
import type { RoutingContext, RoutingDecision } from './routingStrategy.js';
|
||||
import { DefaultStrategy } from './strategies/defaultStrategy.js';
|
||||
import { CompositeStrategy } from './strategies/compositeStrategy.js';
|
||||
import { FallbackStrategy } from './strategies/fallbackStrategy.js';
|
||||
import { OverrideStrategy } from './strategies/overrideStrategy.js';
|
||||
|
||||
vi.mock('../config/config.js');
|
||||
vi.mock('../core/baseLlmClient.js');
|
||||
vi.mock('./strategies/defaultStrategy.js');
|
||||
vi.mock('./strategies/compositeStrategy.js');
|
||||
vi.mock('./strategies/fallbackStrategy.js');
|
||||
vi.mock('./strategies/overrideStrategy.js');
|
||||
|
||||
describe('ModelRouterService', () => {
|
||||
let service: ModelRouterService;
|
||||
let mockConfig: Config;
|
||||
let mockBaseLlmClient: BaseLlmClient;
|
||||
let mockContext: RoutingContext;
|
||||
let mockCompositeStrategy: CompositeStrategy;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
|
||||
mockConfig = new Config({} as never);
|
||||
mockBaseLlmClient = {} as BaseLlmClient;
|
||||
vi.spyOn(mockConfig, 'getBaseLlmClient').mockReturnValue(mockBaseLlmClient);
|
||||
|
||||
mockCompositeStrategy = new CompositeStrategy(
|
||||
[new FallbackStrategy(), new OverrideStrategy(), new DefaultStrategy()],
|
||||
'agent-router',
|
||||
);
|
||||
vi.mocked(CompositeStrategy).mockImplementation(
|
||||
() => mockCompositeStrategy,
|
||||
);
|
||||
|
||||
service = new ModelRouterService(mockConfig);
|
||||
|
||||
mockContext = {
|
||||
history: [],
|
||||
request: [{ text: 'test prompt' }],
|
||||
signal: new AbortController().signal,
|
||||
};
|
||||
});
|
||||
|
||||
it('should initialize with a CompositeStrategy', () => {
|
||||
expect(CompositeStrategy).toHaveBeenCalled();
|
||||
expect(service['strategy']).toBeInstanceOf(CompositeStrategy);
|
||||
});
|
||||
|
||||
it('should initialize the CompositeStrategy with the correct child strategies in order', () => {
|
||||
// This test relies on the mock implementation detail of the constructor
|
||||
const compositeStrategyArgs = vi.mocked(CompositeStrategy).mock.calls[0];
|
||||
const childStrategies = compositeStrategyArgs[0];
|
||||
|
||||
expect(childStrategies.length).toBe(3);
|
||||
expect(childStrategies[0]).toBeInstanceOf(FallbackStrategy);
|
||||
expect(childStrategies[1]).toBeInstanceOf(OverrideStrategy);
|
||||
expect(childStrategies[2]).toBeInstanceOf(DefaultStrategy);
|
||||
expect(compositeStrategyArgs[1]).toBe('agent-router');
|
||||
});
|
||||
|
||||
describe('route()', () => {
|
||||
it('should delegate routing to the composite strategy', async () => {
|
||||
const strategyDecision: RoutingDecision = {
|
||||
model: 'strategy-chosen-model',
|
||||
metadata: {
|
||||
source: 'test-router/fallback',
|
||||
latencyMs: 10,
|
||||
reasoning: 'Strategy reasoning',
|
||||
},
|
||||
};
|
||||
const strategySpy = vi
|
||||
.spyOn(mockCompositeStrategy, 'route')
|
||||
.mockResolvedValue(strategyDecision);
|
||||
|
||||
const decision = await service.route(mockContext);
|
||||
|
||||
expect(strategySpy).toHaveBeenCalledWith(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
expect(decision).toEqual(strategyDecision);
|
||||
});
|
||||
});
|
||||
});
|
||||
54
packages/core/src/routing/modelRouterService.ts
Normal file
54
packages/core/src/routing/modelRouterService.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { Config } from '../config/config.js';
|
||||
import type {
|
||||
RoutingContext,
|
||||
RoutingDecision,
|
||||
TerminalStrategy,
|
||||
} from './routingStrategy.js';
|
||||
import { DefaultStrategy } from './strategies/defaultStrategy.js';
|
||||
import { CompositeStrategy } from './strategies/compositeStrategy.js';
|
||||
import { FallbackStrategy } from './strategies/fallbackStrategy.js';
|
||||
import { OverrideStrategy } from './strategies/overrideStrategy.js';
|
||||
|
||||
/**
|
||||
* A centralized service for making model routing decisions.
|
||||
*/
|
||||
export class ModelRouterService {
|
||||
private config: Config;
|
||||
private strategy: TerminalStrategy;
|
||||
|
||||
constructor(config: Config) {
|
||||
this.config = config;
|
||||
this.strategy = this.initializeDefaultStrategy();
|
||||
}
|
||||
|
||||
private initializeDefaultStrategy(): TerminalStrategy {
|
||||
// Initialize the composite strategy with the desired priority order.
|
||||
// The strategies are ordered in order of highest priority.
|
||||
return new CompositeStrategy(
|
||||
[new FallbackStrategy(), new OverrideStrategy(), new DefaultStrategy()],
|
||||
'agent-router',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines which model to use for a given request context.
|
||||
*
|
||||
* @param context The full context of the request.
|
||||
* @returns A promise that resolves to a RoutingDecision.
|
||||
*/
|
||||
async route(context: RoutingContext): Promise<RoutingDecision> {
|
||||
const decision = await this.strategy.route(
|
||||
context,
|
||||
this.config,
|
||||
this.config.getBaseLlmClient(),
|
||||
);
|
||||
|
||||
return decision;
|
||||
}
|
||||
}
|
||||
76
packages/core/src/routing/routingStrategy.ts
Normal file
76
packages/core/src/routing/routingStrategy.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { Content, PartListUnion } from '@google/genai';
|
||||
import type { BaseLlmClient } from '../core/baseLlmClient.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
|
||||
/**
|
||||
* The output of a routing decision. It specifies which model to use and why.
|
||||
*/
|
||||
export interface RoutingDecision {
|
||||
/** The model identifier string to use for the next API call (e.g., 'gemini-2.5-pro'). */
|
||||
model: string;
|
||||
/**
|
||||
* Metadata about the routing decision for logging purposes.
|
||||
*/
|
||||
metadata: {
|
||||
source: string;
|
||||
latencyMs: number;
|
||||
reasoning: string;
|
||||
error?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The context provided to the router for making a decision.
|
||||
*/
|
||||
export interface RoutingContext {
|
||||
/** The full history of the conversation. */
|
||||
history: Content[];
|
||||
/** The immediate request parts to be processed. */
|
||||
request: PartListUnion;
|
||||
/** An abort signal to cancel an LLM call during routing. */
|
||||
signal: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* The core interface that all routing strategies must implement.
|
||||
* Strategies implementing this interface may decline a request by returning null.
|
||||
*/
|
||||
export interface RoutingStrategy {
|
||||
/** The name of the strategy (e.g., 'fallback', 'override', 'composite'). */
|
||||
readonly name: string;
|
||||
|
||||
/**
|
||||
* Determines which model to use for a given request context.
|
||||
* @param context The full context of the request.
|
||||
* @param config The current configuration.
|
||||
* @param client A reference to the GeminiClient, allowing the strategy to make its own API calls if needed.
|
||||
* @returns A promise that resolves to a RoutingDecision, or null if the strategy is not applicable.
|
||||
*/
|
||||
route(
|
||||
context: RoutingContext,
|
||||
config: Config,
|
||||
baseLlmClient: BaseLlmClient,
|
||||
): Promise<RoutingDecision | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A strategy that is guaranteed to return a decision. It must not return null.
|
||||
* This is used to ensure that a composite chain always terminates.
|
||||
*/
|
||||
export interface TerminalStrategy extends RoutingStrategy {
|
||||
/**
|
||||
* Determines which model to use for a given request context.
|
||||
* @returns A promise that resolves to a RoutingDecision.
|
||||
*/
|
||||
route(
|
||||
context: RoutingContext,
|
||||
config: Config,
|
||||
baseLlmClient: BaseLlmClient,
|
||||
): Promise<RoutingDecision>;
|
||||
}
|
||||
215
packages/core/src/routing/strategies/compositeStrategy.test.ts
Normal file
215
packages/core/src/routing/strategies/compositeStrategy.test.ts
Normal file
@@ -0,0 +1,215 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { CompositeStrategy } from './compositeStrategy.js';
|
||||
import type {
|
||||
RoutingContext,
|
||||
RoutingDecision,
|
||||
RoutingStrategy,
|
||||
TerminalStrategy,
|
||||
} from '../routingStrategy.js';
|
||||
import type { Config } from '../../config/config.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
|
||||
describe('CompositeStrategy', () => {
|
||||
let mockContext: RoutingContext;
|
||||
let mockConfig: Config;
|
||||
let mockBaseLlmClient: BaseLlmClient;
|
||||
let mockStrategy1: RoutingStrategy;
|
||||
let mockStrategy2: RoutingStrategy;
|
||||
let mockTerminalStrategy: TerminalStrategy;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
|
||||
mockContext = {} as RoutingContext;
|
||||
mockConfig = {} as Config;
|
||||
mockBaseLlmClient = {} as BaseLlmClient;
|
||||
|
||||
mockStrategy1 = {
|
||||
name: 'strategy1',
|
||||
route: vi.fn().mockResolvedValue(null),
|
||||
};
|
||||
|
||||
mockStrategy2 = {
|
||||
name: 'strategy2',
|
||||
route: vi.fn().mockResolvedValue(null),
|
||||
};
|
||||
|
||||
mockTerminalStrategy = {
|
||||
name: 'terminal',
|
||||
route: vi.fn().mockResolvedValue({
|
||||
model: 'terminal-model',
|
||||
metadata: {
|
||||
source: 'terminal',
|
||||
latencyMs: 10,
|
||||
reasoning: 'Terminal decision',
|
||||
},
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
it('should try strategies in order and return the first successful decision', async () => {
|
||||
const decision: RoutingDecision = {
|
||||
model: 'strategy2-model',
|
||||
metadata: {
|
||||
source: 'strategy2',
|
||||
latencyMs: 20,
|
||||
reasoning: 'Strategy 2 decided',
|
||||
},
|
||||
};
|
||||
vi.spyOn(mockStrategy2, 'route').mockResolvedValue(decision);
|
||||
|
||||
const composite = new CompositeStrategy(
|
||||
[mockStrategy1, mockStrategy2, mockTerminalStrategy],
|
||||
'test-router',
|
||||
);
|
||||
|
||||
const result = await composite.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
|
||||
expect(mockStrategy1.route).toHaveBeenCalledWith(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
expect(mockStrategy2.route).toHaveBeenCalledWith(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
expect(mockTerminalStrategy.route).not.toHaveBeenCalled();
|
||||
|
||||
expect(result.model).toBe('strategy2-model');
|
||||
expect(result.metadata.source).toBe('test-router/strategy2');
|
||||
});
|
||||
|
||||
it('should fall back to the terminal strategy if no other strategy provides a decision', async () => {
|
||||
const composite = new CompositeStrategy(
|
||||
[mockStrategy1, mockStrategy2, mockTerminalStrategy],
|
||||
'test-router',
|
||||
);
|
||||
|
||||
const result = await composite.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
|
||||
expect(mockStrategy1.route).toHaveBeenCalledTimes(1);
|
||||
expect(mockStrategy2.route).toHaveBeenCalledTimes(1);
|
||||
expect(mockTerminalStrategy.route).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(result.model).toBe('terminal-model');
|
||||
expect(result.metadata.source).toBe('test-router/terminal');
|
||||
});
|
||||
|
||||
it('should handle errors in non-terminal strategies and continue', async () => {
|
||||
const consoleErrorSpy = vi
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => {});
|
||||
vi.spyOn(mockStrategy1, 'route').mockRejectedValue(
|
||||
new Error('Strategy 1 failed'),
|
||||
);
|
||||
|
||||
const composite = new CompositeStrategy(
|
||||
[mockStrategy1, mockTerminalStrategy],
|
||||
'test-router',
|
||||
);
|
||||
|
||||
const result = await composite.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
|
||||
expect(consoleErrorSpy).toHaveBeenCalledWith(
|
||||
"[Routing] Strategy 'strategy1' failed. Continuing to next strategy. Error:",
|
||||
expect.any(Error),
|
||||
);
|
||||
expect(result.model).toBe('terminal-model');
|
||||
consoleErrorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('should re-throw an error from the terminal strategy', async () => {
|
||||
const consoleErrorSpy = vi
|
||||
.spyOn(console, 'error')
|
||||
.mockImplementation(() => {});
|
||||
const terminalError = new Error('Terminal strategy failed');
|
||||
vi.spyOn(mockTerminalStrategy, 'route').mockRejectedValue(terminalError);
|
||||
|
||||
const composite = new CompositeStrategy([mockTerminalStrategy]);
|
||||
|
||||
await expect(
|
||||
composite.route(mockContext, mockConfig, mockBaseLlmClient),
|
||||
).rejects.toThrow(terminalError);
|
||||
|
||||
expect(consoleErrorSpy).toHaveBeenCalledWith(
|
||||
"[Routing] Critical Error: Terminal strategy 'terminal' failed. Routing cannot proceed. Error:",
|
||||
terminalError,
|
||||
);
|
||||
consoleErrorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('should correctly finalize the decision metadata', async () => {
|
||||
const decision: RoutingDecision = {
|
||||
model: 'some-model',
|
||||
metadata: {
|
||||
source: 'child-source',
|
||||
latencyMs: 50,
|
||||
reasoning: 'Child reasoning',
|
||||
},
|
||||
};
|
||||
vi.spyOn(mockStrategy1, 'route').mockResolvedValue(decision);
|
||||
|
||||
const composite = new CompositeStrategy(
|
||||
[mockStrategy1, mockTerminalStrategy],
|
||||
'my-composite',
|
||||
);
|
||||
|
||||
const result = await composite.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
|
||||
expect(result.model).toBe('some-model');
|
||||
expect(result.metadata.source).toBe('my-composite/child-source');
|
||||
expect(result.metadata.reasoning).toBe('Child reasoning');
|
||||
// It should keep the child's latency
|
||||
expect(result.metadata.latencyMs).toBe(50);
|
||||
});
|
||||
|
||||
it('should calculate total latency if child latency is not provided', async () => {
|
||||
const decision: RoutingDecision = {
|
||||
model: 'some-model',
|
||||
metadata: {
|
||||
source: 'child-source',
|
||||
// No latencyMs here
|
||||
latencyMs: 0,
|
||||
reasoning: 'Child reasoning',
|
||||
},
|
||||
};
|
||||
vi.spyOn(mockStrategy1, 'route').mockResolvedValue(decision);
|
||||
|
||||
const composite = new CompositeStrategy(
|
||||
[mockStrategy1, mockTerminalStrategy],
|
||||
'my-composite',
|
||||
);
|
||||
|
||||
const result = await composite.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockBaseLlmClient,
|
||||
);
|
||||
|
||||
expect(result.metadata.latencyMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
});
|
||||
109
packages/core/src/routing/strategies/compositeStrategy.ts
Normal file
109
packages/core/src/routing/strategies/compositeStrategy.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { Config } from '../../config/config.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
import type {
|
||||
RoutingContext,
|
||||
RoutingDecision,
|
||||
RoutingStrategy,
|
||||
TerminalStrategy,
|
||||
} from '../routingStrategy.js';
|
||||
|
||||
/**
|
||||
* A strategy that attempts a list of child strategies in order (Chain of Responsibility).
|
||||
*/
|
||||
export class CompositeStrategy implements TerminalStrategy {
|
||||
readonly name: string;
|
||||
|
||||
private strategies: [...RoutingStrategy[], TerminalStrategy];
|
||||
|
||||
/**
|
||||
* Initializes the CompositeStrategy.
|
||||
* @param strategies The strategies to try, in order of priority. The last strategy must be terminal.
|
||||
* @param name The name of this composite configuration (e.g., 'router' or 'composite').
|
||||
*/
|
||||
constructor(
|
||||
strategies: [...RoutingStrategy[], TerminalStrategy],
|
||||
name: string = 'composite',
|
||||
) {
|
||||
this.strategies = strategies;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
async route(
|
||||
context: RoutingContext,
|
||||
config: Config,
|
||||
baseLlmClient: BaseLlmClient,
|
||||
): Promise<RoutingDecision> {
|
||||
const startTime = performance.now();
|
||||
|
||||
// Separate non-terminal strategies from the terminal one.
|
||||
// This separation allows TypeScript to understand the control flow guarantees.
|
||||
const nonTerminalStrategies = this.strategies.slice(
|
||||
0,
|
||||
-1,
|
||||
) as RoutingStrategy[];
|
||||
const terminalStrategy = this.strategies[
|
||||
this.strategies.length - 1
|
||||
] as TerminalStrategy;
|
||||
|
||||
// Try non-terminal strategies, allowing them to fail gracefully.
|
||||
for (const strategy of nonTerminalStrategies) {
|
||||
try {
|
||||
const decision = await strategy.route(context, config, baseLlmClient);
|
||||
if (decision) {
|
||||
return this.finalizeDecision(decision, startTime);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[Routing] Strategy '${strategy.name}' failed. Continuing to next strategy. Error:`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// If no other strategy matched, execute the terminal strategy.
|
||||
try {
|
||||
const decision = await terminalStrategy.route(
|
||||
context,
|
||||
config,
|
||||
baseLlmClient,
|
||||
);
|
||||
|
||||
return this.finalizeDecision(decision, startTime);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[Routing] Critical Error: Terminal strategy '${terminalStrategy.name}' failed. Routing cannot proceed. Error:`,
|
||||
error,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to enhance the decision metadata with composite information.
|
||||
*/
|
||||
private finalizeDecision(
|
||||
decision: RoutingDecision,
|
||||
startTime: number,
|
||||
): RoutingDecision {
|
||||
const endTime = performance.now();
|
||||
const totalLatency = endTime - startTime;
|
||||
|
||||
// Combine the source paths: composite_name/child_source (e.g. 'router/default')
|
||||
const compositeSource = `${this.name}/${decision.metadata.source}`;
|
||||
|
||||
return {
|
||||
...decision,
|
||||
metadata: {
|
||||
...decision.metadata,
|
||||
source: compositeSource,
|
||||
latencyMs: decision.metadata.latencyMs || totalLatency,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
32
packages/core/src/routing/strategies/defaultStrategy.test.ts
Normal file
32
packages/core/src/routing/strategies/defaultStrategy.test.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { DefaultStrategy } from './defaultStrategy.js';
|
||||
import type { RoutingContext } from '../routingStrategy.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
import { DEFAULT_GEMINI_MODEL } from '../../config/models.js';
|
||||
import type { Config } from '../../config/config.js';
|
||||
|
||||
describe('DefaultStrategy', () => {
|
||||
it('should always route to the default Gemini model', async () => {
|
||||
const strategy = new DefaultStrategy();
|
||||
const mockContext = {} as RoutingContext;
|
||||
const mockConfig = {} as Config;
|
||||
const mockClient = {} as BaseLlmClient;
|
||||
|
||||
const decision = await strategy.route(mockContext, mockConfig, mockClient);
|
||||
|
||||
expect(decision).toEqual({
|
||||
model: DEFAULT_GEMINI_MODEL,
|
||||
metadata: {
|
||||
source: 'default',
|
||||
latencyMs: 0,
|
||||
reasoning: `Routing to default model: ${DEFAULT_GEMINI_MODEL}`,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
33
packages/core/src/routing/strategies/defaultStrategy.ts
Normal file
33
packages/core/src/routing/strategies/defaultStrategy.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { Config } from '../../config/config.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
import type {
|
||||
RoutingContext,
|
||||
RoutingDecision,
|
||||
TerminalStrategy,
|
||||
} from '../routingStrategy.js';
|
||||
import { DEFAULT_GEMINI_MODEL } from '../../config/models.js';
|
||||
|
||||
export class DefaultStrategy implements TerminalStrategy {
|
||||
readonly name = 'default';
|
||||
|
||||
async route(
|
||||
_context: RoutingContext,
|
||||
_config: Config,
|
||||
_baseLlmClient: BaseLlmClient,
|
||||
): Promise<RoutingDecision> {
|
||||
return {
|
||||
model: DEFAULT_GEMINI_MODEL,
|
||||
metadata: {
|
||||
source: this.name,
|
||||
latencyMs: 0,
|
||||
reasoning: `Routing to default model: ${DEFAULT_GEMINI_MODEL}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { FallbackStrategy } from './fallbackStrategy.js';
|
||||
import type { RoutingContext } from '../routingStrategy.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
import type { Config } from '../../config/config.js';
|
||||
import {
|
||||
DEFAULT_GEMINI_MODEL,
|
||||
DEFAULT_GEMINI_FLASH_MODEL,
|
||||
DEFAULT_GEMINI_FLASH_LITE_MODEL,
|
||||
} from '../../config/models.js';
|
||||
|
||||
describe('FallbackStrategy', () => {
|
||||
const strategy = new FallbackStrategy();
|
||||
const mockContext = {} as RoutingContext;
|
||||
const mockClient = {} as BaseLlmClient;
|
||||
|
||||
it('should return null when not in fallback mode', async () => {
|
||||
const mockConfig = {
|
||||
isInFallbackMode: () => false,
|
||||
getModel: () => DEFAULT_GEMINI_MODEL,
|
||||
} as Config;
|
||||
|
||||
const decision = await strategy.route(mockContext, mockConfig, mockClient);
|
||||
expect(decision).toBeNull();
|
||||
});
|
||||
|
||||
describe('when in fallback mode', () => {
|
||||
it('should downgrade a pro model to the flash model', async () => {
|
||||
const mockConfig = {
|
||||
isInFallbackMode: () => true,
|
||||
getModel: () => DEFAULT_GEMINI_MODEL,
|
||||
} as Config;
|
||||
|
||||
const decision = await strategy.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockClient,
|
||||
);
|
||||
|
||||
expect(decision).not.toBeNull();
|
||||
expect(decision?.model).toBe(DEFAULT_GEMINI_FLASH_MODEL);
|
||||
expect(decision?.metadata.source).toBe('fallback');
|
||||
expect(decision?.metadata.reasoning).toContain('In fallback mode');
|
||||
});
|
||||
|
||||
it('should honor a lite model request', async () => {
|
||||
const mockConfig = {
|
||||
isInFallbackMode: () => true,
|
||||
getModel: () => DEFAULT_GEMINI_FLASH_LITE_MODEL,
|
||||
} as Config;
|
||||
|
||||
const decision = await strategy.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockClient,
|
||||
);
|
||||
|
||||
expect(decision).not.toBeNull();
|
||||
expect(decision?.model).toBe(DEFAULT_GEMINI_FLASH_LITE_MODEL);
|
||||
expect(decision?.metadata.source).toBe('fallback');
|
||||
});
|
||||
|
||||
it('should use the flash model if flash is requested', async () => {
|
||||
const mockConfig = {
|
||||
isInFallbackMode: () => true,
|
||||
getModel: () => DEFAULT_GEMINI_FLASH_MODEL,
|
||||
} as Config;
|
||||
|
||||
const decision = await strategy.route(
|
||||
mockContext,
|
||||
mockConfig,
|
||||
mockClient,
|
||||
);
|
||||
|
||||
expect(decision).not.toBeNull();
|
||||
expect(decision?.model).toBe(DEFAULT_GEMINI_FLASH_MODEL);
|
||||
expect(decision?.metadata.source).toBe('fallback');
|
||||
});
|
||||
});
|
||||
});
|
||||
43
packages/core/src/routing/strategies/fallbackStrategy.ts
Normal file
43
packages/core/src/routing/strategies/fallbackStrategy.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { Config } from '../../config/config.js';
|
||||
import { getEffectiveModel } from '../../config/models.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
import type {
|
||||
RoutingContext,
|
||||
RoutingDecision,
|
||||
RoutingStrategy,
|
||||
} from '../routingStrategy.js';
|
||||
|
||||
export class FallbackStrategy implements RoutingStrategy {
|
||||
readonly name = 'fallback';
|
||||
|
||||
async route(
|
||||
_context: RoutingContext,
|
||||
config: Config,
|
||||
_baseLlmClient: BaseLlmClient,
|
||||
): Promise<RoutingDecision | null> {
|
||||
const isInFallbackMode: boolean = config.isInFallbackMode();
|
||||
|
||||
if (!isInFallbackMode) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const effectiveModel = getEffectiveModel(
|
||||
isInFallbackMode,
|
||||
config.getModel(),
|
||||
);
|
||||
return {
|
||||
model: effectiveModel,
|
||||
metadata: {
|
||||
source: this.name,
|
||||
latencyMs: 0,
|
||||
reasoning: `In fallback mode. Using: ${effectiveModel}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { OverrideStrategy } from './overrideStrategy.js';
|
||||
import type { RoutingContext } from '../routingStrategy.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
import type { Config } from '../../config/config.js';
|
||||
|
||||
describe('OverrideStrategy', () => {
|
||||
const strategy = new OverrideStrategy();
|
||||
const mockContext = {} as RoutingContext;
|
||||
const mockClient = {} as BaseLlmClient;
|
||||
|
||||
it('should return null when no override model is specified', async () => {
|
||||
const mockConfig = {
|
||||
getModel: () => '', // Simulate no model override
|
||||
} as Config;
|
||||
|
||||
const decision = await strategy.route(mockContext, mockConfig, mockClient);
|
||||
expect(decision).toBeNull();
|
||||
});
|
||||
|
||||
it('should return a decision with the override model when one is specified', async () => {
|
||||
const overrideModel = 'gemini-2.5-pro-custom';
|
||||
const mockConfig = {
|
||||
getModel: () => overrideModel,
|
||||
} as Config;
|
||||
|
||||
const decision = await strategy.route(mockContext, mockConfig, mockClient);
|
||||
|
||||
expect(decision).not.toBeNull();
|
||||
expect(decision?.model).toBe(overrideModel);
|
||||
expect(decision?.metadata.source).toBe('override');
|
||||
expect(decision?.metadata.reasoning).toContain(
|
||||
'Routing bypassed by forced model directive',
|
||||
);
|
||||
expect(decision?.metadata.reasoning).toContain(overrideModel);
|
||||
});
|
||||
|
||||
it('should handle different override model names', async () => {
|
||||
const overrideModel = 'gemini-2.5-flash-experimental';
|
||||
const mockConfig = {
|
||||
getModel: () => overrideModel,
|
||||
} as Config;
|
||||
|
||||
const decision = await strategy.route(mockContext, mockConfig, mockClient);
|
||||
|
||||
expect(decision).not.toBeNull();
|
||||
expect(decision?.model).toBe(overrideModel);
|
||||
});
|
||||
});
|
||||
40
packages/core/src/routing/strategies/overrideStrategy.ts
Normal file
40
packages/core/src/routing/strategies/overrideStrategy.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { Config } from '../../config/config.js';
|
||||
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
|
||||
import type {
|
||||
RoutingContext,
|
||||
RoutingDecision,
|
||||
RoutingStrategy,
|
||||
} from '../routingStrategy.js';
|
||||
|
||||
/**
|
||||
* Handles cases where the user explicitly specifies a model (override).
|
||||
*/
|
||||
export class OverrideStrategy implements RoutingStrategy {
|
||||
readonly name = 'override';
|
||||
|
||||
async route(
|
||||
_context: RoutingContext,
|
||||
config: Config,
|
||||
_baseLlmClient: BaseLlmClient,
|
||||
): Promise<RoutingDecision | null> {
|
||||
const overrideModel = config.getModel();
|
||||
if (overrideModel) {
|
||||
return {
|
||||
model: overrideModel,
|
||||
metadata: {
|
||||
source: this.name,
|
||||
latencyMs: 0,
|
||||
reasoning: `Routing bypassed by forced model directive. Using: ${overrideModel}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
// No override specified, pass to the next strategy.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user