feat(routing): availability-aware auto-routing with best-effort pro

Adds settings and logic to detect slow/hanging Pro model requests, marking them as temporarily unavailable and automatically triggering a fallback to Flash. Introduces a proTimeoutMinutes and bestEffortPro strategy configuration.
This commit is contained in:
jacob314
2026-04-23 13:50:24 -07:00
parent ff28d55100
commit 19601955fd
15 changed files with 269 additions and 44 deletions
+1
View File
@@ -1044,6 +1044,7 @@ export async function loadCliConfig(
format: (argv.outputFormat ?? settings.output?.format) as OutputFormat,
},
gemmaModelRouter: settings.experimental?.gemmaModelRouter,
autoRouting: settings.model?.autoRouting,
adk: settings.experimental?.adk,
fakeResponses: argv.fakeResponses,
recordResponses: argv.recordResponses,
+40
View File
@@ -1112,6 +1112,46 @@ const SETTINGS_SCHEMA = {
description: 'Skip the next speaker check.',
showInDialog: true,
},
autoRouting: {
type: 'object',
label: 'Auto Routing',
category: 'Model',
requiresRestart: false,
default: {},
description: 'Settings for automatic model routing.',
showInDialog: false,
properties: {
bestEffortPro: {
type: 'boolean',
label: 'Best Effort Pro',
category: 'Model',
requiresRestart: false,
default: false,
description:
'Always prefer the Pro model unless it is unavailable (e.g., due to timeouts or quota), ignoring other routing hints.',
showInDialog: true,
},
proTimeoutMinutes: {
type: 'number',
label: 'Pro Timeout (Minutes)',
category: 'Model',
requiresRestart: false,
default: 5,
description:
'If a Pro request takes longer than this many minutes, it will be marked as temporarily unavailable and fallback to Flash.',
showInDialog: true,
},
proTimeoutFallbackDurationMinutes: {
type: 'number',
label: 'Pro Timeout Fallback Duration (Minutes)',
category: 'Model',
requiresRestart: false,
default: 60,
description: 'How long to route to Flash after Pro times out.',
showInDialog: true,
},
},
},
},
},
@@ -124,6 +124,9 @@ export const createMockConfig = (overrides: Partial<Config> = {}): Config =>
getCompressionThreshold: vi.fn().mockResolvedValue(undefined),
getUserCaching: vi.fn().mockResolvedValue(false),
getNumericalRoutingEnabled: vi.fn().mockResolvedValue(false),
getBestEffortProEnabled: vi.fn().mockResolvedValue(false),
getProTimeoutMinutes: vi.fn().mockResolvedValue(5),
getProTimeoutFallbackDurationMinutes: vi.fn().mockResolvedValue(60),
getClassifierThreshold: vi.fn().mockResolvedValue(undefined),
getBannerTextNoCapacityIssues: vi.fn().mockResolvedValue(''),
getBannerTextCapacityIssues: vi.fn().mockResolvedValue(''),