feat(telemetry): Add telemetry and metrics for model routing (#8518)

packages/core/src/routing/modelRouterService.test.ts

@@ -14,6 +14,8 @@ import { CompositeStrategy } from './strategies/compositeStrategy.js';
 import { FallbackStrategy } from './strategies/fallbackStrategy.js';
 import { OverrideStrategy } from './strategies/overrideStrategy.js';
 import { ClassifierStrategy } from './strategies/classifierStrategy.js';
+import { logModelRouting } from '../telemetry/loggers.js';
+import { ModelRoutingEvent } from '../telemetry/types.js';
 
 vi.mock('../config/config.js');
 vi.mock('../core/baseLlmClient.js');
@@ -22,6 +24,8 @@ vi.mock('./strategies/compositeStrategy.js');
 vi.mock('./strategies/fallbackStrategy.js');
 vi.mock('./strategies/overrideStrategy.js');
 vi.mock('./strategies/classifierStrategy.js');
+vi.mock('../telemetry/loggers.js');
+vi.mock('../telemetry/types.js');
 
 describe('ModelRouterService', () => {
   let service: ModelRouterService;
@@ -78,15 +82,16 @@ describe('ModelRouterService', () => {
   });
 
   describe('route()', () => {
+    const strategyDecision: RoutingDecision = {
+      model: 'strategy-chosen-model',
+      metadata: {
+        source: 'test-router/fallback',
+        latencyMs: 10,
+        reasoning: 'Strategy reasoning',
+      },
+    };
+
     it('should delegate routing to the composite strategy', async () => {
-      const strategyDecision: RoutingDecision = {
-        model: 'strategy-chosen-model',
-        metadata: {
-          source: 'test-router/fallback',
-          latencyMs: 10,
-          reasoning: 'Strategy reasoning',
-        },
-      };
       const strategySpy = vi
         .spyOn(mockCompositeStrategy, 'route')
         .mockResolvedValue(strategyDecision);
@@ -100,5 +105,47 @@ describe('ModelRouterService', () => {
       );
       expect(decision).toEqual(strategyDecision);
     });
+
+    it('should log a telemetry event on a successful decision', async () => {
+      vi.spyOn(mockCompositeStrategy, 'route').mockResolvedValue(
+        strategyDecision,
+      );
+
+      await service.route(mockContext);
+
+      expect(ModelRoutingEvent).toHaveBeenCalledWith(
+        'strategy-chosen-model',
+        'test-router/fallback',
+        10,
+        'Strategy reasoning',
+        false,
+        undefined,
+      );
+      expect(logModelRouting).toHaveBeenCalledWith(
+        mockConfig,
+        expect.any(ModelRoutingEvent),
+      );
+    });
+
+    it('should log a telemetry event and re-throw on a failed decision', async () => {
+      const testError = new Error('Strategy failed');
+      vi.spyOn(mockCompositeStrategy, 'route').mockRejectedValue(testError);
+      vi.spyOn(mockConfig, 'getModel').mockReturnValue('default-model');
+
+      await expect(service.route(mockContext)).rejects.toThrow(testError);
+
+      expect(ModelRoutingEvent).toHaveBeenCalledWith(
+        'default-model',
+        'router-exception',
+        expect.any(Number),
+        'An exception occurred during routing.',
+        true,
+        'Strategy failed',
+      );
+      expect(logModelRouting).toHaveBeenCalledWith(
+        mockConfig,
+        expect.any(ModelRoutingEvent),
+      );
+    });
   });
 });

packages/core/src/routing/modelRouterService.ts

@@ -16,6 +16,9 @@ import { CompositeStrategy } from './strategies/compositeStrategy.js';
 import { FallbackStrategy } from './strategies/fallbackStrategy.js';
 import { OverrideStrategy } from './strategies/overrideStrategy.js';
 
+import { logModelRouting } from '../telemetry/loggers.js';
+import { ModelRoutingEvent } from '../telemetry/types.js';
+
 /**
  * A centralized service for making model routing decisions.
  */
@@ -49,12 +52,55 @@ export class ModelRouterService {
    * @returns A promise that resolves to a RoutingDecision.
    */
   async route(context: RoutingContext): Promise<RoutingDecision> {
-    const decision = await this.strategy.route(
-      context,
-      this.config,
-      this.config.getBaseLlmClient(),
-    );
+    const startTime = Date.now();
+    let decision: RoutingDecision;
 
-    return decision;
+    try {
+      decision = await this.strategy.route(
+        context,
+        this.config,
+        this.config.getBaseLlmClient(),
+      );
+
+      const event = new ModelRoutingEvent(
+        decision.model,
+        decision.metadata.source,
+        decision.metadata.latencyMs,
+        decision.metadata.reasoning,
+        false, // failed
+        undefined, // error_message
+      );
+      logModelRouting(this.config, event);
+
+      return decision;
+    } catch (e) {
+      const failed = true;
+      const error_message = e instanceof Error ? e.message : String(e);
+      // Create a fallback decision for logging purposes
+      // We do not actually route here. This should never happen so we should
+      // fail loudly to catch any issues where this happens.
+      decision = {
+        model: this.config.getModel(),
+        metadata: {
+          source: 'router-exception',
+          latencyMs: Date.now() - startTime,
+          reasoning: 'An exception occurred during routing.',
+          error: error_message,
+        },
+      };
+
+      const event = new ModelRoutingEvent(
+        decision.model,
+        decision.metadata.source,
+        decision.metadata.latencyMs,
+        decision.metadata.reasoning,
+        failed,
+        error_message,
+      );
+
+      logModelRouting(this.config, event);
+
+      throw e;
+    }
   }
 }