diff --git a/packages/core/src/routing/strategies/numericalClassifierStrategy.test.ts b/packages/core/src/routing/strategies/numericalClassifierStrategy.test.ts index b585fefe91..93e75fcdb5 100644 --- a/packages/core/src/routing/strategies/numericalClassifierStrategy.test.ts +++ b/packages/core/src/routing/strategies/numericalClassifierStrategy.test.ts @@ -122,7 +122,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_FLASH_MODEL, metadata: { - source: 'Classifier (Control)', + source: 'NumericalClassifier (Control)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 40 / Threshold: 50'), }, @@ -148,7 +148,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_MODEL, metadata: { - source: 'Classifier (Control)', + source: 'NumericalClassifier (Control)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 60 / Threshold: 50'), }, @@ -174,7 +174,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_FLASH_MODEL, // Routed to Flash because 60 < 80 metadata: { - source: 'Classifier (Strict)', + source: 'NumericalClassifier (Strict)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 60 / Threshold: 80'), }, @@ -200,7 +200,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_MODEL, metadata: { - source: 'Classifier (Strict)', + source: 'NumericalClassifier (Strict)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 90 / Threshold: 80'), }, @@ -228,7 +228,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_FLASH_MODEL, // Score 60 < Threshold 70 metadata: { - source: 'Classifier (Remote)', + source: 'NumericalClassifier (Remote)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 60 / Threshold: 70'), }, @@ -254,7 +254,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_FLASH_MODEL, // Score 40 < Threshold 45.5 metadata: { - source: 'Classifier (Remote)', + source: 'NumericalClassifier (Remote)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 40 / Threshold: 45.5'), }, @@ -280,7 +280,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_MODEL, // Score 35 >= Threshold 30 metadata: { - source: 'Classifier (Remote)', + source: 'NumericalClassifier (Remote)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 35 / Threshold: 30'), }, @@ -308,7 +308,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_FLASH_MODEL, // Score 40 < Default A/B Threshold 50 metadata: { - source: 'Classifier (Control)', + source: 'NumericalClassifier (Control)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 40 / Threshold: 50'), }, @@ -335,7 +335,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_FLASH_MODEL, metadata: { - source: 'Classifier (Control)', + source: 'NumericalClassifier (Control)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 40 / Threshold: 50'), }, @@ -362,7 +362,7 @@ describe('NumericalClassifierStrategy', () => { expect(decision).toEqual({ model: DEFAULT_GEMINI_MODEL, metadata: { - source: 'Classifier (Control)', + source: 'NumericalClassifier (Control)', latencyMs: expect.any(Number), reasoning: expect.stringContaining('Score: 60 / Threshold: 50'), }, diff --git a/packages/core/src/routing/strategies/numericalClassifierStrategy.ts b/packages/core/src/routing/strategies/numericalClassifierStrategy.ts index bcbb8543c2..9bcaebf432 100644 --- a/packages/core/src/routing/strategies/numericalClassifierStrategy.ts +++ b/packages/core/src/routing/strategies/numericalClassifierStrategy.ts @@ -187,7 +187,7 @@ export class NumericalClassifierStrategy implements RoutingStrategy { return { model: selectedModel, metadata: { - source: `Classifier (${groupLabel})`, + source: `NumericalClassifier (${groupLabel})`, latencyMs, reasoning: `[Score: ${score} / Threshold: ${threshold}] ${routerResponse.complexity_reasoning}`, }, diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts index fa7dd705c6..3cad76b491 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts @@ -941,6 +941,38 @@ describe('ClearcutLogger', () => { 'Something went wrong', ]); }); + + it('logs a successful routing event with numerical routing fields', () => { + const { logger } = setup(); + const event = new ModelRoutingEvent( + 'gemini-pro', + 'NumericalClassifier (Strict)', + 123, + '[Score: 90 / Threshold: 80] reasoning', + false, + undefined, + true, + '80', + ); + + logger?.logModelRoutingEvent(event); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.MODEL_ROUTING); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_ROUTING_REASONING, + '[Score: 90 / Threshold: 80] reasoning', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_ROUTING_NUMERICAL_ENABLED, + 'true', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_ROUTING_CLASSIFIER_THRESHOLD, + '80', + ]); + }); }); describe('logAgentStartEvent', () => { diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index 9417bbe983..cf009307c5 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -1234,6 +1234,28 @@ export class ClearcutLogger { }); } + if (event.reasoning && this.config?.getTelemetryLogPromptsEnabled()) { + data.push({ + gemini_cli_key: EventMetadataKey.GEMINI_CLI_ROUTING_REASONING, + value: event.reasoning, + }); + } + + if (event.enable_numerical_routing !== undefined) { + data.push({ + gemini_cli_key: EventMetadataKey.GEMINI_CLI_ROUTING_NUMERICAL_ENABLED, + value: event.enable_numerical_routing.toString(), + }); + } + + if (event.classifier_threshold) { + data.push({ + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_ROUTING_CLASSIFIER_THRESHOLD, + value: event.classifier_threshold, + }); + } + this.enqueueLogEvent(this.createLogEvent(EventNames.MODEL_ROUTING, data)); this.flushIfNeeded(); } diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts index a3b22ce58e..89bf4afb5a 100644 --- a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -7,7 +7,7 @@ // Defines valid event metadata keys for Clearcut logging. export enum EventMetadataKey { // Deleted enums: 24 - // Next ID: 144 + // Next ID: 148 GEMINI_CLI_KEY_UNKNOWN = 0, @@ -542,4 +542,17 @@ export enum EventMetadataKey { // Logs the duration spent in an approval mode in milliseconds. GEMINI_CLI_APPROVAL_MODE_DURATION_MS = 143, + + // ========================================================================== + // Model Routing Event Keys (Cont.) + // ========================================================================== + + // Logs the reasoning for the routing decision. + GEMINI_CLI_ROUTING_REASONING = 145, + + // Logs whether numerical routing was enabled. + GEMINI_CLI_ROUTING_NUMERICAL_ENABLED = 146, + + // Logs the classifier threshold used. + GEMINI_CLI_ROUTING_CLASSIFIER_THRESHOLD = 147, } diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index d584dc8ae7..43d8faeeea 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -1734,6 +1734,37 @@ describe('loggers', () => { ); }); + it('should log the event with numerical routing fields', () => { + const event = new ModelRoutingEvent( + 'gemini-pro', + 'NumericalClassifier (Strict)', + 150, + '[Score: 90 / Threshold: 80] reasoning', + false, + undefined, + true, + '80', + ); + + logModelRouting(mockConfig, event); + + expect( + ClearcutLogger.prototype.logModelRoutingEvent, + ).toHaveBeenCalledWith(event); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Model routing decision. Model: gemini-pro, Source: NumericalClassifier (Strict)', + attributes: { + 'session.id': 'test-session-id', + 'user.email': 'test-user@example.com', + 'installation.id': 'test-installation-id', + ...event, + 'event.name': EVENT_MODEL_ROUTING, + interactive: false, + }, + }); + }); + it('should only log to Clearcut if OTEL SDK is not initialized', () => { vi.spyOn(sdk, 'isTelemetrySdkInitialized').mockReturnValue(false); vi.spyOn(sdk, 'bufferTelemetryEvent').mockImplementation(() => {}); diff --git a/packages/core/src/telemetry/metrics.test.ts b/packages/core/src/telemetry/metrics.test.ts index e027a350ba..f1f7f2d223 100644 --- a/packages/core/src/telemetry/metrics.test.ts +++ b/packages/core/src/telemetry/metrics.test.ts @@ -489,7 +489,7 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); const event = new ModelRoutingEvent( 'gemini-pro', - 'classifier', + 'Classifier', 200, 'test-reason', true, @@ -502,7 +502,7 @@ describe('Telemetry Metrics', () => { 'installation.id': 'test-installation-id', 'user.email': 'test@example.com', 'routing.decision_model': 'gemini-pro', - 'routing.decision_source': 'classifier', + 'routing.decision_source': 'Classifier', 'routing.failed': true, 'routing.reasoning': 'test-reason', }); @@ -513,7 +513,7 @@ describe('Telemetry Metrics', () => { 'installation.id': 'test-installation-id', 'user.email': 'test@example.com', 'routing.decision_model': 'gemini-pro', - 'routing.decision_source': 'classifier', + 'routing.decision_source': 'Classifier', 'routing.failed': true, 'routing.reasoning': 'test-reason', 'routing.error_message': 'test-error',