diff --git a/packages/core/src/telemetry/constants.ts b/packages/core/src/telemetry/constants.ts index bf8fc140fa..4815f8b6dc 100644 --- a/packages/core/src/telemetry/constants.ts +++ b/packages/core/src/telemetry/constants.ts @@ -31,39 +31,7 @@ export const EVENT_CONTENT_RETRY_FAILURE = 'gemini_cli.chat.content_retry_failure'; export const EVENT_FILE_OPERATION = 'gemini_cli.file_operation'; export const EVENT_MODEL_SLASH_COMMAND = 'gemini_cli.slash_command.model'; -export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count'; -export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency'; -export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count'; -export const METRIC_API_REQUEST_LATENCY = 'gemini_cli.api.request.latency'; -export const METRIC_TOKEN_USAGE = 'gemini_cli.token.usage'; -export const METRIC_SESSION_COUNT = 'gemini_cli.session.count'; -export const METRIC_FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count'; -export const METRIC_INVALID_CHUNK_COUNT = 'gemini_cli.chat.invalid_chunk.count'; -export const METRIC_CONTENT_RETRY_COUNT = 'gemini_cli.chat.content_retry.count'; -export const METRIC_CONTENT_RETRY_FAILURE_COUNT = - 'gemini_cli.chat.content_retry_failure.count'; export const EVENT_MODEL_ROUTING = 'gemini_cli.model_routing'; -export const METRIC_MODEL_ROUTING_LATENCY = 'gemini_cli.model_routing.latency'; -export const METRIC_MODEL_ROUTING_FAILURE_COUNT = - 'gemini_cli.model_routing.failure.count'; -export const METRIC_MODEL_SLASH_COMMAND_CALL_COUNT = - 'gemini_cli.slash_command.model.call_count'; - -// Performance Monitoring Metrics -export const METRIC_STARTUP_TIME = 'gemini_cli.startup.duration'; -export const METRIC_MEMORY_USAGE = 'gemini_cli.memory.usage'; -export const METRIC_CPU_USAGE = 'gemini_cli.cpu.usage'; -export const METRIC_TOOL_QUEUE_DEPTH = 'gemini_cli.tool.queue.depth'; -export const METRIC_TOOL_EXECUTION_BREAKDOWN = - 'gemini_cli.tool.execution.breakdown'; -export const METRIC_TOKEN_EFFICIENCY = 'gemini_cli.token.efficiency'; -export const METRIC_API_REQUEST_BREAKDOWN = 'gemini_cli.api.request.breakdown'; -export const METRIC_PERFORMANCE_SCORE = 'gemini_cli.performance.score'; -export const METRIC_REGRESSION_DETECTION = 'gemini_cli.performance.regression'; -export const METRIC_REGRESSION_PERCENTAGE_CHANGE = - 'gemini_cli.performance.regression.percentage_change'; -export const METRIC_BASELINE_COMPARISON = - 'gemini_cli.performance.baseline.comparison'; // Performance Events export const EVENT_STARTUP_PERFORMANCE = 'gemini_cli.startup.performance'; diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index 1997602480..f6b0f00dbf 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -344,16 +344,14 @@ describe('loggers', () => { expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith( mockConfig, - 'test-model', 100, - 200, + { model: 'test-model', status_code: 200 }, ); expect(mockMetrics.recordTokenUsageMetrics).toHaveBeenCalledWith( mockConfig, - 'test-model', 50, - 'output', + { model: 'test-model', type: 'output' }, ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -632,11 +630,13 @@ describe('loggers', () => { expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( mockConfig, - 'test-function', 100, - true, - ToolCallDecision.ACCEPT, - 'native', + { + function_name: 'test-function', + success: true, + decision: ToolCallDecision.ACCEPT, + tool_type: 'native', + }, ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -703,11 +703,13 @@ describe('loggers', () => { expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( mockConfig, - 'test-function', 100, - false, - ToolCallDecision.REJECT, - 'native', + { + function_name: 'test-function', + success: false, + decision: ToolCallDecision.REJECT, + tool_type: 'native', + }, ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -777,11 +779,13 @@ describe('loggers', () => { expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( mockConfig, - 'test-function', 100, - true, - ToolCallDecision.MODIFY, - 'native', + { + function_name: 'test-function', + success: true, + decision: ToolCallDecision.MODIFY, + tool_type: 'native', + }, ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -850,11 +854,13 @@ describe('loggers', () => { expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( mockConfig, - 'test-function', 100, - true, - undefined, - 'native', + { + function_name: 'test-function', + success: true, + decision: undefined, + tool_type: 'native', + }, ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -924,11 +930,13 @@ describe('loggers', () => { expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( mockConfig, - 'test-function', 100, - false, - undefined, - 'native', + { + function_name: 'test-function', + success: false, + decision: undefined, + tool_type: 'native', + }, ); expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ @@ -1085,11 +1093,13 @@ describe('loggers', () => { expect(mockMetrics.recordFileOperationMetric).toHaveBeenCalledWith( mockConfig, - 'read', - 10, - 'text/plain', - '.txt', - 'typescript', + { + operation: 'read', + lines: 10, + mimetype: 'text/plain', + extension: '.txt', + programming_language: 'typescript', + }, ); }); }); diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index 2b0889efd9..c614cd1bf6 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -191,14 +191,12 @@ export function logToolCall(config: Config, event: ToolCallEvent): void { attributes, }; logger.emit(logRecord); - recordToolCallMetrics( - config, - event.function_name, - event.duration_ms, - event.success, - event.decision, - event.tool_type, - ); + recordToolCallMetrics(config, event.duration_ms, { + function_name: event.function_name, + success: event.success, + decision: event.decision, + tool_type: event.tool_type, + }); } export function logToolOutputTruncated( @@ -258,14 +256,13 @@ export function logFileOperation( }; logger.emit(logRecord); - recordFileOperationMetric( - config, - event.operation, - event.lines, - event.mimetype, - event.extension, - event.programming_language, - ); + recordFileOperationMetric(config, { + operation: event.operation, + lines: event.lines, + mimetype: event.mimetype, + extension: event.extension, + programming_language: event.programming_language, + }); } export function logApiRequest(config: Config, event: ApiRequestEvent): void { @@ -364,13 +361,11 @@ export function logApiError(config: Config, event: ApiErrorEvent): void { attributes, }; logger.emit(logRecord); - recordApiErrorMetrics( - config, - event.model, - event.duration_ms, - event.status_code, - event.error_type, - ); + recordApiErrorMetrics(config, event.duration_ms, { + model: event.model, + status_code: event.status_code, + error_type: event.error_type, + }); } export function logApiResponse(config: Config, event: ApiResponseEvent): void { @@ -403,37 +398,30 @@ export function logApiResponse(config: Config, event: ApiResponseEvent): void { attributes, }; logger.emit(logRecord); - recordApiResponseMetrics( - config, - event.model, - event.duration_ms, - event.status_code, - ); - recordTokenUsageMetrics( - config, - event.model, - event.input_token_count, - 'input', - ); - recordTokenUsageMetrics( - config, - event.model, - event.output_token_count, - 'output', - ); - recordTokenUsageMetrics( - config, - event.model, - event.cached_content_token_count, - 'cache', - ); - recordTokenUsageMetrics( - config, - event.model, - event.thoughts_token_count, - 'thought', - ); - recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool'); + recordApiResponseMetrics(config, event.duration_ms, { + model: event.model, + status_code: event.status_code, + }); + recordTokenUsageMetrics(config, event.input_token_count, { + model: event.model, + type: 'input', + }); + recordTokenUsageMetrics(config, event.output_token_count, { + model: event.model, + type: 'output', + }); + recordTokenUsageMetrics(config, event.cached_content_token_count, { + model: event.model, + type: 'cache', + }); + recordTokenUsageMetrics(config, event.thoughts_token_count, { + model: event.model, + type: 'thought', + }); + recordTokenUsageMetrics(config, event.tool_token_count, { + model: event.model, + type: 'tool', + }); } export function logLoopDetected( diff --git a/packages/core/src/telemetry/metrics.test.ts b/packages/core/src/telemetry/metrics.test.ts index 8a19682846..58418d6c02 100644 --- a/packages/core/src/telemetry/metrics.test.ts +++ b/packages/core/src/telemetry/metrics.test.ts @@ -160,13 +160,19 @@ describe('Telemetry Metrics', () => { } as unknown as Config; it('should not record metrics if not initialized', () => { - recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input'); + recordTokenUsageMetricsModule(mockConfig, 100, { + model: 'gemini-pro', + type: 'input', + }); expect(mockCounterAddFn).not.toHaveBeenCalled(); }); it('should record token usage with the correct attributes', () => { initializeMetricsModule(mockConfig); - recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input'); + recordTokenUsageMetricsModule(mockConfig, 100, { + model: 'gemini-pro', + type: 'input', + }); expect(mockCounterAddFn).toHaveBeenCalledTimes(2); expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, { 'session.id': 'test-session-id', @@ -182,28 +188,40 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 50, 'output'); + recordTokenUsageMetricsModule(mockConfig, 50, { + model: 'gemini-pro', + type: 'output', + }); expect(mockCounterAddFn).toHaveBeenCalledWith(50, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'output', }); - recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 25, 'thought'); + recordTokenUsageMetricsModule(mockConfig, 25, { + model: 'gemini-pro', + type: 'thought', + }); expect(mockCounterAddFn).toHaveBeenCalledWith(25, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'thought', }); - recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 75, 'cache'); + recordTokenUsageMetricsModule(mockConfig, 75, { + model: 'gemini-pro', + type: 'cache', + }); expect(mockCounterAddFn).toHaveBeenCalledWith(75, { 'session.id': 'test-session-id', model: 'gemini-pro', type: 'cache', }); - recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 125, 'tool'); + recordTokenUsageMetricsModule(mockConfig, 125, { + model: 'gemini-pro', + type: 'tool', + }); expect(mockCounterAddFn).toHaveBeenCalledWith(125, { 'session.id': 'test-session-id', model: 'gemini-pro', @@ -215,7 +233,10 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordTokenUsageMetricsModule(mockConfig, 'gemini-ultra', 200, 'input'); + recordTokenUsageMetricsModule(mockConfig, 200, { + model: 'gemini-ultra', + type: 'input', + }); expect(mockCounterAddFn).toHaveBeenCalledWith(200, { 'session.id': 'test-session-id', model: 'gemini-ultra', @@ -231,25 +252,23 @@ describe('Telemetry Metrics', () => { } as unknown as Config; it('should not record metrics if not initialized', () => { - recordFileOperationMetricModule( - mockConfig, - FileOperation.CREATE, - 10, - 'text/plain', - 'txt', - ); + recordFileOperationMetricModule(mockConfig, { + operation: FileOperation.CREATE, + lines: 10, + mimetype: 'text/plain', + extension: 'txt', + }); expect(mockCounterAddFn).not.toHaveBeenCalled(); }); it('should record file creation with all attributes', () => { initializeMetricsModule(mockConfig); - recordFileOperationMetricModule( - mockConfig, - FileOperation.CREATE, - 10, - 'text/plain', - 'txt', - ); + recordFileOperationMetricModule(mockConfig, { + operation: FileOperation.CREATE, + lines: 10, + mimetype: 'text/plain', + extension: 'txt', + }); expect(mockCounterAddFn).toHaveBeenCalledTimes(2); expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, { @@ -268,7 +287,9 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordFileOperationMetricModule(mockConfig, FileOperation.READ); + recordFileOperationMetricModule(mockConfig, { + operation: FileOperation.READ, + }); expect(mockCounterAddFn).toHaveBeenCalledWith(1, { 'session.id': 'test-session-id', operation: FileOperation.READ, @@ -279,12 +300,10 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordFileOperationMetricModule( - mockConfig, - FileOperation.UPDATE, - undefined, - 'application/javascript', - ); + recordFileOperationMetricModule(mockConfig, { + operation: FileOperation.UPDATE, + mimetype: 'application/javascript', + }); expect(mockCounterAddFn).toHaveBeenCalledWith(1, { 'session.id': 'test-session-id', operation: FileOperation.UPDATE, @@ -296,13 +315,9 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordFileOperationMetricModule( - mockConfig, - FileOperation.UPDATE, - undefined, - undefined, - undefined, - ); + recordFileOperationMetricModule(mockConfig, { + operation: FileOperation.UPDATE, + }); expect(mockCounterAddFn).toHaveBeenCalledWith(1, { 'session.id': 'test-session-id', @@ -314,14 +329,12 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordFileOperationMetricModule( - mockConfig, - FileOperation.UPDATE, - 10, - 'text/plain', - 'txt', - undefined, - ); + recordFileOperationMetricModule(mockConfig, { + operation: FileOperation.UPDATE, + lines: 10, + mimetype: 'text/plain', + extension: 'txt', + }); expect(mockCounterAddFn).toHaveBeenCalledWith(1, { 'session.id': 'test-session-id', @@ -336,13 +349,9 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordFileOperationMetricModule( - mockConfig, - FileOperation.UPDATE, - undefined, - undefined, - undefined, - ); + recordFileOperationMetricModule(mockConfig, { + operation: FileOperation.UPDATE, + }); expect(mockCounterAddFn).toHaveBeenCalledWith(1, { 'session.id': 'test-session-id', @@ -436,14 +445,12 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfigDisabled); mockHistogramRecordFn.mockClear(); - recordStartupPerformanceModule( - mockConfigDisabled, - 'settings_loading', - 100, - { + recordStartupPerformanceModule(mockConfigDisabled, 100, { + phase: 'settings_loading', + details: { auth_type: 'gemini', }, - ); + }); expect(mockHistogramRecordFn).not.toHaveBeenCalled(); }); @@ -452,10 +459,13 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordStartupPerformanceModule(mockConfig, 'settings_loading', 150, { - auth_type: 'gemini', - telemetry_enabled: true, - settings_sources: 2, + recordStartupPerformanceModule(mockConfig, 150, { + phase: 'settings_loading', + details: { + auth_type: 'gemini', + telemetry_enabled: true, + settings_sources: 2, + }, }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, { @@ -471,7 +481,7 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordStartupPerformanceModule(mockConfig, 'cleanup', 50); + recordStartupPerformanceModule(mockConfig, 50, { phase: 'cleanup' }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(50, { 'session.id': 'test-session-id', @@ -485,15 +495,13 @@ describe('Telemetry Metrics', () => { // Test with realistic floating-point values that performance.now() would return const floatingPointDuration = 123.45678; - recordStartupPerformanceModule( - mockConfig, - 'total_startup', - floatingPointDuration, - { + recordStartupPerformanceModule(mockConfig, floatingPointDuration, { + phase: 'total_startup', + details: { is_tty: true, has_question: false, }, - ); + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith( floatingPointDuration, @@ -512,12 +520,10 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordMemoryUsageModule( - mockConfig, - MemoryMetricType.HEAP_USED, - 15728640, - 'startup', - ); + recordMemoryUsageModule(mockConfig, 15728640, { + memory_type: MemoryMetricType.HEAP_USED, + component: 'startup', + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, { 'session.id': 'test-session-id', @@ -530,24 +536,18 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordMemoryUsageModule( - mockConfig, - MemoryMetricType.HEAP_TOTAL, - 31457280, - 'api_call', - ); - recordMemoryUsageModule( - mockConfig, - MemoryMetricType.EXTERNAL, - 2097152, - 'tool_execution', - ); - recordMemoryUsageModule( - mockConfig, - MemoryMetricType.RSS, - 41943040, - 'memory_monitor', - ); + recordMemoryUsageModule(mockConfig, 31457280, { + memory_type: MemoryMetricType.HEAP_TOTAL, + component: 'api_call', + }); + recordMemoryUsageModule(mockConfig, 2097152, { + memory_type: MemoryMetricType.EXTERNAL, + component: 'tool_execution', + }); + recordMemoryUsageModule(mockConfig, 41943040, { + memory_type: MemoryMetricType.RSS, + component: 'memory_monitor', + }); expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 31457280, { @@ -571,16 +571,13 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordMemoryUsageModule( - mockConfig, - MemoryMetricType.HEAP_USED, - 15728640, - ); + recordMemoryUsageModule(mockConfig, 15728640, { + memory_type: MemoryMetricType.HEAP_USED, + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, { 'session.id': 'test-session-id', memory_type: 'heap_used', - component: undefined, }); }); }); @@ -590,7 +587,9 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordCpuUsageModule(mockConfig, 85.5, 'tool_execution'); + recordCpuUsageModule(mockConfig, 85.5, { + component: 'tool_execution', + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, { 'session.id': 'test-session-id', @@ -602,11 +601,10 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordCpuUsageModule(mockConfig, 42.3); + recordCpuUsageModule(mockConfig, 42.3, {}); expect(mockHistogramRecordFn).toHaveBeenCalledWith(42.3, { 'session.id': 'test-session-id', - component: undefined, }); }); }); @@ -640,12 +638,10 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordToolExecutionBreakdownModule( - mockConfig, - 'Read', - ToolExecutionPhase.VALIDATION, - 25, - ); + recordToolExecutionBreakdownModule(mockConfig, 25, { + function_name: 'Read', + phase: ToolExecutionPhase.VALIDATION, + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(25, { 'session.id': 'test-session-id', @@ -658,24 +654,18 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordToolExecutionBreakdownModule( - mockConfig, - 'Bash', - ToolExecutionPhase.PREPARATION, - 50, - ); - recordToolExecutionBreakdownModule( - mockConfig, - 'Bash', - ToolExecutionPhase.EXECUTION, - 1500, - ); - recordToolExecutionBreakdownModule( - mockConfig, - 'Bash', - ToolExecutionPhase.RESULT_PROCESSING, - 75, - ); + recordToolExecutionBreakdownModule(mockConfig, 50, { + function_name: 'Bash', + phase: ToolExecutionPhase.PREPARATION, + }); + recordToolExecutionBreakdownModule(mockConfig, 1500, { + function_name: 'Bash', + phase: ToolExecutionPhase.EXECUTION, + }); + recordToolExecutionBreakdownModule(mockConfig, 75, { + function_name: 'Bash', + phase: ToolExecutionPhase.RESULT_PROCESSING, + }); expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 50, { @@ -701,13 +691,11 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordTokenEfficiencyModule( - mockConfig, - 'gemini-pro', - 'cache_hit_rate', - 0.85, - 'api_request', - ); + recordTokenEfficiencyModule(mockConfig, 0.85, { + model: 'gemini-pro', + metric: 'cache_hit_rate', + context: 'api_request', + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(0.85, { 'session.id': 'test-session-id', @@ -721,18 +709,15 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordTokenEfficiencyModule( - mockConfig, - 'gemini-pro', - 'tokens_per_operation', - 125.5, - ); + recordTokenEfficiencyModule(mockConfig, 125.5, { + model: 'gemini-pro', + metric: 'tokens_per_operation', + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(125.5, { 'session.id': 'test-session-id', model: 'gemini-pro', metric: 'tokens_per_operation', - context: undefined, }); }); }); @@ -742,12 +727,10 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordApiRequestBreakdownModule( - mockConfig, - 'gemini-pro', - ApiRequestPhase.REQUEST_PREPARATION, - 15, - ); + recordApiRequestBreakdownModule(mockConfig, 15, { + model: 'gemini-pro', + phase: ApiRequestPhase.REQUEST_PREPARATION, + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(15, { 'session.id': 'test-session-id', @@ -760,24 +743,18 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordApiRequestBreakdownModule( - mockConfig, - 'gemini-pro', - ApiRequestPhase.NETWORK_LATENCY, - 250, - ); - recordApiRequestBreakdownModule( - mockConfig, - 'gemini-pro', - ApiRequestPhase.RESPONSE_PROCESSING, - 100, - ); - recordApiRequestBreakdownModule( - mockConfig, - 'gemini-pro', - ApiRequestPhase.TOKEN_PROCESSING, - 50, - ); + recordApiRequestBreakdownModule(mockConfig, 250, { + model: 'gemini-pro', + phase: ApiRequestPhase.NETWORK_LATENCY, + }); + recordApiRequestBreakdownModule(mockConfig, 100, { + model: 'gemini-pro', + phase: ApiRequestPhase.RESPONSE_PROCESSING, + }); + recordApiRequestBreakdownModule(mockConfig, 50, { + model: 'gemini-pro', + phase: ApiRequestPhase.TOKEN_PROCESSING, + }); expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 250, { @@ -803,12 +780,10 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordPerformanceScoreModule( - mockConfig, - 85.5, - 'memory_efficiency', - 80.0, - ); + recordPerformanceScoreModule(mockConfig, 85.5, { + category: 'memory_efficiency', + baseline: 80.0, + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, { 'session.id': 'test-session-id', @@ -821,12 +796,13 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordPerformanceScoreModule(mockConfig, 92.3, 'overall_performance'); + recordPerformanceScoreModule(mockConfig, 92.3, { + category: 'overall_performance', + }); expect(mockHistogramRecordFn).toHaveBeenCalledWith(92.3, { 'session.id': 'test-session-id', category: 'overall_performance', - baseline: undefined, }); }); }); @@ -837,13 +813,12 @@ describe('Telemetry Metrics', () => { mockCounterAddFn.mockClear(); mockHistogramRecordFn.mockClear(); - recordPerformanceRegressionModule( - mockConfig, - 'startup_time', - 1200, - 1000, - 'medium', - ); + recordPerformanceRegressionModule(mockConfig, { + metric: 'startup_time', + current_value: 1200, + baseline_value: 1000, + severity: 'medium', + }); // Verify regression counter expect(mockCounterAddFn).toHaveBeenCalledWith(1, { @@ -869,13 +844,12 @@ describe('Telemetry Metrics', () => { mockCounterAddFn.mockClear(); mockHistogramRecordFn.mockClear(); - recordPerformanceRegressionModule( - mockConfig, - 'memory_usage', - 100, - 0, - 'high', - ); + recordPerformanceRegressionModule(mockConfig, { + metric: 'memory_usage', + current_value: 100, + baseline_value: 0, + severity: 'high', + }); // Verify regression counter still recorded expect(mockCounterAddFn).toHaveBeenCalledWith(1, { @@ -894,20 +868,18 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); - recordPerformanceRegressionModule( - mockConfig, - 'api_latency', - 500, - 400, - 'low', - ); - recordPerformanceRegressionModule( - mockConfig, - 'cpu_usage', - 90, - 70, - 'high', - ); + recordPerformanceRegressionModule(mockConfig, { + metric: 'api_latency', + current_value: 500, + baseline_value: 400, + severity: 'low', + }); + recordPerformanceRegressionModule(mockConfig, { + metric: 'cpu_usage', + current_value: 90, + baseline_value: 70, + severity: 'high', + }); expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, { 'session.id': 'test-session-id', @@ -931,13 +903,12 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordBaselineComparisonModule( - mockConfig, - 'memory_usage', - 120, - 100, - 'performance_tracking', - ); + recordBaselineComparisonModule(mockConfig, { + metric: 'memory_usage', + current_value: 120, + baseline_value: 100, + category: 'performance_tracking', + }); // 20% increase: (120 - 100) / 100 * 100 = 20% expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, { @@ -953,13 +924,12 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordBaselineComparisonModule( - mockConfig, - 'startup_time', - 800, - 1000, - 'optimization', - ); + recordBaselineComparisonModule(mockConfig, { + metric: 'startup_time', + current_value: 800, + baseline_value: 1000, + category: 'optimization', + }); // 20% decrease: (800 - 1000) / 1000 * 100 = -20% expect(mockHistogramRecordFn).toHaveBeenCalledWith(-20, { @@ -981,13 +951,12 @@ describe('Telemetry Metrics', () => { initializeMetricsModule(mockConfig); mockHistogramRecordFn.mockClear(); - recordBaselineComparisonModule( - mockConfig, - 'new_metric', - 50, - 0, - 'testing', - ); + recordBaselineComparisonModule(mockConfig, { + metric: 'new_metric', + current_value: 50, + baseline_value: 0, + category: 'testing', + }); expect(diagSpy).toHaveBeenCalledWith( 'Baseline value is zero, skipping comparison.', diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts index 0d66119f55..9c9085e6fe 100644 --- a/packages/core/src/telemetry/metrics.ts +++ b/packages/core/src/telemetry/metrics.ts @@ -6,38 +6,305 @@ import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api'; import { diag, metrics, ValueType } from '@opentelemetry/api'; -import { - SERVICE_NAME, - METRIC_TOOL_CALL_COUNT, - METRIC_TOOL_CALL_LATENCY, - METRIC_API_REQUEST_COUNT, - METRIC_API_REQUEST_LATENCY, - METRIC_TOKEN_USAGE, - METRIC_SESSION_COUNT, - METRIC_FILE_OPERATION_COUNT, - EVENT_CHAT_COMPRESSION, - METRIC_INVALID_CHUNK_COUNT, - METRIC_CONTENT_RETRY_COUNT, - METRIC_CONTENT_RETRY_FAILURE_COUNT, - METRIC_MODEL_ROUTING_LATENCY, - METRIC_MODEL_ROUTING_FAILURE_COUNT, - METRIC_MODEL_SLASH_COMMAND_CALL_COUNT, - // Performance Monitoring Metrics - METRIC_STARTUP_TIME, - METRIC_MEMORY_USAGE, - METRIC_CPU_USAGE, - METRIC_TOOL_QUEUE_DEPTH, - METRIC_TOOL_EXECUTION_BREAKDOWN, - METRIC_TOKEN_EFFICIENCY, - METRIC_API_REQUEST_BREAKDOWN, - METRIC_PERFORMANCE_SCORE, - METRIC_REGRESSION_DETECTION, - METRIC_REGRESSION_PERCENTAGE_CHANGE, - METRIC_BASELINE_COMPARISON, -} from './constants.js'; +import { SERVICE_NAME, EVENT_CHAT_COMPRESSION } from './constants.js'; import type { Config } from '../config/config.js'; import type { ModelRoutingEvent, ModelSlashCommandEvent } from './types.js'; +const TOOL_CALL_COUNT = 'gemini_cli.tool.call.count'; +const TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency'; +const API_REQUEST_COUNT = 'gemini_cli.api.request.count'; +const API_REQUEST_LATENCY = 'gemini_cli.api.request.latency'; +const TOKEN_USAGE = 'gemini_cli.token.usage'; +const SESSION_COUNT = 'gemini_cli.session.count'; +const FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count'; +const INVALID_CHUNK_COUNT = 'gemini_cli.chat.invalid_chunk.count'; +const CONTENT_RETRY_COUNT = 'gemini_cli.chat.content_retry.count'; +const CONTENT_RETRY_FAILURE_COUNT = + 'gemini_cli.chat.content_retry_failure.count'; +const MODEL_ROUTING_LATENCY = 'gemini_cli.model_routing.latency'; +const MODEL_ROUTING_FAILURE_COUNT = 'gemini_cli.model_routing.failure.count'; +const MODEL_SLASH_COMMAND_CALL_COUNT = + 'gemini_cli.slash_command.model.call_count'; + +// Performance Monitoring Metrics +const STARTUP_TIME = 'gemini_cli.startup.duration'; +const MEMORY_USAGE = 'gemini_cli.memory.usage'; +const CPU_USAGE = 'gemini_cli.cpu.usage'; +const TOOL_QUEUE_DEPTH = 'gemini_cli.tool.queue.depth'; +const TOOL_EXECUTION_BREAKDOWN = 'gemini_cli.tool.execution.breakdown'; +const TOKEN_EFFICIENCY = 'gemini_cli.token.efficiency'; +const API_REQUEST_BREAKDOWN = 'gemini_cli.api.request.breakdown'; +const PERFORMANCE_SCORE = 'gemini_cli.performance.score'; +const REGRESSION_DETECTION = 'gemini_cli.performance.regression'; +const REGRESSION_PERCENTAGE_CHANGE = + 'gemini_cli.performance.regression.percentage_change'; +const BASELINE_COMPARISON = 'gemini_cli.performance.baseline.comparison'; + +const baseMetricDefinition = { + getCommonAttributes: (config: Config): Attributes => ({ + 'session.id': config.getSessionId(), + }), +}; + +const COUNTER_DEFINITIONS = { + [TOOL_CALL_COUNT]: { + description: 'Counts tool calls, tagged by function name and success.', + valueType: ValueType.INT, + assign: (c: Counter) => (toolCallCounter = c), + attributes: {} as { + function_name: string; + success: boolean; + decision?: 'accept' | 'reject' | 'modify' | 'auto_accept'; + tool_type?: 'native' | 'mcp'; + }, + }, + [API_REQUEST_COUNT]: { + description: 'Counts API requests, tagged by model and status.', + valueType: ValueType.INT, + assign: (c: Counter) => (apiRequestCounter = c), + attributes: {} as { + model: string; + status_code?: number | string; + error_type?: string; + }, + }, + [TOKEN_USAGE]: { + description: 'Counts the total number of tokens used.', + valueType: ValueType.INT, + assign: (c: Counter) => (tokenUsageCounter = c), + attributes: {} as { + model: string; + type: 'input' | 'output' | 'thought' | 'cache' | 'tool'; + }, + }, + [SESSION_COUNT]: { + description: 'Count of CLI sessions started.', + valueType: ValueType.INT, + assign: (c: Counter) => (sessionCounter = c), + attributes: {} as Record, + }, + [FILE_OPERATION_COUNT]: { + description: 'Counts file operations (create, read, update).', + valueType: ValueType.INT, + assign: (c: Counter) => (fileOperationCounter = c), + attributes: {} as { + operation: FileOperation; + lines?: number; + mimetype?: string; + extension?: string; + programming_language?: string; + }, + }, + [INVALID_CHUNK_COUNT]: { + description: 'Counts invalid chunks received from a stream.', + valueType: ValueType.INT, + assign: (c: Counter) => (invalidChunkCounter = c), + attributes: {} as Record, + }, + [CONTENT_RETRY_COUNT]: { + description: 'Counts retries due to content errors (e.g., empty stream).', + valueType: ValueType.INT, + assign: (c: Counter) => (contentRetryCounter = c), + attributes: {} as Record, + }, + [CONTENT_RETRY_FAILURE_COUNT]: { + description: 'Counts occurrences of all content retries failing.', + valueType: ValueType.INT, + assign: (c: Counter) => (contentRetryFailureCounter = c), + attributes: {} as Record, + }, + [MODEL_ROUTING_FAILURE_COUNT]: { + description: 'Counts model routing failures.', + valueType: ValueType.INT, + assign: (c: Counter) => (modelRoutingFailureCounter = c), + attributes: {} as { + 'routing.decision_source': string; + 'routing.error_message': string; + }, + }, + [MODEL_SLASH_COMMAND_CALL_COUNT]: { + description: 'Counts model slash command calls.', + valueType: ValueType.INT, + assign: (c: Counter) => (modelSlashCommandCallCounter = c), + attributes: {} as { + 'slash_command.model.model_name': string; + }, + }, + [EVENT_CHAT_COMPRESSION]: { + description: 'Counts chat compression events.', + valueType: ValueType.INT, + assign: (c: Counter) => (chatCompressionCounter = c), + attributes: {} as { + tokens_before: number; + tokens_after: number; + }, + }, +} as const; + +const HISTOGRAM_DEFINITIONS = { + [TOOL_CALL_LATENCY]: { + description: 'Latency of tool calls in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + assign: (h: Histogram) => (toolCallLatencyHistogram = h), + attributes: {} as { + function_name: string; + }, + }, + [API_REQUEST_LATENCY]: { + description: 'Latency of API requests in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + assign: (h: Histogram) => (apiRequestLatencyHistogram = h), + attributes: {} as { + model: string; + }, + }, + [MODEL_ROUTING_LATENCY]: { + description: 'Latency of model routing decisions in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + assign: (h: Histogram) => (modelRoutingLatencyHistogram = h), + attributes: {} as { + 'routing.decision_model': string; + 'routing.decision_source': string; + }, + }, +} as const; + +const PERFORMANCE_COUNTER_DEFINITIONS = { + [REGRESSION_DETECTION]: { + description: 'Performance regression detection events.', + valueType: ValueType.INT, + assign: (c: Counter) => (regressionDetectionCounter = c), + attributes: {} as { + metric: string; + severity: 'low' | 'medium' | 'high'; + current_value: number; + baseline_value: number; + }, + }, +} as const; + +const PERFORMANCE_HISTOGRAM_DEFINITIONS = { + [STARTUP_TIME]: { + description: + 'CLI startup time in milliseconds, broken down by initialization phase.', + unit: 'ms', + valueType: ValueType.DOUBLE, + assign: (h: Histogram) => (startupTimeHistogram = h), + attributes: {} as { + phase: string; + details?: Record; + }, + }, + [MEMORY_USAGE]: { + description: 'Memory usage in bytes.', + unit: 'bytes', + valueType: ValueType.INT, + assign: (h: Histogram) => (memoryUsageGauge = h), + attributes: {} as { + memory_type: MemoryMetricType; + component?: string; + }, + }, + [CPU_USAGE]: { + description: 'CPU usage percentage.', + unit: 'percent', + valueType: ValueType.DOUBLE, + assign: (h: Histogram) => (cpuUsageGauge = h), + attributes: {} as { + component?: string; + }, + }, + [TOOL_QUEUE_DEPTH]: { + description: 'Number of tools in execution queue.', + unit: 'count', + valueType: ValueType.INT, + assign: (h: Histogram) => (toolQueueDepthGauge = h), + attributes: {} as Record, + }, + [TOOL_EXECUTION_BREAKDOWN]: { + description: 'Tool execution time breakdown by phase in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + assign: (h: Histogram) => (toolExecutionBreakdownHistogram = h), + attributes: {} as { + function_name: string; + phase: ToolExecutionPhase; + }, + }, + [TOKEN_EFFICIENCY]: { + description: + 'Token efficiency metrics (tokens per operation, cache hit rate, etc.).', + unit: 'ratio', + valueType: ValueType.DOUBLE, + assign: (h: Histogram) => (tokenEfficiencyHistogram = h), + attributes: {} as { + model: string; + metric: string; + context?: string; + }, + }, + [API_REQUEST_BREAKDOWN]: { + description: 'API request time breakdown by phase in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + assign: (h: Histogram) => (apiRequestBreakdownHistogram = h), + attributes: {} as { + model: string; + phase: ApiRequestPhase; + }, + }, + [PERFORMANCE_SCORE]: { + description: 'Composite performance score (0-100).', + unit: 'score', + valueType: ValueType.DOUBLE, + assign: (h: Histogram) => (performanceScoreGauge = h), + attributes: {} as { + category: string; + baseline?: number; + }, + }, + [REGRESSION_PERCENTAGE_CHANGE]: { + description: + 'Percentage change compared to baseline for detected regressions.', + unit: 'percent', + valueType: ValueType.DOUBLE, + assign: (h: Histogram) => (regressionPercentageChangeHistogram = h), + attributes: {} as { + metric: string; + severity: 'low' | 'medium' | 'high'; + current_value: number; + baseline_value: number; + }, + }, + [BASELINE_COMPARISON]: { + description: + 'Performance comparison to established baseline (percentage change).', + unit: 'percent', + valueType: ValueType.DOUBLE, + assign: (h: Histogram) => (baselineComparisonHistogram = h), + attributes: {} as { + metric: string; + category: string; + current_value: number; + baseline_value: number; + }, + }, +} as const; + +type AllMetricDefs = typeof COUNTER_DEFINITIONS & + typeof HISTOGRAM_DEFINITIONS & + typeof PERFORMANCE_COUNTER_DEFINITIONS & + typeof PERFORMANCE_HISTOGRAM_DEFINITIONS; + +export type MetricDefinitions = { + [K in keyof AllMetricDefs]: { + attributes: AllMetricDefs[K]['attributes']; + }; +}; + export enum FileOperation { CREATE = 'create', READ = 'read', @@ -80,6 +347,7 @@ let toolCallLatencyHistogram: Histogram | undefined; let apiRequestCounter: Counter | undefined; let apiRequestLatencyHistogram: Histogram | undefined; let tokenUsageCounter: Counter | undefined; +let sessionCounter: Counter | undefined; let fileOperationCounter: Counter | undefined; let chatCompressionCounter: Counter | undefined; let invalidChunkCounter: Counter | undefined; @@ -104,12 +372,6 @@ let baselineComparisonHistogram: Histogram | undefined; let isMetricsInitialized = false; let isPerformanceMonitoringEnabled = false; -function getCommonAttributes(config: Config): Attributes { - return { - 'session.id': config.getSessionId(), - }; -} - export function getMeter(): Meter | undefined { if (!cliMeter) { cliMeter = metrics.getMeter(SERVICE_NAME); @@ -124,84 +386,20 @@ export function initializeMetrics(config: Config): void { if (!meter) return; // Initialize core metrics - toolCallCounter = meter.createCounter(METRIC_TOOL_CALL_COUNT, { - description: 'Counts tool calls, tagged by function name and success.', - valueType: ValueType.INT, - }); - toolCallLatencyHistogram = meter.createHistogram(METRIC_TOOL_CALL_LATENCY, { - description: 'Latency of tool calls in milliseconds.', - unit: 'ms', - valueType: ValueType.INT, - }); - apiRequestCounter = meter.createCounter(METRIC_API_REQUEST_COUNT, { - description: 'Counts API requests, tagged by model and status.', - valueType: ValueType.INT, - }); - apiRequestLatencyHistogram = meter.createHistogram( - METRIC_API_REQUEST_LATENCY, - { - description: 'Latency of API requests in milliseconds.', - unit: 'ms', - valueType: ValueType.INT, - }, - ); - tokenUsageCounter = meter.createCounter(METRIC_TOKEN_USAGE, { - description: 'Counts the total number of tokens used.', - valueType: ValueType.INT, - }); - fileOperationCounter = meter.createCounter(METRIC_FILE_OPERATION_COUNT, { - description: 'Counts file operations (create, read, update).', - valueType: ValueType.INT, - }); - chatCompressionCounter = meter.createCounter(EVENT_CHAT_COMPRESSION, { - description: 'Counts chat compression events.', - valueType: ValueType.INT, - }); - - // New counters for content errors - invalidChunkCounter = meter.createCounter(METRIC_INVALID_CHUNK_COUNT, { - description: 'Counts invalid chunks received from a stream.', - valueType: ValueType.INT, - }); - contentRetryCounter = meter.createCounter(METRIC_CONTENT_RETRY_COUNT, { - description: 'Counts retries due to content errors (e.g., empty stream).', - valueType: ValueType.INT, - }); - contentRetryFailureCounter = meter.createCounter( - METRIC_CONTENT_RETRY_FAILURE_COUNT, - { - description: 'Counts occurrences of all content retries failing.', - valueType: ValueType.INT, - }, - ); - modelRoutingLatencyHistogram = meter.createHistogram( - METRIC_MODEL_ROUTING_LATENCY, - { - description: 'Latency of model routing decisions in milliseconds.', - unit: 'ms', - valueType: ValueType.INT, - }, - ); - modelRoutingFailureCounter = meter.createCounter( - METRIC_MODEL_ROUTING_FAILURE_COUNT, - { - description: 'Counts model routing failures.', - valueType: ValueType.INT, - }, - ); - modelSlashCommandCallCounter = meter.createCounter( - METRIC_MODEL_SLASH_COMMAND_CALL_COUNT, - { - description: 'Counts model slash command calls.', - valueType: ValueType.INT, + Object.entries(COUNTER_DEFINITIONS).forEach( + ([name, { description, valueType, assign }]) => { + assign(meter.createCounter(name, { description, valueType })); }, ); - const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, { - description: 'Count of CLI sessions started.', - valueType: ValueType.INT, - }); - sessionCounter.add(1, getCommonAttributes(config)); + Object.entries(HISTOGRAM_DEFINITIONS).forEach( + ([name, { description, unit, valueType, assign }]) => { + assign(meter.createHistogram(name, { description, unit, valueType })); + }, + ); + + // Increment session counter after all metrics are initialized + sessionCounter?.add(1, baseMetricDefinition.getCommonAttributes(config)); // Initialize performance monitoring metrics if enabled initializePerformanceMonitoring(config); @@ -211,59 +409,50 @@ export function initializeMetrics(config: Config): void { export function recordChatCompressionMetrics( config: Config, - args: { tokens_before: number; tokens_after: number }, + attributes: MetricDefinitions[typeof EVENT_CHAT_COMPRESSION]['attributes'], ) { if (!chatCompressionCounter || !isMetricsInitialized) return; chatCompressionCounter.add(1, { - ...getCommonAttributes(config), - ...args, + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }); } export function recordToolCallMetrics( config: Config, - functionName: string, durationMs: number, - success: boolean, - decision?: 'accept' | 'reject' | 'modify' | 'auto_accept', - tool_type?: 'native' | 'mcp', + attributes: MetricDefinitions[typeof TOOL_CALL_COUNT]['attributes'], ): void { if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized) return; const metricAttributes: Attributes = { - ...getCommonAttributes(config), - function_name: functionName, - success, - decision, - tool_type, + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; toolCallCounter.add(1, metricAttributes); toolCallLatencyHistogram.record(durationMs, { - ...getCommonAttributes(config), - function_name: functionName, + ...baseMetricDefinition.getCommonAttributes(config), + function_name: attributes.function_name, }); } export function recordTokenUsageMetrics( config: Config, - model: string, tokenCount: number, - type: 'input' | 'output' | 'thought' | 'cache' | 'tool', + attributes: MetricDefinitions[typeof TOKEN_USAGE]['attributes'], ): void { if (!tokenUsageCounter || !isMetricsInitialized) return; tokenUsageCounter.add(tokenCount, { - ...getCommonAttributes(config), - model, - type, + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }); } export function recordApiResponseMetrics( config: Config, - model: string, durationMs: number, - statusCode?: number | string, + attributes: MetricDefinitions[typeof API_REQUEST_COUNT]['attributes'], ): void { if ( !apiRequestCounter || @@ -272,23 +461,21 @@ export function recordApiResponseMetrics( ) return; const metricAttributes: Attributes = { - ...getCommonAttributes(config), - model, - status_code: statusCode ?? 'ok', + ...baseMetricDefinition.getCommonAttributes(config), + model: attributes.model, + status_code: attributes.status_code ?? 'ok', }; apiRequestCounter.add(1, metricAttributes); apiRequestLatencyHistogram.record(durationMs, { - ...getCommonAttributes(config), - model, + ...baseMetricDefinition.getCommonAttributes(config), + model: attributes.model, }); } export function recordApiErrorMetrics( config: Config, - model: string, durationMs: number, - statusCode?: number | string, - errorType?: string, + attributes: MetricDefinitions[typeof API_REQUEST_COUNT]['attributes'], ): void { if ( !apiRequestCounter || @@ -297,38 +484,27 @@ export function recordApiErrorMetrics( ) return; const metricAttributes: Attributes = { - ...getCommonAttributes(config), - model, - status_code: statusCode ?? 'error', - error_type: errorType ?? 'unknown', + ...baseMetricDefinition.getCommonAttributes(config), + model: attributes.model, + status_code: attributes.status_code ?? 'error', + error_type: attributes.error_type ?? 'unknown', }; apiRequestCounter.add(1, metricAttributes); apiRequestLatencyHistogram.record(durationMs, { - ...getCommonAttributes(config), - model, + ...baseMetricDefinition.getCommonAttributes(config), + model: attributes.model, }); } export function recordFileOperationMetric( config: Config, - operation: FileOperation, - lines?: number, - mimetype?: string, - extension?: string, - programming_language?: string, + attributes: MetricDefinitions[typeof FILE_OPERATION_COUNT]['attributes'], ): void { if (!fileOperationCounter || !isMetricsInitialized) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - operation, - }; - if (lines !== undefined) attributes['lines'] = lines; - if (mimetype !== undefined) attributes['mimetype'] = mimetype; - if (extension !== undefined) attributes['extension'] = extension; - if (programming_language !== undefined) { - attributes['programming_language'] = programming_language; - } - fileOperationCounter.add(1, attributes); + fileOperationCounter.add(1, { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, + }); } // --- New Metric Recording Functions --- @@ -338,7 +514,7 @@ export function recordFileOperationMetric( */ export function recordInvalidChunk(config: Config): void { if (!invalidChunkCounter || !isMetricsInitialized) return; - invalidChunkCounter.add(1, getCommonAttributes(config)); + invalidChunkCounter.add(1, baseMetricDefinition.getCommonAttributes(config)); } /** @@ -346,7 +522,7 @@ export function recordInvalidChunk(config: Config): void { */ export function recordContentRetry(config: Config): void { if (!contentRetryCounter || !isMetricsInitialized) return; - contentRetryCounter.add(1, getCommonAttributes(config)); + contentRetryCounter.add(1, baseMetricDefinition.getCommonAttributes(config)); } /** @@ -354,7 +530,10 @@ export function recordContentRetry(config: Config): void { */ export function recordContentRetryFailure(config: Config): void { if (!contentRetryFailureCounter || !isMetricsInitialized) return; - contentRetryFailureCounter.add(1, getCommonAttributes(config)); + contentRetryFailureCounter.add( + 1, + baseMetricDefinition.getCommonAttributes(config), + ); } export function recordModelSlashCommand( @@ -363,7 +542,7 @@ export function recordModelSlashCommand( ): void { if (!modelSlashCommandCallCounter || !isMetricsInitialized) return; modelSlashCommandCallCounter.add(1, { - ...getCommonAttributes(config), + ...baseMetricDefinition.getCommonAttributes(config), 'slash_command.model.model_name': event.model_name, }); } @@ -380,14 +559,14 @@ export function recordModelRoutingMetrics( return; modelRoutingLatencyHistogram.record(event.routing_latency_ms, { - ...getCommonAttributes(config), + ...baseMetricDefinition.getCommonAttributes(config), 'routing.decision_model': event.decision_model, 'routing.decision_source': event.decision_source, }); if (event.failed) { modelRoutingFailureCounter.add(1, { - ...getCommonAttributes(config), + ...baseMetricDefinition.getCommonAttributes(config), 'routing.decision_source': event.decision_source, 'routing.error_message': event.error_message, }); @@ -406,149 +585,70 @@ export function initializePerformanceMonitoring(config: Config): void { if (!isPerformanceMonitoringEnabled) return; - // Initialize startup time histogram - startupTimeHistogram = meter.createHistogram(METRIC_STARTUP_TIME, { - description: - 'CLI startup time in milliseconds, broken down by initialization phase.', - unit: 'ms', - valueType: ValueType.DOUBLE, - }); - - // Initialize memory usage histogram (using histogram until ObservableGauge is available) - memoryUsageGauge = meter.createHistogram(METRIC_MEMORY_USAGE, { - description: 'Memory usage in bytes.', - unit: 'bytes', - valueType: ValueType.INT, - }); - - // Initialize CPU usage histogram - cpuUsageGauge = meter.createHistogram(METRIC_CPU_USAGE, { - description: 'CPU usage percentage.', - unit: 'percent', - valueType: ValueType.DOUBLE, - }); - - // Initialize tool queue depth histogram - toolQueueDepthGauge = meter.createHistogram(METRIC_TOOL_QUEUE_DEPTH, { - description: 'Number of tools in execution queue.', - valueType: ValueType.INT, - }); - - // Initialize performance breakdowns - toolExecutionBreakdownHistogram = meter.createHistogram( - METRIC_TOOL_EXECUTION_BREAKDOWN, - { - description: 'Tool execution time breakdown by phase in milliseconds.', - unit: 'ms', - valueType: ValueType.INT, + Object.entries(PERFORMANCE_COUNTER_DEFINITIONS).forEach( + ([name, { description, valueType, assign }]) => { + assign(meter.createCounter(name, { description, valueType })); }, ); - tokenEfficiencyHistogram = meter.createHistogram(METRIC_TOKEN_EFFICIENCY, { - description: - 'Token efficiency metrics (tokens per operation, cache hit rate, etc.).', - valueType: ValueType.DOUBLE, - }); - - apiRequestBreakdownHistogram = meter.createHistogram( - METRIC_API_REQUEST_BREAKDOWN, - { - description: 'API request time breakdown by phase in milliseconds.', - unit: 'ms', - valueType: ValueType.INT, - }, - ); - - // Initialize performance score and regression detection - performanceScoreGauge = meter.createHistogram(METRIC_PERFORMANCE_SCORE, { - description: 'Composite performance score (0-100).', - unit: 'score', - valueType: ValueType.DOUBLE, - }); - - regressionDetectionCounter = meter.createCounter( - METRIC_REGRESSION_DETECTION, - { - description: 'Performance regression detection events.', - valueType: ValueType.INT, - }, - ); - - regressionPercentageChangeHistogram = meter.createHistogram( - METRIC_REGRESSION_PERCENTAGE_CHANGE, - { - description: - 'Percentage change compared to baseline for detected regressions.', - unit: 'percent', - valueType: ValueType.DOUBLE, - }, - ); - - baselineComparisonHistogram = meter.createHistogram( - METRIC_BASELINE_COMPARISON, - { - description: - 'Performance comparison to established baseline (percentage change).', - unit: 'percent', - valueType: ValueType.DOUBLE, + Object.entries(PERFORMANCE_HISTOGRAM_DEFINITIONS).forEach( + ([name, { description, unit, valueType, assign }]) => { + assign(meter.createHistogram(name, { description, unit, valueType })); }, ); } export function recordStartupPerformance( config: Config, - phase: string, durationMs: number, - details?: Record, + attributes: MetricDefinitions[typeof STARTUP_TIME]['attributes'], ): void { if (!startupTimeHistogram || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - phase, - ...details, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + phase: attributes.phase, + ...attributes.details, }; - startupTimeHistogram.record(durationMs, attributes); + startupTimeHistogram.record(durationMs, metricAttributes); } export function recordMemoryUsage( config: Config, - memoryType: MemoryMetricType, bytes: number, - component?: string, + attributes: MetricDefinitions[typeof MEMORY_USAGE]['attributes'], ): void { if (!memoryUsageGauge || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - memory_type: memoryType, - component, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - memoryUsageGauge.record(bytes, attributes); + memoryUsageGauge.record(bytes, metricAttributes); } export function recordCpuUsage( config: Config, percentage: number, - component?: string, + attributes: MetricDefinitions[typeof CPU_USAGE]['attributes'], ): void { if (!cpuUsageGauge || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - component, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - cpuUsageGauge.record(percentage, attributes); + cpuUsageGauge.record(percentage, metricAttributes); } export function recordToolQueueDepth(config: Config, queueDepth: number): void { if (!toolQueueDepthGauge || !isPerformanceMonitoringEnabled) return; const attributes: Attributes = { - ...getCommonAttributes(config), + ...baseMetricDefinition.getCommonAttributes(config), }; toolQueueDepthGauge.record(queueDepth, attributes); @@ -556,126 +656,111 @@ export function recordToolQueueDepth(config: Config, queueDepth: number): void { export function recordToolExecutionBreakdown( config: Config, - functionName: string, - phase: ToolExecutionPhase, durationMs: number, + attributes: MetricDefinitions[typeof TOOL_EXECUTION_BREAKDOWN]['attributes'], ): void { if (!toolExecutionBreakdownHistogram || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - function_name: functionName, - phase, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - toolExecutionBreakdownHistogram.record(durationMs, attributes); + toolExecutionBreakdownHistogram.record(durationMs, metricAttributes); } export function recordTokenEfficiency( config: Config, - model: string, - metric: string, value: number, - context?: string, + attributes: MetricDefinitions[typeof TOKEN_EFFICIENCY]['attributes'], ): void { if (!tokenEfficiencyHistogram || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - model, - metric, - context, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - tokenEfficiencyHistogram.record(value, attributes); + tokenEfficiencyHistogram.record(value, metricAttributes); } export function recordApiRequestBreakdown( config: Config, - model: string, - phase: ApiRequestPhase, durationMs: number, + attributes: MetricDefinitions[typeof API_REQUEST_BREAKDOWN]['attributes'], ): void { if (!apiRequestBreakdownHistogram || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - model, - phase, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - apiRequestBreakdownHistogram.record(durationMs, attributes); + apiRequestBreakdownHistogram.record(durationMs, metricAttributes); } export function recordPerformanceScore( config: Config, score: number, - category: string, - baseline?: number, + attributes: MetricDefinitions[typeof PERFORMANCE_SCORE]['attributes'], ): void { if (!performanceScoreGauge || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - category, - baseline, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - performanceScoreGauge.record(score, attributes); + performanceScoreGauge.record(score, metricAttributes); } export function recordPerformanceRegression( config: Config, - metric: string, - currentValue: number, - baselineValue: number, - severity: 'low' | 'medium' | 'high', + attributes: MetricDefinitions[typeof REGRESSION_DETECTION]['attributes'], ): void { if (!regressionDetectionCounter || !isPerformanceMonitoringEnabled) return; - const attributes: Attributes = { - ...getCommonAttributes(config), - metric, - severity, - current_value: currentValue, - baseline_value: baselineValue, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - regressionDetectionCounter.add(1, attributes); + regressionDetectionCounter.add(1, metricAttributes); - if (baselineValue !== 0 && regressionPercentageChangeHistogram) { + if (attributes.baseline_value !== 0 && regressionPercentageChangeHistogram) { const percentageChange = - ((currentValue - baselineValue) / baselineValue) * 100; - regressionPercentageChangeHistogram.record(percentageChange, attributes); + ((attributes.current_value - attributes.baseline_value) / + attributes.baseline_value) * + 100; + regressionPercentageChangeHistogram.record( + percentageChange, + metricAttributes, + ); } } export function recordBaselineComparison( config: Config, - metric: string, - currentValue: number, - baselineValue: number, - category: string, + attributes: MetricDefinitions[typeof BASELINE_COMPARISON]['attributes'], ): void { if (!baselineComparisonHistogram || !isPerformanceMonitoringEnabled) return; - if (baselineValue === 0) { + if (attributes.baseline_value === 0) { diag.warn('Baseline value is zero, skipping comparison.'); return; } const percentageChange = - ((currentValue - baselineValue) / baselineValue) * 100; + ((attributes.current_value - attributes.baseline_value) / + attributes.baseline_value) * + 100; - const attributes: Attributes = { - ...getCommonAttributes(config), - metric, - category, - current_value: currentValue, - baseline_value: baselineValue, + const metricAttributes: Attributes = { + ...baseMetricDefinition.getCommonAttributes(config), + ...attributes, }; - baselineComparisonHistogram.record(percentageChange, attributes); + baselineComparisonHistogram.record(percentageChange, metricAttributes); } // Utility function to check if performance monitoring is enabled