diff --git a/packages/core/src/telemetry/constants.ts b/packages/core/src/telemetry/constants.ts index c1820fffe2..bf8fc140fa 100644 --- a/packages/core/src/telemetry/constants.ts +++ b/packages/core/src/telemetry/constants.ts @@ -48,3 +48,25 @@ export const METRIC_MODEL_ROUTING_FAILURE_COUNT = 'gemini_cli.model_routing.failure.count'; export const METRIC_MODEL_SLASH_COMMAND_CALL_COUNT = 'gemini_cli.slash_command.model.call_count'; + +// Performance Monitoring Metrics +export const METRIC_STARTUP_TIME = 'gemini_cli.startup.duration'; +export const METRIC_MEMORY_USAGE = 'gemini_cli.memory.usage'; +export const METRIC_CPU_USAGE = 'gemini_cli.cpu.usage'; +export const METRIC_TOOL_QUEUE_DEPTH = 'gemini_cli.tool.queue.depth'; +export const METRIC_TOOL_EXECUTION_BREAKDOWN = + 'gemini_cli.tool.execution.breakdown'; +export const METRIC_TOKEN_EFFICIENCY = 'gemini_cli.token.efficiency'; +export const METRIC_API_REQUEST_BREAKDOWN = 'gemini_cli.api.request.breakdown'; +export const METRIC_PERFORMANCE_SCORE = 'gemini_cli.performance.score'; +export const METRIC_REGRESSION_DETECTION = 'gemini_cli.performance.regression'; +export const METRIC_REGRESSION_PERCENTAGE_CHANGE = + 'gemini_cli.performance.regression.percentage_change'; +export const METRIC_BASELINE_COMPARISON = + 'gemini_cli.performance.baseline.comparison'; + +// Performance Events +export const EVENT_STARTUP_PERFORMANCE = 'gemini_cli.startup.performance'; +export const EVENT_MEMORY_USAGE = 'gemini_cli.memory.usage'; +export const EVENT_PERFORMANCE_BASELINE = 'gemini_cli.performance.baseline'; +export const EVENT_PERFORMANCE_REGRESSION = 'gemini_cli.performance.regression'; diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index ba9596a7e1..4a568efab0 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -74,3 +74,33 @@ export { recordUserActivity, isUserActive, } from './activity-detector.js'; +export { + // Core metrics functions + recordToolCallMetrics, + recordTokenUsageMetrics, + recordApiResponseMetrics, + recordApiErrorMetrics, + recordFileOperationMetric, + recordInvalidChunk, + recordContentRetry, + recordContentRetryFailure, + recordModelRoutingMetrics, + // Performance monitoring functions + recordStartupPerformance, + recordMemoryUsage, + recordCpuUsage, + recordToolQueueDepth, + recordToolExecutionBreakdown, + recordTokenEfficiency, + recordApiRequestBreakdown, + recordPerformanceScore, + recordPerformanceRegression, + recordBaselineComparison, + isPerformanceMonitoringActive, + // Performance monitoring types + PerformanceMetricType, + MemoryMetricType, + ToolExecutionPhase, + ApiRequestPhase, + FileOperation, +} from './metrics.js'; diff --git a/packages/core/src/telemetry/metrics.test.ts b/packages/core/src/telemetry/metrics.test.ts index 25a907164a..8a19682846 100644 --- a/packages/core/src/telemetry/metrics.test.ts +++ b/packages/core/src/telemetry/metrics.test.ts @@ -13,7 +13,12 @@ import type { Histogram, } from '@opentelemetry/api'; import type { Config } from '../config/config.js'; -import { FileOperation } from './metrics.js'; +import { + FileOperation, + MemoryMetricType, + ToolExecutionPhase, + ApiRequestPhase, +} from './metrics.js'; import { makeFakeConfig } from '../test-utils/config.js'; import { ModelRoutingEvent } from './types.js'; @@ -50,11 +55,13 @@ function originalOtelMockFactory() { }, ValueType: { INT: 1, + DOUBLE: 2, }, diag: { setLogger: vi.fn(), + warn: vi.fn(), }, - }; + } as const; } vi.mock('@opentelemetry/api'); @@ -65,6 +72,16 @@ describe('Telemetry Metrics', () => { let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric; let recordChatCompressionMetricsModule: typeof import('./metrics.js').recordChatCompressionMetrics; let recordModelRoutingMetricsModule: typeof import('./metrics.js').recordModelRoutingMetrics; + let recordStartupPerformanceModule: typeof import('./metrics.js').recordStartupPerformance; + let recordMemoryUsageModule: typeof import('./metrics.js').recordMemoryUsage; + let recordCpuUsageModule: typeof import('./metrics.js').recordCpuUsage; + let recordToolQueueDepthModule: typeof import('./metrics.js').recordToolQueueDepth; + let recordToolExecutionBreakdownModule: typeof import('./metrics.js').recordToolExecutionBreakdown; + let recordTokenEfficiencyModule: typeof import('./metrics.js').recordTokenEfficiency; + let recordApiRequestBreakdownModule: typeof import('./metrics.js').recordApiRequestBreakdown; + let recordPerformanceScoreModule: typeof import('./metrics.js').recordPerformanceScore; + let recordPerformanceRegressionModule: typeof import('./metrics.js').recordPerformanceRegression; + let recordBaselineComparisonModule: typeof import('./metrics.js').recordBaselineComparison; beforeEach(async () => { vi.resetModules(); @@ -81,6 +98,18 @@ describe('Telemetry Metrics', () => { recordChatCompressionMetricsModule = metricsJsModule.recordChatCompressionMetrics; recordModelRoutingMetricsModule = metricsJsModule.recordModelRoutingMetrics; + recordStartupPerformanceModule = metricsJsModule.recordStartupPerformance; + recordMemoryUsageModule = metricsJsModule.recordMemoryUsage; + recordCpuUsageModule = metricsJsModule.recordCpuUsage; + recordToolQueueDepthModule = metricsJsModule.recordToolQueueDepth; + recordToolExecutionBreakdownModule = + metricsJsModule.recordToolExecutionBreakdown; + recordTokenEfficiencyModule = metricsJsModule.recordTokenEfficiency; + recordApiRequestBreakdownModule = metricsJsModule.recordApiRequestBreakdown; + recordPerformanceScoreModule = metricsJsModule.recordPerformanceScore; + recordPerformanceRegressionModule = + metricsJsModule.recordPerformanceRegression; + recordBaselineComparisonModule = metricsJsModule.recordBaselineComparison; const otelApiModule = await import('@opentelemetry/api'); @@ -127,6 +156,7 @@ describe('Telemetry Metrics', () => { describe('recordTokenUsageMetrics', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => true, } as unknown as Config; it('should not record metrics if not initialized', () => { @@ -197,6 +227,7 @@ describe('Telemetry Metrics', () => { describe('recordFileOperationMetric', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => true, } as unknown as Config; it('should not record metrics if not initialized', () => { @@ -323,6 +354,7 @@ describe('Telemetry Metrics', () => { describe('recordModelRoutingMetrics', () => { const mockConfig = { getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => true, } as unknown as Config; it('should not record metrics if not initialized', () => { @@ -386,4 +418,582 @@ describe('Telemetry Metrics', () => { }); }); }); + + describe('Performance Monitoring Metrics', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => true, + } as unknown as Config; + + describe('recordStartupPerformance', () => { + it('should not record metrics when performance monitoring is disabled', async () => { + // Re-import with performance monitoring disabled by mocking the config + const mockConfigDisabled = { + getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => false, // Disable telemetry to disable performance monitoring + } as unknown as Config; + + initializeMetricsModule(mockConfigDisabled); + mockHistogramRecordFn.mockClear(); + + recordStartupPerformanceModule( + mockConfigDisabled, + 'settings_loading', + 100, + { + auth_type: 'gemini', + }, + ); + + expect(mockHistogramRecordFn).not.toHaveBeenCalled(); + }); + + it('should record startup performance with phase and details', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordStartupPerformanceModule(mockConfig, 'settings_loading', 150, { + auth_type: 'gemini', + telemetry_enabled: true, + settings_sources: 2, + }); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, { + 'session.id': 'test-session-id', + phase: 'settings_loading', + auth_type: 'gemini', + telemetry_enabled: true, + settings_sources: 2, + }); + }); + + it('should record startup performance without details', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordStartupPerformanceModule(mockConfig, 'cleanup', 50); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(50, { + 'session.id': 'test-session-id', + phase: 'cleanup', + }); + }); + + it('should handle floating-point duration values from performance.now()', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + // Test with realistic floating-point values that performance.now() would return + const floatingPointDuration = 123.45678; + recordStartupPerformanceModule( + mockConfig, + 'total_startup', + floatingPointDuration, + { + is_tty: true, + has_question: false, + }, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith( + floatingPointDuration, + { + 'session.id': 'test-session-id', + phase: 'total_startup', + is_tty: true, + has_question: false, + }, + ); + }); + }); + + describe('recordMemoryUsage', () => { + it('should record memory usage for different memory types', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordMemoryUsageModule( + mockConfig, + MemoryMetricType.HEAP_USED, + 15728640, + 'startup', + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, { + 'session.id': 'test-session-id', + memory_type: 'heap_used', + component: 'startup', + }); + }); + + it('should record memory usage for all memory metric types', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordMemoryUsageModule( + mockConfig, + MemoryMetricType.HEAP_TOTAL, + 31457280, + 'api_call', + ); + recordMemoryUsageModule( + mockConfig, + MemoryMetricType.EXTERNAL, + 2097152, + 'tool_execution', + ); + recordMemoryUsageModule( + mockConfig, + MemoryMetricType.RSS, + 41943040, + 'memory_monitor', + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 31457280, { + 'session.id': 'test-session-id', + memory_type: 'heap_total', + component: 'api_call', + }); + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 2097152, { + 'session.id': 'test-session-id', + memory_type: 'external', + component: 'tool_execution', + }); + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 41943040, { + 'session.id': 'test-session-id', + memory_type: 'rss', + component: 'memory_monitor', + }); + }); + + it('should record memory usage without component', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordMemoryUsageModule( + mockConfig, + MemoryMetricType.HEAP_USED, + 15728640, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, { + 'session.id': 'test-session-id', + memory_type: 'heap_used', + component: undefined, + }); + }); + }); + + describe('recordCpuUsage', () => { + it('should record CPU usage percentage', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordCpuUsageModule(mockConfig, 85.5, 'tool_execution'); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, { + 'session.id': 'test-session-id', + component: 'tool_execution', + }); + }); + + it('should record CPU usage without component', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordCpuUsageModule(mockConfig, 42.3); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(42.3, { + 'session.id': 'test-session-id', + component: undefined, + }); + }); + }); + + describe('recordToolQueueDepth', () => { + it('should record tool queue depth', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordToolQueueDepthModule(mockConfig, 3); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(3, { + 'session.id': 'test-session-id', + }); + }); + + it('should record zero queue depth', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordToolQueueDepthModule(mockConfig, 0); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(0, { + 'session.id': 'test-session-id', + }); + }); + }); + + describe('recordToolExecutionBreakdown', () => { + it('should record tool execution breakdown for all phases', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordToolExecutionBreakdownModule( + mockConfig, + 'Read', + ToolExecutionPhase.VALIDATION, + 25, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(25, { + 'session.id': 'test-session-id', + function_name: 'Read', + phase: 'validation', + }); + }); + + it('should record execution breakdown for different phases', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordToolExecutionBreakdownModule( + mockConfig, + 'Bash', + ToolExecutionPhase.PREPARATION, + 50, + ); + recordToolExecutionBreakdownModule( + mockConfig, + 'Bash', + ToolExecutionPhase.EXECUTION, + 1500, + ); + recordToolExecutionBreakdownModule( + mockConfig, + 'Bash', + ToolExecutionPhase.RESULT_PROCESSING, + 75, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 50, { + 'session.id': 'test-session-id', + function_name: 'Bash', + phase: 'preparation', + }); + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 1500, { + 'session.id': 'test-session-id', + function_name: 'Bash', + phase: 'execution', + }); + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 75, { + 'session.id': 'test-session-id', + function_name: 'Bash', + phase: 'result_processing', + }); + }); + }); + + describe('recordTokenEfficiency', () => { + it('should record token efficiency metrics', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordTokenEfficiencyModule( + mockConfig, + 'gemini-pro', + 'cache_hit_rate', + 0.85, + 'api_request', + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(0.85, { + 'session.id': 'test-session-id', + model: 'gemini-pro', + metric: 'cache_hit_rate', + context: 'api_request', + }); + }); + + it('should record token efficiency without context', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordTokenEfficiencyModule( + mockConfig, + 'gemini-pro', + 'tokens_per_operation', + 125.5, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(125.5, { + 'session.id': 'test-session-id', + model: 'gemini-pro', + metric: 'tokens_per_operation', + context: undefined, + }); + }); + }); + + describe('recordApiRequestBreakdown', () => { + it('should record API request breakdown for all phases', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordApiRequestBreakdownModule( + mockConfig, + 'gemini-pro', + ApiRequestPhase.REQUEST_PREPARATION, + 15, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(15, { + 'session.id': 'test-session-id', + model: 'gemini-pro', + phase: 'request_preparation', + }); + }); + + it('should record API request breakdown for different phases', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordApiRequestBreakdownModule( + mockConfig, + 'gemini-pro', + ApiRequestPhase.NETWORK_LATENCY, + 250, + ); + recordApiRequestBreakdownModule( + mockConfig, + 'gemini-pro', + ApiRequestPhase.RESPONSE_PROCESSING, + 100, + ); + recordApiRequestBreakdownModule( + mockConfig, + 'gemini-pro', + ApiRequestPhase.TOKEN_PROCESSING, + 50, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 250, { + 'session.id': 'test-session-id', + model: 'gemini-pro', + phase: 'network_latency', + }); + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 100, { + 'session.id': 'test-session-id', + model: 'gemini-pro', + phase: 'response_processing', + }); + expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 50, { + 'session.id': 'test-session-id', + model: 'gemini-pro', + phase: 'token_processing', + }); + }); + }); + + describe('recordPerformanceScore', () => { + it('should record performance score with category and baseline', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordPerformanceScoreModule( + mockConfig, + 85.5, + 'memory_efficiency', + 80.0, + ); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, { + 'session.id': 'test-session-id', + category: 'memory_efficiency', + baseline: 80.0, + }); + }); + + it('should record performance score without baseline', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordPerformanceScoreModule(mockConfig, 92.3, 'overall_performance'); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(92.3, { + 'session.id': 'test-session-id', + category: 'overall_performance', + baseline: undefined, + }); + }); + }); + + describe('recordPerformanceRegression', () => { + it('should record performance regression with baseline comparison', () => { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + mockHistogramRecordFn.mockClear(); + + recordPerformanceRegressionModule( + mockConfig, + 'startup_time', + 1200, + 1000, + 'medium', + ); + + // Verify regression counter + expect(mockCounterAddFn).toHaveBeenCalledWith(1, { + 'session.id': 'test-session-id', + metric: 'startup_time', + severity: 'medium', + current_value: 1200, + baseline_value: 1000, + }); + + // Verify baseline comparison histogram (20% increase) + expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, { + 'session.id': 'test-session-id', + metric: 'startup_time', + severity: 'medium', + current_value: 1200, + baseline_value: 1000, + }); + }); + + it('should handle zero baseline value gracefully', () => { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + mockHistogramRecordFn.mockClear(); + + recordPerformanceRegressionModule( + mockConfig, + 'memory_usage', + 100, + 0, + 'high', + ); + + // Verify regression counter still recorded + expect(mockCounterAddFn).toHaveBeenCalledWith(1, { + 'session.id': 'test-session-id', + metric: 'memory_usage', + severity: 'high', + current_value: 100, + baseline_value: 0, + }); + + // Verify no baseline comparison due to zero baseline + expect(mockHistogramRecordFn).not.toHaveBeenCalled(); + }); + + it('should record different severity levels', () => { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + + recordPerformanceRegressionModule( + mockConfig, + 'api_latency', + 500, + 400, + 'low', + ); + recordPerformanceRegressionModule( + mockConfig, + 'cpu_usage', + 90, + 70, + 'high', + ); + + expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, { + 'session.id': 'test-session-id', + metric: 'api_latency', + severity: 'low', + current_value: 500, + baseline_value: 400, + }); + expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, { + 'session.id': 'test-session-id', + metric: 'cpu_usage', + severity: 'high', + current_value: 90, + baseline_value: 70, + }); + }); + }); + + describe('recordBaselineComparison', () => { + it('should record baseline comparison with percentage change', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordBaselineComparisonModule( + mockConfig, + 'memory_usage', + 120, + 100, + 'performance_tracking', + ); + + // 20% increase: (120 - 100) / 100 * 100 = 20% + expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, { + 'session.id': 'test-session-id', + metric: 'memory_usage', + category: 'performance_tracking', + current_value: 120, + baseline_value: 100, + }); + }); + + it('should handle negative percentage change (improvement)', () => { + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordBaselineComparisonModule( + mockConfig, + 'startup_time', + 800, + 1000, + 'optimization', + ); + + // 20% decrease: (800 - 1000) / 1000 * 100 = -20% + expect(mockHistogramRecordFn).toHaveBeenCalledWith(-20, { + 'session.id': 'test-session-id', + metric: 'startup_time', + category: 'optimization', + current_value: 800, + baseline_value: 1000, + }); + }); + + it('should skip recording when baseline is zero', async () => { + // Access the actual mocked module + const mockedModule = (await vi.importMock('@opentelemetry/api')) as { + diag: { warn: ReturnType }; + }; + const diagSpy = vi.spyOn(mockedModule.diag, 'warn'); + + initializeMetricsModule(mockConfig); + mockHistogramRecordFn.mockClear(); + + recordBaselineComparisonModule( + mockConfig, + 'new_metric', + 50, + 0, + 'testing', + ); + + expect(diagSpy).toHaveBeenCalledWith( + 'Baseline value is zero, skipping comparison.', + ); + expect(mockHistogramRecordFn).not.toHaveBeenCalled(); + }); + }); + }); }); diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts index 96ca1fd25b..0d66119f55 100644 --- a/packages/core/src/telemetry/metrics.ts +++ b/packages/core/src/telemetry/metrics.ts @@ -5,7 +5,7 @@ */ import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api'; -import { metrics, ValueType } from '@opentelemetry/api'; +import { diag, metrics, ValueType } from '@opentelemetry/api'; import { SERVICE_NAME, METRIC_TOOL_CALL_COUNT, @@ -22,6 +22,18 @@ import { METRIC_MODEL_ROUTING_LATENCY, METRIC_MODEL_ROUTING_FAILURE_COUNT, METRIC_MODEL_SLASH_COMMAND_CALL_COUNT, + // Performance Monitoring Metrics + METRIC_STARTUP_TIME, + METRIC_MEMORY_USAGE, + METRIC_CPU_USAGE, + METRIC_TOOL_QUEUE_DEPTH, + METRIC_TOOL_EXECUTION_BREAKDOWN, + METRIC_TOKEN_EFFICIENCY, + METRIC_API_REQUEST_BREAKDOWN, + METRIC_PERFORMANCE_SCORE, + METRIC_REGRESSION_DETECTION, + METRIC_REGRESSION_PERCENTAGE_CHANGE, + METRIC_BASELINE_COMPARISON, } from './constants.js'; import type { Config } from '../config/config.js'; import type { ModelRoutingEvent, ModelSlashCommandEvent } from './types.js'; @@ -32,6 +44,36 @@ export enum FileOperation { UPDATE = 'update', } +export enum PerformanceMetricType { + STARTUP = 'startup', + MEMORY = 'memory', + CPU = 'cpu', + TOOL_EXECUTION = 'tool_execution', + API_REQUEST = 'api_request', + TOKEN_EFFICIENCY = 'token_efficiency', +} + +export enum MemoryMetricType { + HEAP_USED = 'heap_used', + HEAP_TOTAL = 'heap_total', + EXTERNAL = 'external', + RSS = 'rss', +} + +export enum ToolExecutionPhase { + VALIDATION = 'validation', + PREPARATION = 'preparation', + EXECUTION = 'execution', + RESULT_PROCESSING = 'result_processing', +} + +export enum ApiRequestPhase { + REQUEST_PREPARATION = 'request_preparation', + NETWORK_LATENCY = 'network_latency', + RESPONSE_PROCESSING = 'response_processing', + TOKEN_PROCESSING = 'token_processing', +} + let cliMeter: Meter | undefined; let toolCallCounter: Counter | undefined; let toolCallLatencyHistogram: Histogram | undefined; @@ -46,7 +88,21 @@ let contentRetryFailureCounter: Counter | undefined; let modelRoutingLatencyHistogram: Histogram | undefined; let modelRoutingFailureCounter: Counter | undefined; let modelSlashCommandCallCounter: Counter | undefined; + +// Performance Monitoring Metrics +let startupTimeHistogram: Histogram | undefined; +let memoryUsageGauge: Histogram | undefined; // Using Histogram until ObservableGauge is available +let cpuUsageGauge: Histogram | undefined; +let toolQueueDepthGauge: Histogram | undefined; +let toolExecutionBreakdownHistogram: Histogram | undefined; +let tokenEfficiencyHistogram: Histogram | undefined; +let apiRequestBreakdownHistogram: Histogram | undefined; +let performanceScoreGauge: Histogram | undefined; +let regressionDetectionCounter: Counter | undefined; +let regressionPercentageChangeHistogram: Histogram | undefined; +let baselineComparisonHistogram: Histogram | undefined; let isMetricsInitialized = false; +let isPerformanceMonitoringEnabled = false; function getCommonAttributes(config: Config): Attributes { return { @@ -67,6 +123,7 @@ export function initializeMetrics(config: Config): void { const meter = getMeter(); if (!meter) return; + // Initialize core metrics toolCallCounter = meter.createCounter(METRIC_TOOL_CALL_COUNT, { description: 'Counts tool calls, tagged by function name and success.', valueType: ValueType.INT, @@ -145,6 +202,10 @@ export function initializeMetrics(config: Config): void { valueType: ValueType.INT, }); sessionCounter.add(1, getCommonAttributes(config)); + + // Initialize performance monitoring metrics if enabled + initializePerformanceMonitoring(config); + isMetricsInitialized = true; } @@ -332,3 +393,292 @@ export function recordModelRoutingMetrics( }); } } +// Performance Monitoring Functions + +export function initializePerformanceMonitoring(config: Config): void { + const meter = getMeter(); + if (!meter) return; + + // Check if performance monitoring is enabled in config + // For now, enable performance monitoring when telemetry is enabled + // TODO: Add specific performance monitoring settings to config + isPerformanceMonitoringEnabled = config.getTelemetryEnabled(); + + if (!isPerformanceMonitoringEnabled) return; + + // Initialize startup time histogram + startupTimeHistogram = meter.createHistogram(METRIC_STARTUP_TIME, { + description: + 'CLI startup time in milliseconds, broken down by initialization phase.', + unit: 'ms', + valueType: ValueType.DOUBLE, + }); + + // Initialize memory usage histogram (using histogram until ObservableGauge is available) + memoryUsageGauge = meter.createHistogram(METRIC_MEMORY_USAGE, { + description: 'Memory usage in bytes.', + unit: 'bytes', + valueType: ValueType.INT, + }); + + // Initialize CPU usage histogram + cpuUsageGauge = meter.createHistogram(METRIC_CPU_USAGE, { + description: 'CPU usage percentage.', + unit: 'percent', + valueType: ValueType.DOUBLE, + }); + + // Initialize tool queue depth histogram + toolQueueDepthGauge = meter.createHistogram(METRIC_TOOL_QUEUE_DEPTH, { + description: 'Number of tools in execution queue.', + valueType: ValueType.INT, + }); + + // Initialize performance breakdowns + toolExecutionBreakdownHistogram = meter.createHistogram( + METRIC_TOOL_EXECUTION_BREAKDOWN, + { + description: 'Tool execution time breakdown by phase in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + }, + ); + + tokenEfficiencyHistogram = meter.createHistogram(METRIC_TOKEN_EFFICIENCY, { + description: + 'Token efficiency metrics (tokens per operation, cache hit rate, etc.).', + valueType: ValueType.DOUBLE, + }); + + apiRequestBreakdownHistogram = meter.createHistogram( + METRIC_API_REQUEST_BREAKDOWN, + { + description: 'API request time breakdown by phase in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + }, + ); + + // Initialize performance score and regression detection + performanceScoreGauge = meter.createHistogram(METRIC_PERFORMANCE_SCORE, { + description: 'Composite performance score (0-100).', + unit: 'score', + valueType: ValueType.DOUBLE, + }); + + regressionDetectionCounter = meter.createCounter( + METRIC_REGRESSION_DETECTION, + { + description: 'Performance regression detection events.', + valueType: ValueType.INT, + }, + ); + + regressionPercentageChangeHistogram = meter.createHistogram( + METRIC_REGRESSION_PERCENTAGE_CHANGE, + { + description: + 'Percentage change compared to baseline for detected regressions.', + unit: 'percent', + valueType: ValueType.DOUBLE, + }, + ); + + baselineComparisonHistogram = meter.createHistogram( + METRIC_BASELINE_COMPARISON, + { + description: + 'Performance comparison to established baseline (percentage change).', + unit: 'percent', + valueType: ValueType.DOUBLE, + }, + ); +} + +export function recordStartupPerformance( + config: Config, + phase: string, + durationMs: number, + details?: Record, +): void { + if (!startupTimeHistogram || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + phase, + ...details, + }; + + startupTimeHistogram.record(durationMs, attributes); +} + +export function recordMemoryUsage( + config: Config, + memoryType: MemoryMetricType, + bytes: number, + component?: string, +): void { + if (!memoryUsageGauge || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + memory_type: memoryType, + component, + }; + + memoryUsageGauge.record(bytes, attributes); +} + +export function recordCpuUsage( + config: Config, + percentage: number, + component?: string, +): void { + if (!cpuUsageGauge || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + component, + }; + + cpuUsageGauge.record(percentage, attributes); +} + +export function recordToolQueueDepth(config: Config, queueDepth: number): void { + if (!toolQueueDepthGauge || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + }; + + toolQueueDepthGauge.record(queueDepth, attributes); +} + +export function recordToolExecutionBreakdown( + config: Config, + functionName: string, + phase: ToolExecutionPhase, + durationMs: number, +): void { + if (!toolExecutionBreakdownHistogram || !isPerformanceMonitoringEnabled) + return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + function_name: functionName, + phase, + }; + + toolExecutionBreakdownHistogram.record(durationMs, attributes); +} + +export function recordTokenEfficiency( + config: Config, + model: string, + metric: string, + value: number, + context?: string, +): void { + if (!tokenEfficiencyHistogram || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + model, + metric, + context, + }; + + tokenEfficiencyHistogram.record(value, attributes); +} + +export function recordApiRequestBreakdown( + config: Config, + model: string, + phase: ApiRequestPhase, + durationMs: number, +): void { + if (!apiRequestBreakdownHistogram || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + model, + phase, + }; + + apiRequestBreakdownHistogram.record(durationMs, attributes); +} + +export function recordPerformanceScore( + config: Config, + score: number, + category: string, + baseline?: number, +): void { + if (!performanceScoreGauge || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + category, + baseline, + }; + + performanceScoreGauge.record(score, attributes); +} + +export function recordPerformanceRegression( + config: Config, + metric: string, + currentValue: number, + baselineValue: number, + severity: 'low' | 'medium' | 'high', +): void { + if (!regressionDetectionCounter || !isPerformanceMonitoringEnabled) return; + + const attributes: Attributes = { + ...getCommonAttributes(config), + metric, + severity, + current_value: currentValue, + baseline_value: baselineValue, + }; + + regressionDetectionCounter.add(1, attributes); + + if (baselineValue !== 0 && regressionPercentageChangeHistogram) { + const percentageChange = + ((currentValue - baselineValue) / baselineValue) * 100; + regressionPercentageChangeHistogram.record(percentageChange, attributes); + } +} + +export function recordBaselineComparison( + config: Config, + metric: string, + currentValue: number, + baselineValue: number, + category: string, +): void { + if (!baselineComparisonHistogram || !isPerformanceMonitoringEnabled) return; + + if (baselineValue === 0) { + diag.warn('Baseline value is zero, skipping comparison.'); + return; + } + const percentageChange = + ((currentValue - baselineValue) / baselineValue) * 100; + + const attributes: Attributes = { + ...getCommonAttributes(config), + metric, + category, + current_value: currentValue, + baseline_value: baselineValue, + }; + + baselineComparisonHistogram.record(percentageChange, attributes); +} + +// Utility function to check if performance monitoring is enabled +export function isPerformanceMonitoringActive(): boolean { + return isPerformanceMonitoringEnabled && isMetricsInitialized; +}