MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-03-10 22:21:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

[Part 3/6] feat(telemetry): enhance metrics with performance monitoring APIs (#8113)

Browse Source
This commit is contained in:
Adrian Arribas
2025-09-25 18:12:46 +02:00
committed by GitHub
parent 135d3401cd
commit f80eb71068
4 changed files with 1015 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
packages/core/src/telemetry/constants.ts
View File

@@ -48,3 +48,25 @@ export const METRIC_MODEL_ROUTING_FAILURE_COUNT =
'gemini_cli.model_routing.failure.count';
export const METRIC_MODEL_SLASH_COMMAND_CALL_COUNT =
'gemini_cli.slash_command.model.call_count';
// Performance Monitoring Metrics
export const METRIC_STARTUP_TIME = 'gemini_cli.startup.duration';
export const METRIC_MEMORY_USAGE = 'gemini_cli.memory.usage';
export const METRIC_CPU_USAGE = 'gemini_cli.cpu.usage';
export const METRIC_TOOL_QUEUE_DEPTH = 'gemini_cli.tool.queue.depth';
export const METRIC_TOOL_EXECUTION_BREAKDOWN =
'gemini_cli.tool.execution.breakdown';
export const METRIC_TOKEN_EFFICIENCY = 'gemini_cli.token.efficiency';
export const METRIC_API_REQUEST_BREAKDOWN = 'gemini_cli.api.request.breakdown';
export const METRIC_PERFORMANCE_SCORE = 'gemini_cli.performance.score';
export const METRIC_REGRESSION_DETECTION = 'gemini_cli.performance.regression';
export const METRIC_REGRESSION_PERCENTAGE_CHANGE =
'gemini_cli.performance.regression.percentage_change';
export const METRIC_BASELINE_COMPARISON =
'gemini_cli.performance.baseline.comparison';
// Performance Events
export const EVENT_STARTUP_PERFORMANCE = 'gemini_cli.startup.performance';
export const EVENT_MEMORY_USAGE = 'gemini_cli.memory.usage';
export const EVENT_PERFORMANCE_BASELINE = 'gemini_cli.performance.baseline';
export const EVENT_PERFORMANCE_REGRESSION = 'gemini_cli.performance.regression';

30
packages/core/src/telemetry/index.ts
View File

@@ -74,3 +74,33 @@ export {
recordUserActivity,
isUserActive,
} from './activity-detector.js';
export {
// Core metrics functions
recordToolCallMetrics,
recordTokenUsageMetrics,
recordApiResponseMetrics,
recordApiErrorMetrics,
recordFileOperationMetric,
recordInvalidChunk,
recordContentRetry,
recordContentRetryFailure,
recordModelRoutingMetrics,
// Performance monitoring functions
recordStartupPerformance,
recordMemoryUsage,
recordCpuUsage,
recordToolQueueDepth,
recordToolExecutionBreakdown,
recordTokenEfficiency,
recordApiRequestBreakdown,
recordPerformanceScore,
recordPerformanceRegression,
recordBaselineComparison,
isPerformanceMonitoringActive,
// Performance monitoring types
PerformanceMetricType,
MemoryMetricType,
ToolExecutionPhase,
ApiRequestPhase,
FileOperation,
} from './metrics.js';

614
packages/core/src/telemetry/metrics.test.ts
View File

@@ -13,7 +13,12 @@ import type {
Histogram,
} from '@opentelemetry/api';
import type { Config } from '../config/config.js';
import { FileOperation } from './metrics.js';
import {
FileOperation,
MemoryMetricType,
ToolExecutionPhase,
ApiRequestPhase,
} from './metrics.js';
import { makeFakeConfig } from '../test-utils/config.js';
import { ModelRoutingEvent } from './types.js';
@@ -50,11 +55,13 @@ function originalOtelMockFactory() {
},
ValueType: {
INT: 1,
DOUBLE: 2,
},
diag: {
setLogger: vi.fn(),
warn: vi.fn(),
},
};
} as const;
}
vi.mock('@opentelemetry/api');
@@ -65,6 +72,16 @@ describe('Telemetry Metrics', () => {
let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric;
let recordChatCompressionMetricsModule: typeof import('./metrics.js').recordChatCompressionMetrics;
let recordModelRoutingMetricsModule: typeof import('./metrics.js').recordModelRoutingMetrics;
let recordStartupPerformanceModule: typeof import('./metrics.js').recordStartupPerformance;
let recordMemoryUsageModule: typeof import('./metrics.js').recordMemoryUsage;
let recordCpuUsageModule: typeof import('./metrics.js').recordCpuUsage;
let recordToolQueueDepthModule: typeof import('./metrics.js').recordToolQueueDepth;
let recordToolExecutionBreakdownModule: typeof import('./metrics.js').recordToolExecutionBreakdown;
let recordTokenEfficiencyModule: typeof import('./metrics.js').recordTokenEfficiency;
let recordApiRequestBreakdownModule: typeof import('./metrics.js').recordApiRequestBreakdown;
let recordPerformanceScoreModule: typeof import('./metrics.js').recordPerformanceScore;
let recordPerformanceRegressionModule: typeof import('./metrics.js').recordPerformanceRegression;
let recordBaselineComparisonModule: typeof import('./metrics.js').recordBaselineComparison;
beforeEach(async () => {
vi.resetModules();
@@ -81,6 +98,18 @@ describe('Telemetry Metrics', () => {
recordChatCompressionMetricsModule =
metricsJsModule.recordChatCompressionMetrics;
recordModelRoutingMetricsModule = metricsJsModule.recordModelRoutingMetrics;
recordStartupPerformanceModule = metricsJsModule.recordStartupPerformance;
recordMemoryUsageModule = metricsJsModule.recordMemoryUsage;
recordCpuUsageModule = metricsJsModule.recordCpuUsage;
recordToolQueueDepthModule = metricsJsModule.recordToolQueueDepth;
recordToolExecutionBreakdownModule =
metricsJsModule.recordToolExecutionBreakdown;
recordTokenEfficiencyModule = metricsJsModule.recordTokenEfficiency;
recordApiRequestBreakdownModule = metricsJsModule.recordApiRequestBreakdown;
recordPerformanceScoreModule = metricsJsModule.recordPerformanceScore;
recordPerformanceRegressionModule =
metricsJsModule.recordPerformanceRegression;
recordBaselineComparisonModule = metricsJsModule.recordBaselineComparison;
const otelApiModule = await import('@opentelemetry/api');
@@ -127,6 +156,7 @@ describe('Telemetry Metrics', () => {
describe('recordTokenUsageMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record metrics if not initialized', () => {
@@ -197,6 +227,7 @@ describe('Telemetry Metrics', () => {
describe('recordFileOperationMetric', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record metrics if not initialized', () => {
@@ -323,6 +354,7 @@ describe('Telemetry Metrics', () => {
describe('recordModelRoutingMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record metrics if not initialized', () => {
@@ -386,4 +418,582 @@ describe('Telemetry Metrics', () => {
});
});
});
describe('Performance Monitoring Metrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
describe('recordStartupPerformance', () => {
it('should not record metrics when performance monitoring is disabled', async () => {
// Re-import with performance monitoring disabled by mocking the config
const mockConfigDisabled = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => false, // Disable telemetry to disable performance monitoring
} as unknown as Config;
initializeMetricsModule(mockConfigDisabled);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(
mockConfigDisabled,
'settings_loading',
100,
{
auth_type: 'gemini',
},
);
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record startup performance with phase and details', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(mockConfig, 'settings_loading', 150, {
auth_type: 'gemini',
telemetry_enabled: true,
settings_sources: 2,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, {
'session.id': 'test-session-id',
phase: 'settings_loading',
auth_type: 'gemini',
telemetry_enabled: true,
settings_sources: 2,
});
});
it('should record startup performance without details', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(mockConfig, 'cleanup', 50);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(50, {
'session.id': 'test-session-id',
phase: 'cleanup',
});
});
it('should handle floating-point duration values from performance.now()', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
// Test with realistic floating-point values that performance.now() would return
const floatingPointDuration = 123.45678;
recordStartupPerformanceModule(
mockConfig,
'total_startup',
floatingPointDuration,
{
is_tty: true,
has_question: false,
},
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(
floatingPointDuration,
{
'session.id': 'test-session-id',
phase: 'total_startup',
is_tty: true,
has_question: false,
},
);
});
});
describe('recordMemoryUsage', () => {
it('should record memory usage for different memory types', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.HEAP_USED,
15728640,
'startup',
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, {
'session.id': 'test-session-id',
memory_type: 'heap_used',
component: 'startup',
});
});
it('should record memory usage for all memory metric types', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.HEAP_TOTAL,
31457280,
'api_call',
);
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.EXTERNAL,
2097152,
'tool_execution',
);
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.RSS,
41943040,
'memory_monitor',
);
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 31457280, {
'session.id': 'test-session-id',
memory_type: 'heap_total',
component: 'api_call',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 2097152, {
'session.id': 'test-session-id',
memory_type: 'external',
component: 'tool_execution',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 41943040, {
'session.id': 'test-session-id',
memory_type: 'rss',
component: 'memory_monitor',
});
});
it('should record memory usage without component', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.HEAP_USED,
15728640,
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, {
'session.id': 'test-session-id',
memory_type: 'heap_used',
component: undefined,
});
});
});
describe('recordCpuUsage', () => {
it('should record CPU usage percentage', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordCpuUsageModule(mockConfig, 85.5, 'tool_execution');
expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, {
'session.id': 'test-session-id',
component: 'tool_execution',
});
});
it('should record CPU usage without component', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordCpuUsageModule(mockConfig, 42.3);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(42.3, {
'session.id': 'test-session-id',
component: undefined,
});
});
});
describe('recordToolQueueDepth', () => {
it('should record tool queue depth', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolQueueDepthModule(mockConfig, 3);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(3, {
'session.id': 'test-session-id',
});
});
it('should record zero queue depth', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolQueueDepthModule(mockConfig, 0);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(0, {
'session.id': 'test-session-id',
});
});
});
describe('recordToolExecutionBreakdown', () => {
it('should record tool execution breakdown for all phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolExecutionBreakdownModule(
mockConfig,
'Read',
ToolExecutionPhase.VALIDATION,
25,
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(25, {
'session.id': 'test-session-id',
function_name: 'Read',
phase: 'validation',
});
});
it('should record execution breakdown for different phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolExecutionBreakdownModule(
mockConfig,
'Bash',
ToolExecutionPhase.PREPARATION,
50,
);
recordToolExecutionBreakdownModule(
mockConfig,
'Bash',
ToolExecutionPhase.EXECUTION,
1500,
);
recordToolExecutionBreakdownModule(
mockConfig,
'Bash',
ToolExecutionPhase.RESULT_PROCESSING,
75,
);
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 50, {
'session.id': 'test-session-id',
function_name: 'Bash',
phase: 'preparation',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 1500, {
'session.id': 'test-session-id',
function_name: 'Bash',
phase: 'execution',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 75, {
'session.id': 'test-session-id',
function_name: 'Bash',
phase: 'result_processing',
});
});
});
describe('recordTokenEfficiency', () => {
it('should record token efficiency metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordTokenEfficiencyModule(
mockConfig,
'gemini-pro',
'cache_hit_rate',
0.85,
'api_request',
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(0.85, {
'session.id': 'test-session-id',
model: 'gemini-pro',
metric: 'cache_hit_rate',
context: 'api_request',
});
});
it('should record token efficiency without context', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordTokenEfficiencyModule(
mockConfig,
'gemini-pro',
'tokens_per_operation',
125.5,
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(125.5, {
'session.id': 'test-session-id',
model: 'gemini-pro',
metric: 'tokens_per_operation',
context: undefined,
});
});
});
describe('recordApiRequestBreakdown', () => {
it('should record API request breakdown for all phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.REQUEST_PREPARATION,
15,
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(15, {
'session.id': 'test-session-id',
model: 'gemini-pro',
phase: 'request_preparation',
});
});
it('should record API request breakdown for different phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.NETWORK_LATENCY,
250,
);
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.RESPONSE_PROCESSING,
100,
);
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.TOKEN_PROCESSING,
50,
);
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 250, {
'session.id': 'test-session-id',
model: 'gemini-pro',
phase: 'network_latency',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 100, {
'session.id': 'test-session-id',
model: 'gemini-pro',
phase: 'response_processing',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 50, {
'session.id': 'test-session-id',
model: 'gemini-pro',
phase: 'token_processing',
});
});
});
describe('recordPerformanceScore', () => {
it('should record performance score with category and baseline', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordPerformanceScoreModule(
mockConfig,
85.5,
'memory_efficiency',
80.0,
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, {
'session.id': 'test-session-id',
category: 'memory_efficiency',
baseline: 80.0,
});
});
it('should record performance score without baseline', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordPerformanceScoreModule(mockConfig, 92.3, 'overall_performance');
expect(mockHistogramRecordFn).toHaveBeenCalledWith(92.3, {
'session.id': 'test-session-id',
category: 'overall_performance',
baseline: undefined,
});
});
});
describe('recordPerformanceRegression', () => {
it('should record performance regression with baseline comparison', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordPerformanceRegressionModule(
mockConfig,
'startup_time',
1200,
1000,
'medium',
);
// Verify regression counter
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
metric: 'startup_time',
severity: 'medium',
current_value: 1200,
baseline_value: 1000,
});
// Verify baseline comparison histogram (20% increase)
expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, {
'session.id': 'test-session-id',
metric: 'startup_time',
severity: 'medium',
current_value: 1200,
baseline_value: 1000,
});
});
it('should handle zero baseline value gracefully', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordPerformanceRegressionModule(
mockConfig,
'memory_usage',
100,
0,
'high',
);
// Verify regression counter still recorded
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
metric: 'memory_usage',
severity: 'high',
current_value: 100,
baseline_value: 0,
});
// Verify no baseline comparison due to zero baseline
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record different severity levels', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordPerformanceRegressionModule(
mockConfig,
'api_latency',
500,
400,
'low',
);
recordPerformanceRegressionModule(
mockConfig,
'cpu_usage',
90,
70,
'high',
);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
'session.id': 'test-session-id',
metric: 'api_latency',
severity: 'low',
current_value: 500,
baseline_value: 400,
});
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
metric: 'cpu_usage',
severity: 'high',
current_value: 90,
baseline_value: 70,
});
});
});
describe('recordBaselineComparison', () => {
it('should record baseline comparison with percentage change', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(
mockConfig,
'memory_usage',
120,
100,
'performance_tracking',
);
// 20% increase: (120 - 100) / 100 * 100 = 20%
expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, {
'session.id': 'test-session-id',
metric: 'memory_usage',
category: 'performance_tracking',
current_value: 120,
baseline_value: 100,
});
});
it('should handle negative percentage change (improvement)', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(
mockConfig,
'startup_time',
800,
1000,
'optimization',
);
// 20% decrease: (800 - 1000) / 1000 * 100 = -20%
expect(mockHistogramRecordFn).toHaveBeenCalledWith(-20, {
'session.id': 'test-session-id',
metric: 'startup_time',
category: 'optimization',
current_value: 800,
baseline_value: 1000,
});
});
it('should skip recording when baseline is zero', async () => {
// Access the actual mocked module
const mockedModule = (await vi.importMock('@opentelemetry/api')) as {
diag: { warn: ReturnType<typeof vi.fn> };
};
const diagSpy = vi.spyOn(mockedModule.diag, 'warn');
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(
mockConfig,
'new_metric',
50,
0,
'testing',
);
expect(diagSpy).toHaveBeenCalledWith(
'Baseline value is zero, skipping comparison.',
);
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
});
});
});

352
packages/core/src/telemetry/metrics.ts
View File

@@ -5,7 +5,7 @@
*/
import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api';
import { metrics, ValueType } from '@opentelemetry/api';
import { diag, metrics, ValueType } from '@opentelemetry/api';
import {
SERVICE_NAME,
METRIC_TOOL_CALL_COUNT,
@@ -22,6 +22,18 @@ import {
METRIC_MODEL_ROUTING_LATENCY,
METRIC_MODEL_ROUTING_FAILURE_COUNT,
METRIC_MODEL_SLASH_COMMAND_CALL_COUNT,
// Performance Monitoring Metrics
METRIC_STARTUP_TIME,
METRIC_MEMORY_USAGE,
METRIC_CPU_USAGE,
METRIC_TOOL_QUEUE_DEPTH,
METRIC_TOOL_EXECUTION_BREAKDOWN,
METRIC_TOKEN_EFFICIENCY,
METRIC_API_REQUEST_BREAKDOWN,
METRIC_PERFORMANCE_SCORE,
METRIC_REGRESSION_DETECTION,
METRIC_REGRESSION_PERCENTAGE_CHANGE,
METRIC_BASELINE_COMPARISON,
} from './constants.js';
import type { Config } from '../config/config.js';
import type { ModelRoutingEvent, ModelSlashCommandEvent } from './types.js';
@@ -32,6 +44,36 @@ export enum FileOperation {
UPDATE = 'update',
}
export enum PerformanceMetricType {
STARTUP = 'startup',
MEMORY = 'memory',
CPU = 'cpu',
TOOL_EXECUTION = 'tool_execution',
API_REQUEST = 'api_request',
TOKEN_EFFICIENCY = 'token_efficiency',
}
export enum MemoryMetricType {
HEAP_USED = 'heap_used',
HEAP_TOTAL = 'heap_total',
EXTERNAL = 'external',
RSS = 'rss',
}
export enum ToolExecutionPhase {
VALIDATION = 'validation',
PREPARATION = 'preparation',
EXECUTION = 'execution',
RESULT_PROCESSING = 'result_processing',
}
export enum ApiRequestPhase {
REQUEST_PREPARATION = 'request_preparation',
NETWORK_LATENCY = 'network_latency',
RESPONSE_PROCESSING = 'response_processing',
TOKEN_PROCESSING = 'token_processing',
}
let cliMeter: Meter | undefined;
let toolCallCounter: Counter | undefined;
let toolCallLatencyHistogram: Histogram | undefined;
@@ -46,7 +88,21 @@ let contentRetryFailureCounter: Counter | undefined;
let modelRoutingLatencyHistogram: Histogram | undefined;
let modelRoutingFailureCounter: Counter | undefined;
let modelSlashCommandCallCounter: Counter | undefined;
// Performance Monitoring Metrics
let startupTimeHistogram: Histogram | undefined;
let memoryUsageGauge: Histogram | undefined; // Using Histogram until ObservableGauge is available
let cpuUsageGauge: Histogram | undefined;
let toolQueueDepthGauge: Histogram | undefined;
let toolExecutionBreakdownHistogram: Histogram | undefined;
let tokenEfficiencyHistogram: Histogram | undefined;
let apiRequestBreakdownHistogram: Histogram | undefined;
let performanceScoreGauge: Histogram | undefined;
let regressionDetectionCounter: Counter | undefined;
let regressionPercentageChangeHistogram: Histogram | undefined;
let baselineComparisonHistogram: Histogram | undefined;
let isMetricsInitialized = false;
let isPerformanceMonitoringEnabled = false;
function getCommonAttributes(config: Config): Attributes {
return {
@@ -67,6 +123,7 @@ export function initializeMetrics(config: Config): void {
const meter = getMeter();
if (!meter) return;
// Initialize core metrics
toolCallCounter = meter.createCounter(METRIC_TOOL_CALL_COUNT, {
description: 'Counts tool calls, tagged by function name and success.',
valueType: ValueType.INT,
@@ -145,6 +202,10 @@ export function initializeMetrics(config: Config): void {
valueType: ValueType.INT,
});
sessionCounter.add(1, getCommonAttributes(config));
// Initialize performance monitoring metrics if enabled
initializePerformanceMonitoring(config);
isMetricsInitialized = true;
}
@@ -332,3 +393,292 @@ export function recordModelRoutingMetrics(
});
}
}
// Performance Monitoring Functions
export function initializePerformanceMonitoring(config: Config): void {
const meter = getMeter();
if (!meter) return;
// Check if performance monitoring is enabled in config
// For now, enable performance monitoring when telemetry is enabled
// TODO: Add specific performance monitoring settings to config
isPerformanceMonitoringEnabled = config.getTelemetryEnabled();
if (!isPerformanceMonitoringEnabled) return;
// Initialize startup time histogram
startupTimeHistogram = meter.createHistogram(METRIC_STARTUP_TIME, {
description:
'CLI startup time in milliseconds, broken down by initialization phase.',
unit: 'ms',
valueType: ValueType.DOUBLE,
});
// Initialize memory usage histogram (using histogram until ObservableGauge is available)
memoryUsageGauge = meter.createHistogram(METRIC_MEMORY_USAGE, {
description: 'Memory usage in bytes.',
unit: 'bytes',
valueType: ValueType.INT,
});
// Initialize CPU usage histogram
cpuUsageGauge = meter.createHistogram(METRIC_CPU_USAGE, {
description: 'CPU usage percentage.',
unit: 'percent',
valueType: ValueType.DOUBLE,
});
// Initialize tool queue depth histogram
toolQueueDepthGauge = meter.createHistogram(METRIC_TOOL_QUEUE_DEPTH, {
description: 'Number of tools in execution queue.',
valueType: ValueType.INT,
});
// Initialize performance breakdowns
toolExecutionBreakdownHistogram = meter.createHistogram(
METRIC_TOOL_EXECUTION_BREAKDOWN,
{
description: 'Tool execution time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
},
);
tokenEfficiencyHistogram = meter.createHistogram(METRIC_TOKEN_EFFICIENCY, {
description:
'Token efficiency metrics (tokens per operation, cache hit rate, etc.).',
valueType: ValueType.DOUBLE,
});
apiRequestBreakdownHistogram = meter.createHistogram(
METRIC_API_REQUEST_BREAKDOWN,
{
description: 'API request time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
},
);
// Initialize performance score and regression detection
performanceScoreGauge = meter.createHistogram(METRIC_PERFORMANCE_SCORE, {
description: 'Composite performance score (0-100).',
unit: 'score',
valueType: ValueType.DOUBLE,
});
regressionDetectionCounter = meter.createCounter(
METRIC_REGRESSION_DETECTION,
{
description: 'Performance regression detection events.',
valueType: ValueType.INT,
},
);
regressionPercentageChangeHistogram = meter.createHistogram(
METRIC_REGRESSION_PERCENTAGE_CHANGE,
{
description:
'Percentage change compared to baseline for detected regressions.',
unit: 'percent',
valueType: ValueType.DOUBLE,
},
);
baselineComparisonHistogram = meter.createHistogram(
METRIC_BASELINE_COMPARISON,
{
description:
'Performance comparison to established baseline (percentage change).',
unit: 'percent',
valueType: ValueType.DOUBLE,
},
);
}
export function recordStartupPerformance(
config: Config,
phase: string,
durationMs: number,
details?: Record<string, string | number | boolean>,
): void {
if (!startupTimeHistogram || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
phase,
...details,
};
startupTimeHistogram.record(durationMs, attributes);
}
export function recordMemoryUsage(
config: Config,
memoryType: MemoryMetricType,
bytes: number,
component?: string,
): void {
if (!memoryUsageGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
memory_type: memoryType,
component,
};
memoryUsageGauge.record(bytes, attributes);
}
export function recordCpuUsage(
config: Config,
percentage: number,
component?: string,
): void {
if (!cpuUsageGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
component,
};
cpuUsageGauge.record(percentage, attributes);
}
export function recordToolQueueDepth(config: Config, queueDepth: number): void {
if (!toolQueueDepthGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
};
toolQueueDepthGauge.record(queueDepth, attributes);
}
export function recordToolExecutionBreakdown(
config: Config,
functionName: string,
phase: ToolExecutionPhase,
durationMs: number,
): void {
if (!toolExecutionBreakdownHistogram || !isPerformanceMonitoringEnabled)
return;
const attributes: Attributes = {
...getCommonAttributes(config),
function_name: functionName,
phase,
};
toolExecutionBreakdownHistogram.record(durationMs, attributes);
}
export function recordTokenEfficiency(
config: Config,
model: string,
metric: string,
value: number,
context?: string,
): void {
if (!tokenEfficiencyHistogram || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
model,
metric,
context,
};
tokenEfficiencyHistogram.record(value, attributes);
}
export function recordApiRequestBreakdown(
config: Config,
model: string,
phase: ApiRequestPhase,
durationMs: number,
): void {
if (!apiRequestBreakdownHistogram || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
model,
phase,
};
apiRequestBreakdownHistogram.record(durationMs, attributes);
}
export function recordPerformanceScore(
config: Config,
score: number,
category: string,
baseline?: number,
): void {
if (!performanceScoreGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
category,
baseline,
};
performanceScoreGauge.record(score, attributes);
}
export function recordPerformanceRegression(
config: Config,
metric: string,
currentValue: number,
baselineValue: number,
severity: 'low' | 'medium' | 'high',
): void {
if (!regressionDetectionCounter || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
metric,
severity,
current_value: currentValue,
baseline_value: baselineValue,
};
regressionDetectionCounter.add(1, attributes);
if (baselineValue !== 0 && regressionPercentageChangeHistogram) {
const percentageChange =
((currentValue - baselineValue) / baselineValue) * 100;
regressionPercentageChangeHistogram.record(percentageChange, attributes);
}
}
export function recordBaselineComparison(
config: Config,
metric: string,
currentValue: number,
baselineValue: number,
category: string,
): void {
if (!baselineComparisonHistogram || !isPerformanceMonitoringEnabled) return;
if (baselineValue === 0) {
diag.warn('Baseline value is zero, skipping comparison.');
return;
}
const percentageChange =
((currentValue - baselineValue) / baselineValue) * 100;
const attributes: Attributes = {
...getCommonAttributes(config),
metric,
category,
current_value: currentValue,
baseline_value: baselineValue,
};
baselineComparisonHistogram.record(percentageChange, attributes);
}
// Utility function to check if performance monitoring is enabled
export function isPerformanceMonitoringActive(): boolean {
return isPerformanceMonitoringEnabled && isMetricsInitialized;
}