MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-03-10 22:21:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

Refactor metrics definitions to be easily understandable (#10215)

Browse Source
This commit is contained in:
Christie Warwick (Wilson)
2025-10-01 06:33:47 -07:00
committed by GitHub
parent a80cd28d4c
commit 5c6f006634
5 changed files with 680 additions and 660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
packages/core/src/telemetry/constants.ts
View File

@@ -31,39 +31,7 @@ export const EVENT_CONTENT_RETRY_FAILURE =
'gemini_cli.chat.content_retry_failure';
export const EVENT_FILE_OPERATION = 'gemini_cli.file_operation';
export const EVENT_MODEL_SLASH_COMMAND = 'gemini_cli.slash_command.model';
export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count';
export const METRIC_API_REQUEST_LATENCY = 'gemini_cli.api.request.latency';
export const METRIC_TOKEN_USAGE = 'gemini_cli.token.usage';
export const METRIC_SESSION_COUNT = 'gemini_cli.session.count';
export const METRIC_FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count';
export const METRIC_INVALID_CHUNK_COUNT = 'gemini_cli.chat.invalid_chunk.count';
export const METRIC_CONTENT_RETRY_COUNT = 'gemini_cli.chat.content_retry.count';
export const METRIC_CONTENT_RETRY_FAILURE_COUNT =
'gemini_cli.chat.content_retry_failure.count';
export const EVENT_MODEL_ROUTING = 'gemini_cli.model_routing';
export const METRIC_MODEL_ROUTING_LATENCY = 'gemini_cli.model_routing.latency';
export const METRIC_MODEL_ROUTING_FAILURE_COUNT =
'gemini_cli.model_routing.failure.count';
export const METRIC_MODEL_SLASH_COMMAND_CALL_COUNT =
'gemini_cli.slash_command.model.call_count';
// Performance Monitoring Metrics
export const METRIC_STARTUP_TIME = 'gemini_cli.startup.duration';
export const METRIC_MEMORY_USAGE = 'gemini_cli.memory.usage';
export const METRIC_CPU_USAGE = 'gemini_cli.cpu.usage';
export const METRIC_TOOL_QUEUE_DEPTH = 'gemini_cli.tool.queue.depth';
export const METRIC_TOOL_EXECUTION_BREAKDOWN =
'gemini_cli.tool.execution.breakdown';
export const METRIC_TOKEN_EFFICIENCY = 'gemini_cli.token.efficiency';
export const METRIC_API_REQUEST_BREAKDOWN = 'gemini_cli.api.request.breakdown';
export const METRIC_PERFORMANCE_SCORE = 'gemini_cli.performance.score';
export const METRIC_REGRESSION_DETECTION = 'gemini_cli.performance.regression';
export const METRIC_REGRESSION_PERCENTAGE_CHANGE =
'gemini_cli.performance.regression.percentage_change';
export const METRIC_BASELINE_COMPARISON =
'gemini_cli.performance.baseline.comparison';
// Performance Events
export const EVENT_STARTUP_PERFORMANCE = 'gemini_cli.startup.performance';

68
packages/core/src/telemetry/loggers.test.ts
View File

@@ -344,16 +344,14 @@ describe('loggers', () => {
expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith(
mockConfig,
'test-model',
100,
200,
{ model: 'test-model', status_code: 200 },
);
expect(mockMetrics.recordTokenUsageMetrics).toHaveBeenCalledWith(
mockConfig,
'test-model',
50,
'output',
{ model: 'test-model', type: 'output' },
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -632,11 +630,13 @@ describe('loggers', () => {
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
ToolCallDecision.ACCEPT,
'native',
{
function_name: 'test-function',
success: true,
decision: ToolCallDecision.ACCEPT,
tool_type: 'native',
},
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -703,11 +703,13 @@ describe('loggers', () => {
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
false,
ToolCallDecision.REJECT,
'native',
{
function_name: 'test-function',
success: false,
decision: ToolCallDecision.REJECT,
tool_type: 'native',
},
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -777,11 +779,13 @@ describe('loggers', () => {
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
ToolCallDecision.MODIFY,
'native',
{
function_name: 'test-function',
success: true,
decision: ToolCallDecision.MODIFY,
tool_type: 'native',
},
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -850,11 +854,13 @@ describe('loggers', () => {
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
true,
undefined,
'native',
{
function_name: 'test-function',
success: true,
decision: undefined,
tool_type: 'native',
},
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -924,11 +930,13 @@ describe('loggers', () => {
expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith(
mockConfig,
'test-function',
100,
false,
undefined,
'native',
{
function_name: 'test-function',
success: false,
decision: undefined,
tool_type: 'native',
},
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
@@ -1085,11 +1093,13 @@ describe('loggers', () => {
expect(mockMetrics.recordFileOperationMetric).toHaveBeenCalledWith(
mockConfig,
'read',
10,
'text/plain',
'.txt',
'typescript',
{
operation: 'read',
lines: 10,
mimetype: 'text/plain',
extension: '.txt',
programming_language: 'typescript',
},
);
});
});

96
packages/core/src/telemetry/loggers.ts
View File

@@ -191,14 +191,12 @@ export function logToolCall(config: Config, event: ToolCallEvent): void {
attributes,
};
logger.emit(logRecord);
recordToolCallMetrics(
config,
event.function_name,
event.duration_ms,
event.success,
event.decision,
event.tool_type,
);
recordToolCallMetrics(config, event.duration_ms, {
function_name: event.function_name,
success: event.success,
decision: event.decision,
tool_type: event.tool_type,
});
}
export function logToolOutputTruncated(
@@ -258,14 +256,13 @@ export function logFileOperation(
};
logger.emit(logRecord);
recordFileOperationMetric(
config,
event.operation,
event.lines,
event.mimetype,
event.extension,
event.programming_language,
);
recordFileOperationMetric(config, {
operation: event.operation,
lines: event.lines,
mimetype: event.mimetype,
extension: event.extension,
programming_language: event.programming_language,
});
}
export function logApiRequest(config: Config, event: ApiRequestEvent): void {
@@ -364,13 +361,11 @@ export function logApiError(config: Config, event: ApiErrorEvent): void {
attributes,
};
logger.emit(logRecord);
recordApiErrorMetrics(
config,
event.model,
event.duration_ms,
event.status_code,
event.error_type,
);
recordApiErrorMetrics(config, event.duration_ms, {
model: event.model,
status_code: event.status_code,
error_type: event.error_type,
});
}
export function logApiResponse(config: Config, event: ApiResponseEvent): void {
@@ -403,37 +398,30 @@ export function logApiResponse(config: Config, event: ApiResponseEvent): void {
attributes,
};
logger.emit(logRecord);
recordApiResponseMetrics(
config,
event.model,
event.duration_ms,
event.status_code,
);
recordTokenUsageMetrics(
config,
event.model,
event.input_token_count,
'input',
);
recordTokenUsageMetrics(
config,
event.model,
event.output_token_count,
'output',
);
recordTokenUsageMetrics(
config,
event.model,
event.cached_content_token_count,
'cache',
);
recordTokenUsageMetrics(
config,
event.model,
event.thoughts_token_count,
'thought',
);
recordTokenUsageMetrics(config, event.model, event.tool_token_count, 'tool');
recordApiResponseMetrics(config, event.duration_ms, {
model: event.model,
status_code: event.status_code,
});
recordTokenUsageMetrics(config, event.input_token_count, {
model: event.model,
type: 'input',
});
recordTokenUsageMetrics(config, event.output_token_count, {
model: event.model,
type: 'output',
});
recordTokenUsageMetrics(config, event.cached_content_token_count, {
model: event.model,
type: 'cache',
});
recordTokenUsageMetrics(config, event.thoughts_token_count, {
model: event.model,
type: 'thought',
});
recordTokenUsageMetrics(config, event.tool_token_count, {
model: event.model,
type: 'tool',
});
}
export function logLoopDetected(

407
packages/core/src/telemetry/metrics.test.ts
View File

@@ -160,13 +160,19 @@ describe('Telemetry Metrics', () => {
} as unknown as Config;
it('should not record metrics if not initialized', () => {
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input');
recordTokenUsageMetricsModule(mockConfig, 100, {
model: 'gemini-pro',
type: 'input',
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record token usage with the correct attributes', () => {
initializeMetricsModule(mockConfig);
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input');
recordTokenUsageMetricsModule(mockConfig, 100, {
model: 'gemini-pro',
type: 'input',
});
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
'session.id': 'test-session-id',
@@ -182,28 +188,40 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 50, 'output');
recordTokenUsageMetricsModule(mockConfig, 50, {
model: 'gemini-pro',
type: 'output',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(50, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'output',
});
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 25, 'thought');
recordTokenUsageMetricsModule(mockConfig, 25, {
model: 'gemini-pro',
type: 'thought',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(25, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'thought',
});
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 75, 'cache');
recordTokenUsageMetricsModule(mockConfig, 75, {
model: 'gemini-pro',
type: 'cache',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(75, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'cache',
});
recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 125, 'tool');
recordTokenUsageMetricsModule(mockConfig, 125, {
model: 'gemini-pro',
type: 'tool',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(125, {
'session.id': 'test-session-id',
model: 'gemini-pro',
@@ -215,7 +233,10 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordTokenUsageMetricsModule(mockConfig, 'gemini-ultra', 200, 'input');
recordTokenUsageMetricsModule(mockConfig, 200, {
model: 'gemini-ultra',
type: 'input',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(200, {
'session.id': 'test-session-id',
model: 'gemini-ultra',
@@ -231,25 +252,23 @@ describe('Telemetry Metrics', () => {
} as unknown as Config;
it('should not record metrics if not initialized', () => {
recordFileOperationMetricModule(
mockConfig,
FileOperation.CREATE,
10,
'text/plain',
'txt',
);
recordFileOperationMetricModule(mockConfig, {
operation: FileOperation.CREATE,
lines: 10,
mimetype: 'text/plain',
extension: 'txt',
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record file creation with all attributes', () => {
initializeMetricsModule(mockConfig);
recordFileOperationMetricModule(
mockConfig,
FileOperation.CREATE,
10,
'text/plain',
'txt',
);
recordFileOperationMetricModule(mockConfig, {
operation: FileOperation.CREATE,
lines: 10,
mimetype: 'text/plain',
extension: 'txt',
});
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
@@ -268,7 +287,9 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordFileOperationMetricModule(mockConfig, FileOperation.READ);
recordFileOperationMetricModule(mockConfig, {
operation: FileOperation.READ,
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
operation: FileOperation.READ,
@@ -279,12 +300,10 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordFileOperationMetricModule(
mockConfig,
FileOperation.UPDATE,
undefined,
'application/javascript',
);
recordFileOperationMetricModule(mockConfig, {
operation: FileOperation.UPDATE,
mimetype: 'application/javascript',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
operation: FileOperation.UPDATE,
@@ -296,13 +315,9 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordFileOperationMetricModule(
mockConfig,
FileOperation.UPDATE,
undefined,
undefined,
undefined,
);
recordFileOperationMetricModule(mockConfig, {
operation: FileOperation.UPDATE,
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
@@ -314,14 +329,12 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordFileOperationMetricModule(
mockConfig,
FileOperation.UPDATE,
10,
'text/plain',
'txt',
undefined,
);
recordFileOperationMetricModule(mockConfig, {
operation: FileOperation.UPDATE,
lines: 10,
mimetype: 'text/plain',
extension: 'txt',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
@@ -336,13 +349,9 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordFileOperationMetricModule(
mockConfig,
FileOperation.UPDATE,
undefined,
undefined,
undefined,
);
recordFileOperationMetricModule(mockConfig, {
operation: FileOperation.UPDATE,
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
@@ -436,14 +445,12 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfigDisabled);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(
mockConfigDisabled,
'settings_loading',
100,
{
recordStartupPerformanceModule(mockConfigDisabled, 100, {
phase: 'settings_loading',
details: {
auth_type: 'gemini',
},
);
});
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
@@ -452,10 +459,13 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(mockConfig, 'settings_loading', 150, {
auth_type: 'gemini',
telemetry_enabled: true,
settings_sources: 2,
recordStartupPerformanceModule(mockConfig, 150, {
phase: 'settings_loading',
details: {
auth_type: 'gemini',
telemetry_enabled: true,
settings_sources: 2,
},
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, {
@@ -471,7 +481,7 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(mockConfig, 'cleanup', 50);
recordStartupPerformanceModule(mockConfig, 50, { phase: 'cleanup' });
expect(mockHistogramRecordFn).toHaveBeenCalledWith(50, {
'session.id': 'test-session-id',
@@ -485,15 +495,13 @@ describe('Telemetry Metrics', () => {
// Test with realistic floating-point values that performance.now() would return
const floatingPointDuration = 123.45678;
recordStartupPerformanceModule(
mockConfig,
'total_startup',
floatingPointDuration,
{
recordStartupPerformanceModule(mockConfig, floatingPointDuration, {
phase: 'total_startup',
details: {
is_tty: true,
has_question: false,
},
);
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(
floatingPointDuration,
@@ -512,12 +520,10 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.HEAP_USED,
15728640,
'startup',
);
recordMemoryUsageModule(mockConfig, 15728640, {
memory_type: MemoryMetricType.HEAP_USED,
component: 'startup',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, {
'session.id': 'test-session-id',
@@ -530,24 +536,18 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.HEAP_TOTAL,
31457280,
'api_call',
);
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.EXTERNAL,
2097152,
'tool_execution',
);
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.RSS,
41943040,
'memory_monitor',
);
recordMemoryUsageModule(mockConfig, 31457280, {
memory_type: MemoryMetricType.HEAP_TOTAL,
component: 'api_call',
});
recordMemoryUsageModule(mockConfig, 2097152, {
memory_type: MemoryMetricType.EXTERNAL,
component: 'tool_execution',
});
recordMemoryUsageModule(mockConfig, 41943040, {
memory_type: MemoryMetricType.RSS,
component: 'memory_monitor',
});
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 31457280, {
@@ -571,16 +571,13 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordMemoryUsageModule(
mockConfig,
MemoryMetricType.HEAP_USED,
15728640,
);
recordMemoryUsageModule(mockConfig, 15728640, {
memory_type: MemoryMetricType.HEAP_USED,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(15728640, {
'session.id': 'test-session-id',
memory_type: 'heap_used',
component: undefined,
});
});
});
@@ -590,7 +587,9 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordCpuUsageModule(mockConfig, 85.5, 'tool_execution');
recordCpuUsageModule(mockConfig, 85.5, {
component: 'tool_execution',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, {
'session.id': 'test-session-id',
@@ -602,11 +601,10 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordCpuUsageModule(mockConfig, 42.3);
recordCpuUsageModule(mockConfig, 42.3, {});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(42.3, {
'session.id': 'test-session-id',
component: undefined,
});
});
});
@@ -640,12 +638,10 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolExecutionBreakdownModule(
mockConfig,
'Read',
ToolExecutionPhase.VALIDATION,
25,
);
recordToolExecutionBreakdownModule(mockConfig, 25, {
function_name: 'Read',
phase: ToolExecutionPhase.VALIDATION,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(25, {
'session.id': 'test-session-id',
@@ -658,24 +654,18 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolExecutionBreakdownModule(
mockConfig,
'Bash',
ToolExecutionPhase.PREPARATION,
50,
);
recordToolExecutionBreakdownModule(
mockConfig,
'Bash',
ToolExecutionPhase.EXECUTION,
1500,
);
recordToolExecutionBreakdownModule(
mockConfig,
'Bash',
ToolExecutionPhase.RESULT_PROCESSING,
75,
);
recordToolExecutionBreakdownModule(mockConfig, 50, {
function_name: 'Bash',
phase: ToolExecutionPhase.PREPARATION,
});
recordToolExecutionBreakdownModule(mockConfig, 1500, {
function_name: 'Bash',
phase: ToolExecutionPhase.EXECUTION,
});
recordToolExecutionBreakdownModule(mockConfig, 75, {
function_name: 'Bash',
phase: ToolExecutionPhase.RESULT_PROCESSING,
});
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 50, {
@@ -701,13 +691,11 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordTokenEfficiencyModule(
mockConfig,
'gemini-pro',
'cache_hit_rate',
0.85,
'api_request',
);
recordTokenEfficiencyModule(mockConfig, 0.85, {
model: 'gemini-pro',
metric: 'cache_hit_rate',
context: 'api_request',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(0.85, {
'session.id': 'test-session-id',
@@ -721,18 +709,15 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordTokenEfficiencyModule(
mockConfig,
'gemini-pro',
'tokens_per_operation',
125.5,
);
recordTokenEfficiencyModule(mockConfig, 125.5, {
model: 'gemini-pro',
metric: 'tokens_per_operation',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(125.5, {
'session.id': 'test-session-id',
model: 'gemini-pro',
metric: 'tokens_per_operation',
context: undefined,
});
});
});
@@ -742,12 +727,10 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.REQUEST_PREPARATION,
15,
);
recordApiRequestBreakdownModule(mockConfig, 15, {
model: 'gemini-pro',
phase: ApiRequestPhase.REQUEST_PREPARATION,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(15, {
'session.id': 'test-session-id',
@@ -760,24 +743,18 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.NETWORK_LATENCY,
250,
);
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.RESPONSE_PROCESSING,
100,
);
recordApiRequestBreakdownModule(
mockConfig,
'gemini-pro',
ApiRequestPhase.TOKEN_PROCESSING,
50,
);
recordApiRequestBreakdownModule(mockConfig, 250, {
model: 'gemini-pro',
phase: ApiRequestPhase.NETWORK_LATENCY,
});
recordApiRequestBreakdownModule(mockConfig, 100, {
model: 'gemini-pro',
phase: ApiRequestPhase.RESPONSE_PROCESSING,
});
recordApiRequestBreakdownModule(mockConfig, 50, {
model: 'gemini-pro',
phase: ApiRequestPhase.TOKEN_PROCESSING,
});
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 250, {
@@ -803,12 +780,10 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordPerformanceScoreModule(
mockConfig,
85.5,
'memory_efficiency',
80.0,
);
recordPerformanceScoreModule(mockConfig, 85.5, {
category: 'memory_efficiency',
baseline: 80.0,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, {
'session.id': 'test-session-id',
@@ -821,12 +796,13 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordPerformanceScoreModule(mockConfig, 92.3, 'overall_performance');
recordPerformanceScoreModule(mockConfig, 92.3, {
category: 'overall_performance',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(92.3, {
'session.id': 'test-session-id',
category: 'overall_performance',
baseline: undefined,
});
});
});
@@ -837,13 +813,12 @@ describe('Telemetry Metrics', () => {
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordPerformanceRegressionModule(
mockConfig,
'startup_time',
1200,
1000,
'medium',
);
recordPerformanceRegressionModule(mockConfig, {
metric: 'startup_time',
current_value: 1200,
baseline_value: 1000,
severity: 'medium',
});
// Verify regression counter
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
@@ -869,13 +844,12 @@ describe('Telemetry Metrics', () => {
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordPerformanceRegressionModule(
mockConfig,
'memory_usage',
100,
0,
'high',
);
recordPerformanceRegressionModule(mockConfig, {
metric: 'memory_usage',
current_value: 100,
baseline_value: 0,
severity: 'high',
});
// Verify regression counter still recorded
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
@@ -894,20 +868,18 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordPerformanceRegressionModule(
mockConfig,
'api_latency',
500,
400,
'low',
);
recordPerformanceRegressionModule(
mockConfig,
'cpu_usage',
90,
70,
'high',
);
recordPerformanceRegressionModule(mockConfig, {
metric: 'api_latency',
current_value: 500,
baseline_value: 400,
severity: 'low',
});
recordPerformanceRegressionModule(mockConfig, {
metric: 'cpu_usage',
current_value: 90,
baseline_value: 70,
severity: 'high',
});
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
'session.id': 'test-session-id',
@@ -931,13 +903,12 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(
mockConfig,
'memory_usage',
120,
100,
'performance_tracking',
);
recordBaselineComparisonModule(mockConfig, {
metric: 'memory_usage',
current_value: 120,
baseline_value: 100,
category: 'performance_tracking',
});
// 20% increase: (120 - 100) / 100 * 100 = 20%
expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, {
@@ -953,13 +924,12 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(
mockConfig,
'startup_time',
800,
1000,
'optimization',
);
recordBaselineComparisonModule(mockConfig, {
metric: 'startup_time',
current_value: 800,
baseline_value: 1000,
category: 'optimization',
});
// 20% decrease: (800 - 1000) / 1000 * 100 = -20%
expect(mockHistogramRecordFn).toHaveBeenCalledWith(-20, {
@@ -981,13 +951,12 @@ describe('Telemetry Metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(
mockConfig,
'new_metric',
50,
0,
'testing',
);
recordBaselineComparisonModule(mockConfig, {
metric: 'new_metric',
current_value: 50,
baseline_value: 0,
category: 'testing',
});
expect(diagSpy).toHaveBeenCalledWith(
'Baseline value is zero, skipping comparison.',

737
packages/core/src/telemetry/metrics.ts
View File

@@ -6,38 +6,305 @@
import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api';
import { diag, metrics, ValueType } from '@opentelemetry/api';
import {
SERVICE_NAME,
METRIC_TOOL_CALL_COUNT,
METRIC_TOOL_CALL_LATENCY,
METRIC_API_REQUEST_COUNT,
METRIC_API_REQUEST_LATENCY,
METRIC_TOKEN_USAGE,
METRIC_SESSION_COUNT,
METRIC_FILE_OPERATION_COUNT,
EVENT_CHAT_COMPRESSION,
METRIC_INVALID_CHUNK_COUNT,
METRIC_CONTENT_RETRY_COUNT,
METRIC_CONTENT_RETRY_FAILURE_COUNT,
METRIC_MODEL_ROUTING_LATENCY,
METRIC_MODEL_ROUTING_FAILURE_COUNT,
METRIC_MODEL_SLASH_COMMAND_CALL_COUNT,
// Performance Monitoring Metrics
METRIC_STARTUP_TIME,
METRIC_MEMORY_USAGE,
METRIC_CPU_USAGE,
METRIC_TOOL_QUEUE_DEPTH,
METRIC_TOOL_EXECUTION_BREAKDOWN,
METRIC_TOKEN_EFFICIENCY,
METRIC_API_REQUEST_BREAKDOWN,
METRIC_PERFORMANCE_SCORE,
METRIC_REGRESSION_DETECTION,
METRIC_REGRESSION_PERCENTAGE_CHANGE,
METRIC_BASELINE_COMPARISON,
} from './constants.js';
import { SERVICE_NAME, EVENT_CHAT_COMPRESSION } from './constants.js';
import type { Config } from '../config/config.js';
import type { ModelRoutingEvent, ModelSlashCommandEvent } from './types.js';
const TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
const TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
const API_REQUEST_COUNT = 'gemini_cli.api.request.count';
const API_REQUEST_LATENCY = 'gemini_cli.api.request.latency';
const TOKEN_USAGE = 'gemini_cli.token.usage';
const SESSION_COUNT = 'gemini_cli.session.count';
const FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count';
const INVALID_CHUNK_COUNT = 'gemini_cli.chat.invalid_chunk.count';
const CONTENT_RETRY_COUNT = 'gemini_cli.chat.content_retry.count';
const CONTENT_RETRY_FAILURE_COUNT =
'gemini_cli.chat.content_retry_failure.count';
const MODEL_ROUTING_LATENCY = 'gemini_cli.model_routing.latency';
const MODEL_ROUTING_FAILURE_COUNT = 'gemini_cli.model_routing.failure.count';
const MODEL_SLASH_COMMAND_CALL_COUNT =
'gemini_cli.slash_command.model.call_count';
// Performance Monitoring Metrics
const STARTUP_TIME = 'gemini_cli.startup.duration';
const MEMORY_USAGE = 'gemini_cli.memory.usage';
const CPU_USAGE = 'gemini_cli.cpu.usage';
const TOOL_QUEUE_DEPTH = 'gemini_cli.tool.queue.depth';
const TOOL_EXECUTION_BREAKDOWN = 'gemini_cli.tool.execution.breakdown';
const TOKEN_EFFICIENCY = 'gemini_cli.token.efficiency';
const API_REQUEST_BREAKDOWN = 'gemini_cli.api.request.breakdown';
const PERFORMANCE_SCORE = 'gemini_cli.performance.score';
const REGRESSION_DETECTION = 'gemini_cli.performance.regression';
const REGRESSION_PERCENTAGE_CHANGE =
'gemini_cli.performance.regression.percentage_change';
const BASELINE_COMPARISON = 'gemini_cli.performance.baseline.comparison';
const baseMetricDefinition = {
getCommonAttributes: (config: Config): Attributes => ({
'session.id': config.getSessionId(),
}),
};
const COUNTER_DEFINITIONS = {
[TOOL_CALL_COUNT]: {
description: 'Counts tool calls, tagged by function name and success.',
valueType: ValueType.INT,
assign: (c: Counter) => (toolCallCounter = c),
attributes: {} as {
function_name: string;
success: boolean;
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept';
tool_type?: 'native' | 'mcp';
},
},
[API_REQUEST_COUNT]: {
description: 'Counts API requests, tagged by model and status.',
valueType: ValueType.INT,
assign: (c: Counter) => (apiRequestCounter = c),
attributes: {} as {
model: string;
status_code?: number | string;
error_type?: string;
},
},
[TOKEN_USAGE]: {
description: 'Counts the total number of tokens used.',
valueType: ValueType.INT,
assign: (c: Counter) => (tokenUsageCounter = c),
attributes: {} as {
model: string;
type: 'input' | 'output' | 'thought' | 'cache' | 'tool';
},
},
[SESSION_COUNT]: {
description: 'Count of CLI sessions started.',
valueType: ValueType.INT,
assign: (c: Counter) => (sessionCounter = c),
attributes: {} as Record<string, never>,
},
[FILE_OPERATION_COUNT]: {
description: 'Counts file operations (create, read, update).',
valueType: ValueType.INT,
assign: (c: Counter) => (fileOperationCounter = c),
attributes: {} as {
operation: FileOperation;
lines?: number;
mimetype?: string;
extension?: string;
programming_language?: string;
},
},
[INVALID_CHUNK_COUNT]: {
description: 'Counts invalid chunks received from a stream.',
valueType: ValueType.INT,
assign: (c: Counter) => (invalidChunkCounter = c),
attributes: {} as Record<string, never>,
},
[CONTENT_RETRY_COUNT]: {
description: 'Counts retries due to content errors (e.g., empty stream).',
valueType: ValueType.INT,
assign: (c: Counter) => (contentRetryCounter = c),
attributes: {} as Record<string, never>,
},
[CONTENT_RETRY_FAILURE_COUNT]: {
description: 'Counts occurrences of all content retries failing.',
valueType: ValueType.INT,
assign: (c: Counter) => (contentRetryFailureCounter = c),
attributes: {} as Record<string, never>,
},
[MODEL_ROUTING_FAILURE_COUNT]: {
description: 'Counts model routing failures.',
valueType: ValueType.INT,
assign: (c: Counter) => (modelRoutingFailureCounter = c),
attributes: {} as {
'routing.decision_source': string;
'routing.error_message': string;
},
},
[MODEL_SLASH_COMMAND_CALL_COUNT]: {
description: 'Counts model slash command calls.',
valueType: ValueType.INT,
assign: (c: Counter) => (modelSlashCommandCallCounter = c),
attributes: {} as {
'slash_command.model.model_name': string;
},
},
[EVENT_CHAT_COMPRESSION]: {
description: 'Counts chat compression events.',
valueType: ValueType.INT,
assign: (c: Counter) => (chatCompressionCounter = c),
attributes: {} as {
tokens_before: number;
tokens_after: number;
},
},
} as const;
const HISTOGRAM_DEFINITIONS = {
[TOOL_CALL_LATENCY]: {
description: 'Latency of tool calls in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (toolCallLatencyHistogram = h),
attributes: {} as {
function_name: string;
},
},
[API_REQUEST_LATENCY]: {
description: 'Latency of API requests in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (apiRequestLatencyHistogram = h),
attributes: {} as {
model: string;
},
},
[MODEL_ROUTING_LATENCY]: {
description: 'Latency of model routing decisions in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (modelRoutingLatencyHistogram = h),
attributes: {} as {
'routing.decision_model': string;
'routing.decision_source': string;
},
},
} as const;
const PERFORMANCE_COUNTER_DEFINITIONS = {
[REGRESSION_DETECTION]: {
description: 'Performance regression detection events.',
valueType: ValueType.INT,
assign: (c: Counter) => (regressionDetectionCounter = c),
attributes: {} as {
metric: string;
severity: 'low' | 'medium' | 'high';
current_value: number;
baseline_value: number;
},
},
} as const;
const PERFORMANCE_HISTOGRAM_DEFINITIONS = {
[STARTUP_TIME]: {
description:
'CLI startup time in milliseconds, broken down by initialization phase.',
unit: 'ms',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (startupTimeHistogram = h),
attributes: {} as {
phase: string;
details?: Record<string, string | number | boolean>;
},
},
[MEMORY_USAGE]: {
description: 'Memory usage in bytes.',
unit: 'bytes',
valueType: ValueType.INT,
assign: (h: Histogram) => (memoryUsageGauge = h),
attributes: {} as {
memory_type: MemoryMetricType;
component?: string;
},
},
[CPU_USAGE]: {
description: 'CPU usage percentage.',
unit: 'percent',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (cpuUsageGauge = h),
attributes: {} as {
component?: string;
},
},
[TOOL_QUEUE_DEPTH]: {
description: 'Number of tools in execution queue.',
unit: 'count',
valueType: ValueType.INT,
assign: (h: Histogram) => (toolQueueDepthGauge = h),
attributes: {} as Record<string, never>,
},
[TOOL_EXECUTION_BREAKDOWN]: {
description: 'Tool execution time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (toolExecutionBreakdownHistogram = h),
attributes: {} as {
function_name: string;
phase: ToolExecutionPhase;
},
},
[TOKEN_EFFICIENCY]: {
description:
'Token efficiency metrics (tokens per operation, cache hit rate, etc.).',
unit: 'ratio',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (tokenEfficiencyHistogram = h),
attributes: {} as {
model: string;
metric: string;
context?: string;
},
},
[API_REQUEST_BREAKDOWN]: {
description: 'API request time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (apiRequestBreakdownHistogram = h),
attributes: {} as {
model: string;
phase: ApiRequestPhase;
},
},
[PERFORMANCE_SCORE]: {
description: 'Composite performance score (0-100).',
unit: 'score',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (performanceScoreGauge = h),
attributes: {} as {
category: string;
baseline?: number;
},
},
[REGRESSION_PERCENTAGE_CHANGE]: {
description:
'Percentage change compared to baseline for detected regressions.',
unit: 'percent',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (regressionPercentageChangeHistogram = h),
attributes: {} as {
metric: string;
severity: 'low' | 'medium' | 'high';
current_value: number;
baseline_value: number;
},
},
[BASELINE_COMPARISON]: {
description:
'Performance comparison to established baseline (percentage change).',
unit: 'percent',
valueType: ValueType.DOUBLE,
assign: (h: Histogram) => (baselineComparisonHistogram = h),
attributes: {} as {
metric: string;
category: string;
current_value: number;
baseline_value: number;
},
},
} as const;
type AllMetricDefs = typeof COUNTER_DEFINITIONS &
typeof HISTOGRAM_DEFINITIONS &
typeof PERFORMANCE_COUNTER_DEFINITIONS &
typeof PERFORMANCE_HISTOGRAM_DEFINITIONS;
export type MetricDefinitions = {
[K in keyof AllMetricDefs]: {
attributes: AllMetricDefs[K]['attributes'];
};
};
export enum FileOperation {
CREATE = 'create',
READ = 'read',
@@ -80,6 +347,7 @@ let toolCallLatencyHistogram: Histogram | undefined;
let apiRequestCounter: Counter | undefined;
let apiRequestLatencyHistogram: Histogram | undefined;
let tokenUsageCounter: Counter | undefined;
let sessionCounter: Counter | undefined;
let fileOperationCounter: Counter | undefined;
let chatCompressionCounter: Counter | undefined;
let invalidChunkCounter: Counter | undefined;
@@ -104,12 +372,6 @@ let baselineComparisonHistogram: Histogram | undefined;
let isMetricsInitialized = false;
let isPerformanceMonitoringEnabled = false;
function getCommonAttributes(config: Config): Attributes {
return {
'session.id': config.getSessionId(),
};
}
export function getMeter(): Meter | undefined {
if (!cliMeter) {
cliMeter = metrics.getMeter(SERVICE_NAME);
@@ -124,84 +386,20 @@ export function initializeMetrics(config: Config): void {
if (!meter) return;
// Initialize core metrics
toolCallCounter = meter.createCounter(METRIC_TOOL_CALL_COUNT, {
description: 'Counts tool calls, tagged by function name and success.',
valueType: ValueType.INT,
});
toolCallLatencyHistogram = meter.createHistogram(METRIC_TOOL_CALL_LATENCY, {
description: 'Latency of tool calls in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
});
apiRequestCounter = meter.createCounter(METRIC_API_REQUEST_COUNT, {
description: 'Counts API requests, tagged by model and status.',
valueType: ValueType.INT,
});
apiRequestLatencyHistogram = meter.createHistogram(
METRIC_API_REQUEST_LATENCY,
{
description: 'Latency of API requests in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
},
);
tokenUsageCounter = meter.createCounter(METRIC_TOKEN_USAGE, {
description: 'Counts the total number of tokens used.',
valueType: ValueType.INT,
});
fileOperationCounter = meter.createCounter(METRIC_FILE_OPERATION_COUNT, {
description: 'Counts file operations (create, read, update).',
valueType: ValueType.INT,
});
chatCompressionCounter = meter.createCounter(EVENT_CHAT_COMPRESSION, {
description: 'Counts chat compression events.',
valueType: ValueType.INT,
});
// New counters for content errors
invalidChunkCounter = meter.createCounter(METRIC_INVALID_CHUNK_COUNT, {
description: 'Counts invalid chunks received from a stream.',
valueType: ValueType.INT,
});
contentRetryCounter = meter.createCounter(METRIC_CONTENT_RETRY_COUNT, {
description: 'Counts retries due to content errors (e.g., empty stream).',
valueType: ValueType.INT,
});
contentRetryFailureCounter = meter.createCounter(
METRIC_CONTENT_RETRY_FAILURE_COUNT,
{
description: 'Counts occurrences of all content retries failing.',
valueType: ValueType.INT,
},
);
modelRoutingLatencyHistogram = meter.createHistogram(
METRIC_MODEL_ROUTING_LATENCY,
{
description: 'Latency of model routing decisions in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
},
);
modelRoutingFailureCounter = meter.createCounter(
METRIC_MODEL_ROUTING_FAILURE_COUNT,
{
description: 'Counts model routing failures.',
valueType: ValueType.INT,
},
);
modelSlashCommandCallCounter = meter.createCounter(
METRIC_MODEL_SLASH_COMMAND_CALL_COUNT,
{
description: 'Counts model slash command calls.',
valueType: ValueType.INT,
Object.entries(COUNTER_DEFINITIONS).forEach(
([name, { description, valueType, assign }]) => {
assign(meter.createCounter(name, { description, valueType }));
},
);
const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, {
description: 'Count of CLI sessions started.',
valueType: ValueType.INT,
});
sessionCounter.add(1, getCommonAttributes(config));
Object.entries(HISTOGRAM_DEFINITIONS).forEach(
([name, { description, unit, valueType, assign }]) => {
assign(meter.createHistogram(name, { description, unit, valueType }));
},
);
// Increment session counter after all metrics are initialized
sessionCounter?.add(1, baseMetricDefinition.getCommonAttributes(config));
// Initialize performance monitoring metrics if enabled
initializePerformanceMonitoring(config);
@@ -211,59 +409,50 @@ export function initializeMetrics(config: Config): void {
export function recordChatCompressionMetrics(
config: Config,
args: { tokens_before: number; tokens_after: number },
attributes: MetricDefinitions[typeof EVENT_CHAT_COMPRESSION]['attributes'],
) {
if (!chatCompressionCounter || !isMetricsInitialized) return;
chatCompressionCounter.add(1, {
...getCommonAttributes(config),
...args,
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
});
}
export function recordToolCallMetrics(
config: Config,
functionName: string,
durationMs: number,
success: boolean,
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept',
tool_type?: 'native' | 'mcp',
attributes: MetricDefinitions[typeof TOOL_CALL_COUNT]['attributes'],
): void {
if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized)
return;
const metricAttributes: Attributes = {
...getCommonAttributes(config),
function_name: functionName,
success,
decision,
tool_type,
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
toolCallCounter.add(1, metricAttributes);
toolCallLatencyHistogram.record(durationMs, {
...getCommonAttributes(config),
function_name: functionName,
...baseMetricDefinition.getCommonAttributes(config),
function_name: attributes.function_name,
});
}
export function recordTokenUsageMetrics(
config: Config,
model: string,
tokenCount: number,
type: 'input' | 'output' | 'thought' | 'cache' | 'tool',
attributes: MetricDefinitions[typeof TOKEN_USAGE]['attributes'],
): void {
if (!tokenUsageCounter || !isMetricsInitialized) return;
tokenUsageCounter.add(tokenCount, {
...getCommonAttributes(config),
model,
type,
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
});
}
export function recordApiResponseMetrics(
config: Config,
model: string,
durationMs: number,
statusCode?: number | string,
attributes: MetricDefinitions[typeof API_REQUEST_COUNT]['attributes'],
): void {
if (
!apiRequestCounter ||
@@ -272,23 +461,21 @@ export function recordApiResponseMetrics(
)
return;
const metricAttributes: Attributes = {
...getCommonAttributes(config),
model,
status_code: statusCode ?? 'ok',
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
status_code: attributes.status_code ?? 'ok',
};
apiRequestCounter.add(1, metricAttributes);
apiRequestLatencyHistogram.record(durationMs, {
...getCommonAttributes(config),
model,
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
});
}
export function recordApiErrorMetrics(
config: Config,
model: string,
durationMs: number,
statusCode?: number | string,
errorType?: string,
attributes: MetricDefinitions[typeof API_REQUEST_COUNT]['attributes'],
): void {
if (
!apiRequestCounter ||
@@ -297,38 +484,27 @@ export function recordApiErrorMetrics(
)
return;
const metricAttributes: Attributes = {
...getCommonAttributes(config),
model,
status_code: statusCode ?? 'error',
error_type: errorType ?? 'unknown',
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
status_code: attributes.status_code ?? 'error',
error_type: attributes.error_type ?? 'unknown',
};
apiRequestCounter.add(1, metricAttributes);
apiRequestLatencyHistogram.record(durationMs, {
...getCommonAttributes(config),
model,
...baseMetricDefinition.getCommonAttributes(config),
model: attributes.model,
});
}
export function recordFileOperationMetric(
config: Config,
operation: FileOperation,
lines?: number,
mimetype?: string,
extension?: string,
programming_language?: string,
attributes: MetricDefinitions[typeof FILE_OPERATION_COUNT]['attributes'],
): void {
if (!fileOperationCounter || !isMetricsInitialized) return;
const attributes: Attributes = {
...getCommonAttributes(config),
operation,
};
if (lines !== undefined) attributes['lines'] = lines;
if (mimetype !== undefined) attributes['mimetype'] = mimetype;
if (extension !== undefined) attributes['extension'] = extension;
if (programming_language !== undefined) {
attributes['programming_language'] = programming_language;
}
fileOperationCounter.add(1, attributes);
fileOperationCounter.add(1, {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
});
}
// --- New Metric Recording Functions ---
@@ -338,7 +514,7 @@ export function recordFileOperationMetric(
*/
export function recordInvalidChunk(config: Config): void {
if (!invalidChunkCounter || !isMetricsInitialized) return;
invalidChunkCounter.add(1, getCommonAttributes(config));
invalidChunkCounter.add(1, baseMetricDefinition.getCommonAttributes(config));
}
/**
@@ -346,7 +522,7 @@ export function recordInvalidChunk(config: Config): void {
*/
export function recordContentRetry(config: Config): void {
if (!contentRetryCounter || !isMetricsInitialized) return;
contentRetryCounter.add(1, getCommonAttributes(config));
contentRetryCounter.add(1, baseMetricDefinition.getCommonAttributes(config));
}
/**
@@ -354,7 +530,10 @@ export function recordContentRetry(config: Config): void {
*/
export function recordContentRetryFailure(config: Config): void {
if (!contentRetryFailureCounter || !isMetricsInitialized) return;
contentRetryFailureCounter.add(1, getCommonAttributes(config));
contentRetryFailureCounter.add(
1,
baseMetricDefinition.getCommonAttributes(config),
);
}
export function recordModelSlashCommand(
@@ -363,7 +542,7 @@ export function recordModelSlashCommand(
): void {
if (!modelSlashCommandCallCounter || !isMetricsInitialized) return;
modelSlashCommandCallCounter.add(1, {
...getCommonAttributes(config),
...baseMetricDefinition.getCommonAttributes(config),
'slash_command.model.model_name': event.model_name,
});
}
@@ -380,14 +559,14 @@ export function recordModelRoutingMetrics(
return;
modelRoutingLatencyHistogram.record(event.routing_latency_ms, {
...getCommonAttributes(config),
...baseMetricDefinition.getCommonAttributes(config),
'routing.decision_model': event.decision_model,
'routing.decision_source': event.decision_source,
});
if (event.failed) {
modelRoutingFailureCounter.add(1, {
...getCommonAttributes(config),
...baseMetricDefinition.getCommonAttributes(config),
'routing.decision_source': event.decision_source,
'routing.error_message': event.error_message,
});
@@ -406,149 +585,70 @@ export function initializePerformanceMonitoring(config: Config): void {
if (!isPerformanceMonitoringEnabled) return;
// Initialize startup time histogram
startupTimeHistogram = meter.createHistogram(METRIC_STARTUP_TIME, {
description:
'CLI startup time in milliseconds, broken down by initialization phase.',
unit: 'ms',
valueType: ValueType.DOUBLE,
});
// Initialize memory usage histogram (using histogram until ObservableGauge is available)
memoryUsageGauge = meter.createHistogram(METRIC_MEMORY_USAGE, {
description: 'Memory usage in bytes.',
unit: 'bytes',
valueType: ValueType.INT,
});
// Initialize CPU usage histogram
cpuUsageGauge = meter.createHistogram(METRIC_CPU_USAGE, {
description: 'CPU usage percentage.',
unit: 'percent',
valueType: ValueType.DOUBLE,
});
// Initialize tool queue depth histogram
toolQueueDepthGauge = meter.createHistogram(METRIC_TOOL_QUEUE_DEPTH, {
description: 'Number of tools in execution queue.',
valueType: ValueType.INT,
});
// Initialize performance breakdowns
toolExecutionBreakdownHistogram = meter.createHistogram(
METRIC_TOOL_EXECUTION_BREAKDOWN,
{
description: 'Tool execution time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
Object.entries(PERFORMANCE_COUNTER_DEFINITIONS).forEach(
([name, { description, valueType, assign }]) => {
assign(meter.createCounter(name, { description, valueType }));
},
);
tokenEfficiencyHistogram = meter.createHistogram(METRIC_TOKEN_EFFICIENCY, {
description:
'Token efficiency metrics (tokens per operation, cache hit rate, etc.).',
valueType: ValueType.DOUBLE,
});
apiRequestBreakdownHistogram = meter.createHistogram(
METRIC_API_REQUEST_BREAKDOWN,
{
description: 'API request time breakdown by phase in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
},
);
// Initialize performance score and regression detection
performanceScoreGauge = meter.createHistogram(METRIC_PERFORMANCE_SCORE, {
description: 'Composite performance score (0-100).',
unit: 'score',
valueType: ValueType.DOUBLE,
});
regressionDetectionCounter = meter.createCounter(
METRIC_REGRESSION_DETECTION,
{
description: 'Performance regression detection events.',
valueType: ValueType.INT,
},
);
regressionPercentageChangeHistogram = meter.createHistogram(
METRIC_REGRESSION_PERCENTAGE_CHANGE,
{
description:
'Percentage change compared to baseline for detected regressions.',
unit: 'percent',
valueType: ValueType.DOUBLE,
},
);
baselineComparisonHistogram = meter.createHistogram(
METRIC_BASELINE_COMPARISON,
{
description:
'Performance comparison to established baseline (percentage change).',
unit: 'percent',
valueType: ValueType.DOUBLE,
Object.entries(PERFORMANCE_HISTOGRAM_DEFINITIONS).forEach(
([name, { description, unit, valueType, assign }]) => {
assign(meter.createHistogram(name, { description, unit, valueType }));
},
);
}
export function recordStartupPerformance(
config: Config,
phase: string,
durationMs: number,
details?: Record<string, string | number | boolean>,
attributes: MetricDefinitions[typeof STARTUP_TIME]['attributes'],
): void {
if (!startupTimeHistogram || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
phase,
...details,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
phase: attributes.phase,
...attributes.details,
};
startupTimeHistogram.record(durationMs, attributes);
startupTimeHistogram.record(durationMs, metricAttributes);
}
export function recordMemoryUsage(
config: Config,
memoryType: MemoryMetricType,
bytes: number,
component?: string,
attributes: MetricDefinitions[typeof MEMORY_USAGE]['attributes'],
): void {
if (!memoryUsageGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
memory_type: memoryType,
component,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
memoryUsageGauge.record(bytes, attributes);
memoryUsageGauge.record(bytes, metricAttributes);
}
export function recordCpuUsage(
config: Config,
percentage: number,
component?: string,
attributes: MetricDefinitions[typeof CPU_USAGE]['attributes'],
): void {
if (!cpuUsageGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
component,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
cpuUsageGauge.record(percentage, attributes);
cpuUsageGauge.record(percentage, metricAttributes);
}
export function recordToolQueueDepth(config: Config, queueDepth: number): void {
if (!toolQueueDepthGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
...baseMetricDefinition.getCommonAttributes(config),
};
toolQueueDepthGauge.record(queueDepth, attributes);
@@ -556,126 +656,111 @@ export function recordToolQueueDepth(config: Config, queueDepth: number): void {
export function recordToolExecutionBreakdown(
config: Config,
functionName: string,
phase: ToolExecutionPhase,
durationMs: number,
attributes: MetricDefinitions[typeof TOOL_EXECUTION_BREAKDOWN]['attributes'],
): void {
if (!toolExecutionBreakdownHistogram || !isPerformanceMonitoringEnabled)
return;
const attributes: Attributes = {
...getCommonAttributes(config),
function_name: functionName,
phase,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
toolExecutionBreakdownHistogram.record(durationMs, attributes);
toolExecutionBreakdownHistogram.record(durationMs, metricAttributes);
}
export function recordTokenEfficiency(
config: Config,
model: string,
metric: string,
value: number,
context?: string,
attributes: MetricDefinitions[typeof TOKEN_EFFICIENCY]['attributes'],
): void {
if (!tokenEfficiencyHistogram || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
model,
metric,
context,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
tokenEfficiencyHistogram.record(value, attributes);
tokenEfficiencyHistogram.record(value, metricAttributes);
}
export function recordApiRequestBreakdown(
config: Config,
model: string,
phase: ApiRequestPhase,
durationMs: number,
attributes: MetricDefinitions[typeof API_REQUEST_BREAKDOWN]['attributes'],
): void {
if (!apiRequestBreakdownHistogram || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
model,
phase,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
apiRequestBreakdownHistogram.record(durationMs, attributes);
apiRequestBreakdownHistogram.record(durationMs, metricAttributes);
}
export function recordPerformanceScore(
config: Config,
score: number,
category: string,
baseline?: number,
attributes: MetricDefinitions[typeof PERFORMANCE_SCORE]['attributes'],
): void {
if (!performanceScoreGauge || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
category,
baseline,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
performanceScoreGauge.record(score, attributes);
performanceScoreGauge.record(score, metricAttributes);
}
export function recordPerformanceRegression(
config: Config,
metric: string,
currentValue: number,
baselineValue: number,
severity: 'low' | 'medium' | 'high',
attributes: MetricDefinitions[typeof REGRESSION_DETECTION]['attributes'],
): void {
if (!regressionDetectionCounter || !isPerformanceMonitoringEnabled) return;
const attributes: Attributes = {
...getCommonAttributes(config),
metric,
severity,
current_value: currentValue,
baseline_value: baselineValue,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
regressionDetectionCounter.add(1, attributes);
regressionDetectionCounter.add(1, metricAttributes);
if (baselineValue !== 0 && regressionPercentageChangeHistogram) {
if (attributes.baseline_value !== 0 && regressionPercentageChangeHistogram) {
const percentageChange =
((currentValue - baselineValue) / baselineValue) * 100;
regressionPercentageChangeHistogram.record(percentageChange, attributes);
((attributes.current_value - attributes.baseline_value) /
attributes.baseline_value) *
100;
regressionPercentageChangeHistogram.record(
percentageChange,
metricAttributes,
);
}
}
export function recordBaselineComparison(
config: Config,
metric: string,
currentValue: number,
baselineValue: number,
category: string,
attributes: MetricDefinitions[typeof BASELINE_COMPARISON]['attributes'],
): void {
if (!baselineComparisonHistogram || !isPerformanceMonitoringEnabled) return;
if (baselineValue === 0) {
if (attributes.baseline_value === 0) {
diag.warn('Baseline value is zero, skipping comparison.');
return;
}
const percentageChange =
((currentValue - baselineValue) / baselineValue) * 100;
((attributes.current_value - attributes.baseline_value) /
attributes.baseline_value) *
100;
const attributes: Attributes = {
...getCommonAttributes(config),
metric,
category,
current_value: currentValue,
baseline_value: baselineValue,
const metricAttributes: Attributes = {
...baseMetricDefinition.getCommonAttributes(config),
...attributes,
};
baselineComparisonHistogram.record(percentageChange, attributes);
baselineComparisonHistogram.record(percentageChange, metricAttributes);
}
// Utility function to check if performance monitoring is enabled