Files
gemini-cli/packages/core/src/telemetry/metrics.test.ts

1553 lines
52 KiB
TypeScript

/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest';
import type {
Counter,
Meter,
Attributes,
Context,
Histogram,
} from '@opentelemetry/api';
import type { Config } from '../config/config.js';
import {
FileOperation,
MemoryMetricType,
ToolExecutionPhase,
ApiRequestPhase,
} from './metrics.js';
import { makeFakeConfig } from '../test-utils/config.js';
import {
ModelRoutingEvent,
AgentFinishEvent,
KeychainAvailabilityEvent,
TokenStorageInitializationEvent,
} from './types.js';
import { AgentTerminateMode } from '../agents/types.js';
const mockCounterAddFn: Mock<
(value: number, attributes?: Attributes, context?: Context) => void
> = vi.fn();
const mockHistogramRecordFn: Mock<
(value: number, attributes?: Attributes, context?: Context) => void
> = vi.fn();
const mockCreateCounterFn: Mock<(name: string, options?: unknown) => Counter> =
vi.fn();
const mockCreateHistogramFn: Mock<
(name: string, options?: unknown) => Histogram
> = vi.fn();
const mockCounterInstance: Counter = {
add: mockCounterAddFn,
} as Partial<Counter> as Counter;
const mockHistogramInstance: Histogram = {
record: mockHistogramRecordFn,
} as Partial<Histogram> as Histogram;
const mockMeterInstance: Meter = {
createCounter: mockCreateCounterFn.mockReturnValue(mockCounterInstance),
createHistogram: mockCreateHistogramFn.mockReturnValue(mockHistogramInstance),
} as Partial<Meter> as Meter;
function originalOtelMockFactory() {
return {
metrics: {
getMeter: vi.fn(),
},
ValueType: {
INT: 1,
DOUBLE: 2,
},
diag: {
setLogger: vi.fn(),
warn: vi.fn(),
},
DiagConsoleLogger: vi.fn(),
DiagLogLevel: {
NONE: 0,
INFO: 1,
},
} as const;
}
vi.mock('@opentelemetry/api');
vi.mock('./telemetryAttributes.js');
describe('Telemetry Metrics', () => {
let initializeMetricsModule: typeof import('./metrics.js').initializeMetrics;
let recordTokenUsageMetricsModule: typeof import('./metrics.js').recordTokenUsageMetrics;
let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric;
let recordChatCompressionMetricsModule: typeof import('./metrics.js').recordChatCompressionMetrics;
let recordModelRoutingMetricsModule: typeof import('./metrics.js').recordModelRoutingMetrics;
let recordStartupPerformanceModule: typeof import('./metrics.js').recordStartupPerformance;
let recordMemoryUsageModule: typeof import('./metrics.js').recordMemoryUsage;
let recordCpuUsageModule: typeof import('./metrics.js').recordCpuUsage;
let recordToolQueueDepthModule: typeof import('./metrics.js').recordToolQueueDepth;
let recordToolExecutionBreakdownModule: typeof import('./metrics.js').recordToolExecutionBreakdown;
let recordTokenEfficiencyModule: typeof import('./metrics.js').recordTokenEfficiency;
let recordApiRequestBreakdownModule: typeof import('./metrics.js').recordApiRequestBreakdown;
let recordPerformanceScoreModule: typeof import('./metrics.js').recordPerformanceScore;
let recordPerformanceRegressionModule: typeof import('./metrics.js').recordPerformanceRegression;
let recordBaselineComparisonModule: typeof import('./metrics.js').recordBaselineComparison;
let recordGenAiClientTokenUsageModule: typeof import('./metrics.js').recordGenAiClientTokenUsage;
let recordGenAiClientOperationDurationModule: typeof import('./metrics.js').recordGenAiClientOperationDuration;
let recordFlickerFrameModule: typeof import('./metrics.js').recordFlickerFrame;
let recordExitFailModule: typeof import('./metrics.js').recordExitFail;
let recordAgentRunMetricsModule: typeof import('./metrics.js').recordAgentRunMetrics;
let recordLinesChangedModule: typeof import('./metrics.js').recordLinesChanged;
let recordSlowRenderModule: typeof import('./metrics.js').recordSlowRender;
let recordPlanExecutionModule: typeof import('./metrics.js').recordPlanExecution;
let recordKeychainAvailabilityModule: typeof import('./metrics.js').recordKeychainAvailability;
let recordTokenStorageInitializationModule: typeof import('./metrics.js').recordTokenStorageInitialization;
beforeEach(async () => {
vi.resetModules();
vi.doMock('@opentelemetry/api', () => {
const actualApi = originalOtelMockFactory();
actualApi.metrics.getMeter.mockReturnValue(mockMeterInstance);
return actualApi;
});
const { getCommonAttributes } = await import('./telemetryAttributes.js');
(getCommonAttributes as Mock).mockReturnValue({
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
const metricsJsModule = await import('./metrics.js');
initializeMetricsModule = metricsJsModule.initializeMetrics;
recordTokenUsageMetricsModule = metricsJsModule.recordTokenUsageMetrics;
recordFileOperationMetricModule = metricsJsModule.recordFileOperationMetric;
recordChatCompressionMetricsModule =
metricsJsModule.recordChatCompressionMetrics;
recordModelRoutingMetricsModule = metricsJsModule.recordModelRoutingMetrics;
recordStartupPerformanceModule = metricsJsModule.recordStartupPerformance;
recordMemoryUsageModule = metricsJsModule.recordMemoryUsage;
recordCpuUsageModule = metricsJsModule.recordCpuUsage;
recordToolQueueDepthModule = metricsJsModule.recordToolQueueDepth;
recordToolExecutionBreakdownModule =
metricsJsModule.recordToolExecutionBreakdown;
recordTokenEfficiencyModule = metricsJsModule.recordTokenEfficiency;
recordApiRequestBreakdownModule = metricsJsModule.recordApiRequestBreakdown;
recordPerformanceScoreModule = metricsJsModule.recordPerformanceScore;
recordPerformanceRegressionModule =
metricsJsModule.recordPerformanceRegression;
recordBaselineComparisonModule = metricsJsModule.recordBaselineComparison;
recordGenAiClientTokenUsageModule =
metricsJsModule.recordGenAiClientTokenUsage;
recordGenAiClientOperationDurationModule =
metricsJsModule.recordGenAiClientOperationDuration;
recordFlickerFrameModule = metricsJsModule.recordFlickerFrame;
recordExitFailModule = metricsJsModule.recordExitFail;
recordAgentRunMetricsModule = metricsJsModule.recordAgentRunMetrics;
recordLinesChangedModule = metricsJsModule.recordLinesChanged;
recordSlowRenderModule = metricsJsModule.recordSlowRender;
recordPlanExecutionModule = metricsJsModule.recordPlanExecution;
recordKeychainAvailabilityModule =
metricsJsModule.recordKeychainAvailability;
recordTokenStorageInitializationModule =
metricsJsModule.recordTokenStorageInitialization;
const otelApiModule = await import('@opentelemetry/api');
mockCounterAddFn.mockClear();
mockCreateCounterFn.mockClear();
mockCreateHistogramFn.mockClear();
mockHistogramRecordFn.mockClear();
(otelApiModule.metrics.getMeter as Mock).mockClear();
(otelApiModule.metrics.getMeter as Mock).mockReturnValue(mockMeterInstance);
mockCreateCounterFn.mockReturnValue(mockCounterInstance);
mockCreateHistogramFn.mockReturnValue(mockHistogramInstance);
});
describe('recordFlickerFrame', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordFlickerFrameModule(config);
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records a flicker frame event when initialized', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
recordFlickerFrameModule(config);
// Called for session, then for flicker
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
});
});
describe('recordExitFail', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordExitFailModule(config);
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records a exit fail event when initialized', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
recordExitFailModule(config);
// Called for session, then for exit fail
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
});
});
describe('recordSlowRender', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordSlowRenderModule(config, 123);
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('records a slow render event when initialized', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
recordSlowRenderModule(config, 123);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(123, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
});
});
describe('recordPlanExecution', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordPlanExecutionModule(config, { approval_mode: 'default' });
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records a plan execution event when initialized', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
recordPlanExecutionModule(config, { approval_mode: 'autoEdit' });
// Called for session, then for plan execution
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
approval_mode: 'autoEdit',
});
});
});
describe('initializeMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should apply common attributes including email', () => {
initializeMetricsModule(mockConfig);
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
});
});
describe('recordChatCompressionMetrics', () => {
it('does not record metrics if not initialized', () => {
const lol = makeFakeConfig({});
recordChatCompressionMetricsModule(lol, {
tokens_after: 100,
tokens_before: 200,
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records token compression with the correct attributes', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
recordChatCompressionMetricsModule(config, {
tokens_after: 100,
tokens_before: 200,
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
tokens_after: 100,
tokens_before: 200,
});
});
});
describe('recordTokenUsageMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record metrics if not initialized', () => {
recordTokenUsageMetricsModule(mockConfig, 100, {
model: 'gemini-pro',
type: 'input',
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it.each([
{ type: 'input', tokens: 100, model: 'gemini-pro' },
{ type: 'output', tokens: 50, model: 'gemini-pro' },
{ type: 'thought', tokens: 25, model: 'gemini-pro' },
{ type: 'cache', tokens: 75, model: 'gemini-pro' },
{ type: 'tool', tokens: 125, model: 'gemini-pro' },
{ type: 'input', tokens: 200, model: 'gemini-different-model' },
])(
'should record token usage for $type type with $tokens tokens for model $model',
({ type, tokens, model }) => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordTokenUsageMetricsModule(mockConfig, tokens, {
model,
type: type as 'input' | 'output' | 'thought' | 'cache' | 'tool',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(tokens, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
model,
type,
});
},
);
});
describe('recordLinesChanged metric', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record lines added/removed if not initialized', () => {
recordLinesChangedModule(mockConfig, 10, 'added', {
function_name: 'fn',
});
recordLinesChangedModule(mockConfig, 5, 'removed', {
function_name: 'fn',
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record lines added with function_name after initialization', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordLinesChangedModule(mockConfig, 10, 'added', {
function_name: 'my-fn',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(10, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
type: 'added',
function_name: 'my-fn',
});
});
it('should record lines removed with function_name after initialization', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordLinesChangedModule(mockConfig, 7, 'removed', {
function_name: 'my-fn',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(7, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
type: 'removed',
function_name: 'my-fn',
});
});
});
describe('recordFileOperationMetric', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
type FileOperationTestCase = {
name: string;
initialized: boolean;
attributes: {
operation: FileOperation;
lines?: number;
mimetype?: string;
extension?: string;
};
shouldCall: boolean;
};
it.each<FileOperationTestCase>([
{
name: 'should not record metrics if not initialized',
initialized: false,
attributes: {
operation: FileOperation.CREATE,
lines: 10,
mimetype: 'text/plain',
extension: 'txt',
},
shouldCall: false,
},
{
name: 'should record file creation with all attributes',
initialized: true,
attributes: {
operation: FileOperation.CREATE,
lines: 10,
mimetype: 'text/plain',
extension: 'txt',
},
shouldCall: true,
},
{
name: 'should record file read with minimal attributes',
initialized: true,
attributes: { operation: FileOperation.READ },
shouldCall: true,
},
{
name: 'should record file update with some attributes',
initialized: true,
attributes: {
operation: FileOperation.UPDATE,
mimetype: 'application/javascript',
},
shouldCall: true,
},
{
name: 'should record file update with no optional attributes',
initialized: true,
attributes: { operation: FileOperation.UPDATE },
shouldCall: true,
},
])('$name', ({ initialized, attributes, shouldCall }) => {
if (initialized) {
initializeMetricsModule(mockConfig);
// The session start event also calls the counter.
mockCounterAddFn.mockClear();
}
recordFileOperationMetricModule(mockConfig, attributes);
if (shouldCall) {
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
...attributes,
});
} else {
expect(mockCounterAddFn).not.toHaveBeenCalled();
}
});
});
describe('recordModelRoutingMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record metrics if not initialized', () => {
const event = new ModelRoutingEvent(
'gemini-pro',
'default',
100,
'test-reason',
false,
undefined,
);
recordModelRoutingMetricsModule(mockConfig, event);
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record latency for a successful routing decision', () => {
initializeMetricsModule(mockConfig);
const event = new ModelRoutingEvent(
'gemini-pro',
'default',
150,
'test-reason',
false,
undefined,
);
recordModelRoutingMetricsModule(mockConfig, event);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'routing.decision_model': 'gemini-pro',
'routing.decision_source': 'default',
'routing.failed': false,
'routing.reasoning': 'test-reason',
});
// The session counter is called once on init
expect(mockCounterAddFn).toHaveBeenCalledTimes(1);
});
it('should record latency and failure for a failed routing decision', () => {
initializeMetricsModule(mockConfig);
const event = new ModelRoutingEvent(
'gemini-pro',
'Classifier',
200,
'test-reason',
true,
'test-error',
);
recordModelRoutingMetricsModule(mockConfig, event);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(200, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'routing.decision_model': 'gemini-pro',
'routing.decision_source': 'Classifier',
'routing.failed': true,
'routing.reasoning': 'test-reason',
});
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'routing.decision_model': 'gemini-pro',
'routing.decision_source': 'Classifier',
'routing.failed': true,
'routing.reasoning': 'test-reason',
'routing.error_message': 'test-error',
});
});
});
describe('recordAgentRunMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record metrics if not initialized', () => {
const event = new AgentFinishEvent(
'agent-123',
'TestAgent',
1000,
5,
AgentTerminateMode.GOAL,
);
recordAgentRunMetricsModule(mockConfig, event);
expect(mockCounterAddFn).not.toHaveBeenCalled();
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record agent run metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
const event = new AgentFinishEvent(
'agent-123',
'TestAgent',
1000,
5,
AgentTerminateMode.GOAL,
);
recordAgentRunMetricsModule(mockConfig, event);
// Verify agent run counter
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
agent_name: 'TestAgent',
terminate_reason: 'GOAL',
});
// Verify agent duration histogram
expect(mockHistogramRecordFn).toHaveBeenCalledWith(1000, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
agent_name: 'TestAgent',
});
// Verify agent turns histogram
expect(mockHistogramRecordFn).toHaveBeenCalledWith(5, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
agent_name: 'TestAgent',
});
});
});
describe('OpenTelemetry GenAI Semantic Convention Metrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
describe('recordGenAiClientTokenUsage', () => {
it('should not record metrics when not initialized', () => {
recordGenAiClientTokenUsageModule(mockConfig, 100, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
'gen_ai.token.type': 'input',
});
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record input token usage with correct attributes', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordGenAiClientTokenUsageModule(mockConfig, 150, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
'gen_ai.token.type': 'input',
'gen_ai.request.model': 'gemini-2.0-flash',
'gen_ai.response.model': 'gemini-2.0-flash',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
'gen_ai.token.type': 'input',
'gen_ai.request.model': 'gemini-2.0-flash',
'gen_ai.response.model': 'gemini-2.0-flash',
});
});
it('should record output token usage with correct attributes', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordGenAiClientTokenUsageModule(mockConfig, 75, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.token.type': 'output',
'gen_ai.request.model': 'gemini-pro',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(75, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.token.type': 'output',
'gen_ai.request.model': 'gemini-pro',
});
});
it('should record token usage with optional attributes', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordGenAiClientTokenUsageModule(mockConfig, 200, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.token.type': 'input',
'gen_ai.request.model': 'text-embedding-004',
'server.address': 'aiplatform.googleapis.com',
'server.port': 443,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(200, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.token.type': 'input',
'gen_ai.request.model': 'text-embedding-004',
'server.address': 'aiplatform.googleapis.com',
'server.port': 443,
});
});
});
describe('recordGenAiClientOperationDuration', () => {
it('should not record metrics when not initialized', () => {
recordGenAiClientOperationDurationModule(mockConfig, 2.5, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
});
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record successful operation duration with correct attributes', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordGenAiClientOperationDurationModule(mockConfig, 1.25, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
'gen_ai.request.model': 'gemini-2.0-flash',
'gen_ai.response.model': 'gemini-2.0-flash',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(1.25, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
'gen_ai.request.model': 'gemini-2.0-flash',
'gen_ai.response.model': 'gemini-2.0-flash',
});
});
it('should record failed operation duration with error type', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordGenAiClientOperationDurationModule(mockConfig, 3.75, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.request.model': 'gemini-pro',
'error.type': 'quota_exceeded',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(3.75, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.request.model': 'gemini-pro',
'error.type': 'quota_exceeded',
});
});
it('should record operation duration with server details', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordGenAiClientOperationDurationModule(mockConfig, 0.95, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.request.model': 'gemini-1.5-pro',
'gen_ai.response.model': 'gemini-1.5-pro-001',
'server.address': 'us-central1-aiplatform.googleapis.com',
'server.port': 443,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(0.95, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.request.model': 'gemini-1.5-pro',
'gen_ai.response.model': 'gemini-1.5-pro-001',
'server.address': 'us-central1-aiplatform.googleapis.com',
'server.port': 443,
});
});
it('should handle minimal required attributes', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordGenAiClientOperationDurationModule(mockConfig, 2.1, {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(2.1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
});
});
});
});
describe('Performance Monitoring Metrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
describe('recordStartupPerformance', () => {
it('should not record metrics when performance monitoring is disabled', async () => {
// Re-import with performance monitoring disabled by mocking the config
const mockConfigDisabled = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => false, // Disable telemetry to disable performance monitoring
} as unknown as Config;
initializeMetricsModule(mockConfigDisabled);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(mockConfigDisabled, 100, {
phase: 'settings_loading',
details: {
auth_type: 'gemini',
},
});
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record startup performance with phase and details', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(mockConfig, 150, {
phase: 'settings_loading',
details: {
auth_type: 'gemini',
telemetry_enabled: true,
settings_sources: 2,
},
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
phase: 'settings_loading',
auth_type: 'gemini',
telemetry_enabled: true,
settings_sources: 2,
});
});
it('should record startup performance without details', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordStartupPerformanceModule(mockConfig, 50, { phase: 'cleanup' });
expect(mockHistogramRecordFn).toHaveBeenCalledWith(50, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
phase: 'cleanup',
});
});
it('should handle floating-point duration values from performance.now()', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
// Test with realistic floating-point values that performance.now() would return
const floatingPointDuration = 123.45678;
recordStartupPerformanceModule(mockConfig, floatingPointDuration, {
phase: 'total_startup',
details: {
is_tty: true,
has_question: false,
},
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(
floatingPointDuration,
{
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
phase: 'total_startup',
is_tty: true,
has_question: false,
},
);
});
});
describe('recordMemoryUsage', () => {
it.each([
{
memory_type: MemoryMetricType.HEAP_USED,
component: 'startup',
value: 15728640,
},
{
memory_type: MemoryMetricType.HEAP_TOTAL,
component: 'api_call',
value: 31457280,
},
{
memory_type: MemoryMetricType.EXTERNAL,
component: 'tool_execution',
value: 2097152,
},
{
memory_type: MemoryMetricType.RSS,
component: 'memory_monitor',
value: 41943040,
},
{
memory_type: MemoryMetricType.HEAP_USED,
component: undefined,
value: 15728640,
},
])(
'should record memory usage for $memory_type',
({ memory_type, component, value }) => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordMemoryUsageModule(mockConfig, value, {
memory_type,
component,
});
const expectedAttributes: Record<string, unknown> = {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
memory_type,
};
if (component) {
expectedAttributes['component'] = component;
}
expect(mockHistogramRecordFn).toHaveBeenCalledWith(
value,
expectedAttributes,
);
},
);
});
describe('recordCpuUsage', () => {
it('should record CPU usage percentage', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordCpuUsageModule(mockConfig, 85.5, {
component: 'tool_execution',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
component: 'tool_execution',
});
});
it('should record CPU usage without component', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordCpuUsageModule(mockConfig, 42.3, {});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(42.3, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
});
});
describe('recordToolQueueDepth', () => {
it('should record tool queue depth', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolQueueDepthModule(mockConfig, 3);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(3, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
});
it('should record zero queue depth', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolQueueDepthModule(mockConfig, 0);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(0, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
});
});
});
describe('recordToolExecutionBreakdown', () => {
it('should record tool execution breakdown for all phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolExecutionBreakdownModule(mockConfig, 25, {
function_name: 'Read',
phase: ToolExecutionPhase.VALIDATION,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(25, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
function_name: 'Read',
phase: 'validation',
});
});
it('should record execution breakdown for different phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordToolExecutionBreakdownModule(mockConfig, 50, {
function_name: 'Bash',
phase: ToolExecutionPhase.PREPARATION,
});
recordToolExecutionBreakdownModule(mockConfig, 1500, {
function_name: 'Bash',
phase: ToolExecutionPhase.EXECUTION,
});
recordToolExecutionBreakdownModule(mockConfig, 75, {
function_name: 'Bash',
phase: ToolExecutionPhase.RESULT_PROCESSING,
});
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 50, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
function_name: 'Bash',
phase: 'preparation',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 1500, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
function_name: 'Bash',
phase: 'execution',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 75, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
function_name: 'Bash',
phase: 'result_processing',
});
});
});
describe('recordTokenEfficiency', () => {
it('should record token efficiency metrics', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordTokenEfficiencyModule(mockConfig, 0.85, {
model: 'gemini-pro',
metric: 'cache_hit_rate',
context: 'api_request',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(0.85, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
model: 'gemini-pro',
metric: 'cache_hit_rate',
context: 'api_request',
});
});
it('should record token efficiency without context', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordTokenEfficiencyModule(mockConfig, 125.5, {
model: 'gemini-pro',
metric: 'tokens_per_operation',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(125.5, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
model: 'gemini-pro',
metric: 'tokens_per_operation',
});
});
});
describe('recordApiRequestBreakdown', () => {
it('should record API request breakdown for all phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordApiRequestBreakdownModule(mockConfig, 15, {
model: 'gemini-pro',
phase: ApiRequestPhase.REQUEST_PREPARATION,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(15, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
model: 'gemini-pro',
phase: 'request_preparation',
});
});
it('should record API request breakdown for different phases', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordApiRequestBreakdownModule(mockConfig, 250, {
model: 'gemini-pro',
phase: ApiRequestPhase.NETWORK_LATENCY,
});
recordApiRequestBreakdownModule(mockConfig, 100, {
model: 'gemini-pro',
phase: ApiRequestPhase.RESPONSE_PROCESSING,
});
recordApiRequestBreakdownModule(mockConfig, 50, {
model: 'gemini-pro',
phase: ApiRequestPhase.TOKEN_PROCESSING,
});
expect(mockHistogramRecordFn).toHaveBeenCalledTimes(3); // One for each call
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(1, 250, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
model: 'gemini-pro',
phase: 'network_latency',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(2, 100, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
model: 'gemini-pro',
phase: 'response_processing',
});
expect(mockHistogramRecordFn).toHaveBeenNthCalledWith(3, 50, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
model: 'gemini-pro',
phase: 'token_processing',
});
});
});
describe('recordPerformanceScore', () => {
it('should record performance score with category and baseline', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordPerformanceScoreModule(mockConfig, 85.5, {
category: 'memory_efficiency',
baseline: 80.0,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(85.5, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
category: 'memory_efficiency',
baseline: 80.0,
});
});
it('should record performance score without baseline', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordPerformanceScoreModule(mockConfig, 92.3, {
category: 'overall_performance',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(92.3, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
category: 'overall_performance',
});
});
});
describe('recordPerformanceRegression', () => {
it('should record performance regression with baseline comparison', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordPerformanceRegressionModule(mockConfig, {
metric: 'startup_time',
current_value: 1200,
baseline_value: 1000,
severity: 'medium',
});
// Verify regression counter
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
metric: 'startup_time',
severity: 'medium',
current_value: 1200,
baseline_value: 1000,
});
// Verify baseline comparison histogram (20% increase)
expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
metric: 'startup_time',
severity: 'medium',
current_value: 1200,
baseline_value: 1000,
});
});
it('should handle zero baseline value gracefully', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordPerformanceRegressionModule(mockConfig, {
metric: 'memory_usage',
current_value: 100,
baseline_value: 0,
severity: 'high',
});
// Verify regression counter still recorded
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
metric: 'memory_usage',
severity: 'high',
current_value: 100,
baseline_value: 0,
});
// Verify no baseline comparison due to zero baseline
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record different severity levels', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordPerformanceRegressionModule(mockConfig, {
metric: 'api_latency',
current_value: 500,
baseline_value: 400,
severity: 'low',
});
recordPerformanceRegressionModule(mockConfig, {
metric: 'cpu_usage',
current_value: 90,
baseline_value: 70,
severity: 'high',
});
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
metric: 'api_latency',
severity: 'low',
current_value: 500,
baseline_value: 400,
});
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
metric: 'cpu_usage',
severity: 'high',
current_value: 90,
baseline_value: 70,
});
});
});
describe('recordBaselineComparison', () => {
it('should record baseline comparison with percentage change', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(mockConfig, {
metric: 'memory_usage',
current_value: 120,
baseline_value: 100,
category: 'performance_tracking',
});
// 20% increase: (120 - 100) / 100 * 100 = 20%
expect(mockHistogramRecordFn).toHaveBeenCalledWith(20, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
metric: 'memory_usage',
category: 'performance_tracking',
current_value: 120,
baseline_value: 100,
});
});
it('should handle negative percentage change (improvement)', () => {
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(mockConfig, {
metric: 'startup_time',
current_value: 800,
baseline_value: 1000,
category: 'optimization',
});
// 20% decrease: (800 - 1000) / 1000 * 100 = -20%
expect(mockHistogramRecordFn).toHaveBeenCalledWith(-20, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
metric: 'startup_time',
category: 'optimization',
current_value: 800,
baseline_value: 1000,
});
});
it('should skip recording when baseline is zero', async () => {
// Access the actual mocked module
const mockedModule = (await vi.importMock('@opentelemetry/api')) as {
diag: { warn: ReturnType<typeof vi.fn> };
};
const diagSpy = vi.spyOn(mockedModule.diag, 'warn');
initializeMetricsModule(mockConfig);
mockHistogramRecordFn.mockClear();
recordBaselineComparisonModule(mockConfig, {
metric: 'new_metric',
current_value: 50,
baseline_value: 0,
category: 'testing',
});
expect(diagSpy).toHaveBeenCalledWith(
'Baseline value is zero, skipping comparison.',
);
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
});
describe('recordHookCallMetrics', () => {
let recordHookCallMetricsModule: typeof import('./metrics.js').recordHookCallMetrics;
beforeEach(async () => {
recordHookCallMetricsModule = (await import('./metrics.js'))
.recordHookCallMetrics;
});
it('should record hook call metrics with counter and histogram', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordHookCallMetricsModule(
mockConfig,
'BeforeTool',
'test-hook',
150,
true,
);
// Verify counter recorded
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
hook_event_name: 'BeforeTool',
hook_name: 'test-hook',
success: true,
});
// Verify histogram recorded
expect(mockHistogramRecordFn).toHaveBeenCalledWith(150, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
hook_event_name: 'BeforeTool',
hook_name: 'test-hook',
success: true,
});
});
it('should always sanitize hook names regardless of content', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
// Test with a command that has sensitive information
recordHookCallMetricsModule(
mockConfig,
'BeforeTool',
'/path/to/.gemini/hooks/check-secrets.sh --api-key=abc123',
150,
true,
);
// Verify hook name is sanitized (detailed sanitization tested in hook-call-event.test.ts)
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
hook_event_name: 'BeforeTool',
hook_name: 'check-secrets.sh', // Sanitized
success: true,
});
});
it('should track both success and failure', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
// Success case
recordHookCallMetricsModule(
mockConfig,
'BeforeTool',
'test-hook',
100,
true,
);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(
1,
1,
expect.objectContaining({
hook_event_name: 'BeforeTool',
hook_name: 'test-hook',
success: true,
}),
);
// Failure case
recordHookCallMetricsModule(
mockConfig,
'AfterTool',
'test-hook',
150,
false,
);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(
2,
1,
expect.objectContaining({
hook_event_name: 'AfterTool',
hook_name: 'test-hook',
success: false,
}),
);
});
});
});
describe('Keychain and Token Storage Metrics', () => {
describe('recordKeychainAvailability', () => {
it('should not record metrics if not initialized', () => {
const config = makeFakeConfig({});
const event = new KeychainAvailabilityEvent(true);
recordKeychainAvailabilityModule(config, event);
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record keychain availability when initialized', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
mockCounterAddFn.mockClear();
const event = new KeychainAvailabilityEvent(true);
recordKeychainAvailabilityModule(config, event);
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
available: true,
});
});
});
describe('recordTokenStorageInitialization', () => {
it('should not record metrics if not initialized', () => {
const config = makeFakeConfig({});
const event = new TokenStorageInitializationEvent('hybrid', false);
recordTokenStorageInitializationModule(config, event);
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record token storage initialization when initialized', () => {
const config = makeFakeConfig({});
initializeMetricsModule(config);
mockCounterAddFn.mockClear();
const event = new TokenStorageInitializationEvent('keychain', true);
recordTokenStorageInitializationModule(config, event);
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
type: 'keychain',
forced: true,
});
});
});
});
});