Create line change metrics (#12299)

This commit is contained in:
Jerop Kipruto
2025-10-30 14:55:47 -04:00
committed by GitHub
parent 54fa26ef0e
commit 135d981e3c
5 changed files with 114 additions and 21 deletions

View File

@@ -542,10 +542,6 @@ Measures tool usage and latency.
- `decision` (string: "accept", "reject", "modify", or "auto_accept", if
applicable)
- `tool_type` (string: "mcp" or "native", if applicable)
- `model_added_lines` (Int, optional)
- `model_removed_lines` (Int, optional)
- `user_added_lines` (Int, optional)
- `user_removed_lines` (Int, optional)
- `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency.
- **Attributes**:
@@ -589,6 +585,12 @@ Counts file operations with basic context.
- `extension` (string, optional)
- `programming_language` (string, optional)
- `gemini_cli.lines.changed` (Counter, Int): Number of lines changed (from file
diffs).
- **Attributes**:
- `function_name`
- `type` ("added" or "removed")
##### Chat and Streaming
Resilience counters for compression, invalid chunks, and retries.

View File

@@ -793,12 +793,16 @@ describe('loggers', () => {
const mockMetrics = {
recordToolCallMetrics: vi.fn(),
recordLinesChanged: vi.fn(),
};
beforeEach(() => {
vi.spyOn(metrics, 'recordToolCallMetrics').mockImplementation(
mockMetrics.recordToolCallMetrics,
);
vi.spyOn(metrics, 'recordLinesChanged').mockImplementation(
mockMetrics.recordLinesChanged,
);
mockLogger.emit.mockReset();
});
@@ -895,10 +899,6 @@ describe('loggers', () => {
success: true,
decision: ToolCallDecision.ACCEPT,
tool_type: 'native',
model_added_lines: 1,
model_removed_lines: 2,
user_added_lines: 5,
user_removed_lines: 6,
},
);
@@ -907,6 +907,19 @@ describe('loggers', () => {
'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
expect(mockMetrics.recordLinesChanged).toHaveBeenCalledWith(
mockConfig,
1,
'added',
{ function_name: 'test-function' },
);
expect(mockMetrics.recordLinesChanged).toHaveBeenCalledWith(
mockConfig,
2,
'removed',
{ function_name: 'test-function' },
);
});
it('should log a tool call with a reject decision', () => {
const call: ErroredToolCall = {

View File

@@ -63,6 +63,7 @@ import {
recordTokenUsageMetrics,
recordApiResponseMetrics,
recordAgentRunMetrics,
recordLinesChanged,
} from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import type { UiEvent } from './uiTelemetry.js';
@@ -118,15 +119,22 @@ export function logToolCall(config: Config, event: ToolCallEvent): void {
success: event.success,
decision: event.decision,
tool_type: event.tool_type,
...(event.metadata
? {
model_added_lines: event.metadata['model_added_lines'],
model_removed_lines: event.metadata['model_removed_lines'],
user_added_lines: event.metadata['user_added_lines'],
user_removed_lines: event.metadata['user_removed_lines'],
}
: {}),
});
if (event.metadata) {
const added = event.metadata['model_added_lines'];
if (typeof added === 'number' && added > 0) {
recordLinesChanged(config, added, 'added', {
function_name: event.function_name,
});
}
const removed = event.metadata['model_removed_lines'];
if (typeof removed === 'number' && removed > 0) {
recordLinesChanged(config, removed, 'removed', {
function_name: event.function_name,
});
}
}
}
export function logToolOutputTruncated(

View File

@@ -94,6 +94,7 @@ describe('Telemetry Metrics', () => {
let recordFlickerFrameModule: typeof import('./metrics.js').recordFlickerFrame;
let recordExitFailModule: typeof import('./metrics.js').recordExitFail;
let recordAgentRunMetricsModule: typeof import('./metrics.js').recordAgentRunMetrics;
let recordLinesChangedModule: typeof import('./metrics.js').recordLinesChanged;
beforeEach(async () => {
vi.resetModules();
@@ -136,6 +137,7 @@ describe('Telemetry Metrics', () => {
recordFlickerFrameModule = metricsJsModule.recordFlickerFrame;
recordExitFailModule = metricsJsModule.recordExitFail;
recordAgentRunMetricsModule = metricsJsModule.recordAgentRunMetrics;
recordLinesChangedModule = metricsJsModule.recordLinesChanged;
const otelApiModule = await import('@opentelemetry/api');
@@ -348,6 +350,53 @@ describe('Telemetry Metrics', () => {
});
});
describe('recordLinesChanged metric', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record lines added/removed if not initialized', () => {
recordLinesChangedModule(mockConfig, 10, 'added', {
function_name: 'fn',
});
recordLinesChangedModule(mockConfig, 5, 'removed', {
function_name: 'fn',
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record lines added with function_name after initialization', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordLinesChangedModule(mockConfig, 10, 'added', {
function_name: 'my-fn',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(10, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
type: 'added',
function_name: 'my-fn',
});
});
it('should record lines removed with function_name after initialization', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordLinesChangedModule(mockConfig, 7, 'removed', {
function_name: 'my-fn',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(7, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
type: 'removed',
function_name: 'my-fn',
});
});
});
describe('recordFileOperationMetric', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',

View File

@@ -24,6 +24,7 @@ const API_REQUEST_LATENCY = 'gemini_cli.api.request.latency';
const TOKEN_USAGE = 'gemini_cli.token.usage';
const SESSION_COUNT = 'gemini_cli.session.count';
const FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count';
const LINES_CHANGED = 'gemini_cli.lines.changed';
const INVALID_CHUNK_COUNT = 'gemini_cli.chat.invalid_chunk.count';
const CONTENT_RETRY_COUNT = 'gemini_cli.chat.content_retry.count';
const CONTENT_RETRY_FAILURE_COUNT =
@@ -72,11 +73,6 @@ const COUNTER_DEFINITIONS = {
success: boolean;
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept';
tool_type?: 'native' | 'mcp';
// Optional diff statistics for file-modifying tools
model_added_lines?: number;
model_removed_lines?: number;
user_added_lines?: number;
user_removed_lines?: number;
},
},
[API_REQUEST_COUNT]: {
@@ -116,6 +112,15 @@ const COUNTER_DEFINITIONS = {
programming_language?: string;
},
},
[LINES_CHANGED]: {
description: 'Number of lines changed (from file diffs).',
valueType: ValueType.INT,
assign: (c: Counter) => (linesChangedCounter = c),
attributes: {} as {
function_name?: string;
type: 'added' | 'removed';
},
},
[INVALID_CHUNK_COUNT]: {
description: 'Counts invalid chunks received from a stream.',
valueType: ValueType.INT,
@@ -454,6 +459,7 @@ let apiRequestLatencyHistogram: Histogram | undefined;
let tokenUsageCounter: Counter | undefined;
let sessionCounter: Counter | undefined;
let fileOperationCounter: Counter | undefined;
let linesChangedCounter: Counter | undefined;
let chatCompressionCounter: Counter | undefined;
let invalidChunkCounter: Counter | undefined;
let contentRetryCounter: Counter | undefined;
@@ -621,6 +627,21 @@ export function recordFileOperationMetric(
});
}
export function recordLinesChanged(
config: Config,
lines: number,
changeType: 'added' | 'removed',
attributes?: { function_name?: string },
): void {
if (!linesChangedCounter || !isMetricsInitialized) return;
if (!Number.isFinite(lines) || lines <= 0) return;
linesChangedCounter.add(lines, {
...baseMetricDefinition.getCommonAttributes(config),
type: changeType,
...(attributes ?? {}),
});
}
// --- New Metric Recording Functions ---
/**