Create line change metrics (#12299)

This commit is contained in:
Jerop Kipruto
2025-10-30 14:55:47 -04:00
committed by GitHub
parent 54fa26ef0e
commit 135d981e3c
5 changed files with 114 additions and 21 deletions
+6 -4
View File
@@ -542,10 +542,6 @@ Measures tool usage and latency.
- `decision` (string: "accept", "reject", "modify", or "auto_accept", if - `decision` (string: "accept", "reject", "modify", or "auto_accept", if
applicable) applicable)
- `tool_type` (string: "mcp" or "native", if applicable) - `tool_type` (string: "mcp" or "native", if applicable)
- `model_added_lines` (Int, optional)
- `model_removed_lines` (Int, optional)
- `user_added_lines` (Int, optional)
- `user_removed_lines` (Int, optional)
- `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency. - `gemini_cli.tool.call.latency` (Histogram, ms): Measures tool call latency.
- **Attributes**: - **Attributes**:
@@ -589,6 +585,12 @@ Counts file operations with basic context.
- `extension` (string, optional) - `extension` (string, optional)
- `programming_language` (string, optional) - `programming_language` (string, optional)
- `gemini_cli.lines.changed` (Counter, Int): Number of lines changed (from file
diffs).
- **Attributes**:
- `function_name`
- `type` ("added" or "removed")
##### Chat and Streaming ##### Chat and Streaming
Resilience counters for compression, invalid chunks, and retries. Resilience counters for compression, invalid chunks, and retries.
+17 -4
View File
@@ -793,12 +793,16 @@ describe('loggers', () => {
const mockMetrics = { const mockMetrics = {
recordToolCallMetrics: vi.fn(), recordToolCallMetrics: vi.fn(),
recordLinesChanged: vi.fn(),
}; };
beforeEach(() => { beforeEach(() => {
vi.spyOn(metrics, 'recordToolCallMetrics').mockImplementation( vi.spyOn(metrics, 'recordToolCallMetrics').mockImplementation(
mockMetrics.recordToolCallMetrics, mockMetrics.recordToolCallMetrics,
); );
vi.spyOn(metrics, 'recordLinesChanged').mockImplementation(
mockMetrics.recordLinesChanged,
);
mockLogger.emit.mockReset(); mockLogger.emit.mockReset();
}); });
@@ -895,10 +899,6 @@ describe('loggers', () => {
success: true, success: true,
decision: ToolCallDecision.ACCEPT, decision: ToolCallDecision.ACCEPT,
tool_type: 'native', tool_type: 'native',
model_added_lines: 1,
model_removed_lines: 2,
user_added_lines: 5,
user_removed_lines: 6,
}, },
); );
@@ -907,6 +907,19 @@ describe('loggers', () => {
'event.name': EVENT_TOOL_CALL, 'event.name': EVENT_TOOL_CALL,
'event.timestamp': '2025-01-01T00:00:00.000Z', 'event.timestamp': '2025-01-01T00:00:00.000Z',
}); });
expect(mockMetrics.recordLinesChanged).toHaveBeenCalledWith(
mockConfig,
1,
'added',
{ function_name: 'test-function' },
);
expect(mockMetrics.recordLinesChanged).toHaveBeenCalledWith(
mockConfig,
2,
'removed',
{ function_name: 'test-function' },
);
}); });
it('should log a tool call with a reject decision', () => { it('should log a tool call with a reject decision', () => {
const call: ErroredToolCall = { const call: ErroredToolCall = {
+16 -8
View File
@@ -63,6 +63,7 @@ import {
recordTokenUsageMetrics, recordTokenUsageMetrics,
recordApiResponseMetrics, recordApiResponseMetrics,
recordAgentRunMetrics, recordAgentRunMetrics,
recordLinesChanged,
} from './metrics.js'; } from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js'; import { isTelemetrySdkInitialized } from './sdk.js';
import type { UiEvent } from './uiTelemetry.js'; import type { UiEvent } from './uiTelemetry.js';
@@ -118,15 +119,22 @@ export function logToolCall(config: Config, event: ToolCallEvent): void {
success: event.success, success: event.success,
decision: event.decision, decision: event.decision,
tool_type: event.tool_type, tool_type: event.tool_type,
...(event.metadata
? {
model_added_lines: event.metadata['model_added_lines'],
model_removed_lines: event.metadata['model_removed_lines'],
user_added_lines: event.metadata['user_added_lines'],
user_removed_lines: event.metadata['user_removed_lines'],
}
: {}),
}); });
if (event.metadata) {
const added = event.metadata['model_added_lines'];
if (typeof added === 'number' && added > 0) {
recordLinesChanged(config, added, 'added', {
function_name: event.function_name,
});
}
const removed = event.metadata['model_removed_lines'];
if (typeof removed === 'number' && removed > 0) {
recordLinesChanged(config, removed, 'removed', {
function_name: event.function_name,
});
}
}
} }
export function logToolOutputTruncated( export function logToolOutputTruncated(
@@ -94,6 +94,7 @@ describe('Telemetry Metrics', () => {
let recordFlickerFrameModule: typeof import('./metrics.js').recordFlickerFrame; let recordFlickerFrameModule: typeof import('./metrics.js').recordFlickerFrame;
let recordExitFailModule: typeof import('./metrics.js').recordExitFail; let recordExitFailModule: typeof import('./metrics.js').recordExitFail;
let recordAgentRunMetricsModule: typeof import('./metrics.js').recordAgentRunMetrics; let recordAgentRunMetricsModule: typeof import('./metrics.js').recordAgentRunMetrics;
let recordLinesChangedModule: typeof import('./metrics.js').recordLinesChanged;
beforeEach(async () => { beforeEach(async () => {
vi.resetModules(); vi.resetModules();
@@ -136,6 +137,7 @@ describe('Telemetry Metrics', () => {
recordFlickerFrameModule = metricsJsModule.recordFlickerFrame; recordFlickerFrameModule = metricsJsModule.recordFlickerFrame;
recordExitFailModule = metricsJsModule.recordExitFail; recordExitFailModule = metricsJsModule.recordExitFail;
recordAgentRunMetricsModule = metricsJsModule.recordAgentRunMetrics; recordAgentRunMetricsModule = metricsJsModule.recordAgentRunMetrics;
recordLinesChangedModule = metricsJsModule.recordLinesChanged;
const otelApiModule = await import('@opentelemetry/api'); const otelApiModule = await import('@opentelemetry/api');
@@ -348,6 +350,53 @@ describe('Telemetry Metrics', () => {
}); });
}); });
describe('recordLinesChanged metric', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record lines added/removed if not initialized', () => {
recordLinesChangedModule(mockConfig, 10, 'added', {
function_name: 'fn',
});
recordLinesChangedModule(mockConfig, 5, 'removed', {
function_name: 'fn',
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record lines added with function_name after initialization', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordLinesChangedModule(mockConfig, 10, 'added', {
function_name: 'my-fn',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(10, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
type: 'added',
function_name: 'my-fn',
});
});
it('should record lines removed with function_name after initialization', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordLinesChangedModule(mockConfig, 7, 'removed', {
function_name: 'my-fn',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(7, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
type: 'removed',
function_name: 'my-fn',
});
});
});
describe('recordFileOperationMetric', () => { describe('recordFileOperationMetric', () => {
const mockConfig = { const mockConfig = {
getSessionId: () => 'test-session-id', getSessionId: () => 'test-session-id',
+26 -5
View File
@@ -24,6 +24,7 @@ const API_REQUEST_LATENCY = 'gemini_cli.api.request.latency';
const TOKEN_USAGE = 'gemini_cli.token.usage'; const TOKEN_USAGE = 'gemini_cli.token.usage';
const SESSION_COUNT = 'gemini_cli.session.count'; const SESSION_COUNT = 'gemini_cli.session.count';
const FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count'; const FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count';
const LINES_CHANGED = 'gemini_cli.lines.changed';
const INVALID_CHUNK_COUNT = 'gemini_cli.chat.invalid_chunk.count'; const INVALID_CHUNK_COUNT = 'gemini_cli.chat.invalid_chunk.count';
const CONTENT_RETRY_COUNT = 'gemini_cli.chat.content_retry.count'; const CONTENT_RETRY_COUNT = 'gemini_cli.chat.content_retry.count';
const CONTENT_RETRY_FAILURE_COUNT = const CONTENT_RETRY_FAILURE_COUNT =
@@ -72,11 +73,6 @@ const COUNTER_DEFINITIONS = {
success: boolean; success: boolean;
decision?: 'accept' | 'reject' | 'modify' | 'auto_accept'; decision?: 'accept' | 'reject' | 'modify' | 'auto_accept';
tool_type?: 'native' | 'mcp'; tool_type?: 'native' | 'mcp';
// Optional diff statistics for file-modifying tools
model_added_lines?: number;
model_removed_lines?: number;
user_added_lines?: number;
user_removed_lines?: number;
}, },
}, },
[API_REQUEST_COUNT]: { [API_REQUEST_COUNT]: {
@@ -116,6 +112,15 @@ const COUNTER_DEFINITIONS = {
programming_language?: string; programming_language?: string;
}, },
}, },
[LINES_CHANGED]: {
description: 'Number of lines changed (from file diffs).',
valueType: ValueType.INT,
assign: (c: Counter) => (linesChangedCounter = c),
attributes: {} as {
function_name?: string;
type: 'added' | 'removed';
},
},
[INVALID_CHUNK_COUNT]: { [INVALID_CHUNK_COUNT]: {
description: 'Counts invalid chunks received from a stream.', description: 'Counts invalid chunks received from a stream.',
valueType: ValueType.INT, valueType: ValueType.INT,
@@ -454,6 +459,7 @@ let apiRequestLatencyHistogram: Histogram | undefined;
let tokenUsageCounter: Counter | undefined; let tokenUsageCounter: Counter | undefined;
let sessionCounter: Counter | undefined; let sessionCounter: Counter | undefined;
let fileOperationCounter: Counter | undefined; let fileOperationCounter: Counter | undefined;
let linesChangedCounter: Counter | undefined;
let chatCompressionCounter: Counter | undefined; let chatCompressionCounter: Counter | undefined;
let invalidChunkCounter: Counter | undefined; let invalidChunkCounter: Counter | undefined;
let contentRetryCounter: Counter | undefined; let contentRetryCounter: Counter | undefined;
@@ -621,6 +627,21 @@ export function recordFileOperationMetric(
}); });
} }
export function recordLinesChanged(
config: Config,
lines: number,
changeType: 'added' | 'removed',
attributes?: { function_name?: string },
): void {
if (!linesChangedCounter || !isMetricsInitialized) return;
if (!Number.isFinite(lines) || lines <= 0) return;
linesChangedCounter.add(lines, {
...baseMetricDefinition.getCommonAttributes(config),
type: changeType,
...(attributes ?? {}),
});
}
// --- New Metric Recording Functions --- // --- New Metric Recording Functions ---
/** /**