From 135d3401cdc1e0aedee4a4aae630076e5a03d7c5 Mon Sep 17 00:00:00 2001 From: Shardul Natu <43422294+kiranani@users.noreply.github.com> Date: Thu, 25 Sep 2025 09:09:56 -0700 Subject: [PATCH] add(telemetry): Add character-level edit metrics to Concord (#9145) Co-authored-by: Shnatu Co-authored-by: owenofbrien <86964623+owenofbrien@users.noreply.github.com> --- .../clearcut-logger/clearcut-logger.test.ts | 153 ++++++++++++++++++ .../clearcut-logger/clearcut-logger.ts | 4 + .../clearcut-logger/event-metadata-key.ts | 14 +- 3 files changed, 170 insertions(+), 1 deletion(-) diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts index 694be06b22..0dcd531d21 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts @@ -18,6 +18,7 @@ import type { LogEvent, LogEventEntry } from './clearcut-logger.js'; import { ClearcutLogger, EventNames, TEST_ONLY } from './clearcut-logger.js'; import type { ContentGeneratorConfig } from '../../core/contentGenerator.js'; import { AuthType } from '../../core/contentGenerator.js'; +import type { SuccessfulToolCall } from '../../core/coreToolScheduler.js'; import type { ConfigParameters } from '../../config/config.js'; import { EventMetadataKey } from './event-metadata-key.js'; import { makeFakeConfig } from '../../test-utils/config.js'; @@ -27,6 +28,7 @@ import { UserPromptEvent, makeChatCompressionEvent, ModelRoutingEvent, + ToolCallEvent, } from '../types.js'; import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js'; import { UserAccountManager } from '../../utils/userAccountManager.js'; @@ -36,6 +38,7 @@ import { safeJsonStringify } from '../../utils/safeJsonStringify.js'; interface CustomMatchers { toHaveMetadataValue: ([key, value]: [EventMetadataKey, string]) => R; toHaveEventName: (name: EventNames) => R; + toHaveMetadataKey: (key: EventMetadataKey) => R; } declare module 'vitest' { @@ -72,6 +75,20 @@ expect.extend({ `event ${received} does${isNot ? ' not' : ''} have ${value}}`, }; }, + + toHaveMetadataKey(received: LogEventEntry[], key: EventMetadataKey) { + const { isNot } = this; + const event = JSON.parse(received[0].source_extension_json) as LogEvent; + const metadata = event['event_metadata'][0]; + + const pass = metadata.some((m) => m.gemini_cli_key === key); + + return { + pass, + message: () => + `event ${received} ${isNot ? 'has' : 'does not have'} the metadata key ${key}`, + }; + }, }); vi.mock('../../utils/userAccountManager.js'); @@ -677,4 +694,140 @@ describe('ClearcutLogger', () => { ]); }); }); + + describe('logToolCallEvent', () => { + it('logs an event with all diff metadata', () => { + const { logger } = setup(); + const completedToolCall = { + request: { name: 'test', args: {}, prompt_id: 'prompt-123' }, + response: { + resultDisplay: { + diffStat: { + model_added_lines: 1, + model_removed_lines: 2, + model_added_chars: 3, + model_removed_chars: 4, + user_added_lines: 5, + user_removed_lines: 6, + user_added_chars: 7, + user_removed_chars: 8, + }, + }, + }, + status: 'success', + } as SuccessfulToolCall; + + logger?.logToolCallEvent(new ToolCallEvent(completedToolCall)); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.TOOL_CALL); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES, + '1', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_REMOVED_LINES, + '2', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_ADDED_CHARS, + '3', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_REMOVED_CHARS, + '4', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_USER_ADDED_LINES, + '5', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_USER_REMOVED_LINES, + '6', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_USER_ADDED_CHARS, + '7', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS, + '8', + ]); + }); + + it('logs an event with partial diff metadata', () => { + const { logger } = setup(); + const completedToolCall = { + request: { name: 'test', args: {}, prompt_id: 'prompt-123' }, + response: { + resultDisplay: { + diffStat: { + model_added_lines: 1, + model_removed_lines: 2, + model_added_chars: 3, + model_removed_chars: 4, + }, + }, + }, + status: 'success', + } as SuccessfulToolCall; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + logger?.logToolCallEvent(new ToolCallEvent(completedToolCall as any)); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.TOOL_CALL); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES, + '1', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_REMOVED_LINES, + '2', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_ADDED_CHARS, + '3', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_AI_REMOVED_CHARS, + '4', + ]); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_USER_ADDED_LINES, + ); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_USER_REMOVED_LINES, + ); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_USER_ADDED_CHARS, + ); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS, + ); + }); + + it('does not log diff metadata if diffStat is not present', () => { + const { logger } = setup(); + const completedToolCall = { + request: { name: 'test', args: {}, prompt_id: 'prompt-123' }, + response: { + resultDisplay: {}, + }, + status: 'success', + } as SuccessfulToolCall; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + logger?.logToolCallEvent(new ToolCallEvent(completedToolCall as any)); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.TOOL_CALL); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES, + ); + }); + }); }); diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index c70e2cda51..1ab4f56e16 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -491,8 +491,12 @@ export class ClearcutLogger { const metadataMapping: { [key: string]: EventMetadataKey } = { model_added_lines: EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES, model_removed_lines: EventMetadataKey.GEMINI_CLI_AI_REMOVED_LINES, + model_added_chars: EventMetadataKey.GEMINI_CLI_AI_ADDED_CHARS, + model_removed_chars: EventMetadataKey.GEMINI_CLI_AI_REMOVED_CHARS, user_added_lines: EventMetadataKey.GEMINI_CLI_USER_ADDED_LINES, user_removed_lines: EventMetadataKey.GEMINI_CLI_USER_REMOVED_LINES, + user_added_chars: EventMetadataKey.GEMINI_CLI_USER_ADDED_CHARS, + user_removed_chars: EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS, }; for (const [key, gemini_cli_key] of Object.entries(metadataMapping)) { diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts index c21359eda7..06f0f82a6d 100644 --- a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -233,6 +233,18 @@ export enum EventMetadataKey { // Logs user removed lines in edit/write tool response. GEMINI_CLI_USER_REMOVED_LINES = 50, + // Logs AI added characters in edit/write tool response. + GEMINI_CLI_AI_ADDED_CHARS = 103, + + // Logs AI removed characters in edit/write tool response. + GEMINI_CLI_AI_REMOVED_CHARS = 104, + + // Logs user added characters in edit/write tool response. + GEMINI_CLI_USER_ADDED_CHARS = 105, + + // Logs user removed characters in edit/write tool response. + GEMINI_CLI_USER_REMOVED_CHARS = 106, + // ========================================================================== // Kitty Sequence Overflow Event Keys // =========================================================================== @@ -403,5 +415,5 @@ export enum EventMetadataKey { GEMINI_CLI_ROUTING_DECISION_SOURCE = 101, // Logs an event when the user uses the /model command. - GEMINI_CLI_MODEL_SLASH_COMMAND = 103, + GEMINI_CLI_MODEL_SLASH_COMMAND = 108, }