add(telemetry): Add character-level edit metrics to Concord (#9145)

Co-authored-by: Shnatu <snatu@google.com>
Co-authored-by: owenofbrien <86964623+owenofbrien@users.noreply.github.com>

packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts

@@ -18,6 +18,7 @@ import type { LogEvent, LogEventEntry } from './clearcut-logger.js';
 import { ClearcutLogger, EventNames, TEST_ONLY } from './clearcut-logger.js';
 import type { ContentGeneratorConfig } from '../../core/contentGenerator.js';
 import { AuthType } from '../../core/contentGenerator.js';
+import type { SuccessfulToolCall } from '../../core/coreToolScheduler.js';
 import type { ConfigParameters } from '../../config/config.js';
 import { EventMetadataKey } from './event-metadata-key.js';
 import { makeFakeConfig } from '../../test-utils/config.js';
@@ -27,6 +28,7 @@ import {
   UserPromptEvent,
   makeChatCompressionEvent,
   ModelRoutingEvent,
+  ToolCallEvent,
 } from '../types.js';
 import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js';
 import { UserAccountManager } from '../../utils/userAccountManager.js';
@@ -36,6 +38,7 @@ import { safeJsonStringify } from '../../utils/safeJsonStringify.js';
 interface CustomMatchers<R = unknown> {
   toHaveMetadataValue: ([key, value]: [EventMetadataKey, string]) => R;
   toHaveEventName: (name: EventNames) => R;
+  toHaveMetadataKey: (key: EventMetadataKey) => R;
 }
 
 declare module 'vitest' {
@@ -72,6 +75,20 @@ expect.extend({
         `event ${received} does${isNot ? ' not' : ''} have ${value}}`,
     };
   },
+
+  toHaveMetadataKey(received: LogEventEntry[], key: EventMetadataKey) {
+    const { isNot } = this;
+    const event = JSON.parse(received[0].source_extension_json) as LogEvent;
+    const metadata = event['event_metadata'][0];
+
+    const pass = metadata.some((m) => m.gemini_cli_key === key);
+
+    return {
+      pass,
+      message: () =>
+        `event ${received} ${isNot ? 'has' : 'does not have'} the metadata key ${key}`,
+    };
+  },
 });
 
 vi.mock('../../utils/userAccountManager.js');
@@ -677,4 +694,140 @@ describe('ClearcutLogger', () => {
       ]);
     });
   });
+
+  describe('logToolCallEvent', () => {
+    it('logs an event with all diff metadata', () => {
+      const { logger } = setup();
+      const completedToolCall = {
+        request: { name: 'test', args: {}, prompt_id: 'prompt-123' },
+        response: {
+          resultDisplay: {
+            diffStat: {
+              model_added_lines: 1,
+              model_removed_lines: 2,
+              model_added_chars: 3,
+              model_removed_chars: 4,
+              user_added_lines: 5,
+              user_removed_lines: 6,
+              user_added_chars: 7,
+              user_removed_chars: 8,
+            },
+          },
+        },
+        status: 'success',
+      } as SuccessfulToolCall;
+
+      logger?.logToolCallEvent(new ToolCallEvent(completedToolCall));
+
+      const events = getEvents(logger!);
+      expect(events.length).toBe(1);
+      expect(events[0]).toHaveEventName(EventNames.TOOL_CALL);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
+        '1',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_REMOVED_LINES,
+        '2',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_ADDED_CHARS,
+        '3',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_REMOVED_CHARS,
+        '4',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_USER_ADDED_LINES,
+        '5',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_USER_REMOVED_LINES,
+        '6',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_USER_ADDED_CHARS,
+        '7',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS,
+        '8',
+      ]);
+    });
+
+    it('logs an event with partial diff metadata', () => {
+      const { logger } = setup();
+      const completedToolCall = {
+        request: { name: 'test', args: {}, prompt_id: 'prompt-123' },
+        response: {
+          resultDisplay: {
+            diffStat: {
+              model_added_lines: 1,
+              model_removed_lines: 2,
+              model_added_chars: 3,
+              model_removed_chars: 4,
+            },
+          },
+        },
+        status: 'success',
+      } as SuccessfulToolCall;
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      logger?.logToolCallEvent(new ToolCallEvent(completedToolCall as any));
+
+      const events = getEvents(logger!);
+      expect(events.length).toBe(1);
+      expect(events[0]).toHaveEventName(EventNames.TOOL_CALL);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
+        '1',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_REMOVED_LINES,
+        '2',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_ADDED_CHARS,
+        '3',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_AI_REMOVED_CHARS,
+        '4',
+      ]);
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_USER_ADDED_LINES,
+      );
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_USER_REMOVED_LINES,
+      );
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_USER_ADDED_CHARS,
+      );
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS,
+      );
+    });
+
+    it('does not log diff metadata if diffStat is not present', () => {
+      const { logger } = setup();
+      const completedToolCall = {
+        request: { name: 'test', args: {}, prompt_id: 'prompt-123' },
+        response: {
+          resultDisplay: {},
+        },
+        status: 'success',
+      } as SuccessfulToolCall;
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      logger?.logToolCallEvent(new ToolCallEvent(completedToolCall as any));
+
+      const events = getEvents(logger!);
+      expect(events.length).toBe(1);
+      expect(events[0]).toHaveEventName(EventNames.TOOL_CALL);
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
+      );
+    });
+  });
 });

packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts

@@ -491,8 +491,12 @@ export class ClearcutLogger {
       const metadataMapping: { [key: string]: EventMetadataKey } = {
         model_added_lines: EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
         model_removed_lines: EventMetadataKey.GEMINI_CLI_AI_REMOVED_LINES,
+        model_added_chars: EventMetadataKey.GEMINI_CLI_AI_ADDED_CHARS,
+        model_removed_chars: EventMetadataKey.GEMINI_CLI_AI_REMOVED_CHARS,
         user_added_lines: EventMetadataKey.GEMINI_CLI_USER_ADDED_LINES,
         user_removed_lines: EventMetadataKey.GEMINI_CLI_USER_REMOVED_LINES,
+        user_added_chars: EventMetadataKey.GEMINI_CLI_USER_ADDED_CHARS,
+        user_removed_chars: EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS,
       };
 
       for (const [key, gemini_cli_key] of Object.entries(metadataMapping)) {

packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts

@@ -233,6 +233,18 @@ export enum EventMetadataKey {
   // Logs user removed lines in edit/write tool response.
   GEMINI_CLI_USER_REMOVED_LINES = 50,
 
+  // Logs AI added characters in edit/write tool response.
+  GEMINI_CLI_AI_ADDED_CHARS = 103,
+
+  // Logs AI removed characters in edit/write tool response.
+  GEMINI_CLI_AI_REMOVED_CHARS = 104,
+
+  // Logs user added characters in edit/write tool response.
+  GEMINI_CLI_USER_ADDED_CHARS = 105,
+
+  // Logs user removed characters in edit/write tool response.
+  GEMINI_CLI_USER_REMOVED_CHARS = 106,
+
   // ==========================================================================
   // Kitty Sequence Overflow Event Keys
   // ===========================================================================
@@ -403,5 +415,5 @@ export enum EventMetadataKey {
   GEMINI_CLI_ROUTING_DECISION_SOURCE = 101,
 
   // Logs an event when the user uses the /model command.
-  GEMINI_CLI_MODEL_SLASH_COMMAND = 103,
+  GEMINI_CLI_MODEL_SLASH_COMMAND = 108,
 }