feat(plan): create metrics for usage of AskUser tool (#18820)

Co-authored-by: Jerop Kipruto <jerop@google.com>

docs/cli/telemetry.md

@@ -275,9 +275,9 @@ For local development and debugging, you can capture telemetry data locally:
 The following section describes the structure of logs and metrics generated for
 Gemini CLI.
 
-The `session.id`, `installation.id`, and `user.email` (available only when
+The `session.id`, `installation.id`, `active_approval_mode`, and `user.email`
-authenticated with a Google account) are included as common attributes on all
+(available only when authenticated with a Google account) are included as common
-logs and metrics.
+attributes on all logs and metrics.
 
 ### Logs
 
@@ -360,7 +360,21 @@ Captures tool executions, output truncation, and Edit behavior.
     - `extension_name` (string, if applicable)
     - `extension_id` (string, if applicable)
     - `content_length` (int, if applicable)
-    - `metadata` (if applicable)
+    - `metadata` (if applicable), which includes for the `AskUser` tool:
+      - `ask_user` (object):
+        - `question_types` (array of strings)
+        - `ask_user_dismissed` (boolean)
+        - `ask_user_empty_submission` (boolean)
+        - `ask_user_answer_count` (number)
+      - `diffStat` (if applicable), which includes:
+        - `model_added_lines` (number)
+        - `model_removed_lines` (number)
+        - `model_added_chars` (number)
+        - `model_removed_chars` (number)
+        - `user_added_lines` (number)
+        - `user_removed_lines` (number)
+        - `user_added_chars` (number)
+        - `user_removed_chars` (number)
 
 - `gemini_cli.tool_output_truncated`: Output of a tool call was truncated.
   - **Attributes**:

packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts

@@ -336,6 +336,10 @@ describe('ClearcutLogger', () => {
             gemini_cli_key: EventMetadataKey.GEMINI_CLI_USER_SETTINGS,
             value: logger?.getConfigJson(),
           },
+          {
+            gemini_cli_key: EventMetadataKey.GEMINI_CLI_ACTIVE_APPROVAL_MODE,
+            value: 'default',
+          },
         ]),
       );
     });
@@ -1239,6 +1243,90 @@ describe('ClearcutLogger', () => {
         EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
       );
     });
+
+    it('logs AskUser tool metadata', () => {
+      const { logger } = setup();
+      const completedToolCall = {
+        request: {
+          name: 'ask_user',
+          args: { questions: [] },
+          prompt_id: 'prompt-123',
+        },
+        response: {
+          resultDisplay: 'User answered: ...',
+          data: {
+            ask_user: {
+              question_types: ['choice', 'text'],
+              dismissed: false,
+              empty_submission: false,
+              answer_count: 2,
+            },
+          },
+        },
+        status: 'success',
+      } as unknown as SuccessfulToolCall;
+
+      logger?.logToolCallEvent(new ToolCallEvent(completedToolCall));
+
+      const events = getEvents(logger!);
+      expect(events.length).toBe(1);
+      expect(events[0]).toHaveEventName(EventNames.TOOL_CALL);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES,
+        JSON.stringify(['choice', 'text']),
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED,
+        'false',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION,
+        'false',
+      ]);
+      expect(events[0]).toHaveMetadataValue([
+        EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT,
+        '2',
+      ]);
+    });
+
+    it('does not log AskUser tool metadata for other tools', () => {
+      const { logger } = setup();
+      const completedToolCall = {
+        request: {
+          name: 'some_other_tool',
+          args: {},
+          prompt_id: 'prompt-123',
+        },
+        response: {
+          resultDisplay: 'Result',
+          data: {
+            ask_user_question_types: ['choice', 'text'],
+            ask_user_dismissed: false,
+            ask_user_empty_submission: false,
+            ask_user_answer_count: 2,
+          },
+        },
+        status: 'success',
+      } as unknown as SuccessfulToolCall;
+
+      logger?.logToolCallEvent(new ToolCallEvent(completedToolCall));
+
+      const events = getEvents(logger!);
+      expect(events.length).toBe(1);
+      expect(events[0]).toHaveEventName(EventNames.TOOL_CALL);
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES,
+      );
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED,
+      );
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION,
+      );
+      expect(events[0]).not.toHaveMetadataKey(
+        EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT,
+      );
+    });
   });
 
   describe('flushIfNeeded', () => {

packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts

@@ -56,6 +56,7 @@ import {
   safeJsonStringify,
   safeJsonStringifyBooleanValuesOnly,
 } from '../../utils/safeJsonStringify.js';
+import { ASK_USER_TOOL_NAME } from '../../tools/tool-names.js';
 import { FixedDeque } from 'mnemonist';
 import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js';
 import {
@@ -704,6 +705,29 @@ export class ClearcutLogger {
         user_removed_chars: EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS,
       };
 
+      if (
+        event.function_name === ASK_USER_TOOL_NAME &&
+        event.metadata['ask_user']
+      ) {
+        const askUser = event.metadata['ask_user'];
+        const askUserMapping: { [key: string]: EventMetadataKey } = {
+          question_types: EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES,
+          dismissed: EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED,
+          empty_submission:
+            EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION,
+          answer_count: EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT,
+        };
+
+        for (const [key, gemini_cli_key] of Object.entries(askUserMapping)) {
+          if (askUser[key] !== undefined) {
+            data.push({
+              gemini_cli_key,
+              value: JSON.stringify(askUser[key]),
+            });
+          }
+        }
+      }
+
       for (const [key, gemini_cli_key] of Object.entries(metadataMapping)) {
         if (event.metadata[key] !== undefined) {
           data.push({
@@ -1625,6 +1649,14 @@ export class ClearcutLogger {
         gemini_cli_key: EventMetadataKey.GEMINI_CLI_INTERACTIVE,
         value: this.config?.isInteractive().toString() ?? 'false',
       },
+      {
+        gemini_cli_key: EventMetadataKey.GEMINI_CLI_ACTIVE_APPROVAL_MODE,
+        value:
+          typeof this.config?.getPolicyEngine === 'function' &&
+          typeof this.config.getPolicyEngine()?.getApprovalMode === 'function'
+            ? this.config.getPolicyEngine().getApprovalMode()
+            : '',
+      },
     ];
     if (this.config?.getExperiments()) {
       defaultLogMetadata.push({

packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts

@@ -7,7 +7,7 @@
 // Defines valid event metadata keys for Clearcut logging.
 export enum EventMetadataKey {
   // Deleted enums: 24
-  // Next ID: 152
+  // Next ID: 156
 
   GEMINI_CLI_KEY_UNKNOWN = 0,
 
@@ -577,4 +577,20 @@ export enum EventMetadataKey {
 
   // Logs the total prunable tokens identified at the trigger point.
   GEMINI_CLI_TOOL_OUTPUT_MASKING_TOTAL_PRUNABLE_TOKENS = 151,
+
+  // ==========================================================================
+  // Ask User Stats Event Keys
+  // ==========================================================================
+
+  // Logs the types of questions asked in the ask_user tool.
+  GEMINI_CLI_ASK_USER_QUESTION_TYPES = 152,
+
+  // Logs whether the ask_user dialog was dismissed.
+  GEMINI_CLI_ASK_USER_DISMISSED = 153,
+
+  // Logs whether the ask_user dialog was submitted empty.
+  GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION = 154,
+
+  // Logs the number of questions answered in the ask_user tool.
+  GEMINI_CLI_ASK_USER_ANSWER_COUNT = 155,
 }

packages/core/src/telemetry/loggers.test.ts

@@ -5,6 +5,7 @@
  */
 
 import type {
+  AnyDeclarativeTool,
   AnyToolInvocation,
   CompletedToolCall,
   ContentGeneratorConfig,
@@ -1184,6 +1185,53 @@ describe('loggers', () => {
         { function_name: 'test-function' },
       );
     });
+
+    it('should merge data from response into metadata', () => {
+      const call: CompletedToolCall = {
+        status: 'success',
+        request: {
+          name: 'ask_user',
+          args: { questions: [] },
+          callId: 'test-call-id',
+          isClientInitiated: true,
+          prompt_id: 'prompt-id-1',
+        },
+        response: {
+          callId: 'test-call-id',
+          responseParts: [{ text: 'test-response' }],
+          resultDisplay: 'User answered: ...',
+          error: undefined,
+          errorType: undefined,
+          data: {
+            ask_user: {
+              question_types: ['choice'],
+              dismissed: false,
+            },
+          },
+        },
+        tool: undefined as unknown as AnyDeclarativeTool,
+        invocation: {} as AnyToolInvocation,
+        durationMs: 100,
+        outcome: ToolConfirmationOutcome.ProceedOnce,
+      };
+      const event = new ToolCallEvent(call);
+
+      logToolCall(mockConfig, event);
+
+      expect(mockLogger.emit).toHaveBeenCalledWith({
+        body: 'Tool call: ask_user. Decision: accept. Success: true. Duration: 100ms.',
+        attributes: expect.objectContaining({
+          function_name: 'ask_user',
+          metadata: expect.objectContaining({
+            ask_user: {
+              question_types: ['choice'],
+              dismissed: false,
+            },
+          }),
+        }),
+      });
+    });
+
     it('should log a tool call with a reject decision', () => {
       const call: ErroredToolCall = {
         status: 'error',

packages/core/src/telemetry/types.ts

@@ -304,6 +304,7 @@ export class ToolCallEvent implements BaseTelemetryEvent {
         const diffStat = fileDiff.diffStat;
         if (diffStat) {
           this.metadata = {
+            ...this.metadata,
             model_added_lines: diffStat.model_added_lines,
             model_removed_lines: diffStat.model_removed_lines,
             model_added_chars: diffStat.model_added_chars,
@@ -315,6 +316,10 @@ export class ToolCallEvent implements BaseTelemetryEvent {
           };
         }
       }
+
+      if (call.status === 'success' && call.response.data) {
+        this.metadata = { ...this.metadata, ...call.response.data };
+      }
     } else {
       // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
       this.function_name = function_name as string;

packages/core/src/tools/ask-user.test.ts

@@ -337,6 +337,14 @@ describe('AskUserTool', () => {
       expect(JSON.parse(result.llmContent as string)).toEqual({
         answers: { '0': 'Quick fix (Recommended)' },
       });
+      expect(result.data).toEqual({
+        ask_user: {
+          question_types: [QuestionType.CHOICE],
+          dismissed: false,
+          empty_submission: false,
+          answer_count: 1,
+        },
+      });
     });
 
     it('should display message when user submits without answering', async () => {
@@ -368,6 +376,14 @@ describe('AskUserTool', () => {
         'User submitted without answering questions.',
       );
       expect(JSON.parse(result.llmContent as string)).toEqual({ answers: {} });
+      expect(result.data).toEqual({
+        ask_user: {
+          question_types: [QuestionType.CHOICE],
+          dismissed: false,
+          empty_submission: true,
+          answer_count: 0,
+        },
+      });
     });
 
     it('should handle cancellation', async () => {
@@ -405,6 +421,12 @@ describe('AskUserTool', () => {
       expect(result.llmContent).toBe(
         'User dismissed ask_user dialog without answering.',
       );
+      expect(result.data).toEqual({
+        ask_user: {
+          question_types: [QuestionType.CHOICE],
+          dismissed: true,
+        },
+      });
     });
   });
 });

packages/core/src/tools/ask-user.ts

@@ -192,16 +192,35 @@ export class AskUserInvocation extends BaseToolInvocation<
   }
 
   async execute(_signal: AbortSignal): Promise<ToolResult> {
+    const questionTypes = this.params.questions.map(
+      (q) => q.type ?? QuestionType.CHOICE,
+    );
+
     if (this.confirmationOutcome === ToolConfirmationOutcome.Cancel) {
       return {
         llmContent: 'User dismissed ask_user dialog without answering.',
         returnDisplay: 'User dismissed dialog',
+        data: {
+          ask_user: {
+            question_types: questionTypes,
+            dismissed: true,
+          },
+        },
       };
     }
 
     const answerEntries = Object.entries(this.userAnswers);
     const hasAnswers = answerEntries.length > 0;
 
+    const metrics: Record<string, unknown> = {
+      ask_user: {
+        question_types: questionTypes,
+        dismissed: false,
+        empty_submission: !hasAnswers,
+        answer_count: answerEntries.length,
+      },
+    };
+
     const returnDisplay = hasAnswers
       ? `**User answered:**\n${answerEntries
           .map(([index, answer]) => {
@@ -219,6 +238,7 @@ export class AskUserInvocation extends BaseToolInvocation<
     return {
       llmContent: JSON.stringify({ answers: this.userAnswers }),
       returnDisplay,
+      data: metrics,
     };
   }
 }