diff --git a/docs/cli/telemetry.md b/docs/cli/telemetry.md index 407ba101f2..ca44bccaf0 100644 --- a/docs/cli/telemetry.md +++ b/docs/cli/telemetry.md @@ -275,9 +275,9 @@ For local development and debugging, you can capture telemetry data locally: The following section describes the structure of logs and metrics generated for Gemini CLI. -The `session.id`, `installation.id`, and `user.email` (available only when -authenticated with a Google account) are included as common attributes on all -logs and metrics. +The `session.id`, `installation.id`, `active_approval_mode`, and `user.email` +(available only when authenticated with a Google account) are included as common +attributes on all logs and metrics. ### Logs @@ -360,7 +360,21 @@ Captures tool executions, output truncation, and Edit behavior. - `extension_name` (string, if applicable) - `extension_id` (string, if applicable) - `content_length` (int, if applicable) - - `metadata` (if applicable) + - `metadata` (if applicable), which includes for the `AskUser` tool: + - `ask_user` (object): + - `question_types` (array of strings) + - `ask_user_dismissed` (boolean) + - `ask_user_empty_submission` (boolean) + - `ask_user_answer_count` (number) + - `diffStat` (if applicable), which includes: + - `model_added_lines` (number) + - `model_removed_lines` (number) + - `model_added_chars` (number) + - `model_removed_chars` (number) + - `user_added_lines` (number) + - `user_removed_lines` (number) + - `user_added_chars` (number) + - `user_removed_chars` (number) - `gemini_cli.tool_output_truncated`: Output of a tool call was truncated. - **Attributes**: diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts index 3cad76b491..a5bed404d9 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts @@ -336,6 +336,10 @@ describe('ClearcutLogger', () => { gemini_cli_key: EventMetadataKey.GEMINI_CLI_USER_SETTINGS, value: logger?.getConfigJson(), }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_ACTIVE_APPROVAL_MODE, + value: 'default', + }, ]), ); }); @@ -1239,6 +1243,90 @@ describe('ClearcutLogger', () => { EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES, ); }); + + it('logs AskUser tool metadata', () => { + const { logger } = setup(); + const completedToolCall = { + request: { + name: 'ask_user', + args: { questions: [] }, + prompt_id: 'prompt-123', + }, + response: { + resultDisplay: 'User answered: ...', + data: { + ask_user: { + question_types: ['choice', 'text'], + dismissed: false, + empty_submission: false, + answer_count: 2, + }, + }, + }, + status: 'success', + } as unknown as SuccessfulToolCall; + + logger?.logToolCallEvent(new ToolCallEvent(completedToolCall)); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.TOOL_CALL); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES, + JSON.stringify(['choice', 'text']), + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED, + 'false', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION, + 'false', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT, + '2', + ]); + }); + + it('does not log AskUser tool metadata for other tools', () => { + const { logger } = setup(); + const completedToolCall = { + request: { + name: 'some_other_tool', + args: {}, + prompt_id: 'prompt-123', + }, + response: { + resultDisplay: 'Result', + data: { + ask_user_question_types: ['choice', 'text'], + ask_user_dismissed: false, + ask_user_empty_submission: false, + ask_user_answer_count: 2, + }, + }, + status: 'success', + } as unknown as SuccessfulToolCall; + + logger?.logToolCallEvent(new ToolCallEvent(completedToolCall)); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.TOOL_CALL); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES, + ); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED, + ); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION, + ); + expect(events[0]).not.toHaveMetadataKey( + EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT, + ); + }); }); describe('flushIfNeeded', () => { diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index b63cac58eb..570725318a 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -56,6 +56,7 @@ import { safeJsonStringify, safeJsonStringifyBooleanValuesOnly, } from '../../utils/safeJsonStringify.js'; +import { ASK_USER_TOOL_NAME } from '../../tools/tool-names.js'; import { FixedDeque } from 'mnemonist'; import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js'; import { @@ -704,6 +705,29 @@ export class ClearcutLogger { user_removed_chars: EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS, }; + if ( + event.function_name === ASK_USER_TOOL_NAME && + event.metadata['ask_user'] + ) { + const askUser = event.metadata['ask_user']; + const askUserMapping: { [key: string]: EventMetadataKey } = { + question_types: EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES, + dismissed: EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED, + empty_submission: + EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION, + answer_count: EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT, + }; + + for (const [key, gemini_cli_key] of Object.entries(askUserMapping)) { + if (askUser[key] !== undefined) { + data.push({ + gemini_cli_key, + value: JSON.stringify(askUser[key]), + }); + } + } + } + for (const [key, gemini_cli_key] of Object.entries(metadataMapping)) { if (event.metadata[key] !== undefined) { data.push({ @@ -1625,6 +1649,14 @@ export class ClearcutLogger { gemini_cli_key: EventMetadataKey.GEMINI_CLI_INTERACTIVE, value: this.config?.isInteractive().toString() ?? 'false', }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_ACTIVE_APPROVAL_MODE, + value: + typeof this.config?.getPolicyEngine === 'function' && + typeof this.config.getPolicyEngine()?.getApprovalMode === 'function' + ? this.config.getPolicyEngine().getApprovalMode() + : '', + }, ]; if (this.config?.getExperiments()) { defaultLogMetadata.push({ diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts index 25e6e18d13..8934db5570 100644 --- a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -7,7 +7,7 @@ // Defines valid event metadata keys for Clearcut logging. export enum EventMetadataKey { // Deleted enums: 24 - // Next ID: 152 + // Next ID: 156 GEMINI_CLI_KEY_UNKNOWN = 0, @@ -577,4 +577,20 @@ export enum EventMetadataKey { // Logs the total prunable tokens identified at the trigger point. GEMINI_CLI_TOOL_OUTPUT_MASKING_TOTAL_PRUNABLE_TOKENS = 151, + + // ========================================================================== + // Ask User Stats Event Keys + // ========================================================================== + + // Logs the types of questions asked in the ask_user tool. + GEMINI_CLI_ASK_USER_QUESTION_TYPES = 152, + + // Logs whether the ask_user dialog was dismissed. + GEMINI_CLI_ASK_USER_DISMISSED = 153, + + // Logs whether the ask_user dialog was submitted empty. + GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION = 154, + + // Logs the number of questions answered in the ask_user tool. + GEMINI_CLI_ASK_USER_ANSWER_COUNT = 155, } diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index 16da103244..fd2d1bc221 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -5,6 +5,7 @@ */ import type { + AnyDeclarativeTool, AnyToolInvocation, CompletedToolCall, ContentGeneratorConfig, @@ -1184,6 +1185,53 @@ describe('loggers', () => { { function_name: 'test-function' }, ); }); + + it('should merge data from response into metadata', () => { + const call: CompletedToolCall = { + status: 'success', + request: { + name: 'ask_user', + args: { questions: [] }, + callId: 'test-call-id', + isClientInitiated: true, + prompt_id: 'prompt-id-1', + }, + response: { + callId: 'test-call-id', + responseParts: [{ text: 'test-response' }], + resultDisplay: 'User answered: ...', + error: undefined, + errorType: undefined, + data: { + ask_user: { + question_types: ['choice'], + dismissed: false, + }, + }, + }, + tool: undefined as unknown as AnyDeclarativeTool, + invocation: {} as AnyToolInvocation, + durationMs: 100, + outcome: ToolConfirmationOutcome.ProceedOnce, + }; + const event = new ToolCallEvent(call); + + logToolCall(mockConfig, event); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Tool call: ask_user. Decision: accept. Success: true. Duration: 100ms.', + attributes: expect.objectContaining({ + function_name: 'ask_user', + metadata: expect.objectContaining({ + ask_user: { + question_types: ['choice'], + dismissed: false, + }, + }), + }), + }); + }); + it('should log a tool call with a reject decision', () => { const call: ErroredToolCall = { status: 'error', diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index 0c438764f1..cf0e5f853f 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -304,6 +304,7 @@ export class ToolCallEvent implements BaseTelemetryEvent { const diffStat = fileDiff.diffStat; if (diffStat) { this.metadata = { + ...this.metadata, model_added_lines: diffStat.model_added_lines, model_removed_lines: diffStat.model_removed_lines, model_added_chars: diffStat.model_added_chars, @@ -315,6 +316,10 @@ export class ToolCallEvent implements BaseTelemetryEvent { }; } } + + if (call.status === 'success' && call.response.data) { + this.metadata = { ...this.metadata, ...call.response.data }; + } } else { // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion this.function_name = function_name as string; diff --git a/packages/core/src/tools/ask-user.test.ts b/packages/core/src/tools/ask-user.test.ts index 969a4f7f15..c7d64eae6e 100644 --- a/packages/core/src/tools/ask-user.test.ts +++ b/packages/core/src/tools/ask-user.test.ts @@ -337,6 +337,14 @@ describe('AskUserTool', () => { expect(JSON.parse(result.llmContent as string)).toEqual({ answers: { '0': 'Quick fix (Recommended)' }, }); + expect(result.data).toEqual({ + ask_user: { + question_types: [QuestionType.CHOICE], + dismissed: false, + empty_submission: false, + answer_count: 1, + }, + }); }); it('should display message when user submits without answering', async () => { @@ -368,6 +376,14 @@ describe('AskUserTool', () => { 'User submitted without answering questions.', ); expect(JSON.parse(result.llmContent as string)).toEqual({ answers: {} }); + expect(result.data).toEqual({ + ask_user: { + question_types: [QuestionType.CHOICE], + dismissed: false, + empty_submission: true, + answer_count: 0, + }, + }); }); it('should handle cancellation', async () => { @@ -405,6 +421,12 @@ describe('AskUserTool', () => { expect(result.llmContent).toBe( 'User dismissed ask_user dialog without answering.', ); + expect(result.data).toEqual({ + ask_user: { + question_types: [QuestionType.CHOICE], + dismissed: true, + }, + }); }); }); }); diff --git a/packages/core/src/tools/ask-user.ts b/packages/core/src/tools/ask-user.ts index 071dd1b317..db9103c720 100644 --- a/packages/core/src/tools/ask-user.ts +++ b/packages/core/src/tools/ask-user.ts @@ -192,16 +192,35 @@ export class AskUserInvocation extends BaseToolInvocation< } async execute(_signal: AbortSignal): Promise { + const questionTypes = this.params.questions.map( + (q) => q.type ?? QuestionType.CHOICE, + ); + if (this.confirmationOutcome === ToolConfirmationOutcome.Cancel) { return { llmContent: 'User dismissed ask_user dialog without answering.', returnDisplay: 'User dismissed dialog', + data: { + ask_user: { + question_types: questionTypes, + dismissed: true, + }, + }, }; } const answerEntries = Object.entries(this.userAnswers); const hasAnswers = answerEntries.length > 0; + const metrics: Record = { + ask_user: { + question_types: questionTypes, + dismissed: false, + empty_submission: !hasAnswers, + answer_count: answerEntries.length, + }, + }; + const returnDisplay = hasAnswers ? `**User answered:**\n${answerEntries .map(([index, answer]) => { @@ -219,6 +238,7 @@ export class AskUserInvocation extends BaseToolInvocation< return { llmContent: JSON.stringify({ answers: this.userAnswers }), returnDisplay, + data: metrics, }; } }