MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-04-21 10:34:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(plan): create metrics for usage of AskUser tool (#18820)

Browse Source 
Co-authored-by: Jerop Kipruto <jerop@google.com>
This commit is contained in:
Adib234
2026-02-12 12:46:59 -05:00
committed by GitHub
parent 27a1bae03b
commit 868f43927e
8 changed files with 250 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts
+88
View File
@@ -336,6 +336,10 @@ describe('ClearcutLogger', () => {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_USER_SETTINGS,
value: logger?.getConfigJson(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_ACTIVE_APPROVAL_MODE,
value: 'default',
},
]),
);
});
@@ -1239,6 +1243,90 @@ describe('ClearcutLogger', () => {
EventMetadataKey.GEMINI_CLI_AI_ADDED_LINES,
);
});
it('logs AskUser tool metadata', () => {
const { logger } = setup();
const completedToolCall = {
request: {
name: 'ask_user',
args: { questions: [] },
prompt_id: 'prompt-123',
},
response: {
resultDisplay: 'User answered: ...',
data: {
ask_user: {
question_types: ['choice', 'text'],
dismissed: false,
empty_submission: false,
answer_count: 2,
},
},
},
status: 'success',
} as unknown as SuccessfulToolCall;
logger?.logToolCallEvent(new ToolCallEvent(completedToolCall));
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.TOOL_CALL);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES,
JSON.stringify(['choice', 'text']),
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED,
'false',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION,
'false',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT,
'2',
]);
});
it('does not log AskUser tool metadata for other tools', () => {
const { logger } = setup();
const completedToolCall = {
request: {
name: 'some_other_tool',
args: {},
prompt_id: 'prompt-123',
},
response: {
resultDisplay: 'Result',
data: {
ask_user_question_types: ['choice', 'text'],
ask_user_dismissed: false,
ask_user_empty_submission: false,
ask_user_answer_count: 2,
},
},
status: 'success',
} as unknown as SuccessfulToolCall;
logger?.logToolCallEvent(new ToolCallEvent(completedToolCall));
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.TOOL_CALL);
expect(events[0]).not.toHaveMetadataKey(
EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES,
);
expect(events[0]).not.toHaveMetadataKey(
EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED,
);
expect(events[0]).not.toHaveMetadataKey(
EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION,
);
expect(events[0]).not.toHaveMetadataKey(
EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT,
);
});
});
describe('flushIfNeeded', () => {
packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
+32
View File
@@ -56,6 +56,7 @@ import {
safeJsonStringify,
safeJsonStringifyBooleanValuesOnly,
} from '../../utils/safeJsonStringify.js';
import { ASK_USER_TOOL_NAME } from '../../tools/tool-names.js';
import { FixedDeque } from 'mnemonist';
import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js';
import {
@@ -704,6 +705,29 @@ export class ClearcutLogger {
user_removed_chars: EventMetadataKey.GEMINI_CLI_USER_REMOVED_CHARS,
};
if (
event.function_name === ASK_USER_TOOL_NAME &&
event.metadata['ask_user']
) {
const askUser = event.metadata['ask_user'];
const askUserMapping: { [key: string]: EventMetadataKey } = {
question_types: EventMetadataKey.GEMINI_CLI_ASK_USER_QUESTION_TYPES,
dismissed: EventMetadataKey.GEMINI_CLI_ASK_USER_DISMISSED,
empty_submission:
EventMetadataKey.GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION,
answer_count: EventMetadataKey.GEMINI_CLI_ASK_USER_ANSWER_COUNT,
};
for (const [key, gemini_cli_key] of Object.entries(askUserMapping)) {
if (askUser[key] !== undefined) {
data.push({
gemini_cli_key,
value: JSON.stringify(askUser[key]),
});
}
}
}
for (const [key, gemini_cli_key] of Object.entries(metadataMapping)) {
if (event.metadata[key] !== undefined) {
data.push({
@@ -1625,6 +1649,14 @@ export class ClearcutLogger {
gemini_cli_key: EventMetadataKey.GEMINI_CLI_INTERACTIVE,
value: this.config?.isInteractive().toString() ?? 'false',
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_ACTIVE_APPROVAL_MODE,
value:
typeof this.config?.getPolicyEngine === 'function' &&
typeof this.config.getPolicyEngine()?.getApprovalMode === 'function'
? this.config.getPolicyEngine().getApprovalMode()
: '',
},
];
if (this.config?.getExperiments()) {
defaultLogMetadata.push({
packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts
+17 -1
View File
@@ -7,7 +7,7 @@
// Defines valid event metadata keys for Clearcut logging.
export enum EventMetadataKey {
// Deleted enums: 24
// Next ID: 152
// Next ID: 156
GEMINI_CLI_KEY_UNKNOWN = 0,
@@ -577,4 +577,20 @@ export enum EventMetadataKey {
// Logs the total prunable tokens identified at the trigger point.
GEMINI_CLI_TOOL_OUTPUT_MASKING_TOTAL_PRUNABLE_TOKENS = 151,
// ==========================================================================
// Ask User Stats Event Keys
// ==========================================================================
// Logs the types of questions asked in the ask_user tool.
GEMINI_CLI_ASK_USER_QUESTION_TYPES = 152,
// Logs whether the ask_user dialog was dismissed.
GEMINI_CLI_ASK_USER_DISMISSED = 153,
// Logs whether the ask_user dialog was submitted empty.
GEMINI_CLI_ASK_USER_EMPTY_SUBMISSION = 154,
// Logs the number of questions answered in the ask_user tool.
GEMINI_CLI_ASK_USER_ANSWER_COUNT = 155,
}
packages/core/src/telemetry/loggers.test.ts
+48
View File
@@ -5,6 +5,7 @@
*/
import type {
AnyDeclarativeTool,
AnyToolInvocation,
CompletedToolCall,
ContentGeneratorConfig,
@@ -1184,6 +1185,53 @@ describe('loggers', () => {
{ function_name: 'test-function' },
);
});
it('should merge data from response into metadata', () => {
const call: CompletedToolCall = {
status: 'success',
request: {
name: 'ask_user',
args: { questions: [] },
callId: 'test-call-id',
isClientInitiated: true,
prompt_id: 'prompt-id-1',
},
response: {
callId: 'test-call-id',
responseParts: [{ text: 'test-response' }],
resultDisplay: 'User answered: ...',
error: undefined,
errorType: undefined,
data: {
ask_user: {
question_types: ['choice'],
dismissed: false,
},
},
},
tool: undefined as unknown as AnyDeclarativeTool,
invocation: {} as AnyToolInvocation,
durationMs: 100,
outcome: ToolConfirmationOutcome.ProceedOnce,
};
const event = new ToolCallEvent(call);
logToolCall(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Tool call: ask_user. Decision: accept. Success: true. Duration: 100ms.',
attributes: expect.objectContaining({
function_name: 'ask_user',
metadata: expect.objectContaining({
ask_user: {
question_types: ['choice'],
dismissed: false,
},
}),
}),
});
});
it('should log a tool call with a reject decision', () => {
const call: ErroredToolCall = {
status: 'error',
packages/core/src/telemetry/types.ts
+5
View File
@@ -304,6 +304,7 @@ export class ToolCallEvent implements BaseTelemetryEvent {
const diffStat = fileDiff.diffStat;
if (diffStat) {
this.metadata = {
...this.metadata,
model_added_lines: diffStat.model_added_lines,
model_removed_lines: diffStat.model_removed_lines,
model_added_chars: diffStat.model_added_chars,
@@ -315,6 +316,10 @@ export class ToolCallEvent implements BaseTelemetryEvent {
};
}
}
if (call.status === 'success' && call.response.data) {
this.metadata = { ...this.metadata, ...call.response.data };
}
} else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
this.function_name = function_name as string;
packages/core/src/tools/ask-user.test.ts
+22
View File
@@ -337,6 +337,14 @@ describe('AskUserTool', () => {
expect(JSON.parse(result.llmContent as string)).toEqual({
answers: { '0': 'Quick fix (Recommended)' },
});
expect(result.data).toEqual({
ask_user: {
question_types: [QuestionType.CHOICE],
dismissed: false,
empty_submission: false,
answer_count: 1,
},
});
});
it('should display message when user submits without answering', async () => {
@@ -368,6 +376,14 @@ describe('AskUserTool', () => {
'User submitted without answering questions.',
);
expect(JSON.parse(result.llmContent as string)).toEqual({ answers: {} });
expect(result.data).toEqual({
ask_user: {
question_types: [QuestionType.CHOICE],
dismissed: false,
empty_submission: true,
answer_count: 0,
},
});
});
it('should handle cancellation', async () => {
@@ -405,6 +421,12 @@ describe('AskUserTool', () => {
expect(result.llmContent).toBe(
'User dismissed ask_user dialog without answering.',
);
expect(result.data).toEqual({
ask_user: {
question_types: [QuestionType.CHOICE],
dismissed: true,
},
});
});
});
});
packages/core/src/tools/ask-user.ts
+20
View File
@@ -192,16 +192,35 @@ export class AskUserInvocation extends BaseToolInvocation<
}
async execute(_signal: AbortSignal): Promise<ToolResult> {
const questionTypes = this.params.questions.map(
(q) => q.type ?? QuestionType.CHOICE,
);
if (this.confirmationOutcome === ToolConfirmationOutcome.Cancel) {
return {
llmContent: 'User dismissed ask_user dialog without answering.',
returnDisplay: 'User dismissed dialog',
data: {
ask_user: {
question_types: questionTypes,
dismissed: true,
},
},
};
}
const answerEntries = Object.entries(this.userAnswers);
const hasAnswers = answerEntries.length > 0;
const metrics: Record<string, unknown> = {
ask_user: {
question_types: questionTypes,
dismissed: false,
empty_submission: !hasAnswers,
answer_count: answerEntries.length,
},
};
const returnDisplay = hasAnswers
? `**User answered:**\n${answerEntries
.map(([index, answer]) => {
@@ -219,6 +238,7 @@ export class AskUserInvocation extends BaseToolInvocation<
return {
llmContent: JSON.stringify({ answers: this.userAnswers }),
returnDisplay,
data: metrics,
};
}
}