MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-05-12 21:03:05 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): add browser agent clearcut metrics (#24688)

Browse Source
This commit is contained in:
Gaurav
2026-04-07 15:48:38 +08:00
committed by GitHub
parent 83096c68b0
commit 4c5e887732
10 changed files with 494 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/agents/browser/browserAgentFactory.ts
+7 -7
View File
@@ -32,11 +32,11 @@ import { createAnalyzeScreenshotTool } from './analyzeScreenshot.js';
import { injectAutomationOverlay } from './automationOverlay.js';
import { injectInputBlocker } from './inputBlocker.js';
import { debugLogger } from '../../utils/debugLogger.js';
import { recordBrowserAgentToolDiscovery } from '../../telemetry/metrics.js';
import {
recordBrowserAgentToolDiscovery,
recordBrowserAgentVisionStatus,
recordBrowserAgentCleanup,
} from '../../telemetry/metrics.js';
logBrowserAgentVisionStatus,
logBrowserAgentCleanup,
} from '../../telemetry/loggers.js';
import {
PolicyDecision,
PRIORITY_SUBAGENT_TOOL,
@@ -248,7 +248,7 @@ export async function createBrowserAgentDefinition(
const allTools: AnyDeclarativeTool[] = [...mcpTools];
const visionDisabledReason = getVisionDisabledReason();
recordBrowserAgentVisionStatus(config, {
logBrowserAgentVisionStatus(config, {
enabled: !visionDisabledReason,
disabled_reason: visionDisabledReason?.code,
});
@@ -299,13 +299,13 @@ export async function cleanupBrowserAgent(
const startMs = Date.now();
try {
await browserManager.close();
recordBrowserAgentCleanup(config, Date.now() - startMs, {
logBrowserAgentCleanup(config, Date.now() - startMs, {
session_mode: sessionMode,
success: true,
});
debugLogger.log('Browser agent cleanup complete');
} catch (error) {
recordBrowserAgentCleanup(config, Date.now() - startMs, {
logBrowserAgentCleanup(config, Date.now() - startMs, {
session_mode: sessionMode,
success: false,
});
packages/core/src/agents/browser/browserAgentInvocation.ts
+2 -2
View File
@@ -36,7 +36,7 @@ import {
import type { MessageBus } from '../../confirmation-bus/message-bus.js';
import { createBrowserAgentDefinition } from './browserAgentFactory.js';
import { removeInputBlocker } from './inputBlocker.js';
import { recordBrowserAgentTaskOutcome } from '../../telemetry/metrics.js';
import { logBrowserAgentTaskOutcome } from '../../telemetry/loggers.js';
import {
sanitizeThoughtContent,
sanitizeToolArgs,
@@ -397,7 +397,7 @@ ${output.result}`;
},
};
} finally {
recordBrowserAgentTaskOutcome(this.config, {
logBrowserAgentTaskOutcome(this.config, {
success: taskSuccess,
session_mode: sessionMode,
vision_enabled: visionEnabled,
packages/core/src/agents/browser/browserManager.test.ts
+1
View File
@@ -373,6 +373,7 @@ describe('BrowserManager', () => {
session_mode: 'persistent',
headless: false,
success: true,
tool_count: 4,
},
);
});
packages/core/src/agents/browser/browserManager.ts
+12 -13
View File
@@ -30,7 +30,7 @@ import * as path from 'node:path';
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { injectAutomationOverlay } from './automationOverlay.js';
import { recordBrowserAgentConnection } from '../../telemetry/metrics.js';
import { logBrowserAgentConnection } from '../../telemetry/loggers.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@@ -563,7 +563,9 @@ export class BrowserManager {
// Add optional settings from config.
// Force headless in seatbelt sandbox since Chrome profile/display access
// may be restricted, and the user is running in a sandboxed environment.
if (browserConfig.customConfig.headless || isSeatbeltSandbox) {
const effectiveHeadless =
!!browserConfig.customConfig.headless || isSeatbeltSandbox;
if (effectiveHeadless) {
mcpArgs.push('--headless');
}
if (browserConfig.customConfig.profilePath) {
@@ -667,15 +669,12 @@ export class BrowserManager {
// clear the action counter for each connection
this.actionCounter = 0;
recordBrowserAgentConnection(
this.config,
Date.now() - connectStartMs,
{
session_mode: sessionMode,
headless: !!browserConfig.customConfig.headless,
success: true,
},
);
logBrowserAgentConnection(this.config, Date.now() - connectStartMs, {
session_mode: sessionMode,
headless: effectiveHeadless,
success: true,
tool_count: this.discoveredTools.length,
});
})(),
new Promise<never>((_, reject) => {
timeoutId = setTimeout(
@@ -696,9 +695,9 @@ export class BrowserManager {
error instanceof Error ? error.message : String(error);
const errorType = BrowserManager.classifyConnectionError(rawErrorMessage);
recordBrowserAgentConnection(this.config, Date.now() - connectStartMs, {
logBrowserAgentConnection(this.config, Date.now() - connectStartMs, {
session_mode: sessionMode,
headless: !!browserConfig.customConfig.headless,
headless: effectiveHeadless,
success: false,
error_type: errorType,
});
packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts
+183
View File
@@ -1692,4 +1692,187 @@ describe('ClearcutLogger', () => {
]);
});
});
describe('logBrowserAgentConnectionEvent', () => {
it('logs a successful connection event', () => {
const { logger } = setup();
logger?.logBrowserAgentConnectionEvent({
session_mode: 'isolated',
headless: true,
success: true,
duration_ms: 1500,
});
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_CONNECTION);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE,
'isolated',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS,
'true',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
'true',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS,
'1500',
]);
});
it('logs a failed connection event with error_type', () => {
const { logger } = setup();
logger?.logBrowserAgentConnectionEvent({
session_mode: 'persistent',
headless: false,
success: false,
duration_ms: 30000,
error_type: 'timeout',
});
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
'false',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_ERROR_TYPE,
'timeout',
]);
});
it('logs tool_count when provided', () => {
const { logger } = setup();
logger?.logBrowserAgentConnectionEvent({
session_mode: 'existing',
headless: true,
success: true,
duration_ms: 800,
tool_count: 12,
});
const events = getEvents(logger!);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_TOOL_COUNT,
'12',
]);
});
});
describe('logBrowserAgentVisionStatusEvent', () => {
it('logs vision enabled', () => {
const { logger } = setup();
logger?.logBrowserAgentVisionStatusEvent({ enabled: true });
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_VISION_STATUS);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED,
'true',
]);
});
it('logs vision disabled with reason', () => {
const { logger } = setup();
logger?.logBrowserAgentVisionStatusEvent({
enabled: false,
disabled_reason: 'no_visual_model',
});
const events = getEvents(logger!);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED,
'false',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_DISABLED_REASON,
'no_visual_model',
]);
});
});
describe('logBrowserAgentTaskOutcomeEvent', () => {
it('logs a task outcome event with all attributes', () => {
const { logger } = setup();
logger?.logBrowserAgentTaskOutcomeEvent({
success: true,
session_mode: 'isolated',
vision_enabled: true,
headless: true,
duration_ms: 5000,
});
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_TASK_OUTCOME);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
'true',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE,
'isolated',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED,
'true',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS,
'true',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS,
'5000',
]);
});
});
describe('logBrowserAgentCleanupEvent', () => {
it('logs a cleanup event with all attributes', () => {
const { logger } = setup();
logger?.logBrowserAgentCleanupEvent({
session_mode: 'isolated',
success: true,
duration_ms: 200,
});
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_CLEANUP);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE,
'isolated',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
'true',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS,
'200',
]);
});
it('logs a failed cleanup event', () => {
const { logger } = setup();
logger?.logBrowserAgentCleanupEvent({
session_mode: 'persistent',
success: false,
duration_ms: 5000,
});
const events = getEvents(logger!);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
'false',
]);
});
});
});
packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
+144
View File
@@ -135,6 +135,10 @@ export enum EventNames {
OVERAGE_OPTION_SELECTED = 'overage_option_selected',
EMPTY_WALLET_MENU_SHOWN = 'empty_wallet_menu_shown',
CREDIT_PURCHASE_CLICK = 'credit_purchase_click',
BROWSER_AGENT_CONNECTION = 'browser_agent_connection',
BROWSER_AGENT_VISION_STATUS = 'browser_agent_vision_status',
BROWSER_AGENT_TASK_OUTCOME = 'browser_agent_task_outcome',
BROWSER_AGENT_CLEANUP = 'browser_agent_cleanup',
}
export interface LogResponse {
@@ -1935,6 +1939,146 @@ export class ClearcutLogger {
this.flushIfNeeded();
}
// ==========================================================================
// Browser Agent Events
// ==========================================================================
logBrowserAgentConnectionEvent(attrs: {
session_mode: string;
headless: boolean;
success: boolean;
duration_ms: number;
error_type?: string;
tool_count?: number;
}): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE,
value: attrs.session_mode,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS,
value: attrs.headless.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
value: attrs.success.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS,
value: attrs.duration_ms.toString(),
},
];
if (attrs.error_type) {
data.push({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_ERROR_TYPE,
value: attrs.error_type,
});
}
if (attrs.tool_count !== undefined) {
data.push({
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_TOOL_COUNT,
value: attrs.tool_count.toString(),
});
}
this.enqueueLogEvent(
this.createLogEvent(EventNames.BROWSER_AGENT_CONNECTION, data),
);
this.flushIfNeeded();
}
logBrowserAgentVisionStatusEvent(attrs: {
enabled: boolean;
disabled_reason?: string;
}): void {
const data: EventValue[] = [
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED,
value: attrs.enabled.toString(),
},
];
if (attrs.disabled_reason) {
data.push({
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_DISABLED_REASON,
value: attrs.disabled_reason,
});
}
this.enqueueLogEvent(
this.createLogEvent(EventNames.BROWSER_AGENT_VISION_STATUS, data),
);
this.flushIfNeeded();
}
logBrowserAgentTaskOutcomeEvent(attrs: {
success: boolean;
session_mode: string;
vision_enabled: boolean;
headless: boolean;
duration_ms: number;
}): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
value: attrs.success.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE,
value: attrs.session_mode,
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED,
value: attrs.vision_enabled.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS,
value: attrs.headless.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS,
value: attrs.duration_ms.toString(),
},
];
this.enqueueLogEvent(
this.createLogEvent(EventNames.BROWSER_AGENT_TASK_OUTCOME, data),
);
this.flushIfNeeded();
}
logBrowserAgentCleanupEvent(attrs: {
session_mode: string;
success: boolean;
duration_ms: number;
}): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE,
value: attrs.session_mode,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS,
value: attrs.success.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS,
value: attrs.duration_ms.toString(),
},
];
this.enqueueLogEvent(
this.createLogEvent(EventNames.BROWSER_AGENT_CLEANUP, data),
);
this.flushIfNeeded();
}
/**
* Adds default fields to data, and returns a new data array. This fields
* should exist on all log events.
packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts
+29 -1
View File
@@ -7,7 +7,7 @@
// Defines valid event metadata keys for Clearcut logging.
export enum EventMetadataKey {
// Deleted enums: 24
// Next ID: 195
// Next ID: 203
GEMINI_CLI_KEY_UNKNOWN = 0,
@@ -725,4 +725,32 @@ export enum EventMetadataKey {
// Logs the duration of the onboarding process in milliseconds.
GEMINI_CLI_ONBOARDING_DURATION_MS = 194,
// ==========================================================================
// Browser Agent Event Keys
// ==========================================================================
// Logs the browser agent session mode (persistent, isolated, existing).
GEMINI_CLI_BROWSER_AGENT_SESSION_MODE = 195,
// Logs whether the browser agent ran in headless mode.
GEMINI_CLI_BROWSER_AGENT_HEADLESS = 196,
// Logs whether the browser agent operation was successful.
GEMINI_CLI_BROWSER_AGENT_SUCCESS = 197,
// Logs the error type for a browser agent connection failure.
GEMINI_CLI_BROWSER_AGENT_ERROR_TYPE = 198,
// Logs the duration in milliseconds for a browser agent operation.
GEMINI_CLI_BROWSER_AGENT_DURATION_MS = 199,
// Logs whether vision mode was enabled for the browser agent.
GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED = 200,
// Logs the reason vision mode was disabled for the browser agent.
GEMINI_CLI_BROWSER_AGENT_VISION_DISABLED_REASON = 201,
// Logs the number of tools discovered from the MCP server.
GEMINI_CLI_BROWSER_AGENT_TOOL_COUNT = 202,
}
packages/core/src/telemetry/loggers.ts
+91
View File
@@ -83,6 +83,10 @@ import {
recordInvalidChunk,
recordOnboardingStart,
recordOnboardingSuccess,
recordBrowserAgentConnection,
recordBrowserAgentVisionStatus,
recordBrowserAgentTaskOutcome,
recordBrowserAgentCleanup,
} from './metrics.js';
import { bufferTelemetryEvent } from './sdk.js';
import { uiTelemetryService, type UiEvent } from './uiTelemetry.js';
@@ -939,3 +943,90 @@ export function logBillingEvent(
}
}
}
// ==========================================================================
// Browser Agent Events
// ==========================================================================
export function logBrowserAgentConnection(
config: Config,
durationMs: number,
attributes: {
session_mode: 'persistent' | 'isolated' | 'existing';
headless: boolean;
success: boolean;
error_type?:
| 'profile_locked'
| 'timeout'
| 'connection_refused'
| 'unknown';
tool_count?: number;
},
): void {
ClearcutLogger.getInstance(config)?.logBrowserAgentConnectionEvent({
session_mode: attributes.session_mode,
headless: attributes.headless,
success: attributes.success,
duration_ms: durationMs,
error_type: attributes.error_type,
tool_count: attributes.tool_count,
});
recordBrowserAgentConnection(config, durationMs, attributes);
}
export function logBrowserAgentVisionStatus(
config: Config,
attributes: {
enabled: boolean;
disabled_reason?:
| 'no_visual_model'
| 'missing_visual_tools'
| 'blocked_auth_type';
},
): void {
ClearcutLogger.getInstance(config)?.logBrowserAgentVisionStatusEvent({
enabled: attributes.enabled,
disabled_reason: attributes.disabled_reason,
});
recordBrowserAgentVisionStatus(config, attributes);
}
export function logBrowserAgentTaskOutcome(
config: Config,
attributes: {
success: boolean;
session_mode: 'persistent' | 'isolated' | 'existing';
vision_enabled: boolean;
headless: boolean;
duration_ms: number;
},
): void {
ClearcutLogger.getInstance(config)?.logBrowserAgentTaskOutcomeEvent({
success: attributes.success,
session_mode: attributes.session_mode,
vision_enabled: attributes.vision_enabled,
headless: attributes.headless,
duration_ms: attributes.duration_ms,
});
recordBrowserAgentTaskOutcome(config, attributes);
}
export function logBrowserAgentCleanup(
config: Config,
durationMs: number,
attributes: {
session_mode: 'persistent' | 'isolated' | 'existing';
success: boolean;
},
): void {
ClearcutLogger.getInstance(config)?.logBrowserAgentCleanupEvent({
session_mode: attributes.session_mode,
success: attributes.success,
duration_ms: durationMs,
});
recordBrowserAgentCleanup(config, durationMs, attributes);
}
packages/core/src/telemetry/metrics.test.ts
+23
View File
@@ -1687,6 +1687,29 @@ describe('Telemetry Metrics', () => {
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records tool_count on success when provided', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordBrowserAgentConnectionModule(mockConfig, 1200, {
session_mode: 'isolated',
headless: false,
success: true,
tool_count: 5,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(1200, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'isolated',
headless: false,
success: true,
tool_count: 5,
});
});
it('records connection duration and failure counter on error', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
packages/core/src/telemetry/metrics.ts
+2
View File
@@ -1624,6 +1624,7 @@ export function recordBrowserAgentConnection(
| 'timeout'
| 'connection_refused'
| 'unknown';
tool_count?: number;
},
): void {
if (!isMetricsInitialized) return;
@@ -1635,6 +1636,7 @@ export function recordBrowserAgentConnection(
session_mode: attributes.session_mode,
headless: attributes.headless,
success: attributes.success,
tool_count: attributes.tool_count,
});
if (!attributes.success && browserAgentConnectionFailureCounter) {