diff --git a/packages/core/src/agents/browser/browserAgentFactory.ts b/packages/core/src/agents/browser/browserAgentFactory.ts index b341ce6836..e07f403ba7 100644 --- a/packages/core/src/agents/browser/browserAgentFactory.ts +++ b/packages/core/src/agents/browser/browserAgentFactory.ts @@ -32,11 +32,11 @@ import { createAnalyzeScreenshotTool } from './analyzeScreenshot.js'; import { injectAutomationOverlay } from './automationOverlay.js'; import { injectInputBlocker } from './inputBlocker.js'; import { debugLogger } from '../../utils/debugLogger.js'; +import { recordBrowserAgentToolDiscovery } from '../../telemetry/metrics.js'; import { - recordBrowserAgentToolDiscovery, - recordBrowserAgentVisionStatus, - recordBrowserAgentCleanup, -} from '../../telemetry/metrics.js'; + logBrowserAgentVisionStatus, + logBrowserAgentCleanup, +} from '../../telemetry/loggers.js'; import { PolicyDecision, PRIORITY_SUBAGENT_TOOL, @@ -248,7 +248,7 @@ export async function createBrowserAgentDefinition( const allTools: AnyDeclarativeTool[] = [...mcpTools]; const visionDisabledReason = getVisionDisabledReason(); - recordBrowserAgentVisionStatus(config, { + logBrowserAgentVisionStatus(config, { enabled: !visionDisabledReason, disabled_reason: visionDisabledReason?.code, }); @@ -299,13 +299,13 @@ export async function cleanupBrowserAgent( const startMs = Date.now(); try { await browserManager.close(); - recordBrowserAgentCleanup(config, Date.now() - startMs, { + logBrowserAgentCleanup(config, Date.now() - startMs, { session_mode: sessionMode, success: true, }); debugLogger.log('Browser agent cleanup complete'); } catch (error) { - recordBrowserAgentCleanup(config, Date.now() - startMs, { + logBrowserAgentCleanup(config, Date.now() - startMs, { session_mode: sessionMode, success: false, }); diff --git a/packages/core/src/agents/browser/browserAgentInvocation.ts b/packages/core/src/agents/browser/browserAgentInvocation.ts index f40ea90632..6fb05753ee 100644 --- a/packages/core/src/agents/browser/browserAgentInvocation.ts +++ b/packages/core/src/agents/browser/browserAgentInvocation.ts @@ -36,7 +36,7 @@ import { import type { MessageBus } from '../../confirmation-bus/message-bus.js'; import { createBrowserAgentDefinition } from './browserAgentFactory.js'; import { removeInputBlocker } from './inputBlocker.js'; -import { recordBrowserAgentTaskOutcome } from '../../telemetry/metrics.js'; +import { logBrowserAgentTaskOutcome } from '../../telemetry/loggers.js'; import { sanitizeThoughtContent, sanitizeToolArgs, @@ -397,7 +397,7 @@ ${output.result}`; }, }; } finally { - recordBrowserAgentTaskOutcome(this.config, { + logBrowserAgentTaskOutcome(this.config, { success: taskSuccess, session_mode: sessionMode, vision_enabled: visionEnabled, diff --git a/packages/core/src/agents/browser/browserManager.test.ts b/packages/core/src/agents/browser/browserManager.test.ts index 8ddcf1836d..baabc80bcb 100644 --- a/packages/core/src/agents/browser/browserManager.test.ts +++ b/packages/core/src/agents/browser/browserManager.test.ts @@ -373,6 +373,7 @@ describe('BrowserManager', () => { session_mode: 'persistent', headless: false, success: true, + tool_count: 4, }, ); }); diff --git a/packages/core/src/agents/browser/browserManager.ts b/packages/core/src/agents/browser/browserManager.ts index 08e4cc2ae9..89d54e9c72 100644 --- a/packages/core/src/agents/browser/browserManager.ts +++ b/packages/core/src/agents/browser/browserManager.ts @@ -30,7 +30,7 @@ import * as path from 'node:path'; import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; import { injectAutomationOverlay } from './automationOverlay.js'; -import { recordBrowserAgentConnection } from '../../telemetry/metrics.js'; +import { logBrowserAgentConnection } from '../../telemetry/loggers.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -563,7 +563,9 @@ export class BrowserManager { // Add optional settings from config. // Force headless in seatbelt sandbox since Chrome profile/display access // may be restricted, and the user is running in a sandboxed environment. - if (browserConfig.customConfig.headless || isSeatbeltSandbox) { + const effectiveHeadless = + !!browserConfig.customConfig.headless || isSeatbeltSandbox; + if (effectiveHeadless) { mcpArgs.push('--headless'); } if (browserConfig.customConfig.profilePath) { @@ -667,15 +669,12 @@ export class BrowserManager { // clear the action counter for each connection this.actionCounter = 0; - recordBrowserAgentConnection( - this.config, - Date.now() - connectStartMs, - { - session_mode: sessionMode, - headless: !!browserConfig.customConfig.headless, - success: true, - }, - ); + logBrowserAgentConnection(this.config, Date.now() - connectStartMs, { + session_mode: sessionMode, + headless: effectiveHeadless, + success: true, + tool_count: this.discoveredTools.length, + }); })(), new Promise((_, reject) => { timeoutId = setTimeout( @@ -696,9 +695,9 @@ export class BrowserManager { error instanceof Error ? error.message : String(error); const errorType = BrowserManager.classifyConnectionError(rawErrorMessage); - recordBrowserAgentConnection(this.config, Date.now() - connectStartMs, { + logBrowserAgentConnection(this.config, Date.now() - connectStartMs, { session_mode: sessionMode, - headless: !!browserConfig.customConfig.headless, + headless: effectiveHeadless, success: false, error_type: errorType, }); diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts index de1aaeb32f..292588c7a1 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts @@ -1692,4 +1692,187 @@ describe('ClearcutLogger', () => { ]); }); }); + + describe('logBrowserAgentConnectionEvent', () => { + it('logs a successful connection event', () => { + const { logger } = setup(); + logger?.logBrowserAgentConnectionEvent({ + session_mode: 'isolated', + headless: true, + success: true, + duration_ms: 1500, + }); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_CONNECTION); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE, + 'isolated', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS, + 'true', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + 'true', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS, + '1500', + ]); + }); + + it('logs a failed connection event with error_type', () => { + const { logger } = setup(); + logger?.logBrowserAgentConnectionEvent({ + session_mode: 'persistent', + headless: false, + success: false, + duration_ms: 30000, + error_type: 'timeout', + }); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + 'false', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_ERROR_TYPE, + 'timeout', + ]); + }); + + it('logs tool_count when provided', () => { + const { logger } = setup(); + logger?.logBrowserAgentConnectionEvent({ + session_mode: 'existing', + headless: true, + success: true, + duration_ms: 800, + tool_count: 12, + }); + + const events = getEvents(logger!); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_TOOL_COUNT, + '12', + ]); + }); + }); + + describe('logBrowserAgentVisionStatusEvent', () => { + it('logs vision enabled', () => { + const { logger } = setup(); + logger?.logBrowserAgentVisionStatusEvent({ enabled: true }); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_VISION_STATUS); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED, + 'true', + ]); + }); + + it('logs vision disabled with reason', () => { + const { logger } = setup(); + logger?.logBrowserAgentVisionStatusEvent({ + enabled: false, + disabled_reason: 'no_visual_model', + }); + + const events = getEvents(logger!); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED, + 'false', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_DISABLED_REASON, + 'no_visual_model', + ]); + }); + }); + + describe('logBrowserAgentTaskOutcomeEvent', () => { + it('logs a task outcome event with all attributes', () => { + const { logger } = setup(); + logger?.logBrowserAgentTaskOutcomeEvent({ + success: true, + session_mode: 'isolated', + vision_enabled: true, + headless: true, + duration_ms: 5000, + }); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_TASK_OUTCOME); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + 'true', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE, + 'isolated', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED, + 'true', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS, + 'true', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS, + '5000', + ]); + }); + }); + + describe('logBrowserAgentCleanupEvent', () => { + it('logs a cleanup event with all attributes', () => { + const { logger } = setup(); + logger?.logBrowserAgentCleanupEvent({ + session_mode: 'isolated', + success: true, + duration_ms: 200, + }); + + const events = getEvents(logger!); + expect(events.length).toBe(1); + expect(events[0]).toHaveEventName(EventNames.BROWSER_AGENT_CLEANUP); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE, + 'isolated', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + 'true', + ]); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS, + '200', + ]); + }); + + it('logs a failed cleanup event', () => { + const { logger } = setup(); + logger?.logBrowserAgentCleanupEvent({ + session_mode: 'persistent', + success: false, + duration_ms: 5000, + }); + + const events = getEvents(logger!); + expect(events[0]).toHaveMetadataValue([ + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + 'false', + ]); + }); + }); }); diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index 2915edf712..a5896d57f3 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -135,6 +135,10 @@ export enum EventNames { OVERAGE_OPTION_SELECTED = 'overage_option_selected', EMPTY_WALLET_MENU_SHOWN = 'empty_wallet_menu_shown', CREDIT_PURCHASE_CLICK = 'credit_purchase_click', + BROWSER_AGENT_CONNECTION = 'browser_agent_connection', + BROWSER_AGENT_VISION_STATUS = 'browser_agent_vision_status', + BROWSER_AGENT_TASK_OUTCOME = 'browser_agent_task_outcome', + BROWSER_AGENT_CLEANUP = 'browser_agent_cleanup', } export interface LogResponse { @@ -1935,6 +1939,146 @@ export class ClearcutLogger { this.flushIfNeeded(); } + // ========================================================================== + // Browser Agent Events + // ========================================================================== + + logBrowserAgentConnectionEvent(attrs: { + session_mode: string; + headless: boolean; + success: boolean; + duration_ms: number; + error_type?: string; + tool_count?: number; + }): void { + const data: EventValue[] = [ + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE, + value: attrs.session_mode, + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS, + value: attrs.headless.toString(), + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + value: attrs.success.toString(), + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS, + value: attrs.duration_ms.toString(), + }, + ]; + + if (attrs.error_type) { + data.push({ + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_ERROR_TYPE, + value: attrs.error_type, + }); + } + + if (attrs.tool_count !== undefined) { + data.push({ + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_TOOL_COUNT, + value: attrs.tool_count.toString(), + }); + } + + this.enqueueLogEvent( + this.createLogEvent(EventNames.BROWSER_AGENT_CONNECTION, data), + ); + this.flushIfNeeded(); + } + + logBrowserAgentVisionStatusEvent(attrs: { + enabled: boolean; + disabled_reason?: string; + }): void { + const data: EventValue[] = [ + { + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED, + value: attrs.enabled.toString(), + }, + ]; + + if (attrs.disabled_reason) { + data.push({ + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_DISABLED_REASON, + value: attrs.disabled_reason, + }); + } + + this.enqueueLogEvent( + this.createLogEvent(EventNames.BROWSER_AGENT_VISION_STATUS, data), + ); + this.flushIfNeeded(); + } + + logBrowserAgentTaskOutcomeEvent(attrs: { + success: boolean; + session_mode: string; + vision_enabled: boolean; + headless: boolean; + duration_ms: number; + }): void { + const data: EventValue[] = [ + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + value: attrs.success.toString(), + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE, + value: attrs.session_mode, + }, + { + gemini_cli_key: + EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED, + value: attrs.vision_enabled.toString(), + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_HEADLESS, + value: attrs.headless.toString(), + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS, + value: attrs.duration_ms.toString(), + }, + ]; + + this.enqueueLogEvent( + this.createLogEvent(EventNames.BROWSER_AGENT_TASK_OUTCOME, data), + ); + this.flushIfNeeded(); + } + + logBrowserAgentCleanupEvent(attrs: { + session_mode: string; + success: boolean; + duration_ms: number; + }): void { + const data: EventValue[] = [ + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SESSION_MODE, + value: attrs.session_mode, + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_SUCCESS, + value: attrs.success.toString(), + }, + { + gemini_cli_key: EventMetadataKey.GEMINI_CLI_BROWSER_AGENT_DURATION_MS, + value: attrs.duration_ms.toString(), + }, + ]; + + this.enqueueLogEvent( + this.createLogEvent(EventNames.BROWSER_AGENT_CLEANUP, data), + ); + this.flushIfNeeded(); + } + /** * Adds default fields to data, and returns a new data array. This fields * should exist on all log events. diff --git a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts index b5688a3e65..b9e7b2b75c 100644 --- a/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts +++ b/packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts @@ -7,7 +7,7 @@ // Defines valid event metadata keys for Clearcut logging. export enum EventMetadataKey { // Deleted enums: 24 - // Next ID: 195 + // Next ID: 203 GEMINI_CLI_KEY_UNKNOWN = 0, @@ -725,4 +725,32 @@ export enum EventMetadataKey { // Logs the duration of the onboarding process in milliseconds. GEMINI_CLI_ONBOARDING_DURATION_MS = 194, + + // ========================================================================== + // Browser Agent Event Keys + // ========================================================================== + + // Logs the browser agent session mode (persistent, isolated, existing). + GEMINI_CLI_BROWSER_AGENT_SESSION_MODE = 195, + + // Logs whether the browser agent ran in headless mode. + GEMINI_CLI_BROWSER_AGENT_HEADLESS = 196, + + // Logs whether the browser agent operation was successful. + GEMINI_CLI_BROWSER_AGENT_SUCCESS = 197, + + // Logs the error type for a browser agent connection failure. + GEMINI_CLI_BROWSER_AGENT_ERROR_TYPE = 198, + + // Logs the duration in milliseconds for a browser agent operation. + GEMINI_CLI_BROWSER_AGENT_DURATION_MS = 199, + + // Logs whether vision mode was enabled for the browser agent. + GEMINI_CLI_BROWSER_AGENT_VISION_ENABLED = 200, + + // Logs the reason vision mode was disabled for the browser agent. + GEMINI_CLI_BROWSER_AGENT_VISION_DISABLED_REASON = 201, + + // Logs the number of tools discovered from the MCP server. + GEMINI_CLI_BROWSER_AGENT_TOOL_COUNT = 202, } diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index a33c8ca200..a3c3cb48ee 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -83,6 +83,10 @@ import { recordInvalidChunk, recordOnboardingStart, recordOnboardingSuccess, + recordBrowserAgentConnection, + recordBrowserAgentVisionStatus, + recordBrowserAgentTaskOutcome, + recordBrowserAgentCleanup, } from './metrics.js'; import { bufferTelemetryEvent } from './sdk.js'; import { uiTelemetryService, type UiEvent } from './uiTelemetry.js'; @@ -939,3 +943,90 @@ export function logBillingEvent( } } } + +// ========================================================================== +// Browser Agent Events +// ========================================================================== + +export function logBrowserAgentConnection( + config: Config, + durationMs: number, + attributes: { + session_mode: 'persistent' | 'isolated' | 'existing'; + headless: boolean; + success: boolean; + error_type?: + | 'profile_locked' + | 'timeout' + | 'connection_refused' + | 'unknown'; + tool_count?: number; + }, +): void { + ClearcutLogger.getInstance(config)?.logBrowserAgentConnectionEvent({ + session_mode: attributes.session_mode, + headless: attributes.headless, + success: attributes.success, + duration_ms: durationMs, + error_type: attributes.error_type, + tool_count: attributes.tool_count, + }); + + recordBrowserAgentConnection(config, durationMs, attributes); +} + +export function logBrowserAgentVisionStatus( + config: Config, + attributes: { + enabled: boolean; + disabled_reason?: + | 'no_visual_model' + | 'missing_visual_tools' + | 'blocked_auth_type'; + }, +): void { + ClearcutLogger.getInstance(config)?.logBrowserAgentVisionStatusEvent({ + enabled: attributes.enabled, + disabled_reason: attributes.disabled_reason, + }); + + recordBrowserAgentVisionStatus(config, attributes); +} + +export function logBrowserAgentTaskOutcome( + config: Config, + attributes: { + success: boolean; + session_mode: 'persistent' | 'isolated' | 'existing'; + vision_enabled: boolean; + headless: boolean; + duration_ms: number; + }, +): void { + ClearcutLogger.getInstance(config)?.logBrowserAgentTaskOutcomeEvent({ + success: attributes.success, + session_mode: attributes.session_mode, + vision_enabled: attributes.vision_enabled, + headless: attributes.headless, + duration_ms: attributes.duration_ms, + }); + + recordBrowserAgentTaskOutcome(config, attributes); +} + +export function logBrowserAgentCleanup( + config: Config, + durationMs: number, + attributes: { + session_mode: 'persistent' | 'isolated' | 'existing'; + success: boolean; + }, +): void { + ClearcutLogger.getInstance(config)?.logBrowserAgentCleanupEvent({ + session_mode: attributes.session_mode, + success: attributes.success, + duration_ms: durationMs, + }); + + recordBrowserAgentCleanup(config, durationMs, attributes); +} diff --git a/packages/core/src/telemetry/metrics.test.ts b/packages/core/src/telemetry/metrics.test.ts index c3d16f977e..0bca699b16 100644 --- a/packages/core/src/telemetry/metrics.test.ts +++ b/packages/core/src/telemetry/metrics.test.ts @@ -1687,6 +1687,29 @@ describe('Telemetry Metrics', () => { expect(mockCounterAddFn).not.toHaveBeenCalled(); }); + it('records tool_count on success when provided', () => { + initializeMetricsModule(mockConfig); + mockCounterAddFn.mockClear(); + mockHistogramRecordFn.mockClear(); + + recordBrowserAgentConnectionModule(mockConfig, 1200, { + session_mode: 'isolated', + headless: false, + success: true, + tool_count: 5, + }); + + expect(mockHistogramRecordFn).toHaveBeenCalledWith(1200, { + 'session.id': 'test-session-id', + 'installation.id': 'test-installation-id', + 'user.email': 'test@example.com', + session_mode: 'isolated', + headless: false, + success: true, + tool_count: 5, + }); + }); + it('records connection duration and failure counter on error', () => { initializeMetricsModule(mockConfig); mockCounterAddFn.mockClear(); diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts index 5c2fedfbed..422f0222a5 100644 --- a/packages/core/src/telemetry/metrics.ts +++ b/packages/core/src/telemetry/metrics.ts @@ -1624,6 +1624,7 @@ export function recordBrowserAgentConnection( | 'timeout' | 'connection_refused' | 'unknown'; + tool_count?: number; }, ): void { if (!isMetricsInitialized) return; @@ -1635,6 +1636,7 @@ export function recordBrowserAgentConnection( session_mode: attributes.session_mode, headless: attributes.headless, success: attributes.success, + tool_count: attributes.tool_count, }); if (!attributes.success && browserAgentConnectionFailureCounter) {