Feat/browser agent metrics (#24210)

Co-authored-by: Gaurav Ghosh <gaghosh@google.com>
2026-07-11 02:20:48 -07:00 · 2026-04-03 13:51:09 +05:30
parent 29ac77f7cc
commit 4b735cf134
9 changed files with 1036 additions and 24 deletions
@@ -107,6 +107,11 @@ describe('Telemetry Metrics', () => {
  let recordKeychainAvailabilityModule: typeof import('./metrics.js').recordKeychainAvailability;
  let recordTokenStorageInitializationModule: typeof import('./metrics.js').recordTokenStorageInitialization;
  let recordInvalidChunkModule: typeof import('./metrics.js').recordInvalidChunk;
+  let recordBrowserAgentConnectionModule: typeof import('./metrics.js').recordBrowserAgentConnection;
+  let recordBrowserAgentToolDiscoveryModule: typeof import('./metrics.js').recordBrowserAgentToolDiscovery;
+  let recordBrowserAgentVisionStatusModule: typeof import('./metrics.js').recordBrowserAgentVisionStatus;
+  let recordBrowserAgentTaskOutcomeModule: typeof import('./metrics.js').recordBrowserAgentTaskOutcome;
+  let recordBrowserAgentCleanupModule: typeof import('./metrics.js').recordBrowserAgentCleanup;

  beforeEach(async () => {
    vi.resetModules();
@@ -158,6 +163,15 @@ describe('Telemetry Metrics', () => {
    recordTokenStorageInitializationModule =
      metricsJsModule.recordTokenStorageInitialization;
    recordInvalidChunkModule = metricsJsModule.recordInvalidChunk;
+    recordBrowserAgentConnectionModule =
+      metricsJsModule.recordBrowserAgentConnection;
+    recordBrowserAgentToolDiscoveryModule =
+      metricsJsModule.recordBrowserAgentToolDiscovery;
+    recordBrowserAgentVisionStatusModule =
+      metricsJsModule.recordBrowserAgentVisionStatus;
+    recordBrowserAgentTaskOutcomeModule =
+      metricsJsModule.recordBrowserAgentTaskOutcome;
+    recordBrowserAgentCleanupModule = metricsJsModule.recordBrowserAgentCleanup;

    const otelApiModule = await import('@opentelemetry/api');

@@ -1632,4 +1646,262 @@ describe('Telemetry Metrics', () => {
      });
    });
  });
+
+  describe('Browser Agent Metrics', () => {
+    const mockConfig = {
+      getSessionId: () => 'test-session-id',
+      getTelemetryEnabled: () => true,
+    } as unknown as Config;
+
+    describe('recordBrowserAgentConnection', () => {
+      it('does not record metrics if not initialized', () => {
+        const config = makeFakeConfig({});
+        recordBrowserAgentConnectionModule(config, 1500, {
+          session_mode: 'persistent',
+          headless: true,
+          success: true,
+        });
+        expect(mockHistogramRecordFn).not.toHaveBeenCalled();
+        expect(mockCounterAddFn).not.toHaveBeenCalled();
+      });
+
+      it('records connection duration on success', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+        mockHistogramRecordFn.mockClear();
+
+        recordBrowserAgentConnectionModule(mockConfig, 1200, {
+          session_mode: 'isolated',
+          headless: false,
+          success: true,
+        });
+
+        expect(mockHistogramRecordFn).toHaveBeenCalledWith(1200, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          session_mode: 'isolated',
+          headless: false,
+          success: true,
+        });
+        expect(mockCounterAddFn).not.toHaveBeenCalled();
+      });
+
+      it('records connection duration and failure counter on error', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+        mockHistogramRecordFn.mockClear();
+
+        recordBrowserAgentConnectionModule(mockConfig, 3000, {
+          session_mode: 'existing',
+          headless: true,
+          success: false,
+          error_type: 'timeout',
+        });
+
+        expect(mockHistogramRecordFn).toHaveBeenCalledWith(3000, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          session_mode: 'existing',
+          headless: true,
+          success: false,
+        });
+        expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          session_mode: 'existing',
+          headless: true,
+          error_type: 'timeout',
+        });
+      });
+    });
+
+    describe('recordBrowserAgentToolDiscovery', () => {
+      it('does not record metrics if not initialized', () => {
+        const config = makeFakeConfig({});
+        recordBrowserAgentToolDiscoveryModule(config, 5, [], 'persistent');
+        expect(mockHistogramRecordFn).not.toHaveBeenCalled();
+        expect(mockCounterAddFn).not.toHaveBeenCalled();
+      });
+
+      it('records tool count and missing tools', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+        mockHistogramRecordFn.mockClear();
+
+        recordBrowserAgentToolDiscoveryModule(
+          mockConfig,
+          3,
+          ['click', 'type'],
+          'isolated',
+        );
+
+        expect(mockHistogramRecordFn).toHaveBeenCalledWith(3, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          session_mode: 'isolated',
+        });
+
+        expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
+        expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          tool_name: 'click',
+        });
+        expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          tool_name: 'type',
+        });
+      });
+    });
+
+    describe('recordBrowserAgentVisionStatus', () => {
+      it('does not record metrics if not initialized', () => {
+        const config = makeFakeConfig({});
+        recordBrowserAgentVisionStatusModule(config, { enabled: true });
+        expect(mockCounterAddFn).not.toHaveBeenCalled();
+      });
+
+      it('records vision enabled status', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+
+        recordBrowserAgentVisionStatusModule(mockConfig, { enabled: true });
+
+        expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          enabled: true,
+        });
+      });
+
+      it('records vision disabled status with reason', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+
+        recordBrowserAgentVisionStatusModule(mockConfig, {
+          enabled: false,
+          disabled_reason: 'no_visual_model',
+        });
+
+        expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          enabled: false,
+          disabled_reason: 'no_visual_model',
+        });
+      });
+    });
+
+    describe('recordBrowserAgentTaskOutcome', () => {
+      it('does not record metrics if not initialized', () => {
+        const config = makeFakeConfig({});
+        recordBrowserAgentTaskOutcomeModule(config, {
+          success: true,
+          session_mode: 'persistent',
+          vision_enabled: true,
+          headless: true,
+          duration_ms: 5000,
+        });
+        expect(mockCounterAddFn).not.toHaveBeenCalled();
+        expect(mockHistogramRecordFn).not.toHaveBeenCalled();
+      });
+
+      it('records task outcome and duration', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+        mockHistogramRecordFn.mockClear();
+
+        recordBrowserAgentTaskOutcomeModule(mockConfig, {
+          success: false,
+          session_mode: 'existing',
+          vision_enabled: false,
+          headless: false,
+          duration_ms: 8500,
+        });
+
+        expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          success: false,
+          session_mode: 'existing',
+          vision_enabled: false,
+          headless: false,
+        });
+
+        expect(mockHistogramRecordFn).toHaveBeenCalledWith(8500, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          success: false,
+          session_mode: 'existing',
+        });
+      });
+    });
+
+    describe('recordBrowserAgentCleanup', () => {
+      it('does not record metrics if not initialized', () => {
+        const config = makeFakeConfig({});
+        recordBrowserAgentCleanupModule(config, 100, {
+          session_mode: 'isolated',
+          success: true,
+        });
+        expect(mockHistogramRecordFn).not.toHaveBeenCalled();
+        expect(mockCounterAddFn).not.toHaveBeenCalled();
+      });
+
+      it('records cleanup duration on success', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+        mockHistogramRecordFn.mockClear();
+
+        recordBrowserAgentCleanupModule(mockConfig, 50, {
+          session_mode: 'persistent',
+          success: true,
+        });
+
+        expect(mockHistogramRecordFn).toHaveBeenCalledWith(50, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          session_mode: 'persistent',
+        });
+        expect(mockCounterAddFn).not.toHaveBeenCalled();
+      });
+
+      it('records cleanup duration and failure counter on error', () => {
+        initializeMetricsModule(mockConfig);
+        mockCounterAddFn.mockClear();
+        mockHistogramRecordFn.mockClear();
+
+        recordBrowserAgentCleanupModule(mockConfig, 300, {
+          session_mode: 'existing',
+          success: false,
+        });
+
+        expect(mockHistogramRecordFn).toHaveBeenCalledWith(300, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          session_mode: 'existing',
+        });
+
+        expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
+          'session.id': 'test-session-id',
+          'installation.id': 'test-installation-id',
+          'user.email': 'test@example.com',
+          session_mode: 'existing',
+        });
+      });
+    });
+  });
 });
@@ -63,6 +63,23 @@ const AGENT_RECOVERY_ATTEMPT_COUNT = 'gemini_cli.agent.recovery_attempt.count';
 const AGENT_RECOVERY_ATTEMPT_DURATION =
  'gemini_cli.agent.recovery_attempt.duration';

+// Browser Agent Metrics
+const BROWSER_AGENT_CONNECTION_DURATION =
+  'gemini_cli.browser_agent.connection.duration';
+const BROWSER_AGENT_CONNECTION_FAILURE_COUNT =
+  'gemini_cli.browser_agent.connection.failure.count';
+const BROWSER_AGENT_TOOLS_DISCOVERED =
+  'gemini_cli.browser_agent.tools.discovered';
+const BROWSER_AGENT_TOOLS_MISSING_SEMANTIC =
+  'gemini_cli.browser_agent.tools.missing_semantic';
+const BROWSER_AGENT_VISION_STATUS = 'gemini_cli.browser_agent.vision.status';
+const BROWSER_AGENT_TASK_OUTCOME = 'gemini_cli.browser_agent.task.outcome';
+const BROWSER_AGENT_TASK_DURATION = 'gemini_cli.browser_agent.task.duration';
+const BROWSER_AGENT_CLEANUP_DURATION =
+  'gemini_cli.browser_agent.cleanup.duration';
+const BROWSER_AGENT_CLEANUP_FAILURE_COUNT =
+  'gemini_cli.browser_agent.cleanup.failure.count';
+
 // OpenTelemetry GenAI Semantic Convention Metrics
 const GEN_AI_CLIENT_TOKEN_USAGE = 'gen_ai.client.token.usage';
 const GEN_AI_CLIENT_OPERATION_DURATION = 'gen_ai.client.operation.duration';
@@ -302,6 +319,62 @@ const COUNTER_DEFINITIONS = {
      model: string;
    },
  },
+  [BROWSER_AGENT_CONNECTION_FAILURE_COUNT]: {
+    description: 'Counts browser agent MCP connection failures.',
+    valueType: ValueType.INT,
+    assign: (c: Counter) => (browserAgentConnectionFailureCounter = c),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      session_mode: 'persistent' | 'isolated' | 'existing';
+      headless: boolean;
+      error_type:
+        | 'profile_locked'
+        | 'timeout'
+        | 'connection_refused'
+        | 'unknown';
+    },
+  },
+  [BROWSER_AGENT_TOOLS_MISSING_SEMANTIC]: {
+    description: 'Counts missing required semantic tools discovered from MCP.',
+    valueType: ValueType.INT,
+    assign: (c: Counter) => (browserAgentToolsMissingSemanticCounter = c),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as { tool_name: string },
+  },
+  [BROWSER_AGENT_VISION_STATUS]: {
+    description: 'Counts browser agent invocations by vision status.',
+    valueType: ValueType.INT,
+    assign: (c: Counter) => (browserAgentVisionStatusCounter = c),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      enabled: boolean;
+      disabled_reason?:
+        | 'no_visual_model'
+        | 'missing_visual_tools'
+        | 'blocked_auth_type';
+    },
+  },
+  [BROWSER_AGENT_TASK_OUTCOME]: {
+    description: 'Counts browser agent task outcomes.',
+    valueType: ValueType.INT,
+    assign: (c: Counter) => (browserAgentTaskOutcomeCounter = c),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      success: boolean;
+      session_mode: 'persistent' | 'isolated' | 'existing';
+      vision_enabled: boolean;
+      headless: boolean;
+    },
+  },
+  [BROWSER_AGENT_CLEANUP_FAILURE_COUNT]: {
+    description: 'Counts browser agent cleanup failures.',
+    valueType: ValueType.INT,
+    assign: (c: Counter) => (browserAgentCleanupFailureCounter = c),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      session_mode: 'persistent' | 'isolated' | 'existing';
+    },
+  },
  [EVENT_ONBOARDING_START]: {
    description: 'Counts onboarding started',
    valueType: ValueType.INT,
@@ -431,6 +504,51 @@ const HISTOGRAM_DEFINITIONS = {
      success: boolean;
    },
  },
+  [BROWSER_AGENT_CONNECTION_DURATION]: {
+    description:
+      'Duration of browser agent MCP connection setup in milliseconds.',
+    unit: 'ms',
+    valueType: ValueType.INT,
+    assign: (h: Histogram) => (browserAgentConnectionDurationHistogram = h),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      session_mode: 'persistent' | 'isolated' | 'existing';
+      headless: boolean;
+      success: boolean;
+    },
+  },
+  [BROWSER_AGENT_TOOLS_DISCOVERED]: {
+    description: 'Count of tools discovered from chrome-devtools-mcp.',
+    unit: 'tools',
+    valueType: ValueType.INT,
+    assign: (h: Histogram) => (browserAgentToolsDiscoveredHistogram = h),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      session_mode: 'persistent' | 'isolated' | 'existing';
+    },
+  },
+  [BROWSER_AGENT_TASK_DURATION]: {
+    description:
+      'Full invocation duration of browser agent (connect + run + cleanup) in milliseconds.',
+    unit: 'ms',
+    valueType: ValueType.INT,
+    assign: (h: Histogram) => (browserAgentTaskDurationHistogram = h),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      success: boolean;
+      session_mode: 'persistent' | 'isolated' | 'existing';
+    },
+  },
+  [BROWSER_AGENT_CLEANUP_DURATION]: {
+    description: 'Duration of browser agent cleanup in milliseconds.',
+    unit: 'ms',
+    valueType: ValueType.INT,
+    assign: (h: Histogram) => (browserAgentCleanupDurationHistogram = h),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      session_mode: 'persistent' | 'isolated' | 'existing';
+    },
+  },
  [EVENT_ONBOARDING_DURATION_MS]: {
    description: 'Duration of onboarding in milliseconds.',
    unit: 'ms',
@@ -670,6 +788,16 @@ let onboardingStartCounter: Counter | undefined;
 let onboardingSuccessCounter: Counter | undefined;
 let onboardingDurationHistogram: Histogram | undefined;

+let browserAgentConnectionDurationHistogram: Histogram | undefined;
+let browserAgentConnectionFailureCounter: Counter | undefined;
+let browserAgentToolsDiscoveredHistogram: Histogram | undefined;
+let browserAgentToolsMissingSemanticCounter: Counter | undefined;
+let browserAgentVisionStatusCounter: Counter | undefined;
+let browserAgentTaskOutcomeCounter: Counter | undefined;
+let browserAgentTaskDurationHistogram: Histogram | undefined;
+let browserAgentCleanupDurationHistogram: Histogram | undefined;
+let browserAgentCleanupFailureCounter: Counter | undefined;
+
 // OpenTelemetry GenAI Semantic Convention Metrics
 let genAiClientTokenUsageHistogram: Histogram | undefined;
 let genAiClientOperationDurationHistogram: Histogram | undefined;
@@ -1483,3 +1611,147 @@ export function recordCreditPurchaseClick(
    ...attributes,
  });
 }
+
+export function recordBrowserAgentConnection(
+  config: Config,
+  durationMs: number,
+  attributes: {
+    session_mode: 'persistent' | 'isolated' | 'existing';
+    headless: boolean;
+    success: boolean;
+    error_type?:
+      | 'profile_locked'
+      | 'timeout'
+      | 'connection_refused'
+      | 'unknown';
+  },
+): void {
+  if (!isMetricsInitialized) return;
+  if (!browserAgentConnectionDurationHistogram) return;
+
+  const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
+  browserAgentConnectionDurationHistogram.record(durationMs, {
+    ...commonAttribs,
+    session_mode: attributes.session_mode,
+    headless: attributes.headless,
+    success: attributes.success,
+  });
+
+  if (!attributes.success && browserAgentConnectionFailureCounter) {
+    browserAgentConnectionFailureCounter.add(1, {
+      ...commonAttribs,
+      session_mode: attributes.session_mode,
+      headless: attributes.headless,
+      error_type: attributes.error_type ?? 'unknown',
+    });
+  }
+}
+
+export function recordBrowserAgentToolDiscovery(
+  config: Config,
+  toolCount: number,
+  missingSemanticTools: string[],
+  sessionMode: 'persistent' | 'isolated' | 'existing',
+): void {
+  if (!isMetricsInitialized) return;
+
+  const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
+  if (browserAgentToolsDiscoveredHistogram) {
+    browserAgentToolsDiscoveredHistogram.record(toolCount, {
+      ...commonAttribs,
+      session_mode: sessionMode,
+    });
+  }
+
+  if (browserAgentToolsMissingSemanticCounter) {
+    for (const tool of missingSemanticTools) {
+      browserAgentToolsMissingSemanticCounter.add(1, {
+        ...commonAttribs,
+        tool_name: tool,
+      });
+    }
+  }
+}
+
+export function recordBrowserAgentVisionStatus(
+  config: Config,
+  attributes: {
+    enabled: boolean;
+    disabled_reason?:
+      | 'no_visual_model'
+      | 'missing_visual_tools'
+      | 'blocked_auth_type';
+  },
+): void {
+  if (!isMetricsInitialized || !browserAgentVisionStatusCounter) return;
+
+  const metricAttributes: Record<string, string | number | boolean> = {
+    ...baseMetricDefinition.getCommonAttributes(config),
+    enabled: attributes.enabled,
+  };
+  if (attributes.disabled_reason) {
+    metricAttributes['disabled_reason'] = attributes.disabled_reason;
+  }
+
+  browserAgentVisionStatusCounter.add(1, metricAttributes);
+}
+
+export function recordBrowserAgentTaskOutcome(
+  config: Config,
+  attributes: {
+    success: boolean;
+    session_mode: 'persistent' | 'isolated' | 'existing';
+    vision_enabled: boolean;
+    headless: boolean;
+    duration_ms: number;
+  },
+): void {
+  if (!isMetricsInitialized) return;
+
+  const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
+
+  if (browserAgentTaskOutcomeCounter) {
+    browserAgentTaskOutcomeCounter.add(1, {
+      ...commonAttribs,
+      success: attributes.success,
+      session_mode: attributes.session_mode,
+      vision_enabled: attributes.vision_enabled,
+      headless: attributes.headless,
+    });
+  }
+
+  if (browserAgentTaskDurationHistogram) {
+    browserAgentTaskDurationHistogram.record(attributes.duration_ms, {
+      ...commonAttribs,
+      success: attributes.success,
+      session_mode: attributes.session_mode,
+    });
+  }
+}
+
+export function recordBrowserAgentCleanup(
+  config: Config,
+  durationMs: number,
+  attributes: {
+    session_mode: 'persistent' | 'isolated' | 'existing';
+    success: boolean;
+  },
+): void {
+  if (!isMetricsInitialized) return;
+
+  const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
+
+  if (browserAgentCleanupDurationHistogram) {
+    browserAgentCleanupDurationHistogram.record(durationMs, {
+      ...commonAttribs,
+      session_mode: attributes.session_mode,
+    });
+  }
+
+  if (!attributes.success && browserAgentCleanupFailureCounter) {
+    browserAgentCleanupFailureCounter.add(1, {
+      ...commonAttribs,
+      session_mode: attributes.session_mode,
+    });
+  }
+}