Feat/browser agent metrics (#24210)

Co-authored-by: Gaurav Ghosh <gaghosh@google.com>
This commit is contained in:
Aditya Bijalwan
2026-04-03 13:51:09 +05:30
committed by GitHub
parent e54eecca51
commit 7a70ab9a5d
9 changed files with 1036 additions and 24 deletions
@@ -7,6 +7,7 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import {
createBrowserAgentDefinition,
cleanupBrowserAgent,
resetBrowserSession,
} from './browserAgentFactory.js';
import { injectAutomationOverlay } from './automationOverlay.js';
@@ -15,6 +16,12 @@ import { PolicyDecision, PRIORITY_SUBAGENT_TOOL } from '../../policy/types.js';
import type { Config } from '../../config/config.js';
import type { MessageBus } from '../../confirmation-bus/message-bus.js';
import type { PolicyEngine } from '../../policy/policy-engine.js';
import type { BrowserManager } from './browserManager.js';
import {
recordBrowserAgentToolDiscovery,
recordBrowserAgentVisionStatus,
recordBrowserAgentCleanup,
} from '../../telemetry/metrics.js';
// Create mock browser manager
const mockBrowserManager = {
@@ -58,6 +65,12 @@ vi.mock('../../utils/debugLogger.js', () => ({
},
}));
vi.mock('../../telemetry/metrics.js', () => ({
recordBrowserAgentToolDiscovery: vi.fn(),
recordBrowserAgentVisionStatus: vi.fn(),
recordBrowserAgentCleanup: vi.fn(),
}));
import {
buildBrowserSystemPrompt,
BROWSER_AGENT_NAME,
@@ -224,6 +237,11 @@ describe('browserAgentFactory', () => {
const systemPrompt = definition.promptConfig?.systemPrompt ?? '';
expect(systemPrompt).toContain('analyze_screenshot');
expect(systemPrompt).toContain('VISUAL IDENTIFICATION');
expect(recordBrowserAgentVisionStatus).toHaveBeenCalledWith(
configWithVision,
{ enabled: true, disabled_reason: undefined },
);
});
it('should include analyze_screenshot tool when visualModel is configured', async () => {
@@ -314,6 +332,47 @@ describe('browserAgentFactory', () => {
// Total: 9 MCP + 1 type_text (no analyze_screenshot without visualModel)
expect(definition.toolConfig?.tools).toHaveLength(10);
});
it('should trigger telemetry recording for tool discovery', async () => {
const configWithVision = makeFakeConfig({
agents: {
overrides: { browser_agent: { enabled: true } },
browser: { headless: false, visualModel: 'gemini-2.5-flash-preview' },
},
});
await createBrowserAgentDefinition(configWithVision, mockMessageBus);
expect(recordBrowserAgentToolDiscovery).toHaveBeenCalledWith(
configWithVision,
6, // 6 mock tools from getDiscoveredTools
[], // Empty because all required semantic tools present
'persistent',
);
});
it('should trigger telemetry recording for missing semantic tools', async () => {
mockBrowserManager.getDiscoveredTools.mockResolvedValueOnce([
{ name: 'take_snapshot', description: 'Take snapshot' },
// 'click', 'fill', 'navigate_page' are missing
]);
const configWithVision = makeFakeConfig({
agents: {
overrides: { browser_agent: { enabled: true } },
browser: { headless: false, visualModel: 'gemini-2.5-flash-preview' },
},
});
await createBrowserAgentDefinition(configWithVision, mockMessageBus);
expect(recordBrowserAgentToolDiscovery).toHaveBeenCalledWith(
configWithVision,
1, // 1 mock tool from getDiscoveredTools
['click', 'fill', 'navigate_page'],
'persistent',
);
});
});
describe('resetBrowserSession', () => {
@@ -452,6 +511,72 @@ describe('browserAgentFactory', () => {
);
});
});
describe('cleanupBrowserAgent', () => {
it('should call close on browser manager', async () => {
const mockConfig = makeFakeConfig({});
await cleanupBrowserAgent(
mockBrowserManager as unknown as BrowserManager,
mockConfig,
'persistent',
);
expect(mockBrowserManager.close).toHaveBeenCalled();
});
it('should handle errors during cleanup gracefully', async () => {
const errorManager = {
close: vi.fn().mockRejectedValue(new Error('Close failed')),
} as unknown as BrowserManager;
const mockConfig = makeFakeConfig({});
// Should not throw
await expect(
cleanupBrowserAgent(errorManager, mockConfig, 'persistent'),
).resolves.toBeUndefined();
});
it('should record successful cleanup metrics', async () => {
const mockConfig = makeFakeConfig({});
await cleanupBrowserAgent(
mockBrowserManager as unknown as BrowserManager,
mockConfig,
'isolated',
);
expect(mockBrowserManager.close).toHaveBeenCalled();
expect(recordBrowserAgentCleanup).toHaveBeenCalledWith(
mockConfig,
expect.any(Number),
{
session_mode: 'isolated',
success: true,
},
);
});
it('should record failed cleanup metrics when browserManager.close() throws', async () => {
const mockConfig = makeFakeConfig({});
mockBrowserManager.close.mockRejectedValueOnce(
new Error('Failed to close'),
);
await cleanupBrowserAgent(
mockBrowserManager as unknown as BrowserManager,
mockConfig,
'existing',
);
expect(recordBrowserAgentCleanup).toHaveBeenCalledWith(
mockConfig,
expect.any(Number),
{
session_mode: 'existing',
success: false,
},
);
});
});
});
describe('buildBrowserSystemPrompt', () => {
@@ -32,12 +32,27 @@ import { createAnalyzeScreenshotTool } from './analyzeScreenshot.js';
import { injectAutomationOverlay } from './automationOverlay.js';
import { injectInputBlocker } from './inputBlocker.js';
import { debugLogger } from '../../utils/debugLogger.js';
import {
recordBrowserAgentToolDiscovery,
recordBrowserAgentVisionStatus,
recordBrowserAgentCleanup,
} from '../../telemetry/metrics.js';
import {
PolicyDecision,
PRIORITY_SUBAGENT_TOOL,
type PolicyRule,
} from '../../policy/types.js';
/**
* Structured return type for vision disabled reasons.
* Separates the condition code from the human-readable message.
*/
type VisionDisabledReason =
| { code: 'no_visual_model'; message: string }
| { code: 'missing_visual_tools'; message: string }
| { code: 'blocked_auth_type'; message: string }
| undefined;
/**
* Creates a browser agent definition with MCP tools configured.
*
@@ -57,6 +72,8 @@ export async function createBrowserAgentDefinition(
): Promise<{
definition: LocalAgentDefinition<typeof BrowserTaskResultSchema>;
browserManager: BrowserManager;
visionEnabled: boolean;
sessionMode: 'persistent' | 'isolated' | 'existing';
}> {
debugLogger.log(
'Creating browser agent definition with isolated MCP tools...',
@@ -169,6 +186,20 @@ export async function createBrowserAgentDefinition(
const missingSemanticTools = requiredSemanticTools.filter(
(t) => !availableToolNames.includes(t),
);
const rawSessionMode = browserConfig?.customConfig?.sessionMode;
const sessionMode =
rawSessionMode === 'isolated' || rawSessionMode === 'existing'
? rawSessionMode
: 'persistent';
recordBrowserAgentToolDiscovery(
config,
mcpTools.length,
missingSemanticTools,
sessionMode,
);
if (missingSemanticTools.length > 0) {
debugLogger.warn(
`Semantic tools missing (${missingSemanticTools.join(', ')}). ` +
@@ -182,17 +213,22 @@ export async function createBrowserAgentDefinition(
(t) => !availableToolNames.includes(t),
);
// Check whether vision can be enabled; returns undefined if all gates pass.
function getVisionDisabledReason(): string | undefined {
// Check whether vision can be enabled; returns structured type with code and message.
function getVisionDisabledReason(): VisionDisabledReason {
const browserConfig = config.getBrowserAgentConfig();
if (!browserConfig.customConfig.visualModel) {
return 'No visualModel configured.';
return {
code: 'no_visual_model',
message: 'No visualModel configured.',
};
}
if (missingVisualTools.length > 0) {
return (
`Visual tools missing (${missingVisualTools.join(', ')}). ` +
`The installed chrome-devtools-mcp version may be too old.`
);
return {
code: 'missing_visual_tools',
message:
`Visual tools missing (${missingVisualTools.join(', ')}). ` +
`The installed chrome-devtools-mcp version may be too old.`,
};
}
const authType = config.getContentGeneratorConfig()?.authType;
const blockedAuthTypes = new Set([
@@ -201,7 +237,10 @@ export async function createBrowserAgentDefinition(
AuthType.COMPUTE_ADC,
]);
if (authType && blockedAuthTypes.has(authType)) {
return 'Visual agent model not available for current auth type.';
return {
code: 'blocked_auth_type',
message: 'Visual agent model not available for current auth type.',
};
}
return undefined;
}
@@ -209,8 +248,13 @@ export async function createBrowserAgentDefinition(
const allTools: AnyDeclarativeTool[] = [...mcpTools];
const visionDisabledReason = getVisionDisabledReason();
recordBrowserAgentVisionStatus(config, {
enabled: !visionDisabledReason,
disabled_reason: visionDisabledReason?.code,
});
if (visionDisabledReason) {
debugLogger.log(`Vision disabled: ${visionDisabledReason}`);
debugLogger.log(`Vision disabled: ${visionDisabledReason.message}`);
} else {
allTools.push(
createAnalyzeScreenshotTool(browserManager, config, messageBus),
@@ -232,12 +276,46 @@ export async function createBrowserAgentDefinition(
},
};
return { definition, browserManager };
return {
definition,
browserManager,
visionEnabled: !visionDisabledReason,
sessionMode,
};
}
/**
* Closes all persistent browser sessions and cleans up resources.
*
* @param browserManager The browser manager to clean up
* @param config Runtime configuration
* @param sessionMode The browser session mode
*/
export async function cleanupBrowserAgent(
browserManager: BrowserManager,
config: Config,
sessionMode: 'persistent' | 'isolated' | 'existing',
): Promise<void> {
const startMs = Date.now();
try {
await browserManager.close();
recordBrowserAgentCleanup(config, Date.now() - startMs, {
session_mode: sessionMode,
success: true,
});
debugLogger.log('Browser agent cleanup complete');
} catch (error) {
recordBrowserAgentCleanup(config, Date.now() - startMs, {
session_mode: sessionMode,
success: false,
});
debugLogger.error(
`Error during browser cleanup: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
/**
* Call this on /clear commands and CLI exit to reset browser state.
*/
export async function resetBrowserSession(): Promise<void> {
@@ -26,6 +26,7 @@ vi.mock('../../utils/debugLogger.js', () => ({
vi.mock('./browserAgentFactory.js', () => ({
createBrowserAgentDefinition: vi.fn(),
cleanupBrowserAgent: vi.fn(),
}));
vi.mock('./inputBlocker.js', () => ({
@@ -36,16 +37,24 @@ vi.mock('./automationOverlay.js', () => ({
removeAutomationOverlay: vi.fn(),
}));
vi.mock('../../telemetry/metrics.js', () => ({
recordBrowserAgentTaskOutcome: vi.fn(),
}));
vi.mock('../local-executor.js', () => ({
LocalAgentExecutor: {
create: vi.fn(),
},
}));
import { createBrowserAgentDefinition } from './browserAgentFactory.js';
import {
createBrowserAgentDefinition,
cleanupBrowserAgent,
} from './browserAgentFactory.js';
import { removeInputBlocker } from './inputBlocker.js';
import { removeAutomationOverlay } from './automationOverlay.js';
import { LocalAgentExecutor } from '../local-executor.js';
import { recordBrowserAgentTaskOutcome } from '../../telemetry/metrics.js';
import type { ToolLiveOutput } from '../../tools/tools.js';
describe('BrowserAgentInvocation', () => {
@@ -184,6 +193,8 @@ describe('BrowserAgentInvocation', () => {
toolConfig: { tools: ['analyze_screenshot', 'click'] },
},
browserManager: {} as never,
visionEnabled: true,
sessionMode: 'persistent',
});
mockExecutor = {
@@ -669,11 +680,12 @@ describe('BrowserAgentInvocation', () => {
.map((c) => c[0] as SubagentProgress)
.filter((p) => p.isSubagentProgress);
const allItems = progressCalls.flatMap((p) => p.recentActivity);
const toolA = allItems.find(
const finalActivity =
progressCalls[progressCalls.length - 1].recentActivity;
const toolA = finalActivity.find(
(a) => a.type === 'tool_call' && a.content === 'tool_a',
);
const toolB = allItems.find(
const toolB = finalActivity.find(
(a) => a.type === 'tool_call' && a.content === 'tool_b',
);
@@ -681,6 +693,69 @@ describe('BrowserAgentInvocation', () => {
expect(toolA?.status).toBe('error');
expect(toolB?.status).toBe('error');
});
it('should record successful task outcome metrics', async () => {
const invocation = new BrowserAgentInvocation(
mockConfig,
mockParams,
mockMessageBus,
);
await invocation.execute(new AbortController().signal, vi.fn());
expect(recordBrowserAgentTaskOutcome).toHaveBeenCalledWith(
mockConfig,
expect.objectContaining({
success: true,
session_mode: 'persistent',
vision_enabled: true,
headless: false,
duration_ms: expect.any(Number),
}),
);
});
it('should record failed task outcome metrics', async () => {
vi.mocked(LocalAgentExecutor.create).mockResolvedValue({
run: vi.fn().mockResolvedValue({
result: JSON.stringify({ success: false, foo: 'bar' }),
}),
} as never);
const updateOutput = vi.fn();
const invocation = new BrowserAgentInvocation(
mockConfig,
mockParams,
mockMessageBus,
);
await invocation.execute(new AbortController().signal, updateOutput);
expect(recordBrowserAgentTaskOutcome).toHaveBeenCalledWith(
mockConfig,
expect.objectContaining({
success: false,
session_mode: 'persistent',
vision_enabled: true,
headless: false,
duration_ms: expect.any(Number),
}),
);
});
it('should call cleanupBrowserAgent with correct params', async () => {
const invocation = new BrowserAgentInvocation(
mockConfig,
mockParams,
mockMessageBus,
);
await invocation.execute(new AbortController().signal, vi.fn());
expect(cleanupBrowserAgent).toHaveBeenCalledWith(
expect.anything(),
mockConfig,
'persistent',
);
});
});
describe('cleanup', () => {
@@ -711,6 +786,8 @@ describe('BrowserAgentInvocation', () => {
toolConfig: { tools: [] },
},
browserManager: mockBrowserManager as never,
visionEnabled: true,
sessionMode: 'persistent',
});
const mockExecutor = {
@@ -15,6 +15,7 @@
*/
import { randomUUID } from 'node:crypto';
import { debugLogger } from '../../utils/debugLogger.js';
import type { Config } from '../../config/config.js';
import { type AgentLoopContext } from '../../config/agent-loop-context.js';
import { LocalAgentExecutor } from '../local-executor.js';
@@ -33,8 +34,12 @@ import {
isToolActivityError,
} from '../types.js';
import type { MessageBus } from '../../confirmation-bus/message-bus.js';
import { createBrowserAgentDefinition } from './browserAgentFactory.js';
import {
createBrowserAgentDefinition,
cleanupBrowserAgent,
} from './browserAgentFactory.js';
import { removeInputBlocker } from './inputBlocker.js';
import { recordBrowserAgentTaskOutcome } from '../../telemetry/metrics.js';
import {
sanitizeThoughtContent,
sanitizeToolArgs,
@@ -109,8 +114,12 @@ export class BrowserAgentInvocation extends BaseToolInvocation<
signal: AbortSignal,
updateOutput?: (output: ToolLiveOutput) => void,
): Promise<ToolResult> {
const invocationStartMs = Date.now();
let browserManager;
let recentActivity: SubagentActivityItem[] = [];
let sessionMode: 'persistent' | 'isolated' | 'existing' = 'persistent';
let visionEnabled = false;
let taskSuccess = false;
try {
if (updateOutput) {
@@ -154,6 +163,8 @@ export class BrowserAgentInvocation extends BaseToolInvocation<
);
const { definition } = result;
browserManager = result.browserManager;
visionEnabled = result.visionEnabled;
sessionMode = result.sessionMode;
// Create activity callback for streaming output
const onActivity = (activity: SubagentActivityEvent): void => {
@@ -302,6 +313,19 @@ export class BrowserAgentInvocation extends BaseToolInvocation<
const output = await executor.run(this.params, signal);
try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const parsed = JSON.parse(output.result);
taskSuccess = parsed?.success === true;
} catch (parseError) {
// non-JSON result -> treat as unknown, default false
debugLogger.log(
'Failed to parse browser agent output as JSON:',
parseError,
);
}
const resultContent = `Browser agent finished.
Termination Reason: ${output.terminate_reason}
Result:
@@ -376,6 +400,14 @@ ${output.result}`;
},
};
} finally {
recordBrowserAgentTaskOutcome(this.config, {
success: taskSuccess,
session_mode: sessionMode,
vision_enabled: visionEnabled,
headless: !!this.config.getBrowserAgentConfig().customConfig.headless,
duration_ms: Date.now() - invocationStartMs,
});
// Clean up input blocker, but keep browserManager alive for persistent sessions
if (browserManager) {
await removeInputBlocker(browserManager, signal);
@@ -412,6 +444,7 @@ ${output.result}`;
} catch {
// Ignore errors for removing the overlays.
}
await cleanupBrowserAgent(browserManager, this.config, sessionMode);
}
}
}
@@ -46,6 +46,10 @@ vi.mock('../../utils/debugLogger.js', () => ({
},
}));
vi.mock('../../telemetry/metrics.js', () => ({
recordBrowserAgentConnection: vi.fn(),
}));
// Mock browser consent to always grant consent by default
vi.mock('../../utils/browserConsent.js', () => ({
getBrowserConsentIfNeeded: vi.fn().mockResolvedValue(true),
@@ -78,6 +82,7 @@ vi.mock('node:fs', async (importOriginal) => {
import * as fs from 'node:fs';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import { recordBrowserAgentConnection } from '../../telemetry/metrics.js';
import { getBrowserConsentIfNeeded } from '../../utils/browserConsent.js';
import { debugLogger } from '../../utils/debugLogger.js';
@@ -355,6 +360,21 @@ describe('BrowserManager', () => {
});
describe('MCP connection', () => {
it('should record connection success metrics', async () => {
const manager = new BrowserManager(mockConfig);
await manager.ensureConnection();
expect(recordBrowserAgentConnection).toHaveBeenCalledWith(
mockConfig,
expect.any(Number),
{
session_mode: 'persistent',
headless: false,
success: true,
},
);
});
it('should spawn npx chrome-devtools-mcp with --experimental-vision (persistent mode by default)', async () => {
const manager = new BrowserManager(mockConfig);
await manager.ensureConnection();
@@ -546,6 +566,18 @@ describe('BrowserManager', () => {
await expect(manager.ensureConnection()).rejects.toThrow(
/Failed to connect to existing Chrome instance/,
);
expect(recordBrowserAgentConnection).toHaveBeenCalledWith(
existingConfig,
expect.any(Number),
{
session_mode: 'existing',
headless: false,
success: false,
error_type: 'connection_refused',
},
);
// Create a fresh manager to verify the error message includes remediation steps
const manager2 = new BrowserManager(existingConfig);
await expect(manager2.ensureConnection()).rejects.toThrow(
@@ -576,6 +608,18 @@ describe('BrowserManager', () => {
await expect(manager.ensureConnection()).rejects.toThrow(
/Close all Chrome windows using this profile/,
);
expect(recordBrowserAgentConnection).toHaveBeenCalledWith(
mockConfig,
expect.any(Number),
{
session_mode: 'persistent',
headless: false,
success: false,
error_type: 'profile_locked',
},
);
const manager2 = new BrowserManager(mockConfig);
await expect(manager2.ensureConnection()).rejects.toThrow(
/Set sessionMode to "isolated"/,
@@ -602,6 +646,17 @@ describe('BrowserManager', () => {
await expect(manager.ensureConnection()).rejects.toThrow(
/Chrome is not installed/,
);
expect(recordBrowserAgentConnection).toHaveBeenCalledWith(
mockConfig,
expect.any(Number),
{
session_mode: 'persistent',
headless: false,
success: false,
error_type: 'timeout',
},
);
});
it('should include sessionMode in generic fallback error', async () => {
@@ -622,6 +677,61 @@ describe('BrowserManager', () => {
await expect(manager.ensureConnection()).rejects.toThrow(
/sessionMode: persistent/,
);
expect(recordBrowserAgentConnection).toHaveBeenCalledWith(
mockConfig,
expect.any(Number),
{
session_mode: 'persistent',
headless: false,
success: false,
error_type: 'unknown',
},
);
});
it('should classify non-connection-refused errors in existing mode as unknown', async () => {
vi.mocked(Client).mockImplementation(
() =>
({
connect: vi
.fn()
.mockRejectedValue(new Error('Some unexpected error')),
close: vi.fn().mockResolvedValue(undefined),
listTools: vi.fn(),
callTool: vi.fn(),
}) as unknown as InstanceType<typeof Client>,
);
const existingConfig = makeFakeConfig({
agents: {
overrides: {
browser_agent: {
enabled: true,
},
},
browser: {
sessionMode: 'existing',
},
},
});
const manager = new BrowserManager(existingConfig);
await expect(manager.ensureConnection()).rejects.toThrow(
/Failed to connect to existing Chrome instance/,
);
expect(recordBrowserAgentConnection).toHaveBeenCalledWith(
existingConfig,
expect.any(Number),
{
session_mode: 'existing',
headless: false,
success: false,
error_type: 'unknown',
},
);
});
it('should pass --no-usage-statistics and --no-performance-crux when privacy is disabled', async () => {
@@ -30,6 +30,7 @@ import * as path from 'node:path';
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { injectAutomationOverlay } from './automationOverlay.js';
import { recordBrowserAgentConnection } from '../../telemetry/metrics.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@@ -486,7 +487,11 @@ export class BrowserManager {
// Build args for chrome-devtools-mcp
const browserConfig = this.config.getBrowserAgentConfig();
let sessionMode = browserConfig.customConfig.sessionMode ?? 'persistent';
const rawSessionMode = browserConfig.customConfig.sessionMode;
let sessionMode: 'persistent' | 'isolated' | 'existing' =
rawSessionMode === 'isolated' || rawSessionMode === 'existing'
? rawSessionMode
: 'persistent';
// Detect sandbox environment.
// SANDBOX env var is set to 'sandbox-exec' (seatbelt) or the container
@@ -652,6 +657,7 @@ export class BrowserManager {
sessionMode === 'existing' ? 15_000 : MCP_TIMEOUT_MS;
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const connectStartMs = Date.now();
try {
await Promise.race([
(async () => {
@@ -660,6 +666,16 @@ export class BrowserManager {
await this.discoverTools();
// clear the action counter for each connection
this.actionCounter = 0;
recordBrowserAgentConnection(
this.config,
Date.now() - connectStartMs,
{
session_mode: sessionMode,
headless: !!browserConfig.customConfig.headless,
success: true,
},
);
})(),
new Promise<never>((_, reject) => {
timeoutId = setTimeout(
@@ -676,11 +692,19 @@ export class BrowserManager {
} catch (error) {
await this.close();
const rawErrorMessage =
error instanceof Error ? error.message : String(error);
const errorType = BrowserManager.classifyConnectionError(rawErrorMessage);
recordBrowserAgentConnection(this.config, Date.now() - connectStartMs, {
session_mode: sessionMode,
headless: !!browserConfig.customConfig.headless,
success: false,
error_type: errorType,
});
// Provide error-specific, session-mode-aware remediation
throw this.createConnectionError(
error instanceof Error ? error.message : String(error),
sessionMode,
);
throw this.createConnectionError(rawErrorMessage, sessionMode);
} finally {
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
@@ -688,15 +712,34 @@ export class BrowserManager {
}
}
/**
* Classifies a connection error message into a known error type.
* Shared between connectMcp error recording and createConnectionError
* to ensure consistent error categorization across the browser agent.
*/
private static classifyConnectionError(
message: string,
): 'profile_locked' | 'timeout' | 'connection_refused' | 'unknown' {
const lowerMessage = message.toLowerCase();
if (lowerMessage.includes('already running')) {
return 'profile_locked';
} else if (lowerMessage.includes('timed out')) {
return 'timeout';
} else if (lowerMessage.includes('connection refused')) {
return 'connection_refused';
}
return 'unknown';
}
/**
* Creates an Error with context-specific remediation based on the actual
* error message and the current sessionMode.
*/
private createConnectionError(message: string, sessionMode: string): Error {
const lowerMessage = message.toLowerCase();
const errorType = BrowserManager.classifyConnectionError(message);
// "already running for the current profile" — persistent mode profile lock
if (lowerMessage.includes('already running')) {
if (errorType === 'profile_locked') {
if (sessionMode === 'persistent' || sessionMode === 'isolated') {
return new Error(
`Could not connect to Chrome: ${message}\n\n` +
@@ -716,7 +759,7 @@ export class BrowserManager {
}
// Timeout errors
if (lowerMessage.includes('timed out')) {
if (errorType === 'timeout') {
if (sessionMode === 'existing') {
return new Error(
`Timed out connecting to Chrome: ${message}\n\n` +
+3 -1
View File
@@ -4,6 +4,7 @@
* SPDX-License-Identifier: Apache-2.0
*/
import { randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
@@ -84,7 +85,8 @@ export class ProjectRegistry {
try {
const content = JSON.stringify(data, null, 2);
const tmpPath = `${this.registryPath}.tmp`;
// Use a randomized tmp path to avoid ENOENT crashes when save() is called concurrently
const tmpPath = this.registryPath + '.' + randomUUID() + '.tmp';
await fs.promises.writeFile(tmpPath, content, 'utf8');
await fs.promises.rename(tmpPath, this.registryPath);
} catch (error) {
+272
View File
@@ -107,6 +107,11 @@ describe('Telemetry Metrics', () => {
let recordKeychainAvailabilityModule: typeof import('./metrics.js').recordKeychainAvailability;
let recordTokenStorageInitializationModule: typeof import('./metrics.js').recordTokenStorageInitialization;
let recordInvalidChunkModule: typeof import('./metrics.js').recordInvalidChunk;
let recordBrowserAgentConnectionModule: typeof import('./metrics.js').recordBrowserAgentConnection;
let recordBrowserAgentToolDiscoveryModule: typeof import('./metrics.js').recordBrowserAgentToolDiscovery;
let recordBrowserAgentVisionStatusModule: typeof import('./metrics.js').recordBrowserAgentVisionStatus;
let recordBrowserAgentTaskOutcomeModule: typeof import('./metrics.js').recordBrowserAgentTaskOutcome;
let recordBrowserAgentCleanupModule: typeof import('./metrics.js').recordBrowserAgentCleanup;
beforeEach(async () => {
vi.resetModules();
@@ -158,6 +163,15 @@ describe('Telemetry Metrics', () => {
recordTokenStorageInitializationModule =
metricsJsModule.recordTokenStorageInitialization;
recordInvalidChunkModule = metricsJsModule.recordInvalidChunk;
recordBrowserAgentConnectionModule =
metricsJsModule.recordBrowserAgentConnection;
recordBrowserAgentToolDiscoveryModule =
metricsJsModule.recordBrowserAgentToolDiscovery;
recordBrowserAgentVisionStatusModule =
metricsJsModule.recordBrowserAgentVisionStatus;
recordBrowserAgentTaskOutcomeModule =
metricsJsModule.recordBrowserAgentTaskOutcome;
recordBrowserAgentCleanupModule = metricsJsModule.recordBrowserAgentCleanup;
const otelApiModule = await import('@opentelemetry/api');
@@ -1632,4 +1646,262 @@ describe('Telemetry Metrics', () => {
});
});
});
describe('Browser Agent Metrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
describe('recordBrowserAgentConnection', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordBrowserAgentConnectionModule(config, 1500, {
session_mode: 'persistent',
headless: true,
success: true,
});
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records connection duration on success', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordBrowserAgentConnectionModule(mockConfig, 1200, {
session_mode: 'isolated',
headless: false,
success: true,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(1200, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'isolated',
headless: false,
success: true,
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records connection duration and failure counter on error', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordBrowserAgentConnectionModule(mockConfig, 3000, {
session_mode: 'existing',
headless: true,
success: false,
error_type: 'timeout',
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(3000, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'existing',
headless: true,
success: false,
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'existing',
headless: true,
error_type: 'timeout',
});
});
});
describe('recordBrowserAgentToolDiscovery', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordBrowserAgentToolDiscoveryModule(config, 5, [], 'persistent');
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records tool count and missing tools', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordBrowserAgentToolDiscoveryModule(
mockConfig,
3,
['click', 'type'],
'isolated',
);
expect(mockHistogramRecordFn).toHaveBeenCalledWith(3, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'isolated',
});
expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
tool_name: 'click',
});
expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
tool_name: 'type',
});
});
});
describe('recordBrowserAgentVisionStatus', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordBrowserAgentVisionStatusModule(config, { enabled: true });
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records vision enabled status', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordBrowserAgentVisionStatusModule(mockConfig, { enabled: true });
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
enabled: true,
});
});
it('records vision disabled status with reason', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
recordBrowserAgentVisionStatusModule(mockConfig, {
enabled: false,
disabled_reason: 'no_visual_model',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
enabled: false,
disabled_reason: 'no_visual_model',
});
});
});
describe('recordBrowserAgentTaskOutcome', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordBrowserAgentTaskOutcomeModule(config, {
success: true,
session_mode: 'persistent',
vision_enabled: true,
headless: true,
duration_ms: 5000,
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('records task outcome and duration', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordBrowserAgentTaskOutcomeModule(mockConfig, {
success: false,
session_mode: 'existing',
vision_enabled: false,
headless: false,
duration_ms: 8500,
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
success: false,
session_mode: 'existing',
vision_enabled: false,
headless: false,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(8500, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
success: false,
session_mode: 'existing',
});
});
});
describe('recordBrowserAgentCleanup', () => {
it('does not record metrics if not initialized', () => {
const config = makeFakeConfig({});
recordBrowserAgentCleanupModule(config, 100, {
session_mode: 'isolated',
success: true,
});
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records cleanup duration on success', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordBrowserAgentCleanupModule(mockConfig, 50, {
session_mode: 'persistent',
success: true,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(50, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'persistent',
});
expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('records cleanup duration and failure counter on error', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
recordBrowserAgentCleanupModule(mockConfig, 300, {
session_mode: 'existing',
success: false,
});
expect(mockHistogramRecordFn).toHaveBeenCalledWith(300, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'existing',
});
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
session_mode: 'existing',
});
});
});
});
});
+272
View File
@@ -63,6 +63,23 @@ const AGENT_RECOVERY_ATTEMPT_COUNT = 'gemini_cli.agent.recovery_attempt.count';
const AGENT_RECOVERY_ATTEMPT_DURATION =
'gemini_cli.agent.recovery_attempt.duration';
// Browser Agent Metrics
const BROWSER_AGENT_CONNECTION_DURATION =
'gemini_cli.browser_agent.connection.duration';
const BROWSER_AGENT_CONNECTION_FAILURE_COUNT =
'gemini_cli.browser_agent.connection.failure.count';
const BROWSER_AGENT_TOOLS_DISCOVERED =
'gemini_cli.browser_agent.tools.discovered';
const BROWSER_AGENT_TOOLS_MISSING_SEMANTIC =
'gemini_cli.browser_agent.tools.missing_semantic';
const BROWSER_AGENT_VISION_STATUS = 'gemini_cli.browser_agent.vision.status';
const BROWSER_AGENT_TASK_OUTCOME = 'gemini_cli.browser_agent.task.outcome';
const BROWSER_AGENT_TASK_DURATION = 'gemini_cli.browser_agent.task.duration';
const BROWSER_AGENT_CLEANUP_DURATION =
'gemini_cli.browser_agent.cleanup.duration';
const BROWSER_AGENT_CLEANUP_FAILURE_COUNT =
'gemini_cli.browser_agent.cleanup.failure.count';
// OpenTelemetry GenAI Semantic Convention Metrics
const GEN_AI_CLIENT_TOKEN_USAGE = 'gen_ai.client.token.usage';
const GEN_AI_CLIENT_OPERATION_DURATION = 'gen_ai.client.operation.duration';
@@ -302,6 +319,62 @@ const COUNTER_DEFINITIONS = {
model: string;
},
},
[BROWSER_AGENT_CONNECTION_FAILURE_COUNT]: {
description: 'Counts browser agent MCP connection failures.',
valueType: ValueType.INT,
assign: (c: Counter) => (browserAgentConnectionFailureCounter = c),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
session_mode: 'persistent' | 'isolated' | 'existing';
headless: boolean;
error_type:
| 'profile_locked'
| 'timeout'
| 'connection_refused'
| 'unknown';
},
},
[BROWSER_AGENT_TOOLS_MISSING_SEMANTIC]: {
description: 'Counts missing required semantic tools discovered from MCP.',
valueType: ValueType.INT,
assign: (c: Counter) => (browserAgentToolsMissingSemanticCounter = c),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as { tool_name: string },
},
[BROWSER_AGENT_VISION_STATUS]: {
description: 'Counts browser agent invocations by vision status.',
valueType: ValueType.INT,
assign: (c: Counter) => (browserAgentVisionStatusCounter = c),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
enabled: boolean;
disabled_reason?:
| 'no_visual_model'
| 'missing_visual_tools'
| 'blocked_auth_type';
},
},
[BROWSER_AGENT_TASK_OUTCOME]: {
description: 'Counts browser agent task outcomes.',
valueType: ValueType.INT,
assign: (c: Counter) => (browserAgentTaskOutcomeCounter = c),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
success: boolean;
session_mode: 'persistent' | 'isolated' | 'existing';
vision_enabled: boolean;
headless: boolean;
},
},
[BROWSER_AGENT_CLEANUP_FAILURE_COUNT]: {
description: 'Counts browser agent cleanup failures.',
valueType: ValueType.INT,
assign: (c: Counter) => (browserAgentCleanupFailureCounter = c),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
session_mode: 'persistent' | 'isolated' | 'existing';
},
},
[EVENT_ONBOARDING_START]: {
description: 'Counts onboarding started',
valueType: ValueType.INT,
@@ -431,6 +504,51 @@ const HISTOGRAM_DEFINITIONS = {
success: boolean;
},
},
[BROWSER_AGENT_CONNECTION_DURATION]: {
description:
'Duration of browser agent MCP connection setup in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (browserAgentConnectionDurationHistogram = h),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
session_mode: 'persistent' | 'isolated' | 'existing';
headless: boolean;
success: boolean;
},
},
[BROWSER_AGENT_TOOLS_DISCOVERED]: {
description: 'Count of tools discovered from chrome-devtools-mcp.',
unit: 'tools',
valueType: ValueType.INT,
assign: (h: Histogram) => (browserAgentToolsDiscoveredHistogram = h),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
session_mode: 'persistent' | 'isolated' | 'existing';
},
},
[BROWSER_AGENT_TASK_DURATION]: {
description:
'Full invocation duration of browser agent (connect + run + cleanup) in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (browserAgentTaskDurationHistogram = h),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
success: boolean;
session_mode: 'persistent' | 'isolated' | 'existing';
},
},
[BROWSER_AGENT_CLEANUP_DURATION]: {
description: 'Duration of browser agent cleanup in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (browserAgentCleanupDurationHistogram = h),
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
attributes: {} as {
session_mode: 'persistent' | 'isolated' | 'existing';
},
},
[EVENT_ONBOARDING_DURATION_MS]: {
description: 'Duration of onboarding in milliseconds.',
unit: 'ms',
@@ -670,6 +788,16 @@ let onboardingStartCounter: Counter | undefined;
let onboardingSuccessCounter: Counter | undefined;
let onboardingDurationHistogram: Histogram | undefined;
let browserAgentConnectionDurationHistogram: Histogram | undefined;
let browserAgentConnectionFailureCounter: Counter | undefined;
let browserAgentToolsDiscoveredHistogram: Histogram | undefined;
let browserAgentToolsMissingSemanticCounter: Counter | undefined;
let browserAgentVisionStatusCounter: Counter | undefined;
let browserAgentTaskOutcomeCounter: Counter | undefined;
let browserAgentTaskDurationHistogram: Histogram | undefined;
let browserAgentCleanupDurationHistogram: Histogram | undefined;
let browserAgentCleanupFailureCounter: Counter | undefined;
// OpenTelemetry GenAI Semantic Convention Metrics
let genAiClientTokenUsageHistogram: Histogram | undefined;
let genAiClientOperationDurationHistogram: Histogram | undefined;
@@ -1483,3 +1611,147 @@ export function recordCreditPurchaseClick(
...attributes,
});
}
export function recordBrowserAgentConnection(
config: Config,
durationMs: number,
attributes: {
session_mode: 'persistent' | 'isolated' | 'existing';
headless: boolean;
success: boolean;
error_type?:
| 'profile_locked'
| 'timeout'
| 'connection_refused'
| 'unknown';
},
): void {
if (!isMetricsInitialized) return;
if (!browserAgentConnectionDurationHistogram) return;
const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
browserAgentConnectionDurationHistogram.record(durationMs, {
...commonAttribs,
session_mode: attributes.session_mode,
headless: attributes.headless,
success: attributes.success,
});
if (!attributes.success && browserAgentConnectionFailureCounter) {
browserAgentConnectionFailureCounter.add(1, {
...commonAttribs,
session_mode: attributes.session_mode,
headless: attributes.headless,
error_type: attributes.error_type ?? 'unknown',
});
}
}
export function recordBrowserAgentToolDiscovery(
config: Config,
toolCount: number,
missingSemanticTools: string[],
sessionMode: 'persistent' | 'isolated' | 'existing',
): void {
if (!isMetricsInitialized) return;
const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
if (browserAgentToolsDiscoveredHistogram) {
browserAgentToolsDiscoveredHistogram.record(toolCount, {
...commonAttribs,
session_mode: sessionMode,
});
}
if (browserAgentToolsMissingSemanticCounter) {
for (const tool of missingSemanticTools) {
browserAgentToolsMissingSemanticCounter.add(1, {
...commonAttribs,
tool_name: tool,
});
}
}
}
export function recordBrowserAgentVisionStatus(
config: Config,
attributes: {
enabled: boolean;
disabled_reason?:
| 'no_visual_model'
| 'missing_visual_tools'
| 'blocked_auth_type';
},
): void {
if (!isMetricsInitialized || !browserAgentVisionStatusCounter) return;
const metricAttributes: Record<string, string | number | boolean> = {
...baseMetricDefinition.getCommonAttributes(config),
enabled: attributes.enabled,
};
if (attributes.disabled_reason) {
metricAttributes['disabled_reason'] = attributes.disabled_reason;
}
browserAgentVisionStatusCounter.add(1, metricAttributes);
}
export function recordBrowserAgentTaskOutcome(
config: Config,
attributes: {
success: boolean;
session_mode: 'persistent' | 'isolated' | 'existing';
vision_enabled: boolean;
headless: boolean;
duration_ms: number;
},
): void {
if (!isMetricsInitialized) return;
const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
if (browserAgentTaskOutcomeCounter) {
browserAgentTaskOutcomeCounter.add(1, {
...commonAttribs,
success: attributes.success,
session_mode: attributes.session_mode,
vision_enabled: attributes.vision_enabled,
headless: attributes.headless,
});
}
if (browserAgentTaskDurationHistogram) {
browserAgentTaskDurationHistogram.record(attributes.duration_ms, {
...commonAttribs,
success: attributes.success,
session_mode: attributes.session_mode,
});
}
}
export function recordBrowserAgentCleanup(
config: Config,
durationMs: number,
attributes: {
session_mode: 'persistent' | 'isolated' | 'existing';
success: boolean;
},
): void {
if (!isMetricsInitialized) return;
const commonAttribs = baseMetricDefinition.getCommonAttributes(config);
if (browserAgentCleanupDurationHistogram) {
browserAgentCleanupDurationHistogram.record(durationMs, {
...commonAttribs,
session_mode: attributes.session_mode,
});
}
if (!attributes.success && browserAgentCleanupFailureCounter) {
browserAgentCleanupFailureCounter.add(1, {
...commonAttribs,
session_mode: attributes.session_mode,
});
}
}