mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-04-29 22:44:45 -07:00
feat(browser): implement input blocker overlay during automation (#21132)
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Gaurav <39389231+gsquared94@users.noreply.github.com> Co-authored-by: Gaurav Ghosh <gaghosh@google.com>
This commit is contained in:
@@ -30,6 +30,23 @@ import {
|
||||
import type { MessageBus } from '../../confirmation-bus/message-bus.js';
|
||||
import type { BrowserManager, McpToolCallResult } from './browserManager.js';
|
||||
import { debugLogger } from '../../utils/debugLogger.js';
|
||||
import { suspendInputBlocker, resumeInputBlocker } from './inputBlocker.js';
|
||||
|
||||
/**
|
||||
* Tools that interact with page elements and require the input blocker
|
||||
* overlay to be temporarily SUSPENDED (pointer-events: none) so
|
||||
* chrome-devtools-mcp's interactability checks pass. The overlay
|
||||
* stays in the DOM — only the CSS property toggles, zero flickering.
|
||||
*/
|
||||
const INTERACTIVE_TOOLS = new Set([
|
||||
'click',
|
||||
'click_at',
|
||||
'fill',
|
||||
'fill_form',
|
||||
'hover',
|
||||
'drag',
|
||||
'upload_file',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Tool invocation that dispatches to BrowserManager's isolated MCP client.
|
||||
@@ -43,6 +60,7 @@ class McpToolInvocation extends BaseToolInvocation<
|
||||
protected readonly toolName: string,
|
||||
params: Record<string, unknown>,
|
||||
messageBus: MessageBus,
|
||||
private readonly shouldDisableInput: boolean,
|
||||
) {
|
||||
super(params, messageBus, toolName, toolName);
|
||||
}
|
||||
@@ -78,16 +96,29 @@ class McpToolInvocation extends BaseToolInvocation<
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this specific tool needs the input blocker suspended
|
||||
* (pointer-events toggled to 'none') before execution.
|
||||
*/
|
||||
private get needsBlockerSuspend(): boolean {
|
||||
return this.shouldDisableInput && INTERACTIVE_TOOLS.has(this.toolName);
|
||||
}
|
||||
|
||||
async execute(signal: AbortSignal): Promise<ToolResult> {
|
||||
try {
|
||||
const callToolPromise = this.browserManager.callTool(
|
||||
// Suspend the input blocker for interactive tools so
|
||||
// chrome-devtools-mcp's interactability checks pass.
|
||||
// Only toggles pointer-events CSS — no DOM change, no flicker.
|
||||
if (this.needsBlockerSuspend) {
|
||||
await suspendInputBlocker(this.browserManager);
|
||||
}
|
||||
|
||||
const result: McpToolCallResult = await this.browserManager.callTool(
|
||||
this.toolName,
|
||||
this.params,
|
||||
signal,
|
||||
);
|
||||
|
||||
const result: McpToolCallResult = await callToolPromise;
|
||||
|
||||
// Extract text content from MCP response
|
||||
let textContent = '';
|
||||
if (result.content && Array.isArray(result.content)) {
|
||||
@@ -103,6 +134,11 @@ class McpToolInvocation extends BaseToolInvocation<
|
||||
textContent,
|
||||
);
|
||||
|
||||
// Resume input blocker after interactive tool completes.
|
||||
if (this.needsBlockerSuspend) {
|
||||
await resumeInputBlocker(this.browserManager);
|
||||
}
|
||||
|
||||
if (result.isError) {
|
||||
return {
|
||||
llmContent: `Error: ${processedContent}`,
|
||||
@@ -124,6 +160,11 @@ class McpToolInvocation extends BaseToolInvocation<
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Resume on error path too so the blocker is always restored
|
||||
if (this.needsBlockerSuspend) {
|
||||
await resumeInputBlocker(this.browserManager).catch(() => {});
|
||||
}
|
||||
|
||||
debugLogger.error(`MCP tool ${this.toolName} failed: ${errorMsg}`);
|
||||
return {
|
||||
llmContent: `Error: ${errorMsg}`,
|
||||
@@ -285,6 +326,7 @@ class McpDeclarativeTool extends DeclarativeTool<
|
||||
description: string,
|
||||
parameterSchema: unknown,
|
||||
messageBus: MessageBus,
|
||||
private readonly shouldDisableInput: boolean,
|
||||
) {
|
||||
super(
|
||||
name,
|
||||
@@ -306,6 +348,7 @@ class McpDeclarativeTool extends DeclarativeTool<
|
||||
this.name,
|
||||
params,
|
||||
this.messageBus,
|
||||
this.shouldDisableInput,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -385,12 +428,14 @@ class TypeTextDeclarativeTool extends DeclarativeTool<
|
||||
export async function createMcpDeclarativeTools(
|
||||
browserManager: BrowserManager,
|
||||
messageBus: MessageBus,
|
||||
shouldDisableInput: boolean = false,
|
||||
): Promise<Array<McpDeclarativeTool | TypeTextDeclarativeTool>> {
|
||||
// Get dynamically discovered tools from the MCP server
|
||||
const mcpTools = await browserManager.getDiscoveredTools();
|
||||
|
||||
debugLogger.log(
|
||||
`Creating ${mcpTools.length} declarative tools for browser agent`,
|
||||
`Creating ${mcpTools.length} declarative tools for browser agent` +
|
||||
(shouldDisableInput ? ' (input blocker enabled)' : ''),
|
||||
);
|
||||
|
||||
const tools: Array<McpDeclarativeTool | TypeTextDeclarativeTool> =
|
||||
@@ -407,6 +452,7 @@ export async function createMcpDeclarativeTools(
|
||||
augmentedDescription,
|
||||
schema.parametersJsonSchema,
|
||||
messageBus,
|
||||
shouldDisableInput,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user