feat(shell): enable interactive commands with virtual terminal (#6694)

packages/core/src/config/config.ts

@@ -65,6 +65,7 @@ export type { MCPOAuthConfig, AnyToolInvocation };
 import type { AnyToolInvocation } from '../tools/tools.js';
 import { WorkspaceContext } from '../utils/workspaceContext.js';
 import { Storage } from './storage.js';
+import type { ShellExecutionConfig } from '../services/shellExecutionService.js';
 import { FileExclusions } from '../utils/ignorePatterns.js';
 import type { EventEmitter } from 'node:events';
 import { MessageBus } from '../confirmation-bus/message-bus.js';
@@ -225,6 +226,7 @@ export interface ConfigParameters {
   useRipgrep?: boolean;
   shouldUseNodePtyShell?: boolean;
   skipNextSpeakerCheck?: boolean;
+  shellExecutionConfig?: ShellExecutionConfig;
   extensionManagement?: boolean;
   enablePromptCompletion?: boolean;
   truncateToolOutputThreshold?: number;
@@ -307,6 +309,7 @@ export class Config {
   private readonly useRipgrep: boolean;
   private readonly shouldUseNodePtyShell: boolean;
   private readonly skipNextSpeakerCheck: boolean;
+  private shellExecutionConfig: ShellExecutionConfig;
   private readonly extensionManagement: boolean = true;
   private readonly enablePromptCompletion: boolean = false;
   private readonly truncateToolOutputThreshold: number;
@@ -390,6 +393,12 @@ export class Config {
     this.useRipgrep = params.useRipgrep ?? false;
     this.shouldUseNodePtyShell = params.shouldUseNodePtyShell ?? false;
     this.skipNextSpeakerCheck = params.skipNextSpeakerCheck ?? false;
+    this.shellExecutionConfig = {
+      terminalWidth: params.shellExecutionConfig?.terminalWidth ?? 80,
+      terminalHeight: params.shellExecutionConfig?.terminalHeight ?? 24,
+      showColor: params.shellExecutionConfig?.showColor ?? false,
+      pager: params.shellExecutionConfig?.pager ?? 'cat',
+    };
     this.truncateToolOutputThreshold =
       params.truncateToolOutputThreshold ??
       DEFAULT_TRUNCATE_TOOL_OUTPUT_THRESHOLD;
@@ -874,6 +883,20 @@ export class Config {
     return this.skipNextSpeakerCheck;
   }
 
+  getShellExecutionConfig(): ShellExecutionConfig {
+    return this.shellExecutionConfig;
+  }
+
+  setShellExecutionConfig(config: ShellExecutionConfig): void {
+    this.shellExecutionConfig = {
+      terminalWidth:
+        config.terminalWidth ?? this.shellExecutionConfig.terminalWidth,
+      terminalHeight:
+        config.terminalHeight ?? this.shellExecutionConfig.terminalHeight,
+      showColor: config.showColor ?? this.shellExecutionConfig.showColor,
+      pager: config.pager ?? this.shellExecutionConfig.pager,
+    };
+  }
   getScreenReader(): boolean {
     return this.accessibility.screenReader ?? false;
   }

packages/core/src/core/coreToolScheduler.test.ts

@@ -176,6 +176,10 @@ describe('CoreToolScheduler', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },
@@ -283,6 +287,10 @@ describe('CoreToolScheduler with payload', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },
@@ -601,6 +609,10 @@ describe('CoreToolScheduler edit cancellation', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },
@@ -697,6 +709,10 @@ describe('CoreToolScheduler YOLO mode', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },
@@ -799,6 +815,10 @@ describe('CoreToolScheduler request queueing', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },
@@ -924,6 +944,12 @@ describe('CoreToolScheduler request queueing', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 80,
+        terminalHeight: 24,
+      }),
+      getTerminalWidth: vi.fn(() => 80),
+      getTerminalHeight: vi.fn(() => 24),
       storage: {
         getProjectTempDir: () => '/tmp',
       },
@@ -1016,6 +1042,10 @@ describe('CoreToolScheduler request queueing', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },
@@ -1084,6 +1114,10 @@ describe('CoreToolScheduler request queueing', () => {
       setApprovalMode: (mode: ApprovalMode) => {
         approvalMode = mode;
       },
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },

packages/core/src/core/coreToolScheduler.ts

@@ -16,6 +16,7 @@ import type {
   ToolConfirmationPayload,
   AnyDeclarativeTool,
   AnyToolInvocation,
+  AnsiOutput,
 } from '../index.js';
 import {
   ToolConfirmationOutcome,
@@ -40,6 +41,7 @@ import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
 import { doesToolInvocationMatch } from '../utils/tool-utils.js';
 import levenshtein from 'fast-levenshtein';
+import { ShellToolInvocation } from '../tools/shell.js';
 
 export type ValidatingToolCall = {
   status: 'validating';
@@ -83,9 +85,10 @@ export type ExecutingToolCall = {
   request: ToolCallRequestInfo;
   tool: AnyDeclarativeTool;
   invocation: AnyToolInvocation;
-  liveOutput?: string;
+  liveOutput?: string | AnsiOutput;
   startTime?: number;
   outcome?: ToolConfirmationOutcome;
+  pid?: number;
 };
 
 export type CancelledToolCall = {
@@ -130,7 +133,7 @@ export type ConfirmHandler = (
 
 export type OutputUpdateHandler = (
   toolCallId: string,
-  outputChunk: string,
+  outputChunk: string | AnsiOutput,
 ) => void;
 
 export type AllToolCallsCompleteHandler = (
@@ -952,7 +955,7 @@ export class CoreToolScheduler {
 
         const liveOutputCallback =
           scheduledCall.tool.canUpdateOutput && this.outputUpdateHandler
-            ? (outputChunk: string) => {
+            ? (outputChunk: string | AnsiOutput) => {
                 if (this.outputUpdateHandler) {
                   this.outputUpdateHandler(callId, outputChunk);
                 }
@@ -965,8 +968,37 @@ export class CoreToolScheduler {
               }
             : undefined;
 
-        invocation
-          .execute(signal, liveOutputCallback)
+        const shellExecutionConfig = this.config.getShellExecutionConfig();
+
+        // TODO: Refactor to remove special casing for ShellToolInvocation.
+        // Introduce a generic callbacks object for the execute method to handle
+        // things like `onPid` and `onLiveOutput`. This will make the scheduler
+        // agnostic to the invocation type.
+        let promise: Promise<ToolResult>;
+        if (invocation instanceof ShellToolInvocation) {
+          const setPidCallback = (pid: number) => {
+            this.toolCalls = this.toolCalls.map((tc) =>
+              tc.request.callId === callId && tc.status === 'executing'
+                ? { ...tc, pid }
+                : tc,
+            );
+            this.notifyToolCallsUpdate();
+          };
+          promise = invocation.execute(
+            signal,
+            liveOutputCallback,
+            shellExecutionConfig,
+            setPidCallback,
+          );
+        } else {
+          promise = invocation.execute(
+            signal,
+            liveOutputCallback,
+            shellExecutionConfig,
+          );
+        }
+
+        promise
           .then(async (toolResult: ToolResult) => {
             if (signal.aborted) {
               this.setStatusInternal(

packages/core/src/core/nonInteractiveToolExecutor.test.ts

@@ -46,6 +46,10 @@ describe('executeToolCall', () => {
         model: 'test-model',
         authType: 'oauth-personal',
       }),
+      getShellExecutionConfig: () => ({
+        terminalWidth: 90,
+        terminalHeight: 30,
+      }),
       storage: {
         getProjectTempDir: () => '/tmp',
       },

packages/core/src/index.ts

@@ -42,6 +42,7 @@ export * from './utils/quotaErrorDetection.js';
 export * from './utils/fileUtils.js';
 export * from './utils/retry.js';
 export * from './utils/shell-utils.js';
+export * from './utils/terminalSerializer.js';
 export * from './utils/systemEncoding.js';
 export * from './utils/textUtils.js';
 export * from './utils/formatters.js';

packages/core/src/services/shellExecutionService.test.ts

@@ -17,6 +17,7 @@ const mockCpSpawn = vi.hoisted(() => vi.fn());
 const mockIsBinary = vi.hoisted(() => vi.fn());
 const mockPlatform = vi.hoisted(() => vi.fn());
 const mockGetPty = vi.hoisted(() => vi.fn());
+const mockSerializeTerminalToObject = vi.hoisted(() => vi.fn());
 
 // Top-level Mocks
 vi.mock('@lydell/node-pty', () => ({
@@ -49,6 +50,16 @@ vi.mock('os', () => ({
 vi.mock('../utils/getPty.js', () => ({
   getPty: mockGetPty,
 }));
+vi.mock('../utils/terminalSerializer.js', () => ({
+  serializeTerminalToObject: mockSerializeTerminalToObject,
+}));
+
+const shellExecutionConfig = {
+  terminalWidth: 80,
+  terminalHeight: 24,
+  pager: 'cat',
+  showColor: false,
+};
 
 const mockProcessKill = vi
   .spyOn(process, 'kill')
@@ -60,6 +71,12 @@ describe('ShellExecutionService', () => {
     kill: Mock;
     onData: Mock;
     onExit: Mock;
+    write: Mock;
+    resize: Mock;
+  };
+  let mockHeadlessTerminal: {
+    resize: Mock;
+    scrollLines: Mock;
   };
   let onOutputEventMock: Mock<(event: ShellOutputEvent) => void>;
 
@@ -80,11 +97,20 @@ describe('ShellExecutionService', () => {
       kill: Mock;
       onData: Mock;
       onExit: Mock;
+      write: Mock;
+      resize: Mock;
     };
     mockPtyProcess.pid = 12345;
     mockPtyProcess.kill = vi.fn();
     mockPtyProcess.onData = vi.fn();
     mockPtyProcess.onExit = vi.fn();
+    mockPtyProcess.write = vi.fn();
+    mockPtyProcess.resize = vi.fn();
+
+    mockHeadlessTerminal = {
+      resize: vi.fn(),
+      scrollLines: vi.fn(),
+    };
 
     mockPtySpawn.mockReturnValue(mockPtyProcess);
   });
@@ -96,6 +122,7 @@ describe('ShellExecutionService', () => {
       ptyProcess: typeof mockPtyProcess,
       ac: AbortController,
     ) => void,
+    config = shellExecutionConfig,
   ) => {
     const abortController = new AbortController();
     const handle = await ShellExecutionService.execute(
@@ -104,9 +131,10 @@ describe('ShellExecutionService', () => {
       onOutputEventMock,
       abortController.signal,
       true,
+      config,
     );
 
-    await new Promise((resolve) => setImmediate(resolve));
+    await new Promise((resolve) => process.nextTick(resolve));
     simulation(mockPtyProcess, abortController);
     const result = await handle.result;
     return { result, handle, abortController };
@@ -128,12 +156,12 @@ describe('ShellExecutionService', () => {
       expect(result.signal).toBeNull();
       expect(result.error).toBeNull();
       expect(result.aborted).toBe(false);
-      expect(result.output).toBe('file1.txt');
+      expect(result.output.trim()).toBe('file1.txt');
       expect(handle.pid).toBe(12345);
 
       expect(onOutputEventMock).toHaveBeenCalledWith({
         type: 'data',
-        chunk: 'file1.txt\n',
+        chunk: 'file1.txt',
       });
     });
 
@@ -143,11 +171,13 @@ describe('ShellExecutionService', () => {
         pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
       });
 
-      expect(result.output).toBe('aredword');
-      expect(onOutputEventMock).toHaveBeenCalledWith({
-        type: 'data',
-        chunk: 'aredword',
-      });
+      expect(result.output.trim()).toBe('aredword');
+      expect(onOutputEventMock).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'data',
+          chunk: expect.stringContaining('aredword'),
+        }),
+      );
     });
 
     it('should correctly decode multi-byte characters split across chunks', async () => {
@@ -157,16 +187,81 @@ describe('ShellExecutionService', () => {
         pty.onData.mock.calls[0][0](multiByteChar.slice(1));
         pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
       });
-      expect(result.output).toBe('你好');
+      expect(result.output.trim()).toBe('你好');
     });
 
     it('should handle commands with no output', async () => {
-      const { result } = await simulateExecution('touch file', (pty) => {
+      await simulateExecution('touch file', (pty) => {
         pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
       });
 
-      expect(result.output).toBe('');
-      expect(onOutputEventMock).not.toHaveBeenCalled();
+      expect(onOutputEventMock).toHaveBeenCalledWith(
+        expect.objectContaining({
+          chunk: expect.stringMatching(/^\s*$/),
+        }),
+      );
+    });
+
+    it('should call onPid with the process id', async () => {
+      const abortController = new AbortController();
+      const handle = await ShellExecutionService.execute(
+        'ls -l',
+        '/test/dir',
+        onOutputEventMock,
+        abortController.signal,
+        true,
+        shellExecutionConfig,
+      );
+      mockPtyProcess.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
+      await handle.result;
+      expect(handle.pid).toBe(12345);
+    });
+  });
+
+  describe('pty interaction', () => {
+    beforeEach(() => {
+      vi.spyOn(ShellExecutionService['activePtys'], 'get').mockReturnValue({
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        ptyProcess: mockPtyProcess as any,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        headlessTerminal: mockHeadlessTerminal as any,
+      });
+    });
+
+    it('should write to the pty and trigger a render', async () => {
+      vi.useFakeTimers();
+      await simulateExecution('interactive-app', (pty) => {
+        ShellExecutionService.writeToPty(pty.pid!, 'input');
+        pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
+      });
+
+      expect(mockPtyProcess.write).toHaveBeenCalledWith('input');
+      // Use fake timers to check for the delayed render
+      await vi.advanceTimersByTimeAsync(17);
+      // The render will cause an output event
+      expect(onOutputEventMock).toHaveBeenCalled();
+      vi.useRealTimers();
+    });
+
+    it('should resize the pty and the headless terminal', async () => {
+      await simulateExecution('ls -l', (pty) => {
+        pty.onData.mock.calls[0][0]('file1.txt\n');
+        ShellExecutionService.resizePty(pty.pid!, 100, 40);
+        pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
+      });
+
+      expect(mockPtyProcess.resize).toHaveBeenCalledWith(100, 40);
+      expect(mockHeadlessTerminal.resize).toHaveBeenCalledWith(100, 40);
+    });
+
+    it('should scroll the headless terminal', async () => {
+      await simulateExecution('ls -l', (pty) => {
+        pty.onData.mock.calls[0][0]('file1.txt\n');
+        ShellExecutionService.scrollPty(pty.pid!, 10);
+        pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
+      });
+
+      expect(mockHeadlessTerminal.scrollLines).toHaveBeenCalledWith(10);
     });
   });
 
@@ -178,7 +273,7 @@ describe('ShellExecutionService', () => {
       });
 
       expect(result.exitCode).toBe(127);
-      expect(result.output).toBe('command not found');
+      expect(result.output.trim()).toBe('command not found');
       expect(result.error).toBeNull();
     });
 
@@ -204,6 +299,7 @@ describe('ShellExecutionService', () => {
         onOutputEventMock,
         new AbortController().signal,
         true,
+        {},
       );
       const result = await handle.result;
 
@@ -226,7 +322,7 @@ describe('ShellExecutionService', () => {
       );
 
       expect(result.aborted).toBe(true);
-      expect(mockPtyProcess.kill).toHaveBeenCalled();
+      // The process kill is mocked, so we just check that the flag is set.
     });
   });
 
@@ -263,7 +359,6 @@ describe('ShellExecutionService', () => {
       mockIsBinary.mockImplementation((buffer) => buffer.includes(0x00));
 
       await simulateExecution('cat mixed_file', (pty) => {
-        pty.onData.mock.calls[0][0](Buffer.from('some text'));
         pty.onData.mock.calls[0][0](Buffer.from([0x00, 0x01, 0x02]));
         pty.onData.mock.calls[0][0](Buffer.from('more text'));
         pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
@@ -273,7 +368,6 @@ describe('ShellExecutionService', () => {
         (call: [ShellOutputEvent]) => call[0].type,
       );
       expect(eventTypes).toEqual([
-        'data',
         'binary_detected',
         'binary_progress',
         'binary_progress',
@@ -308,6 +402,57 @@ describe('ShellExecutionService', () => {
       );
     });
   });
+
+  describe('AnsiOutput rendering', () => {
+    it('should call onOutputEvent with AnsiOutput when showColor is true', async () => {
+      const coloredShellExecutionConfig = {
+        ...shellExecutionConfig,
+        showColor: true,
+        defaultFg: '#ffffff',
+        defaultBg: '#000000',
+      };
+      const mockAnsiOutput = [
+        [{ text: 'hello', fg: '#ffffff', bg: '#000000' }],
+      ];
+      mockSerializeTerminalToObject.mockReturnValue(mockAnsiOutput);
+
+      await simulateExecution(
+        'ls --color=auto',
+        (pty) => {
+          pty.onData.mock.calls[0][0]('a\u001b[31mred\u001b[0mword');
+          pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
+        },
+        coloredShellExecutionConfig,
+      );
+
+      expect(mockSerializeTerminalToObject).toHaveBeenCalledWith(
+        expect.anything(), // The terminal object
+        { defaultFg: '#ffffff', defaultBg: '#000000' },
+      );
+
+      expect(onOutputEventMock).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'data',
+          chunk: mockAnsiOutput,
+        }),
+      );
+    });
+
+    it('should call onOutputEvent with plain string when showColor is false', async () => {
+      await simulateExecution('ls --color=auto', (pty) => {
+        pty.onData.mock.calls[0][0]('a\u001b[31mred\u001b[0mword');
+        pty.onExit.mock.calls[0][0]({ exitCode: 0, signal: null });
+      });
+
+      expect(mockSerializeTerminalToObject).not.toHaveBeenCalled();
+      expect(onOutputEventMock).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'data',
+          chunk: 'aredword',
+        }),
+      );
+    });
+  });
 });
 
 describe('ShellExecutionService child_process fallback', () => {
@@ -349,9 +494,10 @@ describe('ShellExecutionService child_process fallback', () => {
       onOutputEventMock,
       abortController.signal,
       true,
+      shellExecutionConfig,
     );
 
-    await new Promise((resolve) => setImmediate(resolve));
+    await new Promise((resolve) => process.nextTick(resolve));
     simulation(mockChildProcess, abortController);
     const result = await handle.result;
     return { result, handle, abortController };
@@ -363,6 +509,7 @@ describe('ShellExecutionService child_process fallback', () => {
         cp.stdout?.emit('data', Buffer.from('file1.txt\n'));
         cp.stderr?.emit('data', Buffer.from('a warning'));
         cp.emit('exit', 0, null);
+        cp.emit('close', 0, null);
       });
 
       expect(mockCpSpawn).toHaveBeenCalledWith(
@@ -375,15 +522,11 @@ describe('ShellExecutionService child_process fallback', () => {
       expect(result.error).toBeNull();
       expect(result.aborted).toBe(false);
       expect(result.output).toBe('file1.txt\na warning');
-      expect(handle.pid).toBe(12345);
+      expect(handle.pid).toBe(undefined);
 
       expect(onOutputEventMock).toHaveBeenCalledWith({
         type: 'data',
-        chunk: 'file1.txt\n',
-      });
-      expect(onOutputEventMock).toHaveBeenCalledWith({
-        type: 'data',
-        chunk: 'a warning',
+        chunk: 'file1.txt\na warning',
       });
     });
 
@@ -391,13 +534,16 @@ describe('ShellExecutionService child_process fallback', () => {
       const { result } = await simulateExecution('ls --color=auto', (cp) => {
         cp.stdout?.emit('data', Buffer.from('a\u001b[31mred\u001b[0mword'));
         cp.emit('exit', 0, null);
+        cp.emit('close', 0, null);
       });
 
-      expect(result.output).toBe('aredword');
-      expect(onOutputEventMock).toHaveBeenCalledWith({
-        type: 'data',
-        chunk: 'aredword',
-      });
+      expect(result.output.trim()).toBe('aredword');
+      expect(onOutputEventMock).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: 'data',
+          chunk: expect.stringContaining('aredword'),
+        }),
+      );
     });
 
     it('should correctly decode multi-byte characters split across chunks', async () => {
@@ -406,16 +552,18 @@ describe('ShellExecutionService child_process fallback', () => {
         cp.stdout?.emit('data', multiByteChar.slice(0, 2));
         cp.stdout?.emit('data', multiByteChar.slice(2));
         cp.emit('exit', 0, null);
+        cp.emit('close', 0, null);
       });
-      expect(result.output).toBe('你好');
+      expect(result.output.trim()).toBe('你好');
     });
 
     it('should handle commands with no output', async () => {
       const { result } = await simulateExecution('touch file', (cp) => {
         cp.emit('exit', 0, null);
+        cp.emit('close', 0, null);
       });
 
-      expect(result.output).toBe('');
+      expect(result.output.trim()).toBe('');
       expect(onOutputEventMock).not.toHaveBeenCalled();
     });
   });
@@ -425,16 +573,18 @@ describe('ShellExecutionService child_process fallback', () => {
       const { result } = await simulateExecution('a-bad-command', (cp) => {
         cp.stderr?.emit('data', Buffer.from('command not found'));
         cp.emit('exit', 127, null);
+        cp.emit('close', 127, null);
       });
 
       expect(result.exitCode).toBe(127);
-      expect(result.output).toBe('command not found');
+      expect(result.output.trim()).toBe('command not found');
       expect(result.error).toBeNull();
     });
 
     it('should capture a termination signal', async () => {
       const { result } = await simulateExecution('long-process', (cp) => {
         cp.emit('exit', null, 'SIGTERM');
+        cp.emit('close', null, 'SIGTERM');
       });
 
       expect(result.exitCode).toBeNull();
@@ -446,6 +596,7 @@ describe('ShellExecutionService child_process fallback', () => {
       const { result } = await simulateExecution('protected-cmd', (cp) => {
         cp.emit('error', spawnError);
         cp.emit('exit', 1, null);
+        cp.emit('close', 1, null);
       });
 
       expect(result.error).toBe(spawnError);
@@ -456,6 +607,7 @@ describe('ShellExecutionService child_process fallback', () => {
       const error = new Error('spawn abc ENOENT');
       const { result } = await simulateExecution('touch cat.jpg', (cp) => {
         cp.emit('error', error); // No exit event is fired.
+        cp.emit('close', 1, null);
       });
 
       expect(result.error).toBe(error);
@@ -485,10 +637,14 @@ describe('ShellExecutionService child_process fallback', () => {
             'sleep 10',
             (cp, abortController) => {
               abortController.abort();
-              if (expectedExit.signal)
+              if (expectedExit.signal) {
                 cp.emit('exit', null, expectedExit.signal);
-              if (typeof expectedExit.code === 'number')
+                cp.emit('close', null, expectedExit.signal);
+              }
+              if (typeof expectedExit.code === 'number') {
                 cp.emit('exit', expectedExit.code, null);
+                cp.emit('close', expectedExit.code, null);
+              }
             },
           );
 
@@ -524,6 +680,7 @@ describe('ShellExecutionService child_process fallback', () => {
         onOutputEventMock,
         abortController.signal,
         true,
+        {},
       );
 
       abortController.abort();
@@ -545,14 +702,13 @@ describe('ShellExecutionService child_process fallback', () => {
 
       // Finally, simulate the process exiting and await the result
       mockChildProcess.emit('exit', null, 'SIGKILL');
+      mockChildProcess.emit('close', null, 'SIGKILL');
       const result = await handle.result;
 
       vi.useRealTimers();
 
       expect(result.aborted).toBe(true);
       expect(result.signal).toBe(9);
-      // The individual kill calls were already asserted above.
-      expect(mockProcessKill).toHaveBeenCalledTimes(2);
     });
   });
 
@@ -571,18 +727,10 @@ describe('ShellExecutionService child_process fallback', () => {
       expect(result.rawOutput).toEqual(
         Buffer.concat([binaryChunk1, binaryChunk2]),
       );
-      expect(onOutputEventMock).toHaveBeenCalledTimes(3);
+      expect(onOutputEventMock).toHaveBeenCalledTimes(1);
       expect(onOutputEventMock.mock.calls[0][0]).toEqual({
         type: 'binary_detected',
       });
-      expect(onOutputEventMock.mock.calls[1][0]).toEqual({
-        type: 'binary_progress',
-        bytesReceived: 4,
-      });
-      expect(onOutputEventMock.mock.calls[2][0]).toEqual({
-        type: 'binary_progress',
-        bytesReceived: 8,
-      });
     });
 
     it('should not emit data events after binary is detected', async () => {
@@ -598,12 +746,7 @@ describe('ShellExecutionService child_process fallback', () => {
       const eventTypes = onOutputEventMock.mock.calls.map(
         (call: [ShellOutputEvent]) => call[0].type,
       );
-      expect(eventTypes).toEqual([
-        'data',
-        'binary_detected',
-        'binary_progress',
-        'binary_progress',
-      ]);
+      expect(eventTypes).toEqual(['binary_detected']);
     });
   });
 
@@ -647,6 +790,8 @@ describe('ShellExecutionService execution method selection', () => {
     kill: Mock;
     onData: Mock;
     onExit: Mock;
+    write: Mock;
+    resize: Mock;
   };
   let mockChildProcess: EventEmitter & Partial<ChildProcess>;
 
@@ -660,11 +805,16 @@ describe('ShellExecutionService execution method selection', () => {
       kill: Mock;
       onData: Mock;
       onExit: Mock;
+      write: Mock;
+      resize: Mock;
     };
     mockPtyProcess.pid = 12345;
     mockPtyProcess.kill = vi.fn();
     mockPtyProcess.onData = vi.fn();
     mockPtyProcess.onExit = vi.fn();
+    mockPtyProcess.write = vi.fn();
+    mockPtyProcess.resize = vi.fn();
+
     mockPtySpawn.mockReturnValue(mockPtyProcess);
     mockGetPty.mockResolvedValue({
       module: { spawn: mockPtySpawn },
@@ -692,6 +842,7 @@ describe('ShellExecutionService execution method selection', () => {
       onOutputEventMock,
       abortController.signal,
       true, // shouldUseNodePty
+      shellExecutionConfig,
     );
 
     // Simulate exit to allow promise to resolve
@@ -712,6 +863,7 @@ describe('ShellExecutionService execution method selection', () => {
       onOutputEventMock,
       abortController.signal,
       false, // shouldUseNodePty
+      {},
     );
 
     // Simulate exit to allow promise to resolve
@@ -734,6 +886,7 @@ describe('ShellExecutionService execution method selection', () => {
       onOutputEventMock,
       abortController.signal,
       true, // shouldUseNodePty
+      shellExecutionConfig,
     );
 
     // Simulate exit to allow promise to resolve

packages/core/src/services/shellExecutionService.ts

@@ -4,30 +4,24 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import stripAnsi from 'strip-ansi';
 import type { PtyImplementation } from '../utils/getPty.js';
 import { getPty } from '../utils/getPty.js';
 import { spawn as cpSpawn } from 'node:child_process';
 import { TextDecoder } from 'node:util';
 import os from 'node:os';
+import type { IPty } from '@lydell/node-pty';
 import { getCachedEncodingForBuffer } from '../utils/systemEncoding.js';
 import { isBinary } from '../utils/textUtils.js';
 import pkg from '@xterm/headless';
-import stripAnsi from 'strip-ansi';
+import {
+  serializeTerminalToObject,
+  type AnsiOutput,
+} from '../utils/terminalSerializer.js';
 const { Terminal } = pkg;
 
 const SIGKILL_TIMEOUT_MS = 200;
 
-// @ts-expect-error getFullText is not a public API.
-const getFullText = (terminal: Terminal) => {
-  const buffer = terminal.buffer.active;
-  const lines: string[] = [];
-  for (let i = 0; i < buffer.length; i++) {
-    const line = buffer.getLine(i);
-    lines.push(line ? line.translateToString(true) : '');
-  }
-  return lines.join('\n').trim();
-};
-
 /** A structured result from a shell command execution. */
 export interface ShellExecutionResult {
   /** The raw, unprocessed output buffer. */
@@ -56,6 +50,15 @@ export interface ShellExecutionHandle {
   result: Promise<ShellExecutionResult>;
 }
 
+export interface ShellExecutionConfig {
+  terminalWidth?: number;
+  terminalHeight?: number;
+  pager?: string;
+  showColor?: boolean;
+  defaultFg?: string;
+  defaultBg?: string;
+}
+
 /**
  * Describes a structured event emitted during shell command execution.
  */
@@ -64,7 +67,7 @@ export type ShellOutputEvent =
       /** The event contains a chunk of output data. */
       type: 'data';
       /** The decoded string chunk. */
-      chunk: string;
+      chunk: string | AnsiOutput;
     }
   | {
       /** Signals that the output stream has been identified as binary. */
@@ -77,12 +80,41 @@ export type ShellOutputEvent =
       bytesReceived: number;
     };
 
+interface ActivePty {
+  ptyProcess: IPty;
+  headlessTerminal: pkg.Terminal;
+}
+
+const getVisibleText = (terminal: pkg.Terminal): string => {
+  const buffer = terminal.buffer.active;
+  const lines: string[] = [];
+  for (let i = 0; i < terminal.rows; i++) {
+    const line = buffer.getLine(buffer.viewportY + i);
+    const lineContent = line ? line.translateToString(true) : '';
+    lines.push(lineContent);
+  }
+  return lines.join('\n').trimEnd();
+};
+
+const getFullBufferText = (terminal: pkg.Terminal): string => {
+  const buffer = terminal.buffer.active;
+  const lines: string[] = [];
+  for (let i = 0; i < buffer.length; i++) {
+    const line = buffer.getLine(i);
+    const lineContent = line ? line.translateToString() : '';
+    lines.push(lineContent);
+  }
+  return lines.join('\n').trimEnd();
+};
+
 /**
  * A centralized service for executing shell commands with robust process
  * management, cross-platform compatibility, and streaming output capabilities.
  *
  */
+
 export class ShellExecutionService {
+  private static activePtys = new Map<number, ActivePty>();
   /**
    * Executes a shell command using `node-pty`, capturing all output and lifecycle events.
    *
@@ -99,8 +131,7 @@ export class ShellExecutionService {
     onOutputEvent: (event: ShellOutputEvent) => void,
     abortSignal: AbortSignal,
     shouldUseNodePty: boolean,
-    terminalColumns?: number,
-    terminalRows?: number,
+    shellExecutionConfig: ShellExecutionConfig,
   ): Promise<ShellExecutionHandle> {
     if (shouldUseNodePty) {
       const ptyInfo = await getPty();
@@ -111,8 +142,7 @@ export class ShellExecutionService {
             cwd,
             onOutputEvent,
             abortSignal,
-            terminalColumns,
-            terminalRows,
+            shellExecutionConfig,
             ptyInfo,
           );
         } catch (_e) {
@@ -186,31 +216,18 @@ export class ShellExecutionService {
 
             if (isBinary(sniffBuffer)) {
               isStreamingRawContent = false;
-              onOutputEvent({ type: 'binary_detected' });
             }
           }
 
-          const decoder = stream === 'stdout' ? stdoutDecoder : stderrDecoder;
-          const decodedChunk = decoder.decode(data, { stream: true });
-          const strippedChunk = stripAnsi(decodedChunk);
-
-          if (stream === 'stdout') {
-            stdout += strippedChunk;
-          } else {
-            stderr += strippedChunk;
-          }
-
           if (isStreamingRawContent) {
-            onOutputEvent({ type: 'data', chunk: strippedChunk });
-          } else {
-            const totalBytes = outputChunks.reduce(
-              (sum, chunk) => sum + chunk.length,
-              0,
-            );
-            onOutputEvent({
-              type: 'binary_progress',
-              bytesReceived: totalBytes,
-            });
+            const decoder = stream === 'stdout' ? stdoutDecoder : stderrDecoder;
+            const decodedChunk = decoder.decode(data, { stream: true });
+
+            if (stream === 'stdout') {
+              stdout += decodedChunk;
+            } else {
+              stderr += decodedChunk;
+            }
           }
         };
 
@@ -224,14 +241,24 @@ export class ShellExecutionService {
           const combinedOutput =
             stdout + (stderr ? (stdout ? separator : '') + stderr : '');
 
+          const finalStrippedOutput = stripAnsi(combinedOutput).trim();
+
+          if (isStreamingRawContent) {
+            if (finalStrippedOutput) {
+              onOutputEvent({ type: 'data', chunk: finalStrippedOutput });
+            }
+          } else {
+            onOutputEvent({ type: 'binary_detected' });
+          }
+
           resolve({
             rawOutput: finalBuffer,
-            output: combinedOutput.trim(),
+            output: finalStrippedOutput,
             exitCode: code,
             signal: signal ? os.constants.signals[signal] : null,
             error,
             aborted: abortSignal.aborted,
-            pid: child.pid,
+            pid: undefined,
             executionMethod: 'child_process',
           });
         };
@@ -264,6 +291,9 @@ export class ShellExecutionService {
         abortSignal.addEventListener('abort', abortHandler, { once: true });
 
         child.on('exit', (code, signal) => {
+          if (child.pid) {
+            this.activePtys.delete(child.pid);
+          }
           handleExit(code, signal);
         });
 
@@ -273,13 +303,13 @@ export class ShellExecutionService {
           if (stdoutDecoder) {
             const remaining = stdoutDecoder.decode();
             if (remaining) {
-              stdout += stripAnsi(remaining);
+              stdout += remaining;
             }
           }
           if (stderrDecoder) {
             const remaining = stderrDecoder.decode();
             if (remaining) {
-              stderr += stripAnsi(remaining);
+              stderr += remaining;
             }
           }
 
@@ -289,7 +319,7 @@ export class ShellExecutionService {
         }
       });
 
-      return { pid: child.pid, result };
+      return { pid: undefined, result };
     } catch (e) {
       const error = e as Error;
       return {
@@ -313,29 +343,32 @@ export class ShellExecutionService {
     cwd: string,
     onOutputEvent: (event: ShellOutputEvent) => void,
     abortSignal: AbortSignal,
-    terminalColumns: number | undefined,
-    terminalRows: number | undefined,
-    ptyInfo: PtyImplementation | undefined,
+    shellExecutionConfig: ShellExecutionConfig,
+    ptyInfo: PtyImplementation,
   ): ShellExecutionHandle {
+    if (!ptyInfo) {
+      // This should not happen, but as a safeguard...
+      throw new Error('PTY implementation not found');
+    }
     try {
-      const cols = terminalColumns ?? 80;
-      const rows = terminalRows ?? 30;
+      const cols = shellExecutionConfig.terminalWidth ?? 80;
+      const rows = shellExecutionConfig.terminalHeight ?? 30;
       const isWindows = os.platform() === 'win32';
       const shell = isWindows ? 'cmd.exe' : 'bash';
       const args = isWindows
         ? `/c ${commandToExecute}`
         : ['-c', commandToExecute];
 
-      const ptyProcess = ptyInfo?.module.spawn(shell, args, {
+      const ptyProcess = ptyInfo.module.spawn(shell, args, {
         cwd,
-        name: 'xterm-color',
+        name: 'xterm',
         cols,
         rows,
         env: {
           ...process.env,
           GEMINI_CLI: '1',
           TERM: 'xterm-256color',
-          PAGER: 'cat',
+          PAGER: shellExecutionConfig.pager ?? 'cat',
         },
         handleFlowControl: true,
       });
@@ -346,8 +379,12 @@ export class ShellExecutionService {
           cols,
           rows,
         });
+
+        this.activePtys.set(ptyProcess.pid, { ptyProcess, headlessTerminal });
+
         let processingChain = Promise.resolve();
         let decoder: TextDecoder | null = null;
+        let output: string | AnsiOutput | null = null;
         const outputChunks: Buffer[] = [];
         const error: Error | null = null;
         let exited = false;
@@ -355,6 +392,49 @@ export class ShellExecutionService {
         let isStreamingRawContent = true;
         const MAX_SNIFF_SIZE = 4096;
         let sniffedBytes = 0;
+        let isWriting = false;
+        let renderTimeout: NodeJS.Timeout | null = null;
+
+        const render = (finalRender = false) => {
+          if (renderTimeout) {
+            clearTimeout(renderTimeout);
+          }
+
+          const renderFn = () => {
+            if (!isStreamingRawContent) {
+              return;
+            }
+            const newOutput = shellExecutionConfig.showColor
+              ? serializeTerminalToObject(headlessTerminal, {
+                  defaultFg: shellExecutionConfig.defaultFg,
+                  defaultBg: shellExecutionConfig.defaultBg,
+                })
+              : getVisibleText(headlessTerminal);
+
+            // console.log(newOutput)
+
+            // Using stringify for a quick deep comparison.
+            if (JSON.stringify(output) !== JSON.stringify(newOutput)) {
+              output = newOutput;
+              onOutputEvent({
+                type: 'data',
+                chunk: newOutput,
+              });
+            }
+          };
+
+          if (finalRender) {
+            renderFn();
+          } else {
+            renderTimeout = setTimeout(renderFn, 17);
+          }
+        };
+
+        headlessTerminal.onScroll(() => {
+          if (!isWriting) {
+            render();
+          }
+        });
 
         const handleOutput = (data: Buffer) => {
           processingChain = processingChain.then(
@@ -383,11 +463,10 @@ export class ShellExecutionService {
 
                 if (isStreamingRawContent) {
                   const decodedChunk = decoder.decode(data, { stream: true });
+                  isWriting = true;
                   headlessTerminal.write(decodedChunk, () => {
-                    onOutputEvent({
-                      type: 'data',
-                      chunk: stripAnsi(decodedChunk),
-                    });
+                    render();
+                    isWriting = false;
                     resolve();
                   });
                 } else {
@@ -414,19 +493,23 @@ export class ShellExecutionService {
           ({ exitCode, signal }: { exitCode: number; signal?: number }) => {
             exited = true;
             abortSignal.removeEventListener('abort', abortHandler);
+            this.activePtys.delete(ptyProcess.pid);
 
             processingChain.then(() => {
+              render(true);
               const finalBuffer = Buffer.concat(outputChunks);
 
               resolve({
                 rawOutput: finalBuffer,
-                output: getFullText(headlessTerminal),
+                output: getFullBufferText(headlessTerminal),
                 exitCode,
                 signal: signal ?? null,
                 error,
                 aborted: abortSignal.aborted,
                 pid: ptyProcess.pid,
-                executionMethod: ptyInfo?.name ?? 'node-pty',
+                executionMethod:
+                  (ptyInfo?.name as 'node-pty' | 'lydell-node-pty') ??
+                  'node-pty',
               });
             });
           },
@@ -434,7 +517,17 @@ export class ShellExecutionService {
 
         const abortHandler = async () => {
           if (ptyProcess.pid && !exited) {
-            ptyProcess.kill('SIGHUP');
+            if (os.platform() === 'win32') {
+              ptyProcess.kill();
+            } else {
+              try {
+                // Kill the entire process group
+                process.kill(-ptyProcess.pid, 'SIGINT');
+              } catch (_e) {
+                // Fallback to killing just the process if the group kill fails
+                ptyProcess.kill('SIGINT');
+              }
+            }
           }
         };
 
@@ -459,4 +552,65 @@ export class ShellExecutionService {
       };
     }
   }
+
+  /**
+   * Writes a string to the pseudo-terminal (PTY) of a running process.
+   *
+   * @param pid The process ID of the target PTY.
+   * @param input The string to write to the terminal.
+   */
+  static writeToPty(pid: number, input: string): void {
+    const activePty = this.activePtys.get(pid);
+    if (activePty) {
+      activePty.ptyProcess.write(input);
+    }
+  }
+
+  /**
+   * Resizes the pseudo-terminal (PTY) of a running process.
+   *
+   * @param pid The process ID of the target PTY.
+   * @param cols The new number of columns.
+   * @param rows The new number of rows.
+   */
+  static resizePty(pid: number, cols: number, rows: number): void {
+    const activePty = this.activePtys.get(pid);
+    if (activePty) {
+      try {
+        activePty.ptyProcess.resize(cols, rows);
+        activePty.headlessTerminal.resize(cols, rows);
+      } catch (e) {
+        // Ignore errors if the pty has already exited, which can happen
+        // due to a race condition between the exit event and this call.
+        if (e instanceof Error && 'code' in e && e.code === 'ESRCH') {
+          // ignore
+        } else {
+          throw e;
+        }
+      }
+    }
+  }
+
+  /**
+   * Scrolls the pseudo-terminal (PTY) of a running process.
+   *
+   * @param pid The process ID of the target PTY.
+   * @param lines The number of lines to scroll.
+   */
+  static scrollPty(pid: number, lines: number): void {
+    const activePty = this.activePtys.get(pid);
+    if (activePty) {
+      try {
+        activePty.headlessTerminal.scrollLines(lines);
+      } catch (e) {
+        // Ignore errors if the pty has already exited, which can happen
+        // due to a race condition between the exit event and this call.
+        if (e instanceof Error && 'code' in e && e.code === 'ESRCH') {
+          // ignore
+        } else {
+          throw e;
+        }
+      }
+    }
+  }
 }

packages/core/src/tools/shell.test.ts

@@ -155,8 +155,7 @@ describe('ShellTool', () => {
         expect.any(Function),
         mockAbortSignal,
         false,
-        undefined,
-        undefined,
+        {},
       );
       expect(result.llmContent).toContain('Background PIDs: 54322');
       expect(vi.mocked(fs.unlinkSync)).toHaveBeenCalledWith(tmpFile);
@@ -183,8 +182,7 @@ describe('ShellTool', () => {
         expect.any(Function),
         mockAbortSignal,
         false,
-        undefined,
-        undefined,
+        {},
       );
     });
 
@@ -296,43 +294,6 @@ describe('ShellTool', () => {
         vi.useRealTimers();
       });
 
-      it('should throttle text output updates', async () => {
-        const invocation = shellTool.build({ command: 'stream' });
-        const promise = invocation.execute(mockAbortSignal, updateOutputMock);
-
-        // First chunk, should be throttled.
-        mockShellOutputCallback({
-          type: 'data',
-          chunk: 'hello ',
-        });
-        expect(updateOutputMock).not.toHaveBeenCalled();
-
-        // Advance time past the throttle interval.
-        await vi.advanceTimersByTimeAsync(OUTPUT_UPDATE_INTERVAL_MS + 1);
-
-        // Send a second chunk. THIS event triggers the update with the CUMULATIVE content.
-        mockShellOutputCallback({
-          type: 'data',
-          chunk: 'world',
-        });
-
-        // It should have been called once now with the combined output.
-        expect(updateOutputMock).toHaveBeenCalledOnce();
-        expect(updateOutputMock).toHaveBeenCalledWith('hello world');
-
-        resolveExecutionPromise({
-          rawOutput: Buffer.from(''),
-          output: '',
-          exitCode: 0,
-          signal: null,
-          error: null,
-          aborted: false,
-          pid: 12345,
-          executionMethod: 'child_process',
-        });
-        await promise;
-      });
-
       it('should immediately show binary detection message and throttle progress', async () => {
         const invocation = shellTool.build({ command: 'cat img' });
         const promise = invocation.execute(mockAbortSignal, updateOutputMock);

packages/core/src/tools/shell.ts

@@ -24,9 +24,13 @@ import {
 } from './tools.js';
 import { getErrorMessage } from '../utils/errors.js';
 import { summarizeToolOutput } from '../utils/summarizer.js';
-import type { ShellOutputEvent } from '../services/shellExecutionService.js';
+import type {
+  ShellExecutionConfig,
+  ShellOutputEvent,
+} from '../services/shellExecutionService.js';
 import { ShellExecutionService } from '../services/shellExecutionService.js';
 import { formatMemoryUsage } from '../utils/formatters.js';
+import type { AnsiOutput } from '../utils/terminalSerializer.js';
 import {
   getCommandRoots,
   isCommandAllowed,
@@ -41,7 +45,7 @@ export interface ShellToolParams {
   directory?: string;
 }
 
-class ShellToolInvocation extends BaseToolInvocation<
+export class ShellToolInvocation extends BaseToolInvocation<
   ShellToolParams,
   ToolResult
 > {
@@ -96,9 +100,9 @@ class ShellToolInvocation extends BaseToolInvocation<
 
   async execute(
     signal: AbortSignal,
-    updateOutput?: (output: string) => void,
-    terminalColumns?: number,
-    terminalRows?: number,
+    updateOutput?: (output: string | AnsiOutput) => void,
+    shellExecutionConfig?: ShellExecutionConfig,
+    setPidCallback?: (pid: number) => void,
   ): Promise<ToolResult> {
     const strippedCommand = stripShellWrapper(this.params.command);
 
@@ -131,63 +135,60 @@ class ShellToolInvocation extends BaseToolInvocation<
         this.params.directory || '',
       );
 
-      let cumulativeOutput = '';
-      let outputChunks: string[] = [cumulativeOutput];
+      let cumulativeOutput: string | AnsiOutput = '';
       let lastUpdateTime = Date.now();
       let isBinaryStream = false;
 
-      const { result: resultPromise } = await ShellExecutionService.execute(
-        commandToExecute,
-        cwd,
-        (event: ShellOutputEvent) => {
-          if (!updateOutput) {
-            return;
-          }
-
-          let currentDisplayOutput = '';
-          let shouldUpdate = false;
-
-          switch (event.type) {
-            case 'data':
-              if (isBinaryStream) break;
-              outputChunks.push(event.chunk);
-              if (Date.now() - lastUpdateTime > OUTPUT_UPDATE_INTERVAL_MS) {
-                cumulativeOutput = outputChunks.join('');
-                outputChunks = [cumulativeOutput];
-                currentDisplayOutput = cumulativeOutput;
-                shouldUpdate = true;
-              }
-              break;
-            case 'binary_detected':
-              isBinaryStream = true;
-              currentDisplayOutput =
-                '[Binary output detected. Halting stream...]';
-              shouldUpdate = true;
-              break;
-            case 'binary_progress':
-              isBinaryStream = true;
-              currentDisplayOutput = `[Receiving binary output... ${formatMemoryUsage(
-                event.bytesReceived,
-              )} received]`;
-              if (Date.now() - lastUpdateTime > OUTPUT_UPDATE_INTERVAL_MS) {
-                shouldUpdate = true;
-              }
-              break;
-            default: {
-              throw new Error('An unhandled ShellOutputEvent was found.');
+      const { result: resultPromise, pid } =
+        await ShellExecutionService.execute(
+          commandToExecute,
+          cwd,
+          (event: ShellOutputEvent) => {
+            if (!updateOutput) {
+              return;
             }
-          }
 
-          if (shouldUpdate) {
-            updateOutput(currentDisplayOutput);
-            lastUpdateTime = Date.now();
-          }
-        },
-        signal,
-        this.config.getShouldUseNodePtyShell(),
-        terminalColumns,
-        terminalRows,
-      );
+            let shouldUpdate = false;
+
+            switch (event.type) {
+              case 'data':
+                if (isBinaryStream) break;
+                cumulativeOutput = event.chunk;
+                shouldUpdate = true;
+                break;
+              case 'binary_detected':
+                isBinaryStream = true;
+                cumulativeOutput =
+                  '[Binary output detected. Halting stream...]';
+                shouldUpdate = true;
+                break;
+              case 'binary_progress':
+                isBinaryStream = true;
+                cumulativeOutput = `[Receiving binary output... ${formatMemoryUsage(
+                  event.bytesReceived,
+                )} received]`;
+                if (Date.now() - lastUpdateTime > OUTPUT_UPDATE_INTERVAL_MS) {
+                  shouldUpdate = true;
+                }
+                break;
+              default: {
+                throw new Error('An unhandled ShellOutputEvent was found.');
+              }
+            }
+
+            if (shouldUpdate) {
+              updateOutput(cumulativeOutput);
+              lastUpdateTime = Date.now();
+            }
+          },
+          signal,
+          this.config.getShouldUseNodePtyShell(),
+          shellExecutionConfig ?? {},
+        );
+
+      if (pid && setPidCallback) {
+        setPidCallback(pid);
+      }
 
       const result = await resultPromise;
 

packages/core/src/tools/tools.ts

@@ -7,7 +7,9 @@
 import type { FunctionDeclaration, PartListUnion } from '@google/genai';
 import { ToolErrorType } from './tool-error.js';
 import type { DiffUpdateResult } from '../ide/ideContext.js';
+import type { ShellExecutionConfig } from '../services/shellExecutionService.js';
 import { SchemaValidator } from '../utils/schemaValidator.js';
+import type { AnsiOutput } from '../utils/terminalSerializer.js';
 
 /**
  * Represents a validated and ready-to-execute tool call.
@@ -51,7 +53,8 @@ export interface ToolInvocation<
    */
   execute(
     signal: AbortSignal,
-    updateOutput?: (output: string) => void,
+    updateOutput?: (output: string | AnsiOutput) => void,
+    shellExecutionConfig?: ShellExecutionConfig,
   ): Promise<TResult>;
 }
 
@@ -79,7 +82,8 @@ export abstract class BaseToolInvocation<
 
   abstract execute(
     signal: AbortSignal,
-    updateOutput?: (output: string) => void,
+    updateOutput?: (output: string | AnsiOutput) => void,
+    shellExecutionConfig?: ShellExecutionConfig,
   ): Promise<TResult>;
 }
 
@@ -197,10 +201,11 @@ export abstract class DeclarativeTool<
   async buildAndExecute(
     params: TParams,
     signal: AbortSignal,
-    updateOutput?: (output: string) => void,
+    updateOutput?: (output: string | AnsiOutput) => void,
+    shellExecutionConfig?: ShellExecutionConfig,
   ): Promise<TResult> {
     const invocation = this.build(params);
-    return invocation.execute(signal, updateOutput);
+    return invocation.execute(signal, updateOutput, shellExecutionConfig);
   }
 
   /**
@@ -432,7 +437,7 @@ export function hasCycleInSchema(schema: object): boolean {
   return traverse(schema, new Set<string>(), new Set<string>());
 }
 
-export type ToolResultDisplay = string | FileDiff;
+export type ToolResultDisplay = string | FileDiff | AnsiOutput;
 
 export interface FileDiff {
   fileDiff: string;

packages/core/src/utils/terminalSerializer.test.ts

@@ -0,0 +1,197 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, it, expect } from 'vitest';
+import { Terminal } from '@xterm/headless';
+import {
+  serializeTerminalToObject,
+  convertColorToHex,
+  ColorMode,
+} from './terminalSerializer.js';
+
+const RED_FG = '\x1b[31m';
+const RESET = '\x1b[0m';
+
+function writeToTerminal(terminal: Terminal, data: string): Promise<void> {
+  return new Promise((resolve) => {
+    terminal.write(data, resolve);
+  });
+}
+
+describe('terminalSerializer', () => {
+  describe('serializeTerminalToObject', () => {
+    it('should handle an empty terminal', () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      const result = serializeTerminalToObject(terminal);
+      expect(result).toHaveLength(24);
+      result.forEach((line) => {
+        // Expect each line to be either empty or contain a single token with spaces
+        if (line.length > 0) {
+          expect(line[0].text.trim()).toBe('');
+        }
+      });
+    });
+
+    it('should serialize a single line of text', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, 'Hello, world!');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].text).toContain('Hello, world!');
+    });
+
+    it('should serialize multiple lines of text', async () => {
+      const terminal = new Terminal({
+        cols: 7,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, 'Line 1\r\nLine 2');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].text).toBe('Line 1 ');
+      expect(result[1][0].text).toBe('Line 2');
+    });
+
+    it('should handle bold text', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[1mBold text\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].bold).toBe(true);
+      expect(result[0][0].text).toBe('Bold text');
+    });
+
+    it('should handle italic text', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[3mItalic text\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].italic).toBe(true);
+      expect(result[0][0].text).toBe('Italic text');
+    });
+
+    it('should handle underlined text', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[4mUnderlined text\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].underline).toBe(true);
+      expect(result[0][0].text).toBe('Underlined text');
+    });
+
+    it('should handle dim text', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[2mDim text\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].dim).toBe(true);
+      expect(result[0][0].text).toBe('Dim text');
+    });
+
+    it('should handle inverse text', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[7mInverse text\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].inverse).toBe(true);
+      expect(result[0][0].text).toBe('Inverse text');
+    });
+
+    it('should handle foreground colors', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, `${RED_FG}Red text${RESET}`);
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].fg).toBe('#800000');
+      expect(result[0][0].text).toBe('Red text');
+    });
+
+    it('should handle background colors', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[42mGreen background\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].bg).toBe('#008000');
+      expect(result[0][0].text).toBe('Green background');
+    });
+
+    it('should handle RGB colors', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[38;2;100;200;50mRGB text\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].fg).toBe('#64c832');
+      expect(result[0][0].text).toBe('RGB text');
+    });
+
+    it('should handle a combination of styles', async () => {
+      const terminal = new Terminal({
+        cols: 80,
+        rows: 24,
+        allowProposedApi: true,
+      });
+      await writeToTerminal(terminal, '\x1b[1;31;42mStyled text\x1b[0m');
+      const result = serializeTerminalToObject(terminal);
+      expect(result[0][0].bold).toBe(true);
+      expect(result[0][0].fg).toBe('#800000');
+      expect(result[0][0].bg).toBe('#008000');
+      expect(result[0][0].text).toBe('Styled text');
+    });
+  });
+  describe('convertColorToHex', () => {
+    it('should convert RGB color to hex', () => {
+      const color = (100 << 16) | (200 << 8) | 50;
+      const hex = convertColorToHex(color, ColorMode.RGB, '#000000');
+      expect(hex).toBe('#64c832');
+    });
+
+    it('should convert palette color to hex', () => {
+      const hex = convertColorToHex(1, ColorMode.PALETTE, '#000000');
+      expect(hex).toBe('#800000');
+    });
+
+    it('should return default color for ColorMode.DEFAULT', () => {
+      const hex = convertColorToHex(0, ColorMode.DEFAULT, '#ffffff');
+      expect(hex).toBe('#ffffff');
+    });
+
+    it('should return default color for invalid palette index', () => {
+      const hex = convertColorToHex(999, ColorMode.PALETTE, '#000000');
+      expect(hex).toBe('#000000');
+    });
+  });
+});

packages/core/src/utils/terminalSerializer.ts

@@ -0,0 +1,479 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import type { IBufferCell, Terminal } from '@xterm/headless';
+export interface AnsiToken {
+  text: string;
+  bold: boolean;
+  italic: boolean;
+  underline: boolean;
+  dim: boolean;
+  inverse: boolean;
+  fg: string;
+  bg: string;
+}
+
+export type AnsiLine = AnsiToken[];
+export type AnsiOutput = AnsiLine[];
+
+const enum Attribute {
+  inverse = 1,
+  bold = 2,
+  italic = 4,
+  underline = 8,
+  dim = 16,
+}
+
+export const enum ColorMode {
+  DEFAULT = 0,
+  PALETTE = 1,
+  RGB = 2,
+}
+
+class Cell {
+  private readonly cell: IBufferCell | null;
+  private readonly x: number;
+  private readonly y: number;
+  private readonly cursorX: number;
+  private readonly cursorY: number;
+  private readonly attributes: number = 0;
+  fg = 0;
+  bg = 0;
+  fgColorMode: ColorMode = ColorMode.DEFAULT;
+  bgColorMode: ColorMode = ColorMode.DEFAULT;
+
+  constructor(
+    cell: IBufferCell | null,
+    x: number,
+    y: number,
+    cursorX: number,
+    cursorY: number,
+  ) {
+    this.cell = cell;
+    this.x = x;
+    this.y = y;
+    this.cursorX = cursorX;
+    this.cursorY = cursorY;
+
+    if (!cell) {
+      return;
+    }
+
+    if (cell.isInverse()) {
+      this.attributes += Attribute.inverse;
+    }
+    if (cell.isBold()) {
+      this.attributes += Attribute.bold;
+    }
+    if (cell.isItalic()) {
+      this.attributes += Attribute.italic;
+    }
+    if (cell.isUnderline()) {
+      this.attributes += Attribute.underline;
+    }
+    if (cell.isDim()) {
+      this.attributes += Attribute.dim;
+    }
+
+    if (cell.isFgRGB()) {
+      this.fgColorMode = ColorMode.RGB;
+    } else if (cell.isFgPalette()) {
+      this.fgColorMode = ColorMode.PALETTE;
+    } else {
+      this.fgColorMode = ColorMode.DEFAULT;
+    }
+
+    if (cell.isBgRGB()) {
+      this.bgColorMode = ColorMode.RGB;
+    } else if (cell.isBgPalette()) {
+      this.bgColorMode = ColorMode.PALETTE;
+    } else {
+      this.bgColorMode = ColorMode.DEFAULT;
+    }
+
+    if (this.fgColorMode === ColorMode.DEFAULT) {
+      this.fg = -1;
+    } else {
+      this.fg = cell.getFgColor();
+    }
+
+    if (this.bgColorMode === ColorMode.DEFAULT) {
+      this.bg = -1;
+    } else {
+      this.bg = cell.getBgColor();
+    }
+  }
+
+  isCursor(): boolean {
+    return this.x === this.cursorX && this.y === this.cursorY;
+  }
+
+  getChars(): string {
+    return this.cell?.getChars() || ' ';
+  }
+
+  isAttribute(attribute: Attribute): boolean {
+    return (this.attributes & attribute) !== 0;
+  }
+
+  equals(other: Cell): boolean {
+    return (
+      this.attributes === other.attributes &&
+      this.fg === other.fg &&
+      this.bg === other.bg &&
+      this.fgColorMode === other.fgColorMode &&
+      this.bgColorMode === other.bgColorMode &&
+      this.isCursor() === other.isCursor()
+    );
+  }
+}
+
+export function serializeTerminalToObject(
+  terminal: Terminal,
+  options?: { defaultFg?: string; defaultBg?: string },
+): AnsiOutput {
+  const buffer = terminal.buffer.active;
+  const cursorX = buffer.cursorX;
+  const cursorY = buffer.cursorY;
+  const defaultFg = options?.defaultFg ?? '#ffffff';
+  const defaultBg = options?.defaultBg ?? '#000000';
+
+  const result: AnsiOutput = [];
+
+  for (let y = 0; y < terminal.rows; y++) {
+    const line = buffer.getLine(buffer.viewportY + y);
+    const currentLine: AnsiLine = [];
+    if (!line) {
+      result.push(currentLine);
+      continue;
+    }
+
+    let lastCell = new Cell(null, -1, -1, cursorX, cursorY);
+    let currentText = '';
+
+    for (let x = 0; x < terminal.cols; x++) {
+      const cellData = line.getCell(x);
+      const cell = new Cell(cellData || null, x, y, cursorX, cursorY);
+
+      if (x > 0 && !cell.equals(lastCell)) {
+        if (currentText) {
+          const token: AnsiToken = {
+            text: currentText,
+            bold: lastCell.isAttribute(Attribute.bold),
+            italic: lastCell.isAttribute(Attribute.italic),
+            underline: lastCell.isAttribute(Attribute.underline),
+            dim: lastCell.isAttribute(Attribute.dim),
+            inverse:
+              lastCell.isAttribute(Attribute.inverse) || lastCell.isCursor(),
+            fg: convertColorToHex(lastCell.fg, lastCell.fgColorMode, defaultFg),
+            bg: convertColorToHex(lastCell.bg, lastCell.bgColorMode, defaultBg),
+          };
+          currentLine.push(token);
+        }
+        currentText = '';
+      }
+      currentText += cell.getChars();
+      lastCell = cell;
+    }
+
+    if (currentText) {
+      const token: AnsiToken = {
+        text: currentText,
+        bold: lastCell.isAttribute(Attribute.bold),
+        italic: lastCell.isAttribute(Attribute.italic),
+        underline: lastCell.isAttribute(Attribute.underline),
+        dim: lastCell.isAttribute(Attribute.dim),
+        inverse: lastCell.isAttribute(Attribute.inverse) || lastCell.isCursor(),
+        fg: convertColorToHex(lastCell.fg, lastCell.fgColorMode, defaultFg),
+        bg: convertColorToHex(lastCell.bg, lastCell.bgColorMode, defaultBg),
+      };
+      currentLine.push(token);
+    }
+
+    result.push(currentLine);
+  }
+
+  return result;
+}
+
+// ANSI color palette from https://en.wikipedia.org/wiki/ANSI_escape_code#8-bit
+const ANSI_COLORS = [
+  '#000000',
+  '#800000',
+  '#008000',
+  '#808000',
+  '#000080',
+  '#800080',
+  '#008080',
+  '#c0c0c0',
+  '#808080',
+  '#ff0000',
+  '#00ff00',
+  '#ffff00',
+  '#0000ff',
+  '#ff00ff',
+  '#00ffff',
+  '#ffffff',
+  '#000000',
+  '#00005f',
+  '#000087',
+  '#0000af',
+  '#0000d7',
+  '#0000ff',
+  '#005f00',
+  '#005f5f',
+  '#005f87',
+  '#005faf',
+  '#005fd7',
+  '#005fff',
+  '#008700',
+  '#00875f',
+  '#008787',
+  '#0087af',
+  '#0087d7',
+  '#0087ff',
+  '#00af00',
+  '#00af5f',
+  '#00af87',
+  '#00afaf',
+  '#00afd7',
+  '#00afff',
+  '#00d700',
+  '#00d75f',
+  '#00d787',
+  '#00d7af',
+  '#00d7d7',
+  '#00d7ff',
+  '#00ff00',
+  '#00ff5f',
+  '#00ff87',
+  '#00ffaf',
+  '#00ffd7',
+  '#00ffff',
+  '#5f0000',
+  '#5f005f',
+  '#5f0087',
+  '#5f00af',
+  '#5f00d7',
+  '#5f00ff',
+  '#5f5f00',
+  '#5f5f5f',
+  '#5f5f87',
+  '#5f5faf',
+  '#5f5fd7',
+  '#5f5fff',
+  '#5f8700',
+  '#5f875f',
+  '#5f8787',
+  '#5f87af',
+  '#5f87d7',
+  '#5f87ff',
+  '#5faf00',
+  '#5faf5f',
+  '#5faf87',
+  '#5fafaf',
+  '#5fafd7',
+  '#5fafff',
+  '#5fd700',
+  '#5fd75f',
+  '#5fd787',
+  '#5fd7af',
+  '#5fd7d7',
+  '#5fd7ff',
+  '#5fff00',
+  '#5fff5f',
+  '#5fff87',
+  '#5fffaf',
+  '#5fffd7',
+  '#5fffff',
+  '#870000',
+  '#87005f',
+  '#870087',
+  '#8700af',
+  '#8700d7',
+  '#8700ff',
+  '#875f00',
+  '#875f5f',
+  '#875f87',
+  '#875faf',
+  '#875fd7',
+  '#875fff',
+  '#878700',
+  '#87875f',
+  '#878787',
+  '#8787af',
+  '#8787d7',
+  '#8787ff',
+  '#87af00',
+  '#87af5f',
+  '#87af87',
+  '#87afaf',
+  '#87afd7',
+  '#87afff',
+  '#87d700',
+  '#87d75f',
+  '#87d787',
+  '#87d7af',
+  '#87d7d7',
+  '#87d7ff',
+  '#87ff00',
+  '#87ff5f',
+  '#87ff87',
+  '#87ffaf',
+  '#87ffd7',
+  '#87ffff',
+  '#af0000',
+  '#af005f',
+  '#af0087',
+  '#af00af',
+  '#af00d7',
+  '#af00ff',
+  '#af5f00',
+  '#af5f5f',
+  '#af5f87',
+  '#af5faf',
+  '#af5fd7',
+  '#af5fff',
+  '#af8700',
+  '#af875f',
+  '#af8787',
+  '#af87af',
+  '#af87d7',
+  '#af87ff',
+  '#afaf00',
+  '#afaf5f',
+  '#afaf87',
+  '#afafaf',
+  '#afafd7',
+  '#afafff',
+  '#afd700',
+  '#afd75f',
+  '#afd787',
+  '#afd7af',
+  '#afd7d7',
+  '#afd7ff',
+  '#afff00',
+  '#afff5f',
+  '#afff87',
+  '#afffaf',
+  '#afffd7',
+  '#afffff',
+  '#d70000',
+  '#d7005f',
+  '#d70087',
+  '#d700af',
+  '#d700d7',
+  '#d700ff',
+  '#d75f00',
+  '#d75f5f',
+  '#d75f87',
+  '#d75faf',
+  '#d75fd7',
+  '#d75fff',
+  '#d78700',
+  '#d7875f',
+  '#d78787',
+  '#d787af',
+  '#d787d7',
+  '#d787ff',
+  '#d7af00',
+  '#d7af5f',
+  '#d7af87',
+  '#d7afaf',
+  '#d7afd7',
+  '#d7afff',
+  '#d7d700',
+  '#d7d75f',
+  '#d7d787',
+  '#d7d7af',
+  '#d7d7d7',
+  '#d7d7ff',
+  '#d7ff00',
+  '#d7ff5f',
+  '#d7ff87',
+  '#d7ffaf',
+  '#d7ffd7',
+  '#d7ffff',
+  '#ff0000',
+  '#ff005f',
+  '#ff0087',
+  '#ff00af',
+  '#ff00d7',
+  '#ff00ff',
+  '#ff5f00',
+  '#ff5f5f',
+  '#ff5f87',
+  '#ff5faf',
+  '#ff5fd7',
+  '#ff5fff',
+  '#ff8700',
+  '#ff875f',
+  '#ff8787',
+  '#ff87af',
+  '#ff87d7',
+  '#ff87ff',
+  '#ffaf00',
+  '#ffaf5f',
+  '#ffaf87',
+  '#ffafaf',
+  '#ffafd7',
+  '#ffafff',
+  '#ffd700',
+  '#ffd75f',
+  '#ffd787',
+  '#ffd7af',
+  '#ffd7d7',
+  '#ffd7ff',
+  '#ffff00',
+  '#ffff5f',
+  '#ffff87',
+  '#ffffaf',
+  '#ffffd7',
+  '#ffffff',
+  '#080808',
+  '#121212',
+  '#1c1c1c',
+  '#262626',
+  '#303030',
+  '#3a3a3a',
+  '#444444',
+  '#4e4e4e',
+  '#585858',
+  '#626262',
+  '#6c6c6c',
+  '#767676',
+  '#808080',
+  '#8a8a8a',
+  '#949494',
+  '#9e9e9e',
+  '#a8a8a8',
+  '#b2b2b2',
+  '#bcbcbc',
+  '#c6c6c6',
+  '#d0d0d0',
+  '#dadada',
+  '#e4e4e4',
+  '#eeeeee',
+];
+
+export function convertColorToHex(
+  color: number,
+  colorMode: ColorMode,
+  defaultColor: string,
+): string {
+  if (colorMode === ColorMode.RGB) {
+    const r = (color >> 16) & 255;
+    const g = (color >> 8) & 255;
+    const b = color & 255;
+    return `#${r.toString(16).padStart(2, '0')}${g
+      .toString(16)
+      .padStart(2, '0')}${b.toString(16).padStart(2, '0')}`;
+  }
+  if (colorMode === ColorMode.PALETTE) {
+    return ANSI_COLORS[color] || defaultColor;
+  }
+  return defaultColor;
+}