From b10bcf49b9a16033af1fbaacc2c0bbc842385a4a Mon Sep 17 00:00:00 2001 From: matt korwel Date: Mon, 23 Mar 2026 13:54:47 -0700 Subject: [PATCH] fix(test): move flaky tests to non-blocking suite (#23259) --- integration-tests/hooks-system.test.ts | 4325 +++++++++++---------- integration-tests/symlink-install.test.ts | 183 +- integration-tests/test-helper.ts | 2 + package.json | 1 + 4 files changed, 2272 insertions(+), 2239 deletions(-) diff --git a/integration-tests/hooks-system.test.ts b/integration-tests/hooks-system.test.ts index 4fe63a3ab6..73a7ca03ab 100644 --- a/integration-tests/hooks-system.test.ts +++ b/integration-tests/hooks-system.test.ts @@ -5,406 +5,413 @@ */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { TestRig, poll, normalizePath } from './test-helper.js'; +import { TestRig, poll, normalizePath, skipFlaky } from './test-helper.js'; import { join } from 'node:path'; import { writeFileSync, existsSync, mkdirSync } from 'node:fs'; import os from 'node:os'; -describe('Hooks System Integration', { timeout: 120000 }, () => { - let rig: TestRig; +describe.skipIf(skipFlaky)( + 'Hooks System Integration', + { timeout: 120000 }, + () => { + let rig: TestRig; - beforeEach(() => { - rig = new TestRig(); - }); - - afterEach(async () => { - if (rig) { - await rig.cleanup(); - } - }); - - describe('Command Hooks - Blocking Behavior', () => { - it('should block tool execution when hook returns block decision', async () => { - rig.setup( - 'should block tool execution when hook returns block decision', - { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.block-tool.responses', - ), - }, - ); - - const scriptPath = rig.createScript( - 'block_hook.cjs', - "console.log(JSON.stringify({decision: 'block', reason: 'File writing blocked by security policy'}));", - ); - - rig.setup( - 'should block tool execution when hook returns block decision', - { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - matcher: 'write_file', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }, - ); - - const result = await rig.run({ - args: 'Create a file called test.txt with content "Hello World"', - }); - - // The hook should block the write_file tool - const toolLogs = rig.readToolLogs(); - const writeFileCalls = toolLogs.filter( - (t) => - t.toolRequest.name === 'write_file' && t.toolRequest.success === true, - ); - - // Tool should not be called due to blocking hook - expect(writeFileCalls).toHaveLength(0); - - // Result should mention the blocking reason - expect(result).toContain('File writing blocked by security policy'); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); + beforeEach(() => { + rig = new TestRig(); }); - it('should block tool execution and use stderr as reason when hook exits with code 2', async () => { - rig.setup( - 'should block tool execution and use stderr as reason when hook exits with code 2', - { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.block-tool.responses', - ), - }, - ); - - const blockMsg = 'File writing blocked by security policy'; - - const scriptPath = rig.createScript( - 'stderr_block_hook.cjs', - `process.stderr.write(JSON.stringify({ decision: 'deny', reason: '${blockMsg}' })); process.exit(2);`, - ); - - rig.setup( - 'should block tool execution and use stderr as reason when hook exits with code 2', - { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - matcher: 'write_file', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`)!, - timeout: 5000, - }, - ], - }, - ], - }, - }, - }, - ); - - const result = await rig.run({ - args: 'Create a file called test.txt with content "Hello World"', - }); - - // The hook should block the write_file tool - const toolLogs = rig.readToolLogs(); - const writeFileCalls = toolLogs.filter( - (t) => - t.toolRequest.name === 'write_file' && t.toolRequest.success === true, - ); - - // Tool should not be called due to blocking hook - expect(writeFileCalls).toHaveLength(0); - - // Result should mention the blocking reason - expect(result).toContain(blockMsg); - - // Verify hook telemetry shows the deny decision - const hookLogs = rig.readHookLogs(); - const blockHook = hookLogs.find( - (log) => - log.hookCall.hook_event_name === 'BeforeTool' && - (log.hookCall.stdout.includes('"decision":"deny"') || - log.hookCall.stderr.includes('"decision":"deny"')), - ); - expect(blockHook).toBeDefined(); - expect(blockHook?.hookCall.stdout + blockHook?.hookCall.stderr).toContain( - blockMsg, - ); + afterEach(async () => { + if (rig) { + await rig.cleanup(); + } }); - it('should allow tool execution when hook returns allow decision', async () => { - rig.setup( - 'should allow tool execution when hook returns allow decision', - { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.allow-tool.responses', - ), - }, - ); - - const scriptPath = rig.createScript( - 'allow_hook.cjs', - "console.log(JSON.stringify({decision: 'allow', reason: 'File writing approved'}));", - ); - - rig.setup( - 'should allow tool execution when hook returns allow decision', - { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - matcher: 'write_file', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, + describe('Command Hooks - Blocking Behavior', () => { + it('should block tool execution when hook returns block decision', async () => { + rig.setup( + 'should block tool execution when hook returns block decision', + { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.block-tool.responses', + ), }, - }, - ); + ); - await rig.run({ - args: 'Create a file called approved.txt with content "Approved content"', - }); + const scriptPath = rig.createScript( + 'block_hook.cjs', + "console.log(JSON.stringify({decision: 'block', reason: 'File writing blocked by security policy'}));", + ); - // The hook should allow the write_file tool - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // File should be created - const fileContent = rig.readFile('approved.txt'); - expect(fileContent).toContain('Approved content'); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - }); - }); - - describe('Command Hooks - Additional Context', () => { - it('should add additional context from AfterTool hooks', async () => { - rig.setup('should add additional context from AfterTool hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.after-tool-context.responses', - ), - }); - - const scriptPath = rig.createScript( - 'after_tool_context.cjs', - "console.log(JSON.stringify({hookSpecificOutput: {hookEventName: 'AfterTool', additionalContext: 'Security scan: File content appears safe'}}));", - ); - - const command = `node "${scriptPath}"`; - rig.setup('should add additional context from AfterTool hooks', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - AfterTool: [ - { - matcher: 'read_file', - sequential: true, - hooks: [ + rig.setup( + 'should block tool execution when hook returns block decision', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ { - type: 'command', - command: normalizePath(command), - timeout: 5000, + matcher: 'write_file', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], }, ], }, - ], - }, - }, - }); - - // Create a test file to read - rig.createFile('test-file.txt', 'This is test content'); - - await rig.run({ - args: 'Read the contents of test-file.txt and tell me what it contains', - }); - - // Should find read_file tool call - const foundReadFile = await rig.waitForToolCall('read_file'); - expect(foundReadFile).toBeTruthy(); - - // Should generate hook telemetry - const hookTelemetryFound = rig.readHookLogs(); - expect(hookTelemetryFound.length).toBeGreaterThan(0); - expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe('AfterTool'); - expect(hookTelemetryFound[0].hookCall.hook_name).toBe( - normalizePath(command), - ); - expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined(); - expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined(); - expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0); - expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined(); - expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined(); - }); - }); - - describe('Command Hooks - Tail Tool Calls', () => { - it('should execute a tail tool call from AfterTool hooks and replace original response', async () => { - // Create a script that acts as the hook. - // It will trigger on "read_file" and issue a tail call to "write_file". - rig.setup('should execute a tail tool call from AfterTool hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.tail-tool-call.responses', - ), - }); - - const hookOutput = { - decision: 'allow', - hookSpecificOutput: { - hookEventName: 'AfterTool', - tailToolCallRequest: { - name: 'write_file', - args: { - file_path: 'tail-called-file.txt', - content: 'Content from tail call', }, }, - }, - }; + ); - const hookScript = `console.log(JSON.stringify(${JSON.stringify( - hookOutput, - )})); process.exit(0);`; + const result = await rig.run({ + args: 'Create a file called test.txt with content "Hello World"', + }); - const scriptPath = join(rig.testDir!, 'tail_call_hook.js'); - writeFileSync(scriptPath, hookScript); - const commandPath = scriptPath.replace(/\\/g, '/'); + // The hook should block the write_file tool + const toolLogs = rig.readToolLogs(); + const writeFileCalls = toolLogs.filter( + (t) => + t.toolRequest.name === 'write_file' && + t.toolRequest.success === true, + ); - rig.setup('should execute a tail tool call from AfterTool hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.tail-tool-call.responses', - ), - settings: { - hooksConfig: { - enabled: true, + // Tool should not be called due to blocking hook + expect(writeFileCalls).toHaveLength(0); + + // Result should mention the blocking reason + expect(result).toContain('File writing blocked by security policy'); + + // Should generate hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); + }); + + it('should block tool execution and use stderr as reason when hook exits with code 2', async () => { + rig.setup( + 'should block tool execution and use stderr as reason when hook exits with code 2', + { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.block-tool.responses', + ), }, - hooks: { - AfterTool: [ - { - matcher: 'read_file', - hooks: [ + ); + + const blockMsg = 'File writing blocked by security policy'; + + const scriptPath = rig.createScript( + 'stderr_block_hook.cjs', + `process.stderr.write(JSON.stringify({ decision: 'deny', reason: '${blockMsg}' })); process.exit(2);`, + ); + + rig.setup( + 'should block tool execution and use stderr as reason when hook exits with code 2', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ { - type: 'command', - command: `node "${commandPath}"`, - timeout: 5000, + matcher: 'write_file', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`)!, + timeout: 5000, + }, + ], }, ], }, - ], + }, }, - }, + ); + + const result = await rig.run({ + args: 'Create a file called test.txt with content "Hello World"', + }); + + // The hook should block the write_file tool + const toolLogs = rig.readToolLogs(); + const writeFileCalls = toolLogs.filter( + (t) => + t.toolRequest.name === 'write_file' && + t.toolRequest.success === true, + ); + + // Tool should not be called due to blocking hook + expect(writeFileCalls).toHaveLength(0); + + // Result should mention the blocking reason + expect(result).toContain(blockMsg); + + // Verify hook telemetry shows the deny decision + const hookLogs = rig.readHookLogs(); + const blockHook = hookLogs.find( + (log) => + log.hookCall.hook_event_name === 'BeforeTool' && + (log.hookCall.stdout.includes('"decision":"deny"') || + log.hookCall.stderr.includes('"decision":"deny"')), + ); + expect(blockHook).toBeDefined(); + expect( + blockHook?.hookCall.stdout + blockHook?.hookCall.stderr, + ).toContain(blockMsg); }); - // Create a test file to trigger the read_file tool - rig.createFile('original.txt', 'Original content'); + it('should allow tool execution when hook returns allow decision', async () => { + rig.setup( + 'should allow tool execution when hook returns allow decision', + { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.allow-tool.responses', + ), + }, + ); - const cliOutput = await rig.run({ - args: 'Read original.txt', // Fake responses should trigger read_file on this + const scriptPath = rig.createScript( + 'allow_hook.cjs', + "console.log(JSON.stringify({decision: 'allow', reason: 'File writing approved'}));", + ); + + rig.setup( + 'should allow tool execution when hook returns allow decision', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ + { + matcher: 'write_file', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }, + ); + + await rig.run({ + args: 'Create a file called approved.txt with content "Approved content"', + }); + + // The hook should allow the write_file tool + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // File should be created + const fileContent = rig.readFile('approved.txt'); + expect(fileContent).toContain('Approved content'); + + // Should generate hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); }); - - // 1. Verify that write_file was called (as a tail call replacing read_file) - // Since read_file was replaced before finalizing, it will not appear in the tool logs. - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // Ensure hook logs are flushed and the final LLM response is received. - // The mock LLM is configured to respond with "Tail call completed successfully." - expect(cliOutput).toContain('Tail call completed successfully.'); - - // Ensure telemetry is written to disk - await rig.waitForTelemetryReady(); - - // Read hook logs to debug - const hookLogs = rig.readHookLogs(); - const relevantHookLog = hookLogs.find( - (l) => l.hookCall.hook_event_name === 'AfterTool', - ); - - expect(relevantHookLog).toBeDefined(); - - // 2. Verify write_file was executed. - // In non-interactive mode, the CLI deduplicates tool execution logs by callId. - // Since a tail call reuses the original callId, "Tool: write_file" is not printed. - // Instead, we verify the side-effect (file creation) and the telemetry log. - - // 3. Verify the tail-called tool actually wrote the file - const modifiedContent = rig.readFile('tail-called-file.txt'); - expect(modifiedContent).toBe('Content from tail call'); - - // 4. Verify telemetry for the final tool call. - // The original 'read_file' call is replaced, so only 'write_file' is finalized and logged. - const toolLogs = rig.readToolLogs(); - const successfulTools = toolLogs.filter((t) => t.toolRequest.success); - expect( - successfulTools.some((t) => t.toolRequest.name === 'write_file'), - ).toBeTruthy(); - // The original request name should be preserved in the log payload if possible, - // but the executed tool name is 'write_file'. }); - }); - describe('BeforeModel Hooks - LLM Request Modification', () => { - it('should modify LLM requests with BeforeModel hooks', async () => { - // Create a hook script that replaces the LLM request with a modified version - // Note: Providing messages in the hook output REPLACES the entire conversation - rig.setup('should modify LLM requests with BeforeModel hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.before-model.responses', - ), + describe('Command Hooks - Additional Context', () => { + it('should add additional context from AfterTool hooks', async () => { + rig.setup('should add additional context from AfterTool hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.after-tool-context.responses', + ), + }); + + const scriptPath = rig.createScript( + 'after_tool_context.cjs', + "console.log(JSON.stringify({hookSpecificOutput: {hookEventName: 'AfterTool', additionalContext: 'Security scan: File content appears safe'}}));", + ); + + const command = `node "${scriptPath}"`; + rig.setup('should add additional context from AfterTool hooks', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + AfterTool: [ + { + matcher: 'read_file', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(command), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + // Create a test file to read + rig.createFile('test-file.txt', 'This is test content'); + + await rig.run({ + args: 'Read the contents of test-file.txt and tell me what it contains', + }); + + // Should find read_file tool call + const foundReadFile = await rig.waitForToolCall('read_file'); + expect(foundReadFile).toBeTruthy(); + + // Should generate hook telemetry + const hookTelemetryFound = rig.readHookLogs(); + expect(hookTelemetryFound.length).toBeGreaterThan(0); + expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe( + 'AfterTool', + ); + expect(hookTelemetryFound[0].hookCall.hook_name).toBe( + normalizePath(command), + ); + expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined(); + expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined(); + expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0); + expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined(); + expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined(); }); - const hookScript = `const fs = require('fs'); + }); + + describe('Command Hooks - Tail Tool Calls', () => { + it('should execute a tail tool call from AfterTool hooks and replace original response', async () => { + // Create a script that acts as the hook. + // It will trigger on "read_file" and issue a tail call to "write_file". + rig.setup('should execute a tail tool call from AfterTool hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.tail-tool-call.responses', + ), + }); + + const hookOutput = { + decision: 'allow', + hookSpecificOutput: { + hookEventName: 'AfterTool', + tailToolCallRequest: { + name: 'write_file', + args: { + file_path: 'tail-called-file.txt', + content: 'Content from tail call', + }, + }, + }, + }; + + const hookScript = `console.log(JSON.stringify(${JSON.stringify( + hookOutput, + )})); process.exit(0);`; + + const scriptPath = join(rig.testDir!, 'tail_call_hook.js'); + writeFileSync(scriptPath, hookScript); + const commandPath = scriptPath.replace(/\\/g, '/'); + + rig.setup('should execute a tail tool call from AfterTool hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.tail-tool-call.responses', + ), + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + AfterTool: [ + { + matcher: 'read_file', + hooks: [ + { + type: 'command', + command: `node "${commandPath}"`, + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + // Create a test file to trigger the read_file tool + rig.createFile('original.txt', 'Original content'); + + const cliOutput = await rig.run({ + args: 'Read original.txt', // Fake responses should trigger read_file on this + }); + + // 1. Verify that write_file was called (as a tail call replacing read_file) + // Since read_file was replaced before finalizing, it will not appear in the tool logs. + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // Ensure hook logs are flushed and the final LLM response is received. + // The mock LLM is configured to respond with "Tail call completed successfully." + expect(cliOutput).toContain('Tail call completed successfully.'); + + // Ensure telemetry is written to disk + await rig.waitForTelemetryReady(); + + // Read hook logs to debug + const hookLogs = rig.readHookLogs(); + const relevantHookLog = hookLogs.find( + (l) => l.hookCall.hook_event_name === 'AfterTool', + ); + + expect(relevantHookLog).toBeDefined(); + + // 2. Verify write_file was executed. + // In non-interactive mode, the CLI deduplicates tool execution logs by callId. + // Since a tail call reuses the original callId, "Tool: write_file" is not printed. + // Instead, we verify the side-effect (file creation) and the telemetry log. + + // 3. Verify the tail-called tool actually wrote the file + const modifiedContent = rig.readFile('tail-called-file.txt'); + expect(modifiedContent).toBe('Content from tail call'); + + // 4. Verify telemetry for the final tool call. + // The original 'read_file' call is replaced, so only 'write_file' is finalized and logged. + const toolLogs = rig.readToolLogs(); + const successfulTools = toolLogs.filter((t) => t.toolRequest.success); + expect( + successfulTools.some((t) => t.toolRequest.name === 'write_file'), + ).toBeTruthy(); + // The original request name should be preserved in the log payload if possible, + // but the executed tool name is 'write_file'. + }); + }); + + describe('BeforeModel Hooks - LLM Request Modification', () => { + it('should modify LLM requests with BeforeModel hooks', async () => { + // Create a hook script that replaces the LLM request with a modified version + // Note: Providing messages in the hook output REPLACES the entire conversation + rig.setup('should modify LLM requests with BeforeModel hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.before-model.responses', + ), + }); + const hookScript = `const fs = require('fs'); console.log(JSON.stringify({ decision: "allow", hookSpecificOutput: { @@ -420,166 +427,169 @@ console.log(JSON.stringify({ } }));`; - const scriptPath = rig.createScript('before_model_hook.cjs', hookScript); + const scriptPath = rig.createScript( + 'before_model_hook.cjs', + hookScript, + ); - rig.setup('should modify LLM requests with BeforeModel hooks', { - settings: { - hooksConfig: { - enabled: true, + rig.setup('should modify LLM requests with BeforeModel hooks', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeModel: [ + { + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], + }, + ], + }, }, - hooks: { - BeforeModel: [ - { - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, - }, + }); + + const result = await rig.run({ args: 'Tell me a story' }); + + // The hook should have replaced the request entirely + // Verify that the model responded to the modified request, not the original + expect(result).toBeDefined(); + expect(result.length).toBeGreaterThan(0); + // The response should contain the expected text from the modified request + expect(result.toLowerCase()).toContain('security hook modified'); + + // Should generate hook telemetry + + // Should generate hook telemetry + const hookTelemetryFound = rig.readHookLogs(); + expect(hookTelemetryFound.length).toBeGreaterThan(0); + expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe( + 'BeforeModel', + ); + expect(hookTelemetryFound[0].hookCall.hook_name).toBe( + `node "${scriptPath}"`, + ); + expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined(); + expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined(); + expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0); + expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined(); + expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined(); }); - const result = await rig.run({ args: 'Tell me a story' }); - - // The hook should have replaced the request entirely - // Verify that the model responded to the modified request, not the original - expect(result).toBeDefined(); - expect(result.length).toBeGreaterThan(0); - // The response should contain the expected text from the modified request - expect(result.toLowerCase()).toContain('security hook modified'); - - // Should generate hook telemetry - - // Should generate hook telemetry - const hookTelemetryFound = rig.readHookLogs(); - expect(hookTelemetryFound.length).toBeGreaterThan(0); - expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe( - 'BeforeModel', - ); - expect(hookTelemetryFound[0].hookCall.hook_name).toBe( - `node "${scriptPath}"`, - ); - expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined(); - expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined(); - expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0); - expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined(); - expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined(); - }); - - it('should block model execution when BeforeModel hook returns deny decision', async () => { - rig.setup( - 'should block model execution when BeforeModel hook returns deny decision', - ); - const hookScript = `console.log(JSON.stringify({ + it('should block model execution when BeforeModel hook returns deny decision', async () => { + rig.setup( + 'should block model execution when BeforeModel hook returns deny decision', + ); + const hookScript = `console.log(JSON.stringify({ decision: "deny", reason: "Model execution blocked by security policy" }));`; - const scriptPath = rig.createScript( - 'before_model_deny_hook.cjs', - hookScript, - ); + const scriptPath = rig.createScript( + 'before_model_deny_hook.cjs', + hookScript, + ); - rig.setup( - 'should block model execution when BeforeModel hook returns deny decision', - { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeModel: [ - { - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], + rig.setup( + 'should block model execution when BeforeModel hook returns deny decision', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeModel: [ + { + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], + }, + ], + }, }, }, - }, - ); + ); - const result = await rig.run({ args: 'Hello' }); + const result = await rig.run({ args: 'Hello' }); - // The hook should have blocked the request - expect(result).toContain('Model execution blocked by security policy'); + // The hook should have blocked the request + expect(result).toContain('Model execution blocked by security policy'); - // Verify no API requests were made to the LLM - const apiRequests = rig.readAllApiRequest(); - expect(apiRequests).toHaveLength(0); - }); + // Verify no API requests were made to the LLM + const apiRequests = rig.readAllApiRequest(); + expect(apiRequests).toHaveLength(0); + }); - it('should block model execution when BeforeModel hook returns block decision', async () => { - rig.setup( - 'should block model execution when BeforeModel hook returns block decision', - ); - const hookScript = `console.log(JSON.stringify({ + it('should block model execution when BeforeModel hook returns block decision', async () => { + rig.setup( + 'should block model execution when BeforeModel hook returns block decision', + ); + const hookScript = `console.log(JSON.stringify({ decision: "block", reason: "Model execution blocked by security policy" }));`; - const scriptPath = rig.createScript( - 'before_model_block_hook.cjs', - hookScript, - ); + const scriptPath = rig.createScript( + 'before_model_block_hook.cjs', + hookScript, + ); - rig.setup( - 'should block model execution when BeforeModel hook returns block decision', - { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeModel: [ - { - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], + rig.setup( + 'should block model execution when BeforeModel hook returns block decision', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeModel: [ + { + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], + }, + ], + }, }, }, - }, - ); + ); - const result = await rig.run({ args: 'Hello' }); + const result = await rig.run({ args: 'Hello' }); - // The hook should have blocked the request - expect(result).toContain('Model execution blocked by security policy'); + // The hook should have blocked the request + expect(result).toContain('Model execution blocked by security policy'); - // Verify no API requests were made to the LLM - const apiRequests = rig.readAllApiRequest(); - expect(apiRequests).toHaveLength(0); + // Verify no API requests were made to the LLM + const apiRequests = rig.readAllApiRequest(); + expect(apiRequests).toHaveLength(0); + }); }); - }); - describe('AfterModel Hooks - LLM Response Modification', () => { - it.skipIf(process.platform === 'win32')( - 'should modify LLM responses with AfterModel hooks', - async () => { - rig.setup('should modify LLM responses with AfterModel hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.after-model.responses', - ), - }); - // Create a hook script that modifies the LLM response - const hookScript = `const fs = require('fs'); + describe('AfterModel Hooks - LLM Response Modification', () => { + it.skipIf(process.platform === 'win32')( + 'should modify LLM responses with AfterModel hooks', + async () => { + rig.setup('should modify LLM responses with AfterModel hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.after-model.responses', + ), + }); + // Create a hook script that modifies the LLM response + const hookScript = `const fs = require('fs'); console.log(JSON.stringify({ hookSpecificOutput: { hookEventName: "AfterModel", @@ -599,15 +609,148 @@ console.log(JSON.stringify({ } }));`; - const scriptPath = rig.createScript('after_model_hook.cjs', hookScript); + const scriptPath = rig.createScript( + 'after_model_hook.cjs', + hookScript, + ); - rig.setup('should modify LLM responses with AfterModel hooks', { + rig.setup('should modify LLM responses with AfterModel hooks', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + AfterModel: [ + { + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + const result = await rig.run({ args: 'What is 2 + 2?' }); + + // The hook should have replaced the model response + expect(result).toContain( + '[FILTERED] Response has been filtered for security compliance', + ); + + // Should generate hook telemetry + const hookTelemetryFound = + await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); + }, + ); + }); + + describe('BeforeToolSelection Hooks - Tool Configuration', () => { + it('should modify tool selection with BeforeToolSelection hooks', async () => { + // 1. Initial setup to establish test directory + rig.setup('BeforeToolSelection Hooks'); + + const toolConfigJson = JSON.stringify({ + decision: 'allow', + hookSpecificOutput: { + hookEventName: 'BeforeToolSelection', + toolConfig: { + mode: 'ANY', + allowedFunctionNames: ['read_file'], + }, + }, + }); + + // Use file-based hook to avoid quoting issues + const hookScript = `console.log(JSON.stringify(${toolConfigJson}));`; + const hookFilename = 'before_tool_selection_hook.js'; + const scriptPath = rig.createScript(hookFilename, hookScript); + + // 2. Final setup with script path + rig.setup('BeforeToolSelection Hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.before-tool-selection.responses', + ), + settings: { + debugMode: true, + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeToolSelection: [ + { + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 60000, + }, + ], + }, + ], + }, + }, + }); + + // Create a test file + rig.createFile('new_file_data.txt', 'test data'); + + await rig.run({ + args: 'Check the content of new_file_data.txt', + }); + + // Verify the hook was called for BeforeToolSelection event + const hookLogs = rig.readHookLogs(); + const beforeToolSelectionHook = hookLogs.find( + (log) => log.hookCall.hook_event_name === 'BeforeToolSelection', + ); + expect(beforeToolSelectionHook).toBeDefined(); + expect(beforeToolSelectionHook?.hookCall.success).toBe(true); + + // Verify hook telemetry shows it modified the config + expect( + JSON.stringify(beforeToolSelectionHook?.hookCall.hook_output), + ).toContain('read_file'); + }); + }); + + describe('BeforeAgent Hooks - Prompt Augmentation', () => { + it('should augment prompts with BeforeAgent hooks', async () => { + // Create a hook script that adds context to the prompt + const hookScript = `const fs = require('fs'); +console.log(JSON.stringify({ + decision: "allow", + hookSpecificOutput: { + hookEventName: "BeforeAgent", + additionalContext: "SYSTEM INSTRUCTION: You are in a secure environment. Always mention security compliance in your responses." + } +}));`; + + rig.setup('should augment prompts with BeforeAgent hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.before-agent.responses', + ), + }); + + const scriptPath = rig.createScript( + 'before_agent_hook.cjs', + hookScript, + ); + + rig.setup('should augment prompts with BeforeAgent hooks', { settings: { hooksConfig: { enabled: true, }, hooks: { - AfterModel: [ + BeforeAgent: [ { hooks: [ { @@ -622,335 +765,210 @@ console.log(JSON.stringify({ }, }); - const result = await rig.run({ args: 'What is 2 + 2?' }); + const result = await rig.run({ args: 'Hello, how are you?' }); - // The hook should have replaced the model response - expect(result).toContain( - '[FILTERED] Response has been filtered for security compliance', - ); + // The hook should have added security context, which should influence the response + expect(result).toContain('security'); // Should generate hook telemetry const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); expect(hookTelemetryFound).toBeTruthy(); - }, - ); - }); - - describe('BeforeToolSelection Hooks - Tool Configuration', () => { - it('should modify tool selection with BeforeToolSelection hooks', async () => { - // 1. Initial setup to establish test directory - rig.setup('BeforeToolSelection Hooks'); - - const toolConfigJson = JSON.stringify({ - decision: 'allow', - hookSpecificOutput: { - hookEventName: 'BeforeToolSelection', - toolConfig: { - mode: 'ANY', - allowedFunctionNames: ['read_file'], - }, - }, }); - - // Use file-based hook to avoid quoting issues - const hookScript = `console.log(JSON.stringify(${toolConfigJson}));`; - const hookFilename = 'before_tool_selection_hook.js'; - const scriptPath = rig.createScript(hookFilename, hookScript); - - // 2. Final setup with script path - rig.setup('BeforeToolSelection Hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.before-tool-selection.responses', - ), - settings: { - debugMode: true, - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeToolSelection: [ - { - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 60000, - }, - ], - }, - ], - }, - }, - }); - - // Create a test file - rig.createFile('new_file_data.txt', 'test data'); - - await rig.run({ - args: 'Check the content of new_file_data.txt', - }); - - // Verify the hook was called for BeforeToolSelection event - const hookLogs = rig.readHookLogs(); - const beforeToolSelectionHook = hookLogs.find( - (log) => log.hookCall.hook_event_name === 'BeforeToolSelection', - ); - expect(beforeToolSelectionHook).toBeDefined(); - expect(beforeToolSelectionHook?.hookCall.success).toBe(true); - - // Verify hook telemetry shows it modified the config - expect( - JSON.stringify(beforeToolSelectionHook?.hookCall.hook_output), - ).toContain('read_file'); }); - }); - describe('BeforeAgent Hooks - Prompt Augmentation', () => { - it('should augment prompts with BeforeAgent hooks', async () => { - // Create a hook script that adds context to the prompt - const hookScript = `const fs = require('fs'); -console.log(JSON.stringify({ - decision: "allow", - hookSpecificOutput: { - hookEventName: "BeforeAgent", - additionalContext: "SYSTEM INSTRUCTION: You are in a secure environment. Always mention security compliance in your responses." - } -}));`; + describe('Notification Hooks - Permission Handling', () => { + it('should handle notification hooks for tool permissions', async () => { + rig.setup('should handle notification hooks for tool permissions', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.notification.responses', + ), + }); - rig.setup('should augment prompts with BeforeAgent hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.before-agent.responses', - ), - }); - - const scriptPath = rig.createScript('before_agent_hook.cjs', hookScript); - - rig.setup('should augment prompts with BeforeAgent hooks', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeAgent: [ - { - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - const result = await rig.run({ args: 'Hello, how are you?' }); - - // The hook should have added security context, which should influence the response - expect(result).toContain('security'); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - }); - }); - - describe('Notification Hooks - Permission Handling', () => { - it('should handle notification hooks for tool permissions', async () => { - rig.setup('should handle notification hooks for tool permissions', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.notification.responses', - ), - }); - - // Create script file for hook - const scriptPath = rig.createScript( - 'notification_hook.cjs', - "console.log(JSON.stringify({suppressOutput: false, systemMessage: 'Permission request logged by security hook'}));", - ); - - const hookCommand = `node "${scriptPath}"`; - - rig.setup('should handle notification hooks for tool permissions', { - settings: { - // Configure tools to enable hooks and require confirmation to trigger notifications - tools: { - approval: 'ASK', // Disable YOLO mode to show permission prompts - confirmationRequired: ['run_shell_command'], - }, - hooksConfig: { - enabled: true, - }, - hooks: { - Notification: [ - { - matcher: 'ToolPermission', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(hookCommand), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - const run = await rig.runInteractive({ approvalMode: 'default' }); - - // Send prompt that will trigger a permission request - await run.type('Run the command "echo test"'); - await run.type('\r'); - - // Wait for permission prompt to appear - await run.expectText('Allow', 10000); - - // Approve the permission - await run.type('y'); - await run.type('\r'); - - // Wait for command to execute - await run.expectText('test', 10000); - - // Should find the shell command execution - const foundShellCommand = await rig.waitForToolCall('run_shell_command'); - expect(foundShellCommand).toBeTruthy(); - - // Verify Notification hook executed - const hookLogs = rig.readHookLogs(); - const notificationLog = hookLogs.find( - (log) => - log.hookCall.hook_event_name === 'Notification' && - log.hookCall.hook_name === normalizePath(hookCommand), - ); - - expect(notificationLog).toBeDefined(); - if (notificationLog) { - expect(notificationLog.hookCall.exit_code).toBe(0); - expect(notificationLog.hookCall.stdout).toContain( - 'Permission request logged by security hook', + // Create script file for hook + const scriptPath = rig.createScript( + 'notification_hook.cjs', + "console.log(JSON.stringify({suppressOutput: false, systemMessage: 'Permission request logged by security hook'}));", ); - // Verify hook input contains notification details - const hookInputStr = - typeof notificationLog.hookCall.hook_input === 'string' - ? notificationLog.hookCall.hook_input - : JSON.stringify(notificationLog.hookCall.hook_input); - const hookInput = JSON.parse(hookInputStr) as Record; + const hookCommand = `node "${scriptPath}"`; - // Should have notification type (uses snake_case) - expect(hookInput['notification_type']).toBe('ToolPermission'); - - // Should have message - expect(hookInput['message']).toBeDefined(); - - // Should have details with tool info - expect(hookInput['details']).toBeDefined(); - const details = hookInput['details'] as Record; - // For 'exec' type confirmations, details contains: type, title, command, rootCommand - expect(details['type']).toBe('exec'); - expect(details['command']).toBeDefined(); - expect(details['title']).toBeDefined(); - } - }); - }); - - describe('Sequential Hook Execution', () => { - it('should execute hooks sequentially when configured', async () => { - rig.setup('should execute hooks sequentially when configured', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.sequential-execution.responses', - ), - }); - - // Create script files for hooks - const hook1Path = rig.createScript( - 'seq_hook1.cjs', - "console.log(JSON.stringify({decision: 'allow', hookSpecificOutput: {hookEventName: 'BeforeAgent', additionalContext: 'Step 1: Initial validation passed.'}}));", - ); - const hook2Path = rig.createScript( - 'seq_hook2.cjs', - "console.log(JSON.stringify({decision: 'allow', hookSpecificOutput: {hookEventName: 'BeforeAgent', additionalContext: 'Step 2: Security check completed.'}}));", - ); - - const hook1Command = `node "${hook1Path}"`; - const hook2Command = `node "${hook2Path}"`; - - rig.setup('should execute hooks sequentially when configured', { - settings: { - hooksConfig: { - enabled: true, + rig.setup('should handle notification hooks for tool permissions', { + settings: { + // Configure tools to enable hooks and require confirmation to trigger notifications + tools: { + approval: 'ASK', // Disable YOLO mode to show permission prompts + confirmationRequired: ['run_shell_command'], + }, + hooksConfig: { + enabled: true, + }, + hooks: { + Notification: [ + { + matcher: 'ToolPermission', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(hookCommand), + timeout: 5000, + }, + ], + }, + ], + }, }, - hooks: { - BeforeAgent: [ - { - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(hook1Command), - timeout: 5000, - }, - { - type: 'command', - command: normalizePath(hook2Command), - timeout: 5000, - }, - ], - }, - ], - }, - }, + }); + + const run = await rig.runInteractive({ approvalMode: 'default' }); + + // Send prompt that will trigger a permission request + await run.type('Run the command "echo test"'); + await run.type('\r'); + + // Wait for permission prompt to appear + await run.expectText('Allow', 10000); + + // Approve the permission + await run.type('y'); + await run.type('\r'); + + // Wait for command to execute + await run.expectText('test', 10000); + + // Should find the shell command execution + const foundShellCommand = + await rig.waitForToolCall('run_shell_command'); + expect(foundShellCommand).toBeTruthy(); + + // Verify Notification hook executed + const hookLogs = rig.readHookLogs(); + const notificationLog = hookLogs.find( + (log) => + log.hookCall.hook_event_name === 'Notification' && + log.hookCall.hook_name === normalizePath(hookCommand), + ); + + expect(notificationLog).toBeDefined(); + if (notificationLog) { + expect(notificationLog.hookCall.exit_code).toBe(0); + expect(notificationLog.hookCall.stdout).toContain( + 'Permission request logged by security hook', + ); + + // Verify hook input contains notification details + const hookInputStr = + typeof notificationLog.hookCall.hook_input === 'string' + ? notificationLog.hookCall.hook_input + : JSON.stringify(notificationLog.hookCall.hook_input); + const hookInput = JSON.parse(hookInputStr) as Record; + + // Should have notification type (uses snake_case) + expect(hookInput['notification_type']).toBe('ToolPermission'); + + // Should have message + expect(hookInput['message']).toBeDefined(); + + // Should have details with tool info + expect(hookInput['details']).toBeDefined(); + const details = hookInput['details'] as Record; + // For 'exec' type confirmations, details contains: type, title, command, rootCommand + expect(details['type']).toBe('exec'); + expect(details['command']).toBeDefined(); + expect(details['title']).toBeDefined(); + } }); - - await rig.run({ args: 'Hello, please help me with a task' }); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - - // Verify both hooks executed - const hookLogs = rig.readHookLogs(); - const hook1Log = hookLogs.find( - (log) => log.hookCall.hook_name === normalizePath(hook1Command), - ); - const hook2Log = hookLogs.find( - (log) => log.hookCall.hook_name === normalizePath(hook2Command), - ); - - expect(hook1Log).toBeDefined(); - expect(hook1Log?.hookCall.exit_code).toBe(0); - expect(hook1Log?.hookCall.stdout).toContain( - 'Step 1: Initial validation passed', - ); - - expect(hook2Log).toBeDefined(); - expect(hook2Log?.hookCall.exit_code).toBe(0); - expect(hook2Log?.hookCall.stdout).toContain( - 'Step 2: Security check completed', - ); }); - }); - describe('Hook Input/Output Validation', () => { - it('should provide correct input format to hooks', async () => { - rig.setup('should provide correct input format to hooks', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.input-validation.responses', - ), + describe('Sequential Hook Execution', () => { + it('should execute hooks sequentially when configured', async () => { + rig.setup('should execute hooks sequentially when configured', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.sequential-execution.responses', + ), + }); + + // Create script files for hooks + const hook1Path = rig.createScript( + 'seq_hook1.cjs', + "console.log(JSON.stringify({decision: 'allow', hookSpecificOutput: {hookEventName: 'BeforeAgent', additionalContext: 'Step 1: Initial validation passed.'}}));", + ); + const hook2Path = rig.createScript( + 'seq_hook2.cjs', + "console.log(JSON.stringify({decision: 'allow', hookSpecificOutput: {hookEventName: 'BeforeAgent', additionalContext: 'Step 2: Security check completed.'}}));", + ); + + const hook1Command = `node "${hook1Path}"`; + const hook2Command = `node "${hook2Path}"`; + + rig.setup('should execute hooks sequentially when configured', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeAgent: [ + { + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(hook1Command), + timeout: 5000, + }, + { + type: 'command', + command: normalizePath(hook2Command), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + await rig.run({ args: 'Hello, please help me with a task' }); + + // Should generate hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); + + // Verify both hooks executed + const hookLogs = rig.readHookLogs(); + const hook1Log = hookLogs.find( + (log) => log.hookCall.hook_name === normalizePath(hook1Command), + ); + const hook2Log = hookLogs.find( + (log) => log.hookCall.hook_name === normalizePath(hook2Command), + ); + + expect(hook1Log).toBeDefined(); + expect(hook1Log?.hookCall.exit_code).toBe(0); + expect(hook1Log?.hookCall.stdout).toContain( + 'Step 1: Initial validation passed', + ); + + expect(hook2Log).toBeDefined(); + expect(hook2Log?.hookCall.exit_code).toBe(0); + expect(hook2Log?.hookCall.stdout).toContain( + 'Step 2: Security check completed', + ); }); - // Create a hook script that validates the input format - const hookScript = `const fs = require('fs'); + }); + + describe('Hook Input/Output Validation', () => { + it('should provide correct input format to hooks', async () => { + rig.setup('should provide correct input format to hooks', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.input-validation.responses', + ), + }); + // Create a hook script that validates the input format + const hookScript = `const fs = require('fs'); const input = fs.readFileSync(0, 'utf-8'); try { const json = JSON.parse(input); @@ -964,69 +982,12 @@ try { console.log(JSON.stringify({decision: "block", reason: "Invalid JSON"})); }`; - const scriptPath = rig.createScript( - 'input_validation_hook.cjs', - hookScript, - ); + const scriptPath = rig.createScript( + 'input_validation_hook.cjs', + hookScript, + ); - rig.setup('should provide correct input format to hooks', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - await rig.run({ - args: 'Create a file called input-test.txt with content "test"', - }); - - // Hook should validate input format successfully - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // Check that the file was created (hook allowed it) - const fileContent = rig.readFile('input-test.txt'); - expect(fileContent).toContain('test'); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - }); - - it('should treat mixed stdout (text + JSON) as system message and allow execution when exit code is 0', async () => { - rig.setup( - 'should treat mixed stdout (text + JSON) as system message and allow execution when exit code is 0', - { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.allow-tool.responses', - ), - }, - ); - - // Create script file for hook - const scriptPath = rig.createScript( - 'pollution_hook.cjs', - "console.log('Pollution'); console.log(JSON.stringify({decision: 'deny', reason: 'Should be ignored'}));", - ); - - rig.setup( - 'should treat mixed stdout (text + JSON) as system message and allow execution when exit code is 0', - { + rig.setup('should provide correct input format to hooks', { settings: { hooksConfig: { enabled: true, @@ -1034,13 +995,9 @@ try { hooks: { BeforeTool: [ { - matcher: 'write_file', - sequential: true, hooks: [ { type: 'command', - // Output plain text then JSON. - // This breaks JSON parsing, so it falls back to 'allow' with the whole stdout as systemMessage. command: normalizePath(`node "${scriptPath}"`), timeout: 5000, }, @@ -1049,341 +1006,402 @@ try { ], }, }, - }, - ); + }); - const result = await rig.run({ - args: 'Create a file called approved.txt with content "Approved content"', + await rig.run({ + args: 'Create a file called input-test.txt with content "test"', + }); + + // Hook should validate input format successfully + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // Check that the file was created (hook allowed it) + const fileContent = rig.readFile('input-test.txt'); + expect(fileContent).toContain('test'); + + // Should generate hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); }); - // The hook logic fails to parse JSON, so it allows the tool. - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // The entire stdout (including the JSON part) becomes the systemMessage - expect(result).toContain('Pollution'); - expect(result).toContain('Should be ignored'); - }); - }); - - describe('Multiple Event Types', () => { - it('should handle hooks for all major event types', async () => { - rig.setup('should handle hooks for all major event types', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.multiple-events.responses', - ), - }); - - // Create script files for hooks - const btPath = rig.createScript( - 'bt_hook.cjs', - "console.log(JSON.stringify({decision: 'allow', systemMessage: 'BeforeTool: File operation logged'}));", - ); - const atPath = rig.createScript( - 'at_hook.cjs', - "console.log(JSON.stringify({hookSpecificOutput: {hookEventName: 'AfterTool', additionalContext: 'AfterTool: Operation completed successfully'}}));", - ); - const baPath = rig.createScript( - 'ba_hook.cjs', - "console.log(JSON.stringify({decision: 'allow', hookSpecificOutput: {hookEventName: 'BeforeAgent', additionalContext: 'BeforeAgent: User request processed'}}));", - ); - - const beforeToolCommand = `node "${btPath}"`; - const afterToolCommand = `node "${atPath}"`; - const beforeAgentCommand = `node "${baPath}"`; - - rig.setup('should handle hooks for all major event types', { - settings: { - hooksConfig: { - enabled: true, + it('should treat mixed stdout (text + JSON) as system message and allow execution when exit code is 0', async () => { + rig.setup( + 'should treat mixed stdout (text + JSON) as system message and allow execution when exit code is 0', + { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.allow-tool.responses', + ), }, - hooks: { - BeforeAgent: [ - { - hooks: [ - { - type: 'command', - command: normalizePath(beforeAgentCommand), - timeout: 5000, - }, - ], - }, - ], - BeforeTool: [ - { - matcher: 'write_file', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(beforeToolCommand), - timeout: 5000, - }, - ], - }, - ], - AfterTool: [ - { - matcher: 'write_file', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(afterToolCommand), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - const result = await rig.run({ - args: - 'Create a file called multi-event-test.txt with content ' + - '"testing multiple events", and then please reply with ' + - 'everything I say just after this:"', - }); - - // Should execute write_file tool - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // File should be created - const fileContent = rig.readFile('multi-event-test.txt'); - expect(fileContent).toContain('testing multiple events'); - - // Result should contain context from all hooks - expect(result).toContain('BeforeTool: File operation logged'); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - - // Verify all three hooks executed - const hookLogs = rig.readHookLogs(); - const beforeAgentLog = hookLogs.find( - (log) => log.hookCall.hook_name === normalizePath(beforeAgentCommand), - ); - const beforeToolLog = hookLogs.find( - (log) => log.hookCall.hook_name === normalizePath(beforeToolCommand), - ); - const afterToolLog = hookLogs.find( - (log) => log.hookCall.hook_name === normalizePath(afterToolCommand), - ); - - expect(beforeAgentLog).toBeDefined(); - expect(beforeAgentLog?.hookCall.exit_code).toBe(0); - expect(beforeAgentLog?.hookCall.stdout).toContain( - 'BeforeAgent: User request processed', - ); - - expect(beforeToolLog).toBeDefined(); - expect(beforeToolLog?.hookCall.exit_code).toBe(0); - expect(beforeToolLog?.hookCall.stdout).toContain( - 'BeforeTool: File operation logged', - ); - - expect(afterToolLog).toBeDefined(); - expect(afterToolLog?.hookCall.exit_code).toBe(0); - expect(afterToolLog?.hookCall.stdout).toContain( - 'AfterTool: Operation completed successfully', - ); - }); - }); - - describe('Hook Error Handling', () => { - it('should handle hook failures gracefully', async () => { - rig.setup('should handle hook failures gracefully', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.error-handling.responses', - ), - }); - // Create script files for hooks - const failingPath = join(rig.testDir!, 'fail_hook.cjs'); - writeFileSync(failingPath, 'process.exit(1);'); - const workingPath = join(rig.testDir!, 'work_hook.cjs'); - writeFileSync( - workingPath, - "console.log(JSON.stringify({decision: 'allow', reason: 'Working hook succeeded'}));", - ); - - // Failing hook: exits with non-zero code - const failingCommand = `node "${failingPath}"`; - // Working hook: returns success with JSON - const workingCommand = `node "${workingPath}"`; - - rig.setup('should handle hook failures gracefully', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - hooks: [ - { - type: 'command', - command: normalizePath(failingCommand), - timeout: 5000, - }, - { - type: 'command', - command: normalizePath(workingCommand), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - await rig.run({ - args: 'Create a file called error-test.txt with content "testing error handling"', - }); - - // Despite one hook failing, the working hook should still allow the operation - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // File should be created - const fileContent = rig.readFile('error-test.txt'); - expect(fileContent).toContain('testing error handling'); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - }); - }); - - describe('Hook Telemetry and Observability', () => { - it('should generate telemetry events for hook executions', async () => { - rig.setup('should generate telemetry events for hook executions', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.telemetry.responses', - ), - }); - - // Create script file for hook - const scriptPath = rig.createScript( - 'telemetry_hook.cjs', - "console.log(JSON.stringify({decision: 'allow', reason: 'Telemetry test hook'}));", - ); - - const hookCommand = `node "${scriptPath}"`; - - rig.setup('should generate telemetry events for hook executions', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - hooks: [ - { - type: 'command', - command: normalizePath(hookCommand), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - await rig.run({ args: 'Create a file called telemetry-test.txt' }); - - // Should execute the tool - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // Should generate hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - }); - }); - - describe('Session Lifecycle Hooks', () => { - it('should fire SessionStart hook on app startup', async () => { - rig.setup('should fire SessionStart hook on app startup', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.session-startup.responses', - ), - }); - - // Create script file for hook - const scriptPath = rig.createScript( - 'session_start_hook.cjs', - "console.log(JSON.stringify({decision: 'allow', systemMessage: 'Session starting on startup'}));", - ); - - const sessionStartCommand = `node "${scriptPath}"`; - - rig.setup('should fire SessionStart hook on app startup', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - SessionStart: [ - { - matcher: 'startup', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(sessionStartCommand), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - // Run a simple query - the SessionStart hook will fire during app initialization - await rig.run({ args: 'Say hello' }); - - // Verify hook executed with correct parameters - const hookLogs = rig.readHookLogs(); - const sessionStartLog = hookLogs.find( - (log) => log.hookCall.hook_event_name === 'SessionStart', - ); - - expect(sessionStartLog).toBeDefined(); - if (sessionStartLog) { - expect(sessionStartLog.hookCall.hook_name).toBe( - normalizePath(sessionStartCommand), ); - expect(sessionStartLog.hookCall.exit_code).toBe(0); - expect(sessionStartLog.hookCall.hook_input).toBeDefined(); - // hook_input is a string that needs to be parsed - const hookInputStr = - typeof sessionStartLog.hookCall.hook_input === 'string' - ? sessionStartLog.hookCall.hook_input - : JSON.stringify(sessionStartLog.hookCall.hook_input); - const hookInput = JSON.parse(hookInputStr) as Record; - - expect(hookInput['source']).toBe('startup'); - expect(sessionStartLog.hookCall.stdout).toContain( - 'Session starting on startup', + // Create script file for hook + const scriptPath = rig.createScript( + 'pollution_hook.cjs', + "console.log('Pollution'); console.log(JSON.stringify({decision: 'deny', reason: 'Should be ignored'}));", ); - } + + rig.setup( + 'should treat mixed stdout (text + JSON) as system message and allow execution when exit code is 0', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ + { + matcher: 'write_file', + sequential: true, + hooks: [ + { + type: 'command', + // Output plain text then JSON. + // This breaks JSON parsing, so it falls back to 'allow' with the whole stdout as systemMessage. + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }, + ); + + const result = await rig.run({ + args: 'Create a file called approved.txt with content "Approved content"', + }); + + // The hook logic fails to parse JSON, so it allows the tool. + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // The entire stdout (including the JSON part) becomes the systemMessage + expect(result).toContain('Pollution'); + expect(result).toContain('Should be ignored'); + }); }); - it('should fire SessionStart hook and inject context', async () => { - // Create hook script that outputs JSON with additionalContext - const hookScript = `const fs = require('fs'); + describe('Multiple Event Types', () => { + it('should handle hooks for all major event types', async () => { + rig.setup('should handle hooks for all major event types', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.multiple-events.responses', + ), + }); + + // Create script files for hooks + const btPath = rig.createScript( + 'bt_hook.cjs', + "console.log(JSON.stringify({decision: 'allow', systemMessage: 'BeforeTool: File operation logged'}));", + ); + const atPath = rig.createScript( + 'at_hook.cjs', + "console.log(JSON.stringify({hookSpecificOutput: {hookEventName: 'AfterTool', additionalContext: 'AfterTool: Operation completed successfully'}}));", + ); + const baPath = rig.createScript( + 'ba_hook.cjs', + "console.log(JSON.stringify({decision: 'allow', hookSpecificOutput: {hookEventName: 'BeforeAgent', additionalContext: 'BeforeAgent: User request processed'}}));", + ); + + const beforeToolCommand = `node "${btPath}"`; + const afterToolCommand = `node "${atPath}"`; + const beforeAgentCommand = `node "${baPath}"`; + + rig.setup('should handle hooks for all major event types', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeAgent: [ + { + hooks: [ + { + type: 'command', + command: normalizePath(beforeAgentCommand), + timeout: 5000, + }, + ], + }, + ], + BeforeTool: [ + { + matcher: 'write_file', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(beforeToolCommand), + timeout: 5000, + }, + ], + }, + ], + AfterTool: [ + { + matcher: 'write_file', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(afterToolCommand), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + const result = await rig.run({ + args: + 'Create a file called multi-event-test.txt with content ' + + '"testing multiple events", and then please reply with ' + + 'everything I say just after this:"', + }); + + // Should execute write_file tool + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // File should be created + const fileContent = rig.readFile('multi-event-test.txt'); + expect(fileContent).toContain('testing multiple events'); + + // Result should contain context from all hooks + expect(result).toContain('BeforeTool: File operation logged'); + + // Should generate hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); + + // Verify all three hooks executed + const hookLogs = rig.readHookLogs(); + const beforeAgentLog = hookLogs.find( + (log) => log.hookCall.hook_name === normalizePath(beforeAgentCommand), + ); + const beforeToolLog = hookLogs.find( + (log) => log.hookCall.hook_name === normalizePath(beforeToolCommand), + ); + const afterToolLog = hookLogs.find( + (log) => log.hookCall.hook_name === normalizePath(afterToolCommand), + ); + + expect(beforeAgentLog).toBeDefined(); + expect(beforeAgentLog?.hookCall.exit_code).toBe(0); + expect(beforeAgentLog?.hookCall.stdout).toContain( + 'BeforeAgent: User request processed', + ); + + expect(beforeToolLog).toBeDefined(); + expect(beforeToolLog?.hookCall.exit_code).toBe(0); + expect(beforeToolLog?.hookCall.stdout).toContain( + 'BeforeTool: File operation logged', + ); + + expect(afterToolLog).toBeDefined(); + expect(afterToolLog?.hookCall.exit_code).toBe(0); + expect(afterToolLog?.hookCall.stdout).toContain( + 'AfterTool: Operation completed successfully', + ); + }); + }); + + describe('Hook Error Handling', () => { + it('should handle hook failures gracefully', async () => { + rig.setup('should handle hook failures gracefully', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.error-handling.responses', + ), + }); + // Create script files for hooks + const failingPath = join(rig.testDir!, 'fail_hook.cjs'); + writeFileSync(failingPath, 'process.exit(1);'); + const workingPath = join(rig.testDir!, 'work_hook.cjs'); + writeFileSync( + workingPath, + "console.log(JSON.stringify({decision: 'allow', reason: 'Working hook succeeded'}));", + ); + + // Failing hook: exits with non-zero code + const failingCommand = `node "${failingPath}"`; + // Working hook: returns success with JSON + const workingCommand = `node "${workingPath}"`; + + rig.setup('should handle hook failures gracefully', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ + { + hooks: [ + { + type: 'command', + command: normalizePath(failingCommand), + timeout: 5000, + }, + { + type: 'command', + command: normalizePath(workingCommand), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + await rig.run({ + args: 'Create a file called error-test.txt with content "testing error handling"', + }); + + // Despite one hook failing, the working hook should still allow the operation + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // File should be created + const fileContent = rig.readFile('error-test.txt'); + expect(fileContent).toContain('testing error handling'); + + // Should generate hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); + }); + }); + + describe('Hook Telemetry and Observability', () => { + it('should generate telemetry events for hook executions', async () => { + rig.setup('should generate telemetry events for hook executions', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.telemetry.responses', + ), + }); + + // Create script file for hook + const scriptPath = rig.createScript( + 'telemetry_hook.cjs', + "console.log(JSON.stringify({decision: 'allow', reason: 'Telemetry test hook'}));", + ); + + const hookCommand = `node "${scriptPath}"`; + + rig.setup('should generate telemetry events for hook executions', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ + { + hooks: [ + { + type: 'command', + command: normalizePath(hookCommand), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + await rig.run({ args: 'Create a file called telemetry-test.txt' }); + + // Should execute the tool + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // Should generate hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); + }); + }); + + describe('Session Lifecycle Hooks', () => { + it('should fire SessionStart hook on app startup', async () => { + rig.setup('should fire SessionStart hook on app startup', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.session-startup.responses', + ), + }); + + // Create script file for hook + const scriptPath = rig.createScript( + 'session_start_hook.cjs', + "console.log(JSON.stringify({decision: 'allow', systemMessage: 'Session starting on startup'}));", + ); + + const sessionStartCommand = `node "${scriptPath}"`; + + rig.setup('should fire SessionStart hook on app startup', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + SessionStart: [ + { + matcher: 'startup', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(sessionStartCommand), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + // Run a simple query - the SessionStart hook will fire during app initialization + await rig.run({ args: 'Say hello' }); + + // Verify hook executed with correct parameters + const hookLogs = rig.readHookLogs(); + const sessionStartLog = hookLogs.find( + (log) => log.hookCall.hook_event_name === 'SessionStart', + ); + + expect(sessionStartLog).toBeDefined(); + if (sessionStartLog) { + expect(sessionStartLog.hookCall.hook_name).toBe( + normalizePath(sessionStartCommand), + ); + expect(sessionStartLog.hookCall.exit_code).toBe(0); + expect(sessionStartLog.hookCall.hook_input).toBeDefined(); + + // hook_input is a string that needs to be parsed + const hookInputStr = + typeof sessionStartLog.hookCall.hook_input === 'string' + ? sessionStartLog.hookCall.hook_input + : JSON.stringify(sessionStartLog.hookCall.hook_input); + const hookInput = JSON.parse(hookInputStr) as Record; + + expect(hookInput['source']).toBe('startup'); + expect(sessionStartLog.hookCall.stdout).toContain( + 'Session starting on startup', + ); + } + }); + + it('should fire SessionStart hook and inject context', async () => { + // Create hook script that outputs JSON with additionalContext + const hookScript = `const fs = require('fs'); console.log(JSON.stringify({ decision: 'allow', systemMessage: 'Context injected via SessionStart hook', @@ -1393,104 +1411,19 @@ console.log(JSON.stringify({ } }));`; - rig.setup('should fire SessionStart hook and inject context', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.session-startup.responses', - ), - }); - - const scriptPath = rig.createScript( - 'session_start_context_hook.cjs', - hookScript, - ); - - rig.setup('should fire SessionStart hook and inject context', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - SessionStart: [ - { - matcher: 'startup', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - // Run a query - the SessionStart hook will fire during app initialization - const result = await rig.run({ args: 'Who are you?' }); - - // Check if systemMessage was displayed (in stderr, which rig.run captures) - expect(result).toContain('Context injected via SessionStart hook'); - - // Check if additionalContext influenced the model response - // Note: We use fake responses, but the rig records interactions. - // If we are using fake responses, the model won't actually respond unless we provide a fake response for the injected context. - // But the test rig setup uses 'hooks-system.session-startup.responses'. - // If I'm adding a new test, I might need to generate new fake responses or expect the context to be sent to the model (verify API logs). - - // Verify hook executed - const hookLogs = rig.readHookLogs(); - const sessionStartLog = hookLogs.find( - (log) => log.hookCall.hook_event_name === 'SessionStart', - ); - - expect(sessionStartLog).toBeDefined(); - - // Verify the API request contained the injected context - // rig.readAllApiRequest() gives us telemetry on API requests. - const apiRequests = rig.readAllApiRequest(); - // We expect at least one API request - expect(apiRequests.length).toBeGreaterThan(0); - - // The injected context should be in the request text - // For non-interactive mode, I prepended it to input: "context\n\ninput" - // The telemetry `request_text` should contain it. - const requestText = apiRequests[0].attributes?.request_text || ''; - expect(requestText).toContain('protocol droid'); - }); - - it('should fire SessionStart hook and display systemMessage in interactive mode', async () => { - // Create hook script that outputs JSON with systemMessage and additionalContext - const hookScript = `const fs = require('fs'); -console.log(JSON.stringify({ - decision: 'allow', - systemMessage: 'Interactive Session Start Message', - hookSpecificOutput: { - hookEventName: 'SessionStart', - additionalContext: 'The user is a Jedi Master.' - } -}));`; - - rig.setup( - 'should fire SessionStart hook and display systemMessage in interactive mode', - { + rig.setup('should fire SessionStart hook and inject context', { fakeResponsesPath: join( import.meta.dirname, 'hooks-system.session-startup.responses', ), - }, - ); + }); - const scriptPath = rig.createScript( - 'session_start_interactive_hook.cjs', - hookScript, - ); + const scriptPath = rig.createScript( + 'session_start_context_hook.cjs', + hookScript, + ); - rig.setup( - 'should fire SessionStart hook and display systemMessage in interactive mode', - { + rig.setup('should fire SessionStart hook and inject context', { settings: { hooksConfig: { enabled: true, @@ -1511,70 +1444,418 @@ console.log(JSON.stringify({ ], }, }, - }, - ); + }); - const run = await rig.runInteractive(); + // Run a query - the SessionStart hook will fire during app initialization + const result = await rig.run({ args: 'Who are you?' }); - // Verify systemMessage is displayed - await run.expectText('Interactive Session Start Message', 10000); + // Check if systemMessage was displayed (in stderr, which rig.run captures) + expect(result).toContain('Context injected via SessionStart hook'); - // Send a prompt to establish a session and trigger an API call - await run.sendKeys('Hello'); - await run.type('\r'); + // Check if additionalContext influenced the model response + // Note: We use fake responses, but the rig records interactions. + // If we are using fake responses, the model won't actually respond unless we provide a fake response for the injected context. + // But the test rig setup uses 'hooks-system.session-startup.responses'. + // If I'm adding a new test, I might need to generate new fake responses or expect the context to be sent to the model (verify API logs). - // Wait for response to ensure API call happened - await run.expectText('Hello', 15000); + // Verify hook executed + const hookLogs = rig.readHookLogs(); + const sessionStartLog = hookLogs.find( + (log) => log.hookCall.hook_event_name === 'SessionStart', + ); - // Wait for telemetry to be written to disk - await rig.waitForTelemetryReady(); + expect(sessionStartLog).toBeDefined(); - // Verify the API request contained the injected context - // We may need to poll for API requests as they are written asynchronously - const pollResult = await poll( - () => { - const apiRequests = rig.readAllApiRequest(); - return apiRequests.length > 0; - }, - 15000, - 500, - ); + // Verify the API request contained the injected context + // rig.readAllApiRequest() gives us telemetry on API requests. + const apiRequests = rig.readAllApiRequest(); + // We expect at least one API request + expect(apiRequests.length).toBeGreaterThan(0); - expect(pollResult).toBe(true); + // The injected context should be in the request text + // For non-interactive mode, I prepended it to input: "context\n\ninput" + // The telemetry `request_text` should contain it. + const requestText = apiRequests[0].attributes?.request_text || ''; + expect(requestText).toContain('protocol droid'); + }); - const apiRequests = rig.readAllApiRequest(); - // The injected context should be in the request_text of the API request - const requestText = apiRequests[0].attributes?.request_text || ''; - expect(requestText).toContain('Jedi Master'); + it('should fire SessionStart hook and display systemMessage in interactive mode', async () => { + // Create hook script that outputs JSON with systemMessage and additionalContext + const hookScript = `const fs = require('fs'); +console.log(JSON.stringify({ + decision: 'allow', + systemMessage: 'Interactive Session Start Message', + hookSpecificOutput: { + hookEventName: 'SessionStart', + additionalContext: 'The user is a Jedi Master.' + } +}));`; + + rig.setup( + 'should fire SessionStart hook and display systemMessage in interactive mode', + { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.session-startup.responses', + ), + }, + ); + + const scriptPath = rig.createScript( + 'session_start_interactive_hook.cjs', + hookScript, + ); + + rig.setup( + 'should fire SessionStart hook and display systemMessage in interactive mode', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + SessionStart: [ + { + matcher: 'startup', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(`node "${scriptPath}"`), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }, + ); + + const run = await rig.runInteractive(); + + // Verify systemMessage is displayed + await run.expectText('Interactive Session Start Message', 10000); + + // Send a prompt to establish a session and trigger an API call + await run.sendKeys('Hello'); + await run.type('\r'); + + // Wait for response to ensure API call happened + await run.expectText('Hello', 15000); + + // Wait for telemetry to be written to disk + await rig.waitForTelemetryReady(); + + // Verify the API request contained the injected context + // We may need to poll for API requests as they are written asynchronously + const pollResult = await poll( + () => { + const apiRequests = rig.readAllApiRequest(); + return apiRequests.length > 0; + }, + 15000, + 500, + ); + + expect(pollResult).toBe(true); + + const apiRequests = rig.readAllApiRequest(); + // The injected context should be in the request_text of the API request + const requestText = apiRequests[0].attributes?.request_text || ''; + expect(requestText).toContain('Jedi Master'); + }); + + it('should fire SessionEnd and SessionStart hooks on /clear command', async () => { + rig.setup( + 'should fire SessionEnd and SessionStart hooks on /clear command', + { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.session-clear.responses', + ), + }, + ); + + // Create script files for hooks + const endScriptPath = rig.createScript( + 'session_end_clear.cjs', + "console.log(JSON.stringify({decision: 'allow', systemMessage: 'Session ending due to clear'}));", + ); + const startScriptPath = rig.createScript( + 'session_start_clear.cjs', + "console.log(JSON.stringify({decision: 'allow', systemMessage: 'Session starting after clear'}));", + ); + + const sessionEndCommand = `node "${endScriptPath}"`; + const sessionStartCommand = `node "${startScriptPath}"`; + + rig.setup( + 'should fire SessionEnd and SessionStart hooks on /clear command', + { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + SessionEnd: [ + { + matcher: '*', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(sessionEndCommand), + timeout: 5000, + }, + ], + }, + ], + SessionStart: [ + { + matcher: '*', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(sessionStartCommand), + timeout: 5000, + }, + ], + }, + ], + }, + }, + }, + ); + + const run = await rig.runInteractive(); + + // Send an initial prompt to establish a session + await run.sendKeys('Say hello'); + await run.type('\r'); + + // Wait for the response + await run.expectText('Hello', 10000); + + // Execute /clear command multiple times to generate more hook events + // This makes the test more robust by creating multiple start/stop cycles + const numClears = 3; + for (let i = 0; i < numClears; i++) { + await run.sendKeys('/clear'); + await run.type('\r'); + + // Wait a bit for clear to complete + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Send a prompt to establish an active session before next clear + await run.sendKeys('Say hello'); + await run.type('\r'); + + // Wait for response + await run.expectText('Hello', 10000); + } + + // Wait for all clears to complete + // BatchLogRecordProcessor exports telemetry every 10 seconds by default + // Use generous wait time across all platforms (CI, Docker, Mac, Linux) + await new Promise((resolve) => setTimeout(resolve, 15000)); + + // Wait for telemetry to be written to disk + await rig.waitForTelemetryReady(); + + // Wait for hook telemetry events to be flushed to disk + // In interactive mode, telemetry may be buffered, so we need to poll for the events + // We execute multiple clears to generate more hook events (total: 1 + numClears * 2) + // But we only require >= 1 hooks to pass, making the test more permissive + const expectedMinHooks = 1; // SessionStart (startup), SessionEnd (clear), SessionStart (clear) + const pollResult = await poll( + () => { + const hookLogs = rig.readHookLogs(); + return hookLogs.length >= expectedMinHooks; + }, + 90000, // 90 second timeout for all platforms + 1000, // check every 1s to reduce I/O overhead + ); + + // If polling failed, log diagnostic info + if (!pollResult) { + const hookLogs = rig.readHookLogs(); + const hookEvents = hookLogs.map( + (log) => log.hookCall.hook_event_name, + ); + console.error( + `Polling timeout after 90000ms: Expected >= ${expectedMinHooks} hooks, got ${hookLogs.length}`, + ); + console.error( + 'Hooks found:', + hookEvents.length > 0 ? hookEvents.join(', ') : 'NONE', + ); + console.error('Full hook logs:', JSON.stringify(hookLogs, null, 2)); + } + + // Verify hooks executed + const hookLogs = rig.readHookLogs(); + + // Diagnostic: Log which hooks we actually got + const hookEvents = hookLogs.map((log) => log.hookCall.hook_event_name); + if (hookLogs.length < expectedMinHooks) { + console.error( + `TEST FAILURE: Expected >= ${expectedMinHooks} hooks, got ${hookLogs.length}: [${hookEvents.length > 0 ? hookEvents.join(', ') : 'NONE'}]`, + ); + } + + expect(hookLogs.length).toBeGreaterThanOrEqual(expectedMinHooks); + + // Find SessionEnd hook log + const sessionEndLog = hookLogs.find( + (log) => + log.hookCall.hook_event_name === 'SessionEnd' && + log.hookCall.hook_name === normalizePath(sessionEndCommand), + ); + // Because the flakiness of the test, we relax this check + // expect(sessionEndLog).toBeDefined(); + if (sessionEndLog) { + expect(sessionEndLog.hookCall.exit_code).toBe(0); + expect(sessionEndLog.hookCall.stdout).toContain( + 'Session ending due to clear', + ); + + // Verify hook input contains reason + const hookInputStr = + typeof sessionEndLog.hookCall.hook_input === 'string' + ? sessionEndLog.hookCall.hook_input + : JSON.stringify(sessionEndLog.hookCall.hook_input); + const hookInput = JSON.parse(hookInputStr) as Record; + expect(hookInput['reason']).toBe('clear'); + } + + // Find SessionStart hook log after clear + const sessionStartAfterClearLogs = hookLogs.filter( + (log) => + log.hookCall.hook_event_name === 'SessionStart' && + log.hookCall.hook_name === normalizePath(sessionStartCommand), + ); + // Should have at least one SessionStart from after clear + // Because the flakiness of the test, we relax this check + // expect(sessionStartAfterClearLogs.length).toBeGreaterThanOrEqual(1); + + const sessionStartLog = sessionStartAfterClearLogs.find((log) => { + const hookInputStr = + typeof log.hookCall.hook_input === 'string' + ? log.hookCall.hook_input + : JSON.stringify(log.hookCall.hook_input); + const hookInput = JSON.parse(hookInputStr) as Record; + return hookInput['source'] === 'clear'; + }); + + // Because the flakiness of the test, we relax this check + // expect(sessionStartLog).toBeDefined(); + if (sessionStartLog) { + expect(sessionStartLog.hookCall.exit_code).toBe(0); + expect(sessionStartLog.hookCall.stdout).toContain( + 'Session starting after clear', + ); + } + }); }); - it('should fire SessionEnd and SessionStart hooks on /clear command', async () => { - rig.setup( - 'should fire SessionEnd and SessionStart hooks on /clear command', - { + describe('Compression Hooks', () => { + it('should fire PreCompress hook on automatic compression', async () => { + rig.setup('should fire PreCompress hook on automatic compression', { fakeResponsesPath: join( import.meta.dirname, - 'hooks-system.session-clear.responses', + 'hooks-system.compress-auto.responses', ), - }, - ); + }); - // Create script files for hooks - const endScriptPath = rig.createScript( - 'session_end_clear.cjs', - "console.log(JSON.stringify({decision: 'allow', systemMessage: 'Session ending due to clear'}));", - ); - const startScriptPath = rig.createScript( - 'session_start_clear.cjs', - "console.log(JSON.stringify({decision: 'allow', systemMessage: 'Session starting after clear'}));", - ); + // Create script file for hook + const scriptPath = rig.createScript( + 'pre_compress_hook.cjs', + "console.log(JSON.stringify({decision: 'allow', systemMessage: 'PreCompress hook executed for automatic compression'}));", + ); - const sessionEndCommand = `node "${endScriptPath}"`; - const sessionStartCommand = `node "${startScriptPath}"`; + const preCompressCommand = `node "${scriptPath}"`; - rig.setup( - 'should fire SessionEnd and SessionStart hooks on /clear command', - { + rig.setup('should fire PreCompress hook on automatic compression', { + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + PreCompress: [ + { + matcher: 'auto', + sequential: true, + hooks: [ + { + type: 'command', + command: normalizePath(preCompressCommand), + timeout: 5000, + }, + ], + }, + ], + }, + // Configure automatic compression with a very low threshold + // This will trigger auto-compression after the first response + contextCompression: { + // enabled: true, + targetTokenCount: 10, // Very low threshold to trigger compression + }, + }, + }); + + // Run a simple query that will trigger automatic compression + await rig.run({ args: 'Say hello in exactly 5 words' }); + + // Verify hook executed with correct parameters + const hookLogs = rig.readHookLogs(); + const preCompressLog = hookLogs.find( + (log) => log.hookCall.hook_event_name === 'PreCompress', + ); + + expect(preCompressLog).toBeDefined(); + if (preCompressLog) { + expect(preCompressLog.hookCall.hook_name).toBe( + normalizePath(preCompressCommand), + ); + expect(preCompressLog.hookCall.exit_code).toBe(0); + expect(preCompressLog.hookCall.hook_input).toBeDefined(); + + // hook_input is a string that needs to be parsed + const hookInputStr = + typeof preCompressLog.hookCall.hook_input === 'string' + ? preCompressLog.hookCall.hook_input + : JSON.stringify(preCompressLog.hookCall.hook_input); + const hookInput = JSON.parse(hookInputStr) as Record; + + expect(hookInput['trigger']).toBe('auto'); + expect(preCompressLog.hookCall.stdout).toContain( + 'PreCompress hook executed for automatic compression', + ); + } + }); + }); + + describe('SessionEnd on Exit', () => { + it('should fire SessionEnd hook on graceful exit in non-interactive mode', async () => { + rig.setup('should fire SessionEnd hook on graceful exit', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.session-startup.responses', + ), + }); + + // Create script file for hook + const scriptPath = rig.createScript( + 'session_end_exit.cjs', + "console.log(JSON.stringify({decision: 'allow', systemMessage: 'SessionEnd hook executed on exit'}));", + ); + + const sessionEndCommand = `node "${scriptPath}"`; + + rig.setup('should fire SessionEnd hook on graceful exit', { settings: { hooksConfig: { enabled: true, @@ -1582,7 +1863,7 @@ console.log(JSON.stringify({ hooks: { SessionEnd: [ { - matcher: '*', + matcher: 'exit', sequential: true, hooks: [ { @@ -1593,711 +1874,287 @@ console.log(JSON.stringify({ ], }, ], - SessionStart: [ + }, + }, + }); + + // Run in non-interactive mode with a simple prompt + await rig.run({ args: 'Hello' }); + + // The process should exit gracefully, firing the SessionEnd hook + // Wait for telemetry to be written to disk + await rig.waitForTelemetryReady(); + + // Poll for the hook log to appear + const isCI = process.env['CI'] === 'true'; + const pollTimeout = isCI ? 30000 : 10000; + const pollResult = await poll( + () => { + const hookLogs = rig.readHookLogs(); + return hookLogs.some( + (log) => log.hookCall.hook_event_name === 'SessionEnd', + ); + }, + pollTimeout, + 200, + ); + + if (!pollResult) { + const hookLogs = rig.readHookLogs(); + console.error( + 'Polling timeout: Expected SessionEnd hook, got:', + JSON.stringify(hookLogs, null, 2), + ); + } + + expect(pollResult).toBe(true); + + const hookLogs = rig.readHookLogs(); + const sessionEndLog = hookLogs.find( + (log) => log.hookCall.hook_event_name === 'SessionEnd', + ); + + expect(sessionEndLog).toBeDefined(); + if (sessionEndLog) { + expect(sessionEndLog.hookCall.hook_name).toBe( + normalizePath(sessionEndCommand), + ); + expect(sessionEndLog.hookCall.exit_code).toBe(0); + expect(sessionEndLog.hookCall.hook_input).toBeDefined(); + + const hookInputStr = + typeof sessionEndLog.hookCall.hook_input === 'string' + ? sessionEndLog.hookCall.hook_input + : JSON.stringify(sessionEndLog.hookCall.hook_input); + const hookInput = JSON.parse(hookInputStr) as Record; + + expect(hookInput['reason']).toBe('exit'); + expect(sessionEndLog.hookCall.stdout).toContain( + 'SessionEnd hook executed', + ); + } + }); + }); + + describe('Hook Disabling', () => { + it('should not execute hooks disabled in settings file', async () => { + const enabledMsg = 'EXECUTION_ALLOWED_BY_HOOK_A'; + const disabledMsg = 'EXECUTION_BLOCKED_BY_HOOK_B'; + + const enabledJson = JSON.stringify({ + decision: 'allow', + systemMessage: enabledMsg, + }); + const disabledJson = JSON.stringify({ + decision: 'block', + reason: disabledMsg, + }); + + const enabledScript = `console.log(JSON.stringify(${enabledJson}));`; + const disabledScript = `console.log(JSON.stringify(${disabledJson}));`; + const enabledFilename = 'enabled_hook.js'; + const disabledFilename = 'disabled_hook.js'; + const enabledCmd = `node ${enabledFilename}`; + const disabledCmd = `node ${disabledFilename}`; + + // 3. Final setup with full settings + rig.setup('Hook Disabling Settings', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.disabled-via-settings.responses', + ), + settings: { + hooksConfig: { + enabled: true, + disabled: ['hook-b'], + }, + hooks: { + BeforeTool: [ { - matcher: '*', + hooks: [ + { + type: 'command', + name: 'hook-a', + command: enabledCmd, + timeout: 60000, + }, + { + type: 'command', + name: 'hook-b', + command: disabledCmd, + timeout: 60000, + }, + ], + }, + ], + }, + }, + }); + + rig.createScript(enabledFilename, enabledScript); + rig.createScript(disabledFilename, disabledScript); + + await rig.run({ + args: 'Create a file called disabled-test.txt with content "test"', + }); + + // Tool should execute (enabled hook allows it) + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // Check hook telemetry - only enabled hook should have executed + const hookLogs = rig.readHookLogs(); + const enabledHookLog = hookLogs.find((log) => + JSON.stringify(log.hookCall.hook_output).includes(enabledMsg), + ); + const disabledHookLog = hookLogs.find((log) => + JSON.stringify(log.hookCall.hook_output).includes(disabledMsg), + ); + + expect(enabledHookLog).toBeDefined(); + expect(disabledHookLog).toBeUndefined(); + }); + + it('should respect disabled hooks across multiple operations', async () => { + const activeMsg = 'MULTIPLE_OPS_ENABLED_HOOK'; + const disabledMsg = 'MULTIPLE_OPS_DISABLED_HOOK'; + + const activeJson = JSON.stringify({ + decision: 'allow', + systemMessage: activeMsg, + }); + const disabledJson = JSON.stringify({ + decision: 'block', + reason: disabledMsg, + }); + + const activeScript = `console.log(JSON.stringify(${activeJson}));`; + const disabledScript = `console.log(JSON.stringify(${disabledJson}));`; + const activeFilename = 'active_hook.js'; + const disabledFilename = 'disabled_hook.js'; + const activeCmd = `node ${activeFilename}`; + const disabledCmd = `node ${disabledFilename}`; + + // 3. Final setup with full settings + rig.setup('Hook Disabling Multiple Ops', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.disabled-via-command.responses', + ), + settings: { + hooksConfig: { + enabled: true, + disabled: ['multi-hook-disabled'], + }, + hooks: { + BeforeTool: [ + { + hooks: [ + { + type: 'command', + name: 'multi-hook-active', + command: activeCmd, + timeout: 60000, + }, + { + type: 'command', + name: 'multi-hook-disabled', + command: disabledCmd, + timeout: 60000, + }, + ], + }, + ], + }, + }, + }); + + rig.createScript(activeFilename, activeScript); + rig.createScript(disabledFilename, disabledScript); + + // First run - only active hook should execute + await rig.run({ + args: 'Create a file called first-run.txt with "test1"', + }); + + // Tool should execute (active hook allows it) + const foundWriteFile1 = await rig.waitForToolCall('write_file'); + expect(foundWriteFile1).toBeTruthy(); + + // Check hook telemetry - only active hook should have executed + const hookLogs1 = rig.readHookLogs(); + const activeHookLog1 = hookLogs1.find((log) => + JSON.stringify(log.hookCall.hook_output).includes(activeMsg), + ); + const disabledHookLog1 = hookLogs1.find((log) => + JSON.stringify(log.hookCall.hook_output).includes(disabledMsg), + ); + + expect(activeHookLog1).toBeDefined(); + expect(disabledHookLog1).toBeUndefined(); + + // Second run - verify disabled hook stays disabled + await rig.run({ + args: 'Create a file called second-run.txt with "test2"', + }); + + const foundWriteFile2 = await rig.waitForToolCall('write_file'); + expect(foundWriteFile2).toBeTruthy(); + + // Verify disabled hook still hasn't executed + const hookLogs2 = rig.readHookLogs(); + const disabledHookLog2 = hookLogs2.find((log) => + JSON.stringify(log.hookCall.hook_output).includes(disabledMsg), + ); + expect(disabledHookLog2).toBeUndefined(); + }); + }); + + describe('BeforeTool Hooks - Input Override', () => { + it('should override tool input parameters via BeforeTool hook', async () => { + // 1. First setup to get the test directory and prepare the hook script + rig.setup('should override tool input parameters via BeforeTool hook'); + + // Create a hook script that overrides the tool input + const hookOutput = { + decision: 'allow', + hookSpecificOutput: { + hookEventName: 'BeforeTool', + tool_input: { + file_path: 'modified.txt', + content: 'modified content', + }, + }, + }; + + const hookScript = `process.stdout.write(JSON.stringify(${JSON.stringify( + hookOutput, + )}));`; + + const scriptPath = rig.createScript( + 'input_override_hook.js', + hookScript, + ); + + // 2. Full setup with settings and fake responses + rig.setup('should override tool input parameters via BeforeTool hook', { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.input-modification.responses', + ), + settings: { + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ + { + matcher: 'write_file', sequential: true, hooks: [ { type: 'command', - command: normalizePath(sessionStartCommand), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }, - ); - - const run = await rig.runInteractive(); - - // Send an initial prompt to establish a session - await run.sendKeys('Say hello'); - await run.type('\r'); - - // Wait for the response - await run.expectText('Hello', 10000); - - // Execute /clear command multiple times to generate more hook events - // This makes the test more robust by creating multiple start/stop cycles - const numClears = 3; - for (let i = 0; i < numClears; i++) { - await run.sendKeys('/clear'); - await run.type('\r'); - - // Wait a bit for clear to complete - await new Promise((resolve) => setTimeout(resolve, 2000)); - - // Send a prompt to establish an active session before next clear - await run.sendKeys('Say hello'); - await run.type('\r'); - - // Wait for response - await run.expectText('Hello', 10000); - } - - // Wait for all clears to complete - // BatchLogRecordProcessor exports telemetry every 10 seconds by default - // Use generous wait time across all platforms (CI, Docker, Mac, Linux) - await new Promise((resolve) => setTimeout(resolve, 15000)); - - // Wait for telemetry to be written to disk - await rig.waitForTelemetryReady(); - - // Wait for hook telemetry events to be flushed to disk - // In interactive mode, telemetry may be buffered, so we need to poll for the events - // We execute multiple clears to generate more hook events (total: 1 + numClears * 2) - // But we only require >= 1 hooks to pass, making the test more permissive - const expectedMinHooks = 1; // SessionStart (startup), SessionEnd (clear), SessionStart (clear) - const pollResult = await poll( - () => { - const hookLogs = rig.readHookLogs(); - return hookLogs.length >= expectedMinHooks; - }, - 90000, // 90 second timeout for all platforms - 1000, // check every 1s to reduce I/O overhead - ); - - // If polling failed, log diagnostic info - if (!pollResult) { - const hookLogs = rig.readHookLogs(); - const hookEvents = hookLogs.map((log) => log.hookCall.hook_event_name); - console.error( - `Polling timeout after 90000ms: Expected >= ${expectedMinHooks} hooks, got ${hookLogs.length}`, - ); - console.error( - 'Hooks found:', - hookEvents.length > 0 ? hookEvents.join(', ') : 'NONE', - ); - console.error('Full hook logs:', JSON.stringify(hookLogs, null, 2)); - } - - // Verify hooks executed - const hookLogs = rig.readHookLogs(); - - // Diagnostic: Log which hooks we actually got - const hookEvents = hookLogs.map((log) => log.hookCall.hook_event_name); - if (hookLogs.length < expectedMinHooks) { - console.error( - `TEST FAILURE: Expected >= ${expectedMinHooks} hooks, got ${hookLogs.length}: [${hookEvents.length > 0 ? hookEvents.join(', ') : 'NONE'}]`, - ); - } - - expect(hookLogs.length).toBeGreaterThanOrEqual(expectedMinHooks); - - // Find SessionEnd hook log - const sessionEndLog = hookLogs.find( - (log) => - log.hookCall.hook_event_name === 'SessionEnd' && - log.hookCall.hook_name === normalizePath(sessionEndCommand), - ); - // Because the flakiness of the test, we relax this check - // expect(sessionEndLog).toBeDefined(); - if (sessionEndLog) { - expect(sessionEndLog.hookCall.exit_code).toBe(0); - expect(sessionEndLog.hookCall.stdout).toContain( - 'Session ending due to clear', - ); - - // Verify hook input contains reason - const hookInputStr = - typeof sessionEndLog.hookCall.hook_input === 'string' - ? sessionEndLog.hookCall.hook_input - : JSON.stringify(sessionEndLog.hookCall.hook_input); - const hookInput = JSON.parse(hookInputStr) as Record; - expect(hookInput['reason']).toBe('clear'); - } - - // Find SessionStart hook log after clear - const sessionStartAfterClearLogs = hookLogs.filter( - (log) => - log.hookCall.hook_event_name === 'SessionStart' && - log.hookCall.hook_name === normalizePath(sessionStartCommand), - ); - // Should have at least one SessionStart from after clear - // Because the flakiness of the test, we relax this check - // expect(sessionStartAfterClearLogs.length).toBeGreaterThanOrEqual(1); - - const sessionStartLog = sessionStartAfterClearLogs.find((log) => { - const hookInputStr = - typeof log.hookCall.hook_input === 'string' - ? log.hookCall.hook_input - : JSON.stringify(log.hookCall.hook_input); - const hookInput = JSON.parse(hookInputStr) as Record; - return hookInput['source'] === 'clear'; - }); - - // Because the flakiness of the test, we relax this check - // expect(sessionStartLog).toBeDefined(); - if (sessionStartLog) { - expect(sessionStartLog.hookCall.exit_code).toBe(0); - expect(sessionStartLog.hookCall.stdout).toContain( - 'Session starting after clear', - ); - } - }); - }); - - describe('Compression Hooks', () => { - it('should fire PreCompress hook on automatic compression', async () => { - rig.setup('should fire PreCompress hook on automatic compression', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.compress-auto.responses', - ), - }); - - // Create script file for hook - const scriptPath = rig.createScript( - 'pre_compress_hook.cjs', - "console.log(JSON.stringify({decision: 'allow', systemMessage: 'PreCompress hook executed for automatic compression'}));", - ); - - const preCompressCommand = `node "${scriptPath}"`; - - rig.setup('should fire PreCompress hook on automatic compression', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - PreCompress: [ - { - matcher: 'auto', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(preCompressCommand), - timeout: 5000, - }, - ], - }, - ], - }, - // Configure automatic compression with a very low threshold - // This will trigger auto-compression after the first response - contextCompression: { - // enabled: true, - targetTokenCount: 10, // Very low threshold to trigger compression - }, - }, - }); - - // Run a simple query that will trigger automatic compression - await rig.run({ args: 'Say hello in exactly 5 words' }); - - // Verify hook executed with correct parameters - const hookLogs = rig.readHookLogs(); - const preCompressLog = hookLogs.find( - (log) => log.hookCall.hook_event_name === 'PreCompress', - ); - - expect(preCompressLog).toBeDefined(); - if (preCompressLog) { - expect(preCompressLog.hookCall.hook_name).toBe( - normalizePath(preCompressCommand), - ); - expect(preCompressLog.hookCall.exit_code).toBe(0); - expect(preCompressLog.hookCall.hook_input).toBeDefined(); - - // hook_input is a string that needs to be parsed - const hookInputStr = - typeof preCompressLog.hookCall.hook_input === 'string' - ? preCompressLog.hookCall.hook_input - : JSON.stringify(preCompressLog.hookCall.hook_input); - const hookInput = JSON.parse(hookInputStr) as Record; - - expect(hookInput['trigger']).toBe('auto'); - expect(preCompressLog.hookCall.stdout).toContain( - 'PreCompress hook executed for automatic compression', - ); - } - }); - }); - - describe('SessionEnd on Exit', () => { - it('should fire SessionEnd hook on graceful exit in non-interactive mode', async () => { - rig.setup('should fire SessionEnd hook on graceful exit', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.session-startup.responses', - ), - }); - - // Create script file for hook - const scriptPath = rig.createScript( - 'session_end_exit.cjs', - "console.log(JSON.stringify({decision: 'allow', systemMessage: 'SessionEnd hook executed on exit'}));", - ); - - const sessionEndCommand = `node "${scriptPath}"`; - - rig.setup('should fire SessionEnd hook on graceful exit', { - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - SessionEnd: [ - { - matcher: 'exit', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(sessionEndCommand), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - // Run in non-interactive mode with a simple prompt - await rig.run({ args: 'Hello' }); - - // The process should exit gracefully, firing the SessionEnd hook - // Wait for telemetry to be written to disk - await rig.waitForTelemetryReady(); - - // Poll for the hook log to appear - const isCI = process.env['CI'] === 'true'; - const pollTimeout = isCI ? 30000 : 10000; - const pollResult = await poll( - () => { - const hookLogs = rig.readHookLogs(); - return hookLogs.some( - (log) => log.hookCall.hook_event_name === 'SessionEnd', - ); - }, - pollTimeout, - 200, - ); - - if (!pollResult) { - const hookLogs = rig.readHookLogs(); - console.error( - 'Polling timeout: Expected SessionEnd hook, got:', - JSON.stringify(hookLogs, null, 2), - ); - } - - expect(pollResult).toBe(true); - - const hookLogs = rig.readHookLogs(); - const sessionEndLog = hookLogs.find( - (log) => log.hookCall.hook_event_name === 'SessionEnd', - ); - - expect(sessionEndLog).toBeDefined(); - if (sessionEndLog) { - expect(sessionEndLog.hookCall.hook_name).toBe( - normalizePath(sessionEndCommand), - ); - expect(sessionEndLog.hookCall.exit_code).toBe(0); - expect(sessionEndLog.hookCall.hook_input).toBeDefined(); - - const hookInputStr = - typeof sessionEndLog.hookCall.hook_input === 'string' - ? sessionEndLog.hookCall.hook_input - : JSON.stringify(sessionEndLog.hookCall.hook_input); - const hookInput = JSON.parse(hookInputStr) as Record; - - expect(hookInput['reason']).toBe('exit'); - expect(sessionEndLog.hookCall.stdout).toContain( - 'SessionEnd hook executed', - ); - } - }); - }); - - describe('Hook Disabling', () => { - it('should not execute hooks disabled in settings file', async () => { - const enabledMsg = 'EXECUTION_ALLOWED_BY_HOOK_A'; - const disabledMsg = 'EXECUTION_BLOCKED_BY_HOOK_B'; - - const enabledJson = JSON.stringify({ - decision: 'allow', - systemMessage: enabledMsg, - }); - const disabledJson = JSON.stringify({ - decision: 'block', - reason: disabledMsg, - }); - - const enabledScript = `console.log(JSON.stringify(${enabledJson}));`; - const disabledScript = `console.log(JSON.stringify(${disabledJson}));`; - const enabledFilename = 'enabled_hook.js'; - const disabledFilename = 'disabled_hook.js'; - const enabledCmd = `node ${enabledFilename}`; - const disabledCmd = `node ${disabledFilename}`; - - // 3. Final setup with full settings - rig.setup('Hook Disabling Settings', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.disabled-via-settings.responses', - ), - settings: { - hooksConfig: { - enabled: true, - disabled: ['hook-b'], - }, - hooks: { - BeforeTool: [ - { - hooks: [ - { - type: 'command', - name: 'hook-a', - command: enabledCmd, - timeout: 60000, - }, - { - type: 'command', - name: 'hook-b', - command: disabledCmd, - timeout: 60000, - }, - ], - }, - ], - }, - }, - }); - - rig.createScript(enabledFilename, enabledScript); - rig.createScript(disabledFilename, disabledScript); - - await rig.run({ - args: 'Create a file called disabled-test.txt with content "test"', - }); - - // Tool should execute (enabled hook allows it) - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // Check hook telemetry - only enabled hook should have executed - const hookLogs = rig.readHookLogs(); - const enabledHookLog = hookLogs.find((log) => - JSON.stringify(log.hookCall.hook_output).includes(enabledMsg), - ); - const disabledHookLog = hookLogs.find((log) => - JSON.stringify(log.hookCall.hook_output).includes(disabledMsg), - ); - - expect(enabledHookLog).toBeDefined(); - expect(disabledHookLog).toBeUndefined(); - }); - - it('should respect disabled hooks across multiple operations', async () => { - const activeMsg = 'MULTIPLE_OPS_ENABLED_HOOK'; - const disabledMsg = 'MULTIPLE_OPS_DISABLED_HOOK'; - - const activeJson = JSON.stringify({ - decision: 'allow', - systemMessage: activeMsg, - }); - const disabledJson = JSON.stringify({ - decision: 'block', - reason: disabledMsg, - }); - - const activeScript = `console.log(JSON.stringify(${activeJson}));`; - const disabledScript = `console.log(JSON.stringify(${disabledJson}));`; - const activeFilename = 'active_hook.js'; - const disabledFilename = 'disabled_hook.js'; - const activeCmd = `node ${activeFilename}`; - const disabledCmd = `node ${disabledFilename}`; - - // 3. Final setup with full settings - rig.setup('Hook Disabling Multiple Ops', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.disabled-via-command.responses', - ), - settings: { - hooksConfig: { - enabled: true, - disabled: ['multi-hook-disabled'], - }, - hooks: { - BeforeTool: [ - { - hooks: [ - { - type: 'command', - name: 'multi-hook-active', - command: activeCmd, - timeout: 60000, - }, - { - type: 'command', - name: 'multi-hook-disabled', - command: disabledCmd, - timeout: 60000, - }, - ], - }, - ], - }, - }, - }); - - rig.createScript(activeFilename, activeScript); - rig.createScript(disabledFilename, disabledScript); - - // First run - only active hook should execute - await rig.run({ - args: 'Create a file called first-run.txt with "test1"', - }); - - // Tool should execute (active hook allows it) - const foundWriteFile1 = await rig.waitForToolCall('write_file'); - expect(foundWriteFile1).toBeTruthy(); - - // Check hook telemetry - only active hook should have executed - const hookLogs1 = rig.readHookLogs(); - const activeHookLog1 = hookLogs1.find((log) => - JSON.stringify(log.hookCall.hook_output).includes(activeMsg), - ); - const disabledHookLog1 = hookLogs1.find((log) => - JSON.stringify(log.hookCall.hook_output).includes(disabledMsg), - ); - - expect(activeHookLog1).toBeDefined(); - expect(disabledHookLog1).toBeUndefined(); - - // Second run - verify disabled hook stays disabled - await rig.run({ - args: 'Create a file called second-run.txt with "test2"', - }); - - const foundWriteFile2 = await rig.waitForToolCall('write_file'); - expect(foundWriteFile2).toBeTruthy(); - - // Verify disabled hook still hasn't executed - const hookLogs2 = rig.readHookLogs(); - const disabledHookLog2 = hookLogs2.find((log) => - JSON.stringify(log.hookCall.hook_output).includes(disabledMsg), - ); - expect(disabledHookLog2).toBeUndefined(); - }); - }); - - describe('BeforeTool Hooks - Input Override', () => { - it('should override tool input parameters via BeforeTool hook', async () => { - // 1. First setup to get the test directory and prepare the hook script - rig.setup('should override tool input parameters via BeforeTool hook'); - - // Create a hook script that overrides the tool input - const hookOutput = { - decision: 'allow', - hookSpecificOutput: { - hookEventName: 'BeforeTool', - tool_input: { - file_path: 'modified.txt', - content: 'modified content', - }, - }, - }; - - const hookScript = `process.stdout.write(JSON.stringify(${JSON.stringify( - hookOutput, - )}));`; - - const scriptPath = rig.createScript('input_override_hook.js', hookScript); - - // 2. Full setup with settings and fake responses - rig.setup('should override tool input parameters via BeforeTool hook', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.input-modification.responses', - ), - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - matcher: 'write_file', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - // Run the agent. The fake response will attempt to call write_file with - // file_path="original.txt" and content="original content" - await rig.run({ - args: 'Create a file called original.txt with content "original content"', - }); - - // 1. Verify that 'modified.txt' was created with 'modified content' (Override successful) - const modifiedContent = rig.readFile('modified.txt'); - expect(modifiedContent).toBe('modified content'); - - // 2. Verify that 'original.txt' was NOT created (Override replaced original) - let originalExists = false; - try { - rig.readFile('original.txt'); - originalExists = true; - } catch { - originalExists = false; - } - expect(originalExists).toBe(false); - - // 3. Verify hook telemetry - const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); - expect(hookTelemetryFound).toBeTruthy(); - - const hookLogs = rig.readHookLogs(); - expect(hookLogs.length).toBe(1); - expect(hookLogs[0].hookCall.hook_name).toContain( - 'input_override_hook.js', - ); - - // 4. Verify that the agent didn't try to work-around the hook input change - const toolLogs = rig.readToolLogs(); - expect(toolLogs.length).toBe(1); - expect(toolLogs[0].toolRequest.name).toBe('write_file'); - expect(JSON.parse(toolLogs[0].toolRequest.args).file_path).toBe( - 'modified.txt', - ); - }); - }); - - describe('BeforeTool Hooks - Stop Execution', () => { - it('should stop agent execution via BeforeTool hook', async () => { - // Create a hook script that stops execution - const hookOutput = { - continue: false, - reason: 'Emergency Stop triggered by hook', - hookSpecificOutput: { - hookEventName: 'BeforeTool', - }, - }; - - const hookScript = `console.log(JSON.stringify(${JSON.stringify( - hookOutput, - )}));`; - - rig.setup('should stop agent execution via BeforeTool hook'); - const scriptPath = rig.createScript( - 'before_tool_stop_hook.js', - hookScript, - ); - - rig.setup('should stop agent execution via BeforeTool hook', { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.before-tool-stop.responses', - ), - settings: { - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - matcher: 'write_file', - sequential: true, - hooks: [ - { - type: 'command', - command: normalizePath(`node "${scriptPath}"`), - timeout: 5000, - }, - ], - }, - ], - }, - }, - }); - - const result = await rig.run({ - args: 'Use write_file to create test.txt', - }); - - // The hook should have stopped execution message (returned from tool) - expect(result).toContain( - 'Agent execution stopped by hook: Emergency Stop triggered by hook', - ); - - // Tool should NOT be called successfully (it was blocked/stopped) - const toolLogs = rig.readToolLogs(); - const writeFileCalls = toolLogs.filter( - (t) => - t.toolRequest.name === 'write_file' && t.toolRequest.success === true, - ); - expect(writeFileCalls).toHaveLength(0); - }); - }); - - describe('Hooks "ask" Decision Integration', () => { - it( - 'should force confirmation prompt when hook returns "ask" decision even in YOLO mode', - { timeout: 60000 }, - async () => { - const testName = - 'should force confirmation prompt when hook returns "ask" decision even in YOLO mode'; - - // 1. Setup hook script that returns 'ask' decision - const hookOutput = { - decision: 'ask', - systemMessage: 'Confirmation forced by security hook', - hookSpecificOutput: { - hookEventName: 'BeforeTool', - }, - }; - - const hookScript = `console.log(JSON.stringify(${JSON.stringify( - hookOutput, - )}));`; - - // Create script path predictably - const scriptPath = join(os.tmpdir(), 'gemini-cli-tests-ask-hook.js'); - writeFileSync(scriptPath, hookScript); - - // 2. Setup rig with YOLO mode enabled but with the 'ask' hook - rig.setup(testName, { - fakeResponsesPath: join( - import.meta.dirname, - 'hooks-system.allow-tool.responses', - ), - settings: { - debugMode: true, - tools: { - approval: 'yolo', - }, - general: { - enableAutoUpdateNotification: false, - }, - hooksConfig: { - enabled: true, - }, - hooks: { - BeforeTool: [ - { - matcher: 'write_file', - hooks: [ - { - type: 'command', - command: `node "${scriptPath}"`, + command: normalizePath(`node "${scriptPath}"`), timeout: 5000, }, ], @@ -2307,59 +2164,52 @@ console.log(JSON.stringify({ }, }); - // Bypass terminal setup prompt and other startup banners - const stateDir = join(rig.homeDir!, '.gemini'); - if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true }); - writeFileSync( - join(stateDir, 'state.json'), - JSON.stringify({ - terminalSetupPromptShown: true, - hasSeenScreenReaderNudge: true, - tipsShown: 100, - }), + // Run the agent. The fake response will attempt to call write_file with + // file_path="original.txt" and content="original content" + await rig.run({ + args: 'Create a file called original.txt with content "original content"', + }); + + // 1. Verify that 'modified.txt' was created with 'modified content' (Override successful) + const modifiedContent = rig.readFile('modified.txt'); + expect(modifiedContent).toBe('modified content'); + + // 2. Verify that 'original.txt' was NOT created (Override replaced original) + let originalExists = false; + try { + rig.readFile('original.txt'); + originalExists = true; + } catch { + originalExists = false; + } + expect(originalExists).toBe(false); + + // 3. Verify hook telemetry + const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call'); + expect(hookTelemetryFound).toBeTruthy(); + + const hookLogs = rig.readHookLogs(); + expect(hookLogs.length).toBe(1); + expect(hookLogs[0].hookCall.hook_name).toContain( + 'input_override_hook.js', ); - // 3. Run interactive and verify prompt appears despite YOLO mode - const run = await rig.runInteractive(); + // 4. Verify that the agent didn't try to work-around the hook input change + const toolLogs = rig.readToolLogs(); + expect(toolLogs.length).toBe(1); + expect(toolLogs[0].toolRequest.name).toBe('write_file'); + expect(JSON.parse(toolLogs[0].toolRequest.args).file_path).toBe( + 'modified.txt', + ); + }); + }); - // Wait for prompt to appear - await run.expectText('Type your message', 30000); - - // Send prompt that will trigger write_file - await run.type('Create a file called ask-test.txt with content "test"'); - await run.type('\r'); - - // Wait for the FORCED confirmation prompt to appear - // It should contain the system message from the hook - await run.expectText('Confirmation forced by security hook', 30000); - await run.expectText('Allow', 5000); - - // 4. Approve the permission - await run.type('y'); - await run.type('\r'); - - // Wait for command to execute - await run.expectText('approved.txt', 30000); - - // Should find the tool call - const foundWriteFile = await rig.waitForToolCall('write_file'); - expect(foundWriteFile).toBeTruthy(); - - // File should be created - const fileContent = rig.readFile('approved.txt'); - expect(fileContent).toBe('Approved content'); - }, - ); - - it( - 'should allow cancelling when hook forces "ask" decision', - { timeout: 60000 }, - async () => { - const testName = - 'should allow cancelling when hook forces "ask" decision'; + describe('BeforeTool Hooks - Stop Execution', () => { + it('should stop agent execution via BeforeTool hook', async () => { + // Create a hook script that stops execution const hookOutput = { - decision: 'ask', - systemMessage: 'Confirmation forced for cancellation test', + continue: false, + reason: 'Emergency Stop triggered by hook', hookSpecificOutput: { hookEventName: 'BeforeTool', }, @@ -2369,25 +2219,18 @@ console.log(JSON.stringify({ hookOutput, )}));`; - const scriptPath = join( - os.tmpdir(), - 'gemini-cli-tests-ask-cancel-hook.js', + rig.setup('should stop agent execution via BeforeTool hook'); + const scriptPath = rig.createScript( + 'before_tool_stop_hook.js', + hookScript, ); - writeFileSync(scriptPath, hookScript); - rig.setup(testName, { + rig.setup('should stop agent execution via BeforeTool hook', { fakeResponsesPath: join( import.meta.dirname, - 'hooks-system.allow-tool.responses', + 'hooks-system.before-tool-stop.responses', ), settings: { - debugMode: true, - tools: { - approval: 'yolo', - }, - general: { - enableAutoUpdateNotification: false, - }, hooksConfig: { enabled: true, }, @@ -2395,10 +2238,11 @@ console.log(JSON.stringify({ BeforeTool: [ { matcher: 'write_file', + sequential: true, hooks: [ { type: 'command', - command: `node "${scriptPath}"`, + command: normalizePath(`node "${scriptPath}"`), timeout: 5000, }, ], @@ -2408,41 +2252,16 @@ console.log(JSON.stringify({ }, }); - // Bypass terminal setup prompt and other startup banners - const stateDir = join(rig.homeDir!, '.gemini'); - if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true }); - writeFileSync( - join(stateDir, 'state.json'), - JSON.stringify({ - terminalSetupPromptShown: true, - hasSeenScreenReaderNudge: true, - tipsShown: 100, - }), + const result = await rig.run({ + args: 'Use write_file to create test.txt', + }); + + // The hook should have stopped execution message (returned from tool) + expect(result).toContain( + 'Agent execution stopped by hook: Emergency Stop triggered by hook', ); - const run = await rig.runInteractive(); - - // Wait for prompt to appear - await run.expectText('Type your message', 30000); - - await run.type( - 'Create a file called cancel-test.txt with content "test"', - ); - await run.type('\r'); - - await run.expectText( - 'Confirmation forced for cancellation test', - 30000, - ); - - // 4. Deny the permission using option 4 - await run.type('4'); - await run.type('\r'); - - // Wait for cancellation message - await run.expectText('Cancelled', 15000); - - // Tool should NOT be called successfully + // Tool should NOT be called successfully (it was blocked/stopped) const toolLogs = rig.readToolLogs(); const writeFileCalls = toolLogs.filter( (t) => @@ -2450,7 +2269,215 @@ console.log(JSON.stringify({ t.toolRequest.success === true, ); expect(writeFileCalls).toHaveLength(0); - }, - ); - }); -}); + }); + }); + + describe('Hooks "ask" Decision Integration', () => { + it( + 'should force confirmation prompt when hook returns "ask" decision even in YOLO mode', + { timeout: 60000 }, + async () => { + const testName = + 'should force confirmation prompt when hook returns "ask" decision even in YOLO mode'; + + // 1. Setup hook script that returns 'ask' decision + const hookOutput = { + decision: 'ask', + systemMessage: 'Confirmation forced by security hook', + hookSpecificOutput: { + hookEventName: 'BeforeTool', + }, + }; + + const hookScript = `console.log(JSON.stringify(${JSON.stringify( + hookOutput, + )}));`; + + // Create script path predictably + const scriptPath = join(os.tmpdir(), 'gemini-cli-tests-ask-hook.js'); + writeFileSync(scriptPath, hookScript); + + // 2. Setup rig with YOLO mode enabled but with the 'ask' hook + rig.setup(testName, { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.allow-tool.responses', + ), + settings: { + debugMode: true, + tools: { + approval: 'yolo', + }, + general: { + enableAutoUpdateNotification: false, + }, + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ + { + matcher: 'write_file', + hooks: [ + { + type: 'command', + command: `node "${scriptPath}"`, + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + // Bypass terminal setup prompt and other startup banners + const stateDir = join(rig.homeDir!, '.gemini'); + if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true }); + writeFileSync( + join(stateDir, 'state.json'), + JSON.stringify({ + terminalSetupPromptShown: true, + hasSeenScreenReaderNudge: true, + tipsShown: 100, + }), + ); + + // 3. Run interactive and verify prompt appears despite YOLO mode + const run = await rig.runInteractive(); + + // Wait for prompt to appear + await run.expectText('Type your message', 30000); + + // Send prompt that will trigger write_file + await run.type( + 'Create a file called ask-test.txt with content "test"', + ); + await run.type('\r'); + + // Wait for the FORCED confirmation prompt to appear + // It should contain the system message from the hook + await run.expectText('Confirmation forced by security hook', 30000); + await run.expectText('Allow', 5000); + + // 4. Approve the permission + await run.type('y'); + await run.type('\r'); + + // Wait for command to execute + await run.expectText('approved.txt', 30000); + + // Should find the tool call + const foundWriteFile = await rig.waitForToolCall('write_file'); + expect(foundWriteFile).toBeTruthy(); + + // File should be created + const fileContent = rig.readFile('approved.txt'); + expect(fileContent).toBe('Approved content'); + }, + ); + + it( + 'should allow cancelling when hook forces "ask" decision', + { timeout: 60000 }, + async () => { + const testName = + 'should allow cancelling when hook forces "ask" decision'; + const hookOutput = { + decision: 'ask', + systemMessage: 'Confirmation forced for cancellation test', + hookSpecificOutput: { + hookEventName: 'BeforeTool', + }, + }; + + const hookScript = `console.log(JSON.stringify(${JSON.stringify( + hookOutput, + )}));`; + + const scriptPath = join( + os.tmpdir(), + 'gemini-cli-tests-ask-cancel-hook.js', + ); + writeFileSync(scriptPath, hookScript); + + rig.setup(testName, { + fakeResponsesPath: join( + import.meta.dirname, + 'hooks-system.allow-tool.responses', + ), + settings: { + debugMode: true, + tools: { + approval: 'yolo', + }, + general: { + enableAutoUpdateNotification: false, + }, + hooksConfig: { + enabled: true, + }, + hooks: { + BeforeTool: [ + { + matcher: 'write_file', + hooks: [ + { + type: 'command', + command: `node "${scriptPath}"`, + timeout: 5000, + }, + ], + }, + ], + }, + }, + }); + + // Bypass terminal setup prompt and other startup banners + const stateDir = join(rig.homeDir!, '.gemini'); + if (!existsSync(stateDir)) mkdirSync(stateDir, { recursive: true }); + writeFileSync( + join(stateDir, 'state.json'), + JSON.stringify({ + terminalSetupPromptShown: true, + hasSeenScreenReaderNudge: true, + tipsShown: 100, + }), + ); + + const run = await rig.runInteractive(); + + // Wait for prompt to appear + await run.expectText('Type your message', 30000); + + await run.type( + 'Create a file called cancel-test.txt with content "test"', + ); + await run.type('\r'); + + await run.expectText( + 'Confirmation forced for cancellation test', + 30000, + ); + + // 4. Deny the permission using option 4 + await run.type('4'); + await run.type('\r'); + + // Wait for cancellation message + await run.expectText('Cancelled', 15000); + + // Tool should NOT be called successfully + const toolLogs = rig.readToolLogs(); + const writeFileCalls = toolLogs.filter( + (t) => + t.toolRequest.name === 'write_file' && + t.toolRequest.success === true, + ); + expect(writeFileCalls).toHaveLength(0); + }, + ); + }); + }, +); diff --git a/integration-tests/symlink-install.test.ts b/integration-tests/symlink-install.test.ts index be4a5ac398..c98db98029 100644 --- a/integration-tests/symlink-install.test.ts +++ b/integration-tests/symlink-install.test.ts @@ -5,7 +5,7 @@ */ import { describe, expect, it, beforeEach, afterEach } from 'vitest'; -import { TestRig, InteractiveRun } from './test-helper.js'; +import { TestRig, InteractiveRun, skipFlaky } from './test-helper.js'; import * as fs from 'node:fs'; import * as os from 'node:os'; import { @@ -33,104 +33,107 @@ const otherExtension = `{ "version": "6.6.6" }`; -describe('extension symlink install spoofing protection', () => { - let rig: TestRig; +describe.skipIf(skipFlaky)( + 'extension symlink install spoofing protection', + () => { + let rig: TestRig; - beforeEach(() => { - rig = new TestRig(); - }); - - afterEach(async () => await rig.cleanup()); - - it('canonicalizes the trust path and prevents symlink spoofing', async () => { - // Enable folder trust for this test - rig.setup('symlink spoofing test', { - settings: { - security: { - folderTrust: { - enabled: true, - }, - }, - }, + beforeEach(() => { + rig = new TestRig(); }); - const realExtPath = join(rig.testDir!, 'real-extension'); - mkdirSync(realExtPath); - writeFileSync(join(realExtPath, 'gemini-extension.json'), extension); + afterEach(async () => await rig.cleanup()); - const maliciousExtPath = join( - os.tmpdir(), - `malicious-extension-${Date.now()}`, - ); - mkdirSync(maliciousExtPath); - writeFileSync( - join(maliciousExtPath, 'gemini-extension.json'), - otherExtension, - ); - - const symlinkPath = join(rig.testDir!, 'symlink-extension'); - symlinkSync(realExtPath, symlinkPath); - - // Function to run a command with a PTY to avoid headless mode - const runPty = (args: string[]) => { - const ptyProcess = pty.spawn(process.execPath, [BUNDLE_PATH, ...args], { - name: 'xterm-color', - cols: 80, - rows: 80, - cwd: rig.testDir!, - env: { - ...process.env, - GEMINI_CLI_HOME: rig.homeDir!, - GEMINI_CLI_INTEGRATION_TEST: 'true', - GEMINI_PTY_INFO: 'node-pty', + it('canonicalizes the trust path and prevents symlink spoofing', async () => { + // Enable folder trust for this test + rig.setup('symlink spoofing test', { + settings: { + security: { + folderTrust: { + enabled: true, + }, + }, }, }); - return new InteractiveRun(ptyProcess); - }; - // 1. Install via symlink, trust it - const run1 = runPty(['extensions', 'install', symlinkPath]); - await run1.expectText('Do you want to trust this folder', 30000); - await run1.type('y\r'); - await run1.expectText('trust this workspace', 30000); - await run1.type('y\r'); - await run1.expectText('Do you want to continue', 30000); - await run1.type('y\r'); - await run1.expectText('installed successfully', 30000); - await run1.kill(); + const realExtPath = join(rig.testDir!, 'real-extension'); + mkdirSync(realExtPath); + writeFileSync(join(realExtPath, 'gemini-extension.json'), extension); - // 2. Verify trustedFolders.json contains the REAL path, not the symlink path - const trustedFoldersPath = join( - rig.homeDir!, - GEMINI_DIR, - 'trustedFolders.json', - ); - // Wait for file to be written - let attempts = 0; - while (!fs.existsSync(trustedFoldersPath) && attempts < 50) { - await new Promise((resolve) => setTimeout(resolve, 100)); - attempts++; - } + const maliciousExtPath = join( + os.tmpdir(), + `malicious-extension-${Date.now()}`, + ); + mkdirSync(maliciousExtPath); + writeFileSync( + join(maliciousExtPath, 'gemini-extension.json'), + otherExtension, + ); - const trustedFolders = JSON.parse( - readFileSync(trustedFoldersPath, 'utf-8'), - ); - const trustedPaths = Object.keys(trustedFolders); - const canonicalRealExtPath = fs.realpathSync(realExtPath); + const symlinkPath = join(rig.testDir!, 'symlink-extension'); + symlinkSync(realExtPath, symlinkPath); - expect(trustedPaths).toContain(canonicalRealExtPath); - expect(trustedPaths).not.toContain(symlinkPath); + // Function to run a command with a PTY to avoid headless mode + const runPty = (args: string[]) => { + const ptyProcess = pty.spawn(process.execPath, [BUNDLE_PATH, ...args], { + name: 'xterm-color', + cols: 80, + rows: 80, + cwd: rig.testDir!, + env: { + ...process.env, + GEMINI_CLI_HOME: rig.homeDir!, + GEMINI_CLI_INTEGRATION_TEST: 'true', + GEMINI_PTY_INFO: 'node-pty', + }, + }); + return new InteractiveRun(ptyProcess); + }; - // 3. Swap the symlink to point to the malicious extension - unlinkSync(symlinkPath); - symlinkSync(maliciousExtPath, symlinkPath); + // 1. Install via symlink, trust it + const run1 = runPty(['extensions', 'install', symlinkPath]); + await run1.expectText('Do you want to trust this folder', 30000); + await run1.type('y\r'); + await run1.expectText('trust this workspace', 30000); + await run1.type('y\r'); + await run1.expectText('Do you want to continue', 30000); + await run1.type('y\r'); + await run1.expectText('installed successfully', 30000); + await run1.kill(); - // 4. Try to install again via the same symlink path. - // It should NOT be trusted because the real path changed. - const run2 = runPty(['extensions', 'install', symlinkPath]); - await run2.expectText('Do you want to trust this folder', 30000); - await run2.type('n\r'); - await run2.expectText('Installation aborted', 30000); - await run2.kill(); - }, 60000); -}); + // 2. Verify trustedFolders.json contains the REAL path, not the symlink path + const trustedFoldersPath = join( + rig.homeDir!, + GEMINI_DIR, + 'trustedFolders.json', + ); + // Wait for file to be written + let attempts = 0; + while (!fs.existsSync(trustedFoldersPath) && attempts < 50) { + await new Promise((resolve) => setTimeout(resolve, 100)); + attempts++; + } + + const trustedFolders = JSON.parse( + readFileSync(trustedFoldersPath, 'utf-8'), + ); + const trustedPaths = Object.keys(trustedFolders); + const canonicalRealExtPath = fs.realpathSync(realExtPath); + + expect(trustedPaths).toContain(canonicalRealExtPath); + expect(trustedPaths).not.toContain(symlinkPath); + + // 3. Swap the symlink to point to the malicious extension + unlinkSync(symlinkPath); + symlinkSync(maliciousExtPath, symlinkPath); + + // 4. Try to install again via the same symlink path. + // It should NOT be trusted because the real path changed. + const run2 = runPty(['extensions', 'install', symlinkPath]); + await run2.expectText('Do you want to trust this folder', 30000); + await run2.type('n\r'); + await run2.expectText('Installation aborted', 30000); + await run2.kill(); + }, 60000); + }, +); diff --git a/integration-tests/test-helper.ts b/integration-tests/test-helper.ts index a4546a2cd3..5f205ae997 100644 --- a/integration-tests/test-helper.ts +++ b/integration-tests/test-helper.ts @@ -6,3 +6,5 @@ export * from '@google/gemini-cli-test-utils'; export { normalizePath } from '@google/gemini-cli-test-utils'; + +export const skipFlaky = !process.env['RUN_FLAKY_INTEGRATION']; diff --git a/package.json b/package.json index 414f9341ac..d66132c066 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "test:all_evals": "cross-env RUN_EVALS=1 vitest run --config evals/vitest.config.ts", "test:e2e": "cross-env VERBOSE=true KEEP_OUTPUT=true npm run test:integration:sandbox:none", "test:integration:all": "npm run test:integration:sandbox:none && npm run test:integration:sandbox:docker && npm run test:integration:sandbox:podman", + "test:integration:flaky": "cross-env RUN_FLAKY_INTEGRATION=1 npm run test:integration:sandbox:none", "test:integration:sandbox:none": "cross-env GEMINI_SANDBOX=false vitest run --root ./integration-tests", "test:integration:sandbox:docker": "cross-env GEMINI_SANDBOX=docker npm run build:sandbox && cross-env GEMINI_SANDBOX=docker vitest run --root ./integration-tests", "test:integration:sandbox:podman": "cross-env GEMINI_SANDBOX=podman vitest run --root ./integration-tests",