Files
gemini-cli/integration-tests/hooks-system.test.ts

924 lines
30 KiB
TypeScript

/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { TestRig } from './test-helper.js';
import { join } from 'node:path';
import { writeFileSync } from 'node:fs';
describe('Hooks System Integration', () => {
let rig: TestRig;
beforeEach(() => {
rig = new TestRig();
});
afterEach(async () => {
if (rig) {
await rig.cleanup();
}
});
describe('Command Hooks - Blocking Behavior', () => {
it('should block tool execution when hook returns block decision', async () => {
await rig.setup(
'should block tool execution when hook returns block decision',
{
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.block-tool.responses',
),
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeTool: [
{
matcher: 'write_file',
sequential: true,
hooks: [
{
type: 'command',
command:
'echo "{\\"decision\\": \\"block\\", \\"reason\\": \\"File writing blocked by security policy\\"}"',
timeout: 5000,
},
],
},
],
},
},
},
);
const prompt = 'Create a file called test.txt with content "Hello World"';
const result = await rig.run(prompt);
// The hook should block the write_file tool
const toolLogs = rig.readToolLogs();
const writeFileCalls = toolLogs.filter(
(t) =>
t.toolRequest.name === 'write_file' && t.toolRequest.success === true,
);
// Tool should not be called due to blocking hook
expect(writeFileCalls).toHaveLength(0);
// Result should mention the blocking reason
expect(result).toContain('File writing blocked by security policy');
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
it('should allow tool execution when hook returns allow decision', async () => {
await rig.setup(
'should allow tool execution when hook returns allow decision',
{
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.allow-tool.responses',
),
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeTool: [
{
matcher: 'write_file',
hooks: [
{
type: 'command',
command:
'echo "{\\"decision\\": \\"allow\\", \\"reason\\": \\"File writing approved\\"}"',
timeout: 5000,
},
],
},
],
},
},
},
);
const prompt =
'Create a file called approved.txt with content "Approved content"';
await rig.run(prompt);
// The hook should allow the write_file tool
const foundWriteFile = await rig.waitForToolCall('write_file');
expect(foundWriteFile).toBeTruthy();
// File should be created
const fileContent = rig.readFile('approved.txt');
expect(fileContent).toContain('Approved content');
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
});
describe('Command Hooks - Additional Context', () => {
it('should add additional context from AfterTool hooks', async () => {
const command =
'echo "{\\"hookSpecificOutput\\": {\\"hookEventName\\": \\"AfterTool\\", \\"additionalContext\\": \\"Security scan: File content appears safe\\"}}"';
await rig.setup('should add additional context from AfterTool hooks', {
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.after-tool-context.responses',
),
settings: {
tools: {
enableHooks: true,
},
hooks: {
AfterTool: [
{
matcher: 'read_file',
hooks: [
{
type: 'command',
command: command,
timeout: 5000,
},
],
},
],
},
},
});
// Create a test file to read
rig.createFile('test-file.txt', 'This is test content');
const prompt =
'Read the contents of test-file.txt and tell me what it contains';
await rig.run(prompt);
// Should find read_file tool call
const foundReadFile = await rig.waitForToolCall('read_file');
expect(foundReadFile).toBeTruthy();
// Should generate hook telemetry
const hookTelemetryFound = rig.readHookLogs();
expect(hookTelemetryFound.length).toBeGreaterThan(0);
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe('AfterTool');
expect(hookTelemetryFound[0].hookCall.hook_name).toBe(command);
expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined();
expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined();
expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0);
expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined();
expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined();
});
});
describe('BeforeModel Hooks - LLM Request Modification', () => {
it('should modify LLM requests with BeforeModel hooks', async () => {
// Create a hook script that replaces the LLM request with a modified version
// Note: Providing messages in the hook output REPLACES the entire conversation
await rig.setup('should modify LLM requests with BeforeModel hooks', {
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.before-model.responses',
),
});
const hookScript = `#!/bin/bash
echo '{
"decision": "allow",
"hookSpecificOutput": {
"hookEventName": "BeforeModel",
"llm_request": {
"messages": [
{
"role": "user",
"content": "Please respond with exactly: The security hook modified this request successfully."
}
]
}
}
}'`;
const scriptPath = join(rig.testDir!, 'before_model_hook.sh');
writeFileSync(scriptPath, hookScript);
// Make executable
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${scriptPath}"`);
await rig.setup('should modify LLM requests with BeforeModel hooks', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeModel: [
{
hooks: [
{
type: 'command',
command: scriptPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt = 'Tell me a story';
const result = await rig.run(prompt);
// The hook should have replaced the request entirely
// Verify that the model responded to the modified request, not the original
expect(result).toBeDefined();
expect(result.length).toBeGreaterThan(0);
// The response should contain the expected text from the modified request
expect(result.toLowerCase()).toContain('security hook modified');
// Should generate hook telemetry
// Should generate hook telemetry
const hookTelemetryFound = rig.readHookLogs();
expect(hookTelemetryFound.length).toBeGreaterThan(0);
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe(
'BeforeModel',
);
expect(hookTelemetryFound[0].hookCall.hook_name).toBe(scriptPath);
expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined();
expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined();
expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0);
expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined();
expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined();
});
});
describe('AfterModel Hooks - LLM Response Modification', () => {
it('should modify LLM responses with AfterModel hooks', async () => {
await rig.setup('should modify LLM responses with AfterModel hooks', {
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.after-model.responses',
),
});
// Create a hook script that modifies the LLM response
const hookScript = `#!/bin/bash
echo '{
"hookSpecificOutput": {
"hookEventName": "AfterModel",
"llm_response": {
"candidates": [
{
"content": {
"role": "model",
"parts": [
"[FILTERED] Response has been filtered for security compliance."
]
},
"finishReason": "STOP"
}
]
}
}
}'`;
const scriptPath = join(rig.testDir!, 'after_model_hook.sh');
writeFileSync(scriptPath, hookScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${scriptPath}"`);
await rig.setup('should modify LLM responses with AfterModel hooks', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
AfterModel: [
{
hooks: [
{
type: 'command',
command: scriptPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt = 'What is 2 + 2?';
const result = await rig.run(prompt);
// The hook should have replaced the model response
expect(result).toContain(
'[FILTERED] Response has been filtered for security compliance',
);
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
});
describe('BeforeToolSelection Hooks - Tool Configuration', () => {
it('should modify tool selection with BeforeToolSelection hooks', async () => {
await rig.setup(
'should modify tool selection with BeforeToolSelection hooks',
{
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.before-tool-selection.responses',
),
},
);
// Create a hook script that restricts available tools
const hookScript = `#!/bin/bash
echo '{
"hookSpecificOutput": {
"hookEventName": "BeforeToolSelection",
"toolConfig": {
"mode": "ANY",
"allowedFunctionNames": ["read_file", "run_shell_command"]
}
}
}'`;
const scriptPath = join(rig.testDir!, 'before_tool_selection_hook.sh');
writeFileSync(scriptPath, hookScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${scriptPath}"`);
await rig.setup(
'should modify tool selection with BeforeToolSelection hooks',
{
settings: {
debugMode: true,
tools: {
enableHooks: true,
},
hooks: {
BeforeToolSelection: [
{
hooks: [
{
type: 'command',
command: scriptPath,
timeout: 5000,
},
],
},
],
},
},
},
);
// Create a test file
rig.createFile('new_file_data.txt', 'test data');
const prompt =
'Check the content of new_file_data.txt, after that run echo command to see the content';
await rig.run(prompt);
// Should use read_file (allowed) but not run_shell_command (not in allowed list)
const foundReadFile = await rig.waitForToolCall('read_file');
expect(foundReadFile).toBeTruthy();
// Should generate hook telemetry indicating the hook was called
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
// Verify the hook was called for BeforeToolSelection event
const hookLogs = rig.readHookLogs();
const beforeToolSelectionHook = hookLogs.find(
(log) => log.hookCall.hook_event_name === 'BeforeToolSelection',
);
expect(beforeToolSelectionHook).toBeDefined();
expect(beforeToolSelectionHook?.hookCall.success).toBe(true);
});
});
describe('BeforeAgent Hooks - Prompt Augmentation', () => {
it('should augment prompts with BeforeAgent hooks', async () => {
await rig.setup('should augment prompts with BeforeAgent hooks', {
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.before-agent.responses',
),
});
// Create a hook script that adds context to the prompt
const hookScript = `#!/bin/bash
echo '{
"decision": "allow",
"hookSpecificOutput": {
"hookEventName": "BeforeAgent",
"additionalContext": "SYSTEM INSTRUCTION: You are in a secure environment. Always mention security compliance in your responses."
}
}'`;
const scriptPath = join(rig.testDir!, 'before_agent_hook.sh');
writeFileSync(scriptPath, hookScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${scriptPath}"`);
await rig.setup('should augment prompts with BeforeAgent hooks', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeAgent: [
{
hooks: [
{
type: 'command',
command: scriptPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt = 'Hello, how are you?';
const result = await rig.run(prompt);
// The hook should have added security context, which should influence the response
expect(result).toContain('security');
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
});
describe.skip('Notification Hooks - Permission Handling', () => {
it('should handle notification hooks for tool permissions', async () => {
await rig.setup('should handle notification hooks for tool permissions');
// Create a hook script that logs notification events
const hookScript = `#!/bin/bash
echo '{
"suppressOutput": false,
"systemMessage": "Permission request logged by security hook"
}'`;
const scriptPath = join(rig.testDir!, 'notification_hook.sh');
writeFileSync(scriptPath, hookScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${scriptPath}"`);
await rig.setup('should handle notification hooks for tool permissions', {
settings: {
// Configure tools to enable hooks and require confirmation to trigger notifications
tools: {
enableHooks: true,
confirmationRequired: ['run_shell_command'],
},
hooks: {
Notification: [
{
matcher: 'ToolPermission',
hooks: [
{
type: 'command',
command: scriptPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt =
'Run the command "echo test" (this should trigger a permission prompt)';
// Use stdin to automatically approve the permission
await rig.run({
prompt,
stdin: 'y\n', // Approve the permission
});
// Should find the shell command execution
const foundShellCommand = await rig.waitForToolCall('run_shell_command');
expect(foundShellCommand).toBeTruthy();
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
});
describe('Sequential Hook Execution', () => {
// Note: This test checks telemetry for hook context in API requests,
// which behaves differently with mocked responses. Keeping real LLM calls.
it('should execute hooks sequentially when configured', async () => {
await rig.setup('should execute hooks sequentially when configured');
// Create two hooks that modify the input sequentially
const hook1Script = `#!/bin/bash
echo '{
"decision": "allow",
"hookSpecificOutput": {
"hookEventName": "BeforeAgent",
"additionalContext": "Step 1: Initial validation passed."
}
}'`;
const hook2Script = `#!/bin/bash
echo '{
"decision": "allow",
"hookSpecificOutput": {
"hookEventName": "BeforeAgent",
"additionalContext": "Step 2: Security check completed."
}
}'`;
const script1Path = join(rig.testDir!, 'sequential_hook1.sh');
const script2Path = join(rig.testDir!, 'sequential_hook2.sh');
writeFileSync(script1Path, hook1Script);
writeFileSync(script2Path, hook2Script);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${script1Path}"`);
execSync(`chmod +x "${script2Path}"`);
await rig.setup('should execute hooks sequentially when configured', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeAgent: [
{
sequential: true,
hooks: [
{
type: 'command',
command: script1Path,
timeout: 5000,
},
{
type: 'command',
command: script2Path,
timeout: 5000,
},
],
},
],
},
},
});
const prompt = 'Hello, please help me with a task';
await rig.run(prompt);
// Should generate hook telemetry
let hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
hookTelemetryFound = await rig.waitForTelemetryEvent('api_request');
const apiRequests = rig.readAllApiRequest();
const apiRequestsTexts = apiRequests
?.filter(
(request) =>
'attributes' in request &&
typeof request['attributes'] === 'object' &&
request['attributes'] !== null &&
'request_text' in request['attributes'] &&
typeof request['attributes']['request_text'] === 'string',
)
.map((request) => request['attributes']['request_text']);
expect(apiRequestsTexts).toBeDefined();
let hasBeforeAgentHookContext = false;
let hasAfterToolHookContext = false;
for (const requestText of apiRequestsTexts) {
if (requestText.includes('Step 1: Initial validation passed')) {
hasBeforeAgentHookContext = true;
}
if (requestText.includes('Step 2: Security check completed')) {
hasAfterToolHookContext = true;
}
}
expect(hasBeforeAgentHookContext).toBeTruthy();
expect(hasAfterToolHookContext).toBeTruthy();
});
});
describe('Hook Input/Output Validation', () => {
it('should provide correct input format to hooks', async () => {
await rig.setup('should provide correct input format to hooks', {
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.input-validation.responses',
),
});
// Create a hook script that validates the input format
const hookScript = `#!/bin/bash
# Read JSON input from stdin
input=$(cat)
# Check for required fields
if echo "$input" | jq -e '.session_id and .cwd and .hook_event_name and .timestamp and .tool_name and .tool_input' > /dev/null 2>&1; then
echo '{"decision": "allow", "reason": "Input format is correct"}'
exit 0
else
echo '{"decision": "block", "reason": "Input format is invalid"}'
exit 0
fi`;
const scriptPath = join(rig.testDir!, 'input_validation_hook.sh');
writeFileSync(scriptPath, hookScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${scriptPath}"`);
await rig.setup('should provide correct input format to hooks', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeTool: [
{
hooks: [
{
type: 'command',
command: scriptPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt = 'Create a file called input-test.txt with content "test"';
await rig.run(prompt);
// Hook should validate input format successfully
const foundWriteFile = await rig.waitForToolCall('write_file');
expect(foundWriteFile).toBeTruthy();
// Check that the file was created (hook allowed it)
const fileContent = rig.readFile('input-test.txt');
expect(fileContent).toContain('test');
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
});
describe('Multiple Event Types', () => {
// Note: This test checks telemetry for hook context in API requests,
// which behaves differently with mocked responses. Keeping real LLM calls.
it('should handle hooks for all major event types', async () => {
await rig.setup('should handle hooks for all major event types');
// Create hook scripts for different events
const beforeToolScript = `#!/bin/bash
echo '{"decision": "allow", "systemMessage": "BeforeTool: File operation logged"}'`;
const afterToolScript = `#!/bin/bash
echo '{"hookSpecificOutput": {"hookEventName": "AfterTool", "additionalContext": "AfterTool: Operation completed successfully"}}'`;
const beforeAgentScript = `#!/bin/bash
echo '{"decision": "allow", "hookSpecificOutput": {"hookEventName": "BeforeAgent", "additionalContext": "BeforeAgent: User request processed"}}'`;
const beforeToolPath = join(rig.testDir!, 'before_tool.sh');
const afterToolPath = join(rig.testDir!, 'after_tool.sh');
const beforeAgentPath = join(rig.testDir!, 'before_agent.sh');
writeFileSync(beforeToolPath, beforeToolScript);
writeFileSync(afterToolPath, afterToolScript);
writeFileSync(beforeAgentPath, beforeAgentScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${beforeToolPath}"`);
execSync(`chmod +x "${afterToolPath}"`);
execSync(`chmod +x "${beforeAgentPath}"`);
await rig.setup('should handle hooks for all major event types', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeAgent: [
{
hooks: [
{
type: 'command',
command: beforeAgentPath,
timeout: 5000,
},
],
},
],
BeforeTool: [
{
matcher: 'write_file',
hooks: [
{
type: 'command',
command: beforeToolPath,
timeout: 5000,
},
],
},
],
AfterTool: [
{
matcher: 'write_file',
hooks: [
{
type: 'command',
command: afterToolPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt =
'Create a file called multi-event-test.txt with content ' +
'"testing multiple events", and then please reply with ' +
'everything I say just after this:"';
const result = await rig.run(prompt);
// Should execute write_file tool
const foundWriteFile = await rig.waitForToolCall('write_file');
expect(foundWriteFile).toBeTruthy();
// File should be created
const fileContent = rig.readFile('multi-event-test.txt');
expect(fileContent).toContain('testing multiple events');
// Result should contain context from all hooks
expect(result).toContain('BeforeTool: File operation logged');
// Should generate hook telemetry
let hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
hookTelemetryFound = await rig.waitForTelemetryEvent('api_request');
const apiRequests = rig.readAllApiRequest();
const apiRequestsTexts = apiRequests
?.filter(
(request) =>
'attributes' in request &&
typeof request['attributes'] === 'object' &&
request['attributes'] !== null &&
'request_text' in request['attributes'] &&
typeof request['attributes']['request_text'] === 'string',
)
.map((request) => request['attributes']['request_text']);
expect(apiRequestsTexts).toBeDefined();
let hasBeforeAgentHookContext = false;
let hasAfterToolHookContext = false;
for (const requestText of apiRequestsTexts) {
if (requestText.includes('BeforeAgent: User request processed')) {
hasBeforeAgentHookContext = true;
}
if (
requestText.includes('AfterTool: Operation completed successfully')
) {
hasAfterToolHookContext = true;
}
}
expect(hasBeforeAgentHookContext).toBeTruthy();
expect(hasAfterToolHookContext).toBeTruthy();
});
});
describe('Hook Error Handling', () => {
it('should handle hook failures gracefully', async () => {
await rig.setup('should handle hook failures gracefully', {
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.error-handling.responses',
),
});
// Create a hook script that fails
const failingHookScript = `#!/bin/bash
echo "Hook encountered an error" >&2
exit 1`;
const workingHookScript = `#!/bin/bash
echo '{"decision": "allow", "reason": "Working hook succeeded"}'`;
const failingPath = join(rig.testDir!, 'failing_hook.sh');
const workingPath = join(rig.testDir!, 'working_hook.sh');
writeFileSync(failingPath, failingHookScript);
writeFileSync(workingPath, workingHookScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${failingPath}"`);
execSync(`chmod +x "${workingPath}"`);
await rig.setup('should handle hook failures gracefully', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeTool: [
{
hooks: [
{
type: 'command',
command: failingPath,
timeout: 5000,
},
{
type: 'command',
command: workingPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt =
'Create a file called error-test.txt with content "testing error handling"';
await rig.run(prompt);
// Despite one hook failing, the working hook should still allow the operation
const foundWriteFile = await rig.waitForToolCall('write_file');
expect(foundWriteFile).toBeTruthy();
// File should be created
const fileContent = rig.readFile('error-test.txt');
expect(fileContent).toContain('testing error handling');
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
});
describe('Hook Telemetry and Observability', () => {
it('should generate telemetry events for hook executions', async () => {
await rig.setup('should generate telemetry events for hook executions', {
fakeResponsesPath: join(
import.meta.dirname,
'hooks-system.telemetry.responses',
),
});
const hookScript = `#!/bin/bash
echo '{"decision": "allow", "reason": "Telemetry test hook"}'`;
const scriptPath = join(rig.testDir!, 'telemetry_hook.sh');
writeFileSync(scriptPath, hookScript);
const { execSync } = await import('node:child_process');
execSync(`chmod +x "${scriptPath}"`);
await rig.setup('should generate telemetry events for hook executions', {
settings: {
tools: {
enableHooks: true,
},
hooks: {
BeforeTool: [
{
hooks: [
{
type: 'command',
command: scriptPath,
timeout: 5000,
},
],
},
],
},
},
});
const prompt = 'Create a file called telemetry-test.txt';
await rig.run(prompt);
// Should execute the tool
const foundWriteFile = await rig.waitForToolCall('write_file');
expect(foundWriteFile).toBeTruthy();
// Should generate hook telemetry
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
expect(hookTelemetryFound).toBeTruthy();
});
});
});