mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-05-01 15:34:29 -07:00
feat(hooks): Hooks Comprehensive Integration Testing (#9112)
This commit is contained in:
Edilmo Palencia
@@ -0,0 +1,923 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { TestRig } from './test-helper.js';
|
||||
import { join } from 'node:path';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
|
||||
describe('Hooks System Integration', () => {
|
||||
let rig: TestRig;
|
||||
|
||||
beforeEach(() => {
|
||||
rig = new TestRig();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (rig) {
|
||||
await rig.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
describe('Command Hooks - Blocking Behavior', () => {
|
||||
it('should block tool execution when hook returns block decision', async () => {
|
||||
await rig.setup(
|
||||
'should block tool execution when hook returns block decision',
|
||||
{
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.block-tool.responses',
|
||||
),
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeTool: [
|
||||
{
|
||||
matcher: 'write_file',
|
||||
sequential: true,
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command:
|
||||
'echo "{\\"decision\\": \\"block\\", \\"reason\\": \\"File writing blocked by security policy\\"}"',
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const prompt = 'Create a file called test.txt with content "Hello World"';
|
||||
const result = await rig.run(prompt);
|
||||
|
||||
// The hook should block the write_file tool
|
||||
const toolLogs = rig.readToolLogs();
|
||||
const writeFileCalls = toolLogs.filter(
|
||||
(t) =>
|
||||
t.toolRequest.name === 'write_file' && t.toolRequest.success === true,
|
||||
);
|
||||
|
||||
// Tool should not be called due to blocking hook
|
||||
expect(writeFileCalls).toHaveLength(0);
|
||||
|
||||
// Result should mention the blocking reason
|
||||
expect(result).toContain('File writing blocked by security policy');
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
|
||||
it('should allow tool execution when hook returns allow decision', async () => {
|
||||
await rig.setup(
|
||||
'should allow tool execution when hook returns allow decision',
|
||||
{
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.allow-tool.responses',
|
||||
),
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeTool: [
|
||||
{
|
||||
matcher: 'write_file',
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command:
|
||||
'echo "{\\"decision\\": \\"allow\\", \\"reason\\": \\"File writing approved\\"}"',
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const prompt =
|
||||
'Create a file called approved.txt with content "Approved content"';
|
||||
await rig.run(prompt);
|
||||
|
||||
// The hook should allow the write_file tool
|
||||
const foundWriteFile = await rig.waitForToolCall('write_file');
|
||||
expect(foundWriteFile).toBeTruthy();
|
||||
|
||||
// File should be created
|
||||
const fileContent = rig.readFile('approved.txt');
|
||||
expect(fileContent).toContain('Approved content');
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Command Hooks - Additional Context', () => {
|
||||
it('should add additional context from AfterTool hooks', async () => {
|
||||
const command =
|
||||
'echo "{\\"hookSpecificOutput\\": {\\"hookEventName\\": \\"AfterTool\\", \\"additionalContext\\": \\"Security scan: File content appears safe\\"}}"';
|
||||
await rig.setup('should add additional context from AfterTool hooks', {
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.after-tool-context.responses',
|
||||
),
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
AfterTool: [
|
||||
{
|
||||
matcher: 'read_file',
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: command,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Create a test file to read
|
||||
rig.createFile('test-file.txt', 'This is test content');
|
||||
|
||||
const prompt =
|
||||
'Read the contents of test-file.txt and tell me what it contains';
|
||||
await rig.run(prompt);
|
||||
|
||||
// Should find read_file tool call
|
||||
const foundReadFile = await rig.waitForToolCall('read_file');
|
||||
expect(foundReadFile).toBeTruthy();
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = rig.readHookLogs();
|
||||
expect(hookTelemetryFound.length).toBeGreaterThan(0);
|
||||
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe('AfterTool');
|
||||
expect(hookTelemetryFound[0].hookCall.hook_name).toBe(command);
|
||||
expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined();
|
||||
expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined();
|
||||
expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0);
|
||||
expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined();
|
||||
expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('BeforeModel Hooks - LLM Request Modification', () => {
|
||||
it('should modify LLM requests with BeforeModel hooks', async () => {
|
||||
// Create a hook script that replaces the LLM request with a modified version
|
||||
// Note: Providing messages in the hook output REPLACES the entire conversation
|
||||
await rig.setup('should modify LLM requests with BeforeModel hooks', {
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.before-model.responses',
|
||||
),
|
||||
});
|
||||
const hookScript = `#!/bin/bash
|
||||
echo '{
|
||||
"decision": "allow",
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "BeforeModel",
|
||||
"llm_request": {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Please respond with exactly: The security hook modified this request successfully."
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}'`;
|
||||
|
||||
const scriptPath = join(rig.testDir!, 'before_model_hook.sh');
|
||||
writeFileSync(scriptPath, hookScript);
|
||||
// Make executable
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${scriptPath}"`);
|
||||
|
||||
await rig.setup('should modify LLM requests with BeforeModel hooks', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeModel: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: scriptPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt = 'Tell me a story';
|
||||
const result = await rig.run(prompt);
|
||||
|
||||
// The hook should have replaced the request entirely
|
||||
// Verify that the model responded to the modified request, not the original
|
||||
expect(result).toBeDefined();
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
// The response should contain the expected text from the modified request
|
||||
expect(result.toLowerCase()).toContain('security hook modified');
|
||||
|
||||
// Should generate hook telemetry
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = rig.readHookLogs();
|
||||
expect(hookTelemetryFound.length).toBeGreaterThan(0);
|
||||
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe(
|
||||
'BeforeModel',
|
||||
);
|
||||
expect(hookTelemetryFound[0].hookCall.hook_name).toBe(scriptPath);
|
||||
expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined();
|
||||
expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined();
|
||||
expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0);
|
||||
expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined();
|
||||
expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('AfterModel Hooks - LLM Response Modification', () => {
|
||||
it('should modify LLM responses with AfterModel hooks', async () => {
|
||||
await rig.setup('should modify LLM responses with AfterModel hooks', {
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.after-model.responses',
|
||||
),
|
||||
});
|
||||
// Create a hook script that modifies the LLM response
|
||||
const hookScript = `#!/bin/bash
|
||||
echo '{
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "AfterModel",
|
||||
"llm_response": {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"role": "model",
|
||||
"parts": [
|
||||
"[FILTERED] Response has been filtered for security compliance."
|
||||
]
|
||||
},
|
||||
"finishReason": "STOP"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}'`;
|
||||
|
||||
const scriptPath = join(rig.testDir!, 'after_model_hook.sh');
|
||||
writeFileSync(scriptPath, hookScript);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${scriptPath}"`);
|
||||
|
||||
await rig.setup('should modify LLM responses with AfterModel hooks', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
AfterModel: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: scriptPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt = 'What is 2 + 2?';
|
||||
const result = await rig.run(prompt);
|
||||
|
||||
// The hook should have replaced the model response
|
||||
expect(result).toContain(
|
||||
'[FILTERED] Response has been filtered for security compliance',
|
||||
);
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('BeforeToolSelection Hooks - Tool Configuration', () => {
|
||||
it('should modify tool selection with BeforeToolSelection hooks', async () => {
|
||||
await rig.setup(
|
||||
'should modify tool selection with BeforeToolSelection hooks',
|
||||
{
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.before-tool-selection.responses',
|
||||
),
|
||||
},
|
||||
);
|
||||
// Create a hook script that restricts available tools
|
||||
const hookScript = `#!/bin/bash
|
||||
echo '{
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "BeforeToolSelection",
|
||||
"toolConfig": {
|
||||
"mode": "ANY",
|
||||
"allowedFunctionNames": ["read_file", "run_shell_command"]
|
||||
}
|
||||
}
|
||||
}'`;
|
||||
|
||||
const scriptPath = join(rig.testDir!, 'before_tool_selection_hook.sh');
|
||||
writeFileSync(scriptPath, hookScript);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${scriptPath}"`);
|
||||
|
||||
await rig.setup(
|
||||
'should modify tool selection with BeforeToolSelection hooks',
|
||||
{
|
||||
settings: {
|
||||
debugMode: true,
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeToolSelection: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: scriptPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Create a test file
|
||||
rig.createFile('new_file_data.txt', 'test data');
|
||||
|
||||
const prompt =
|
||||
'Check the content of new_file_data.txt, after that run echo command to see the content';
|
||||
await rig.run(prompt);
|
||||
|
||||
// Should use read_file (allowed) but not run_shell_command (not in allowed list)
|
||||
const foundReadFile = await rig.waitForToolCall('read_file');
|
||||
expect(foundReadFile).toBeTruthy();
|
||||
|
||||
// Should generate hook telemetry indicating the hook was called
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
|
||||
// Verify the hook was called for BeforeToolSelection event
|
||||
const hookLogs = rig.readHookLogs();
|
||||
const beforeToolSelectionHook = hookLogs.find(
|
||||
(log) => log.hookCall.hook_event_name === 'BeforeToolSelection',
|
||||
);
|
||||
expect(beforeToolSelectionHook).toBeDefined();
|
||||
expect(beforeToolSelectionHook?.hookCall.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('BeforeAgent Hooks - Prompt Augmentation', () => {
|
||||
it('should augment prompts with BeforeAgent hooks', async () => {
|
||||
await rig.setup('should augment prompts with BeforeAgent hooks', {
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.before-agent.responses',
|
||||
),
|
||||
});
|
||||
// Create a hook script that adds context to the prompt
|
||||
const hookScript = `#!/bin/bash
|
||||
echo '{
|
||||
"decision": "allow",
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "BeforeAgent",
|
||||
"additionalContext": "SYSTEM INSTRUCTION: You are in a secure environment. Always mention security compliance in your responses."
|
||||
}
|
||||
}'`;
|
||||
|
||||
const scriptPath = join(rig.testDir!, 'before_agent_hook.sh');
|
||||
writeFileSync(scriptPath, hookScript);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${scriptPath}"`);
|
||||
|
||||
await rig.setup('should augment prompts with BeforeAgent hooks', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeAgent: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: scriptPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt = 'Hello, how are you?';
|
||||
const result = await rig.run(prompt);
|
||||
|
||||
// The hook should have added security context, which should influence the response
|
||||
expect(result).toContain('security');
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe.skip('Notification Hooks - Permission Handling', () => {
|
||||
it('should handle notification hooks for tool permissions', async () => {
|
||||
await rig.setup('should handle notification hooks for tool permissions');
|
||||
// Create a hook script that logs notification events
|
||||
const hookScript = `#!/bin/bash
|
||||
echo '{
|
||||
"suppressOutput": false,
|
||||
"systemMessage": "Permission request logged by security hook"
|
||||
}'`;
|
||||
|
||||
const scriptPath = join(rig.testDir!, 'notification_hook.sh');
|
||||
writeFileSync(scriptPath, hookScript);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${scriptPath}"`);
|
||||
|
||||
await rig.setup('should handle notification hooks for tool permissions', {
|
||||
settings: {
|
||||
// Configure tools to enable hooks and require confirmation to trigger notifications
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
confirmationRequired: ['run_shell_command'],
|
||||
},
|
||||
hooks: {
|
||||
Notification: [
|
||||
{
|
||||
matcher: 'ToolPermission',
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: scriptPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt =
|
||||
'Run the command "echo test" (this should trigger a permission prompt)';
|
||||
|
||||
// Use stdin to automatically approve the permission
|
||||
await rig.run({
|
||||
prompt,
|
||||
stdin: 'y\n', // Approve the permission
|
||||
});
|
||||
|
||||
// Should find the shell command execution
|
||||
const foundShellCommand = await rig.waitForToolCall('run_shell_command');
|
||||
expect(foundShellCommand).toBeTruthy();
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Sequential Hook Execution', () => {
|
||||
// Note: This test checks telemetry for hook context in API requests,
|
||||
// which behaves differently with mocked responses. Keeping real LLM calls.
|
||||
it('should execute hooks sequentially when configured', async () => {
|
||||
await rig.setup('should execute hooks sequentially when configured');
|
||||
// Create two hooks that modify the input sequentially
|
||||
const hook1Script = `#!/bin/bash
|
||||
echo '{
|
||||
"decision": "allow",
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "BeforeAgent",
|
||||
"additionalContext": "Step 1: Initial validation passed."
|
||||
}
|
||||
}'`;
|
||||
|
||||
const hook2Script = `#!/bin/bash
|
||||
echo '{
|
||||
"decision": "allow",
|
||||
"hookSpecificOutput": {
|
||||
"hookEventName": "BeforeAgent",
|
||||
"additionalContext": "Step 2: Security check completed."
|
||||
}
|
||||
}'`;
|
||||
|
||||
const script1Path = join(rig.testDir!, 'sequential_hook1.sh');
|
||||
const script2Path = join(rig.testDir!, 'sequential_hook2.sh');
|
||||
|
||||
writeFileSync(script1Path, hook1Script);
|
||||
writeFileSync(script2Path, hook2Script);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${script1Path}"`);
|
||||
execSync(`chmod +x "${script2Path}"`);
|
||||
|
||||
await rig.setup('should execute hooks sequentially when configured', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeAgent: [
|
||||
{
|
||||
sequential: true,
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: script1Path,
|
||||
timeout: 5000,
|
||||
},
|
||||
{
|
||||
type: 'command',
|
||||
command: script2Path,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt = 'Hello, please help me with a task';
|
||||
await rig.run(prompt);
|
||||
|
||||
// Should generate hook telemetry
|
||||
let hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
hookTelemetryFound = await rig.waitForTelemetryEvent('api_request');
|
||||
const apiRequests = rig.readAllApiRequest();
|
||||
const apiRequestsTexts = apiRequests
|
||||
?.filter(
|
||||
(request) =>
|
||||
'attributes' in request &&
|
||||
typeof request['attributes'] === 'object' &&
|
||||
request['attributes'] !== null &&
|
||||
'request_text' in request['attributes'] &&
|
||||
typeof request['attributes']['request_text'] === 'string',
|
||||
)
|
||||
.map((request) => request['attributes']['request_text']);
|
||||
expect(apiRequestsTexts).toBeDefined();
|
||||
let hasBeforeAgentHookContext = false;
|
||||
let hasAfterToolHookContext = false;
|
||||
for (const requestText of apiRequestsTexts) {
|
||||
if (requestText.includes('Step 1: Initial validation passed')) {
|
||||
hasBeforeAgentHookContext = true;
|
||||
}
|
||||
if (requestText.includes('Step 2: Security check completed')) {
|
||||
hasAfterToolHookContext = true;
|
||||
}
|
||||
}
|
||||
expect(hasBeforeAgentHookContext).toBeTruthy();
|
||||
expect(hasAfterToolHookContext).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Hook Input/Output Validation', () => {
|
||||
it('should provide correct input format to hooks', async () => {
|
||||
await rig.setup('should provide correct input format to hooks', {
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.input-validation.responses',
|
||||
),
|
||||
});
|
||||
// Create a hook script that validates the input format
|
||||
const hookScript = `#!/bin/bash
|
||||
# Read JSON input from stdin
|
||||
input=$(cat)
|
||||
|
||||
# Check for required fields
|
||||
if echo "$input" | jq -e '.session_id and .cwd and .hook_event_name and .timestamp and .tool_name and .tool_input' > /dev/null 2>&1; then
|
||||
echo '{"decision": "allow", "reason": "Input format is correct"}'
|
||||
exit 0
|
||||
else
|
||||
echo '{"decision": "block", "reason": "Input format is invalid"}'
|
||||
exit 0
|
||||
fi`;
|
||||
|
||||
const scriptPath = join(rig.testDir!, 'input_validation_hook.sh');
|
||||
writeFileSync(scriptPath, hookScript);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${scriptPath}"`);
|
||||
|
||||
await rig.setup('should provide correct input format to hooks', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeTool: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: scriptPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt = 'Create a file called input-test.txt with content "test"';
|
||||
await rig.run(prompt);
|
||||
|
||||
// Hook should validate input format successfully
|
||||
const foundWriteFile = await rig.waitForToolCall('write_file');
|
||||
expect(foundWriteFile).toBeTruthy();
|
||||
|
||||
// Check that the file was created (hook allowed it)
|
||||
const fileContent = rig.readFile('input-test.txt');
|
||||
expect(fileContent).toContain('test');
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Multiple Event Types', () => {
|
||||
// Note: This test checks telemetry for hook context in API requests,
|
||||
// which behaves differently with mocked responses. Keeping real LLM calls.
|
||||
it('should handle hooks for all major event types', async () => {
|
||||
await rig.setup('should handle hooks for all major event types');
|
||||
// Create hook scripts for different events
|
||||
const beforeToolScript = `#!/bin/bash
|
||||
echo '{"decision": "allow", "systemMessage": "BeforeTool: File operation logged"}'`;
|
||||
|
||||
const afterToolScript = `#!/bin/bash
|
||||
echo '{"hookSpecificOutput": {"hookEventName": "AfterTool", "additionalContext": "AfterTool: Operation completed successfully"}}'`;
|
||||
|
||||
const beforeAgentScript = `#!/bin/bash
|
||||
echo '{"decision": "allow", "hookSpecificOutput": {"hookEventName": "BeforeAgent", "additionalContext": "BeforeAgent: User request processed"}}'`;
|
||||
|
||||
const beforeToolPath = join(rig.testDir!, 'before_tool.sh');
|
||||
const afterToolPath = join(rig.testDir!, 'after_tool.sh');
|
||||
const beforeAgentPath = join(rig.testDir!, 'before_agent.sh');
|
||||
|
||||
writeFileSync(beforeToolPath, beforeToolScript);
|
||||
writeFileSync(afterToolPath, afterToolScript);
|
||||
writeFileSync(beforeAgentPath, beforeAgentScript);
|
||||
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${beforeToolPath}"`);
|
||||
execSync(`chmod +x "${afterToolPath}"`);
|
||||
execSync(`chmod +x "${beforeAgentPath}"`);
|
||||
|
||||
await rig.setup('should handle hooks for all major event types', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeAgent: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: beforeAgentPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
BeforeTool: [
|
||||
{
|
||||
matcher: 'write_file',
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: beforeToolPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
AfterTool: [
|
||||
{
|
||||
matcher: 'write_file',
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: afterToolPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt =
|
||||
'Create a file called multi-event-test.txt with content ' +
|
||||
'"testing multiple events", and then please reply with ' +
|
||||
'everything I say just after this:"';
|
||||
const result = await rig.run(prompt);
|
||||
|
||||
// Should execute write_file tool
|
||||
const foundWriteFile = await rig.waitForToolCall('write_file');
|
||||
expect(foundWriteFile).toBeTruthy();
|
||||
|
||||
// File should be created
|
||||
const fileContent = rig.readFile('multi-event-test.txt');
|
||||
expect(fileContent).toContain('testing multiple events');
|
||||
|
||||
// Result should contain context from all hooks
|
||||
expect(result).toContain('BeforeTool: File operation logged');
|
||||
|
||||
// Should generate hook telemetry
|
||||
let hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
hookTelemetryFound = await rig.waitForTelemetryEvent('api_request');
|
||||
const apiRequests = rig.readAllApiRequest();
|
||||
const apiRequestsTexts = apiRequests
|
||||
?.filter(
|
||||
(request) =>
|
||||
'attributes' in request &&
|
||||
typeof request['attributes'] === 'object' &&
|
||||
request['attributes'] !== null &&
|
||||
'request_text' in request['attributes'] &&
|
||||
typeof request['attributes']['request_text'] === 'string',
|
||||
)
|
||||
.map((request) => request['attributes']['request_text']);
|
||||
expect(apiRequestsTexts).toBeDefined();
|
||||
let hasBeforeAgentHookContext = false;
|
||||
let hasAfterToolHookContext = false;
|
||||
for (const requestText of apiRequestsTexts) {
|
||||
if (requestText.includes('BeforeAgent: User request processed')) {
|
||||
hasBeforeAgentHookContext = true;
|
||||
}
|
||||
if (
|
||||
requestText.includes('AfterTool: Operation completed successfully')
|
||||
) {
|
||||
hasAfterToolHookContext = true;
|
||||
}
|
||||
}
|
||||
expect(hasBeforeAgentHookContext).toBeTruthy();
|
||||
expect(hasAfterToolHookContext).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Hook Error Handling', () => {
|
||||
it('should handle hook failures gracefully', async () => {
|
||||
await rig.setup('should handle hook failures gracefully', {
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.error-handling.responses',
|
||||
),
|
||||
});
|
||||
// Create a hook script that fails
|
||||
const failingHookScript = `#!/bin/bash
|
||||
echo "Hook encountered an error" >&2
|
||||
exit 1`;
|
||||
|
||||
const workingHookScript = `#!/bin/bash
|
||||
echo '{"decision": "allow", "reason": "Working hook succeeded"}'`;
|
||||
|
||||
const failingPath = join(rig.testDir!, 'failing_hook.sh');
|
||||
const workingPath = join(rig.testDir!, 'working_hook.sh');
|
||||
|
||||
writeFileSync(failingPath, failingHookScript);
|
||||
writeFileSync(workingPath, workingHookScript);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${failingPath}"`);
|
||||
execSync(`chmod +x "${workingPath}"`);
|
||||
|
||||
await rig.setup('should handle hook failures gracefully', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeTool: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: failingPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
{
|
||||
type: 'command',
|
||||
command: workingPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt =
|
||||
'Create a file called error-test.txt with content "testing error handling"';
|
||||
await rig.run(prompt);
|
||||
|
||||
// Despite one hook failing, the working hook should still allow the operation
|
||||
const foundWriteFile = await rig.waitForToolCall('write_file');
|
||||
expect(foundWriteFile).toBeTruthy();
|
||||
|
||||
// File should be created
|
||||
const fileContent = rig.readFile('error-test.txt');
|
||||
expect(fileContent).toContain('testing error handling');
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Hook Telemetry and Observability', () => {
|
||||
it('should generate telemetry events for hook executions', async () => {
|
||||
await rig.setup('should generate telemetry events for hook executions', {
|
||||
fakeResponsesPath: join(
|
||||
import.meta.dirname,
|
||||
'hooks-system.telemetry.responses',
|
||||
),
|
||||
});
|
||||
const hookScript = `#!/bin/bash
|
||||
echo '{"decision": "allow", "reason": "Telemetry test hook"}'`;
|
||||
|
||||
const scriptPath = join(rig.testDir!, 'telemetry_hook.sh');
|
||||
writeFileSync(scriptPath, hookScript);
|
||||
const { execSync } = await import('node:child_process');
|
||||
execSync(`chmod +x "${scriptPath}"`);
|
||||
|
||||
await rig.setup('should generate telemetry events for hook executions', {
|
||||
settings: {
|
||||
tools: {
|
||||
enableHooks: true,
|
||||
},
|
||||
hooks: {
|
||||
BeforeTool: [
|
||||
{
|
||||
hooks: [
|
||||
{
|
||||
type: 'command',
|
||||
command: scriptPath,
|
||||
timeout: 5000,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const prompt = 'Create a file called telemetry-test.txt';
|
||||
await rig.run(prompt);
|
||||
|
||||
// Should execute the tool
|
||||
const foundWriteFile = await rig.waitForToolCall('write_file');
|
||||
expect(foundWriteFile).toBeTruthy();
|
||||
|
||||
// Should generate hook telemetry
|
||||
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
||||
expect(hookTelemetryFound).toBeTruthy();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user