mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-03-10 14:10:37 -07:00
924 lines
30 KiB
TypeScript
924 lines
30 KiB
TypeScript
/**
|
|
* @license
|
|
* Copyright 2025 Google LLC
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
import { TestRig } from './test-helper.js';
|
|
import { join } from 'node:path';
|
|
import { writeFileSync } from 'node:fs';
|
|
|
|
describe('Hooks System Integration', () => {
|
|
let rig: TestRig;
|
|
|
|
beforeEach(() => {
|
|
rig = new TestRig();
|
|
});
|
|
|
|
afterEach(async () => {
|
|
if (rig) {
|
|
await rig.cleanup();
|
|
}
|
|
});
|
|
|
|
describe('Command Hooks - Blocking Behavior', () => {
|
|
it('should block tool execution when hook returns block decision', async () => {
|
|
await rig.setup(
|
|
'should block tool execution when hook returns block decision',
|
|
{
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.block-tool.responses',
|
|
),
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeTool: [
|
|
{
|
|
matcher: 'write_file',
|
|
sequential: true,
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command:
|
|
'echo "{\\"decision\\": \\"block\\", \\"reason\\": \\"File writing blocked by security policy\\"}"',
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
);
|
|
|
|
const prompt = 'Create a file called test.txt with content "Hello World"';
|
|
const result = await rig.run(prompt);
|
|
|
|
// The hook should block the write_file tool
|
|
const toolLogs = rig.readToolLogs();
|
|
const writeFileCalls = toolLogs.filter(
|
|
(t) =>
|
|
t.toolRequest.name === 'write_file' && t.toolRequest.success === true,
|
|
);
|
|
|
|
// Tool should not be called due to blocking hook
|
|
expect(writeFileCalls).toHaveLength(0);
|
|
|
|
// Result should mention the blocking reason
|
|
expect(result).toContain('File writing blocked by security policy');
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
|
|
it('should allow tool execution when hook returns allow decision', async () => {
|
|
await rig.setup(
|
|
'should allow tool execution when hook returns allow decision',
|
|
{
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.allow-tool.responses',
|
|
),
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeTool: [
|
|
{
|
|
matcher: 'write_file',
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command:
|
|
'echo "{\\"decision\\": \\"allow\\", \\"reason\\": \\"File writing approved\\"}"',
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
);
|
|
|
|
const prompt =
|
|
'Create a file called approved.txt with content "Approved content"';
|
|
await rig.run(prompt);
|
|
|
|
// The hook should allow the write_file tool
|
|
const foundWriteFile = await rig.waitForToolCall('write_file');
|
|
expect(foundWriteFile).toBeTruthy();
|
|
|
|
// File should be created
|
|
const fileContent = rig.readFile('approved.txt');
|
|
expect(fileContent).toContain('Approved content');
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('Command Hooks - Additional Context', () => {
|
|
it('should add additional context from AfterTool hooks', async () => {
|
|
const command =
|
|
'echo "{\\"hookSpecificOutput\\": {\\"hookEventName\\": \\"AfterTool\\", \\"additionalContext\\": \\"Security scan: File content appears safe\\"}}"';
|
|
await rig.setup('should add additional context from AfterTool hooks', {
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.after-tool-context.responses',
|
|
),
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
AfterTool: [
|
|
{
|
|
matcher: 'read_file',
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: command,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
// Create a test file to read
|
|
rig.createFile('test-file.txt', 'This is test content');
|
|
|
|
const prompt =
|
|
'Read the contents of test-file.txt and tell me what it contains';
|
|
await rig.run(prompt);
|
|
|
|
// Should find read_file tool call
|
|
const foundReadFile = await rig.waitForToolCall('read_file');
|
|
expect(foundReadFile).toBeTruthy();
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = rig.readHookLogs();
|
|
expect(hookTelemetryFound.length).toBeGreaterThan(0);
|
|
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe('AfterTool');
|
|
expect(hookTelemetryFound[0].hookCall.hook_name).toBe(command);
|
|
expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined();
|
|
expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined();
|
|
expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0);
|
|
expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined();
|
|
expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined();
|
|
});
|
|
});
|
|
|
|
describe('BeforeModel Hooks - LLM Request Modification', () => {
|
|
it('should modify LLM requests with BeforeModel hooks', async () => {
|
|
// Create a hook script that replaces the LLM request with a modified version
|
|
// Note: Providing messages in the hook output REPLACES the entire conversation
|
|
await rig.setup('should modify LLM requests with BeforeModel hooks', {
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.before-model.responses',
|
|
),
|
|
});
|
|
const hookScript = `#!/bin/bash
|
|
echo '{
|
|
"decision": "allow",
|
|
"hookSpecificOutput": {
|
|
"hookEventName": "BeforeModel",
|
|
"llm_request": {
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "Please respond with exactly: The security hook modified this request successfully."
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}'`;
|
|
|
|
const scriptPath = join(rig.testDir!, 'before_model_hook.sh');
|
|
writeFileSync(scriptPath, hookScript);
|
|
// Make executable
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${scriptPath}"`);
|
|
|
|
await rig.setup('should modify LLM requests with BeforeModel hooks', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeModel: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: scriptPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt = 'Tell me a story';
|
|
const result = await rig.run(prompt);
|
|
|
|
// The hook should have replaced the request entirely
|
|
// Verify that the model responded to the modified request, not the original
|
|
expect(result).toBeDefined();
|
|
expect(result.length).toBeGreaterThan(0);
|
|
// The response should contain the expected text from the modified request
|
|
expect(result.toLowerCase()).toContain('security hook modified');
|
|
|
|
// Should generate hook telemetry
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = rig.readHookLogs();
|
|
expect(hookTelemetryFound.length).toBeGreaterThan(0);
|
|
expect(hookTelemetryFound[0].hookCall.hook_event_name).toBe(
|
|
'BeforeModel',
|
|
);
|
|
expect(hookTelemetryFound[0].hookCall.hook_name).toBe(scriptPath);
|
|
expect(hookTelemetryFound[0].hookCall.hook_input).toBeDefined();
|
|
expect(hookTelemetryFound[0].hookCall.hook_output).toBeDefined();
|
|
expect(hookTelemetryFound[0].hookCall.exit_code).toBe(0);
|
|
expect(hookTelemetryFound[0].hookCall.stdout).toBeDefined();
|
|
expect(hookTelemetryFound[0].hookCall.stderr).toBeDefined();
|
|
});
|
|
});
|
|
|
|
describe('AfterModel Hooks - LLM Response Modification', () => {
|
|
it('should modify LLM responses with AfterModel hooks', async () => {
|
|
await rig.setup('should modify LLM responses with AfterModel hooks', {
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.after-model.responses',
|
|
),
|
|
});
|
|
// Create a hook script that modifies the LLM response
|
|
const hookScript = `#!/bin/bash
|
|
echo '{
|
|
"hookSpecificOutput": {
|
|
"hookEventName": "AfterModel",
|
|
"llm_response": {
|
|
"candidates": [
|
|
{
|
|
"content": {
|
|
"role": "model",
|
|
"parts": [
|
|
"[FILTERED] Response has been filtered for security compliance."
|
|
]
|
|
},
|
|
"finishReason": "STOP"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}'`;
|
|
|
|
const scriptPath = join(rig.testDir!, 'after_model_hook.sh');
|
|
writeFileSync(scriptPath, hookScript);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${scriptPath}"`);
|
|
|
|
await rig.setup('should modify LLM responses with AfterModel hooks', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
AfterModel: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: scriptPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt = 'What is 2 + 2?';
|
|
const result = await rig.run(prompt);
|
|
|
|
// The hook should have replaced the model response
|
|
expect(result).toContain(
|
|
'[FILTERED] Response has been filtered for security compliance',
|
|
);
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('BeforeToolSelection Hooks - Tool Configuration', () => {
|
|
it('should modify tool selection with BeforeToolSelection hooks', async () => {
|
|
await rig.setup(
|
|
'should modify tool selection with BeforeToolSelection hooks',
|
|
{
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.before-tool-selection.responses',
|
|
),
|
|
},
|
|
);
|
|
// Create a hook script that restricts available tools
|
|
const hookScript = `#!/bin/bash
|
|
echo '{
|
|
"hookSpecificOutput": {
|
|
"hookEventName": "BeforeToolSelection",
|
|
"toolConfig": {
|
|
"mode": "ANY",
|
|
"allowedFunctionNames": ["read_file", "run_shell_command"]
|
|
}
|
|
}
|
|
}'`;
|
|
|
|
const scriptPath = join(rig.testDir!, 'before_tool_selection_hook.sh');
|
|
writeFileSync(scriptPath, hookScript);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${scriptPath}"`);
|
|
|
|
await rig.setup(
|
|
'should modify tool selection with BeforeToolSelection hooks',
|
|
{
|
|
settings: {
|
|
debugMode: true,
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeToolSelection: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: scriptPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
);
|
|
|
|
// Create a test file
|
|
rig.createFile('new_file_data.txt', 'test data');
|
|
|
|
const prompt =
|
|
'Check the content of new_file_data.txt, after that run echo command to see the content';
|
|
await rig.run(prompt);
|
|
|
|
// Should use read_file (allowed) but not run_shell_command (not in allowed list)
|
|
const foundReadFile = await rig.waitForToolCall('read_file');
|
|
expect(foundReadFile).toBeTruthy();
|
|
|
|
// Should generate hook telemetry indicating the hook was called
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
|
|
// Verify the hook was called for BeforeToolSelection event
|
|
const hookLogs = rig.readHookLogs();
|
|
const beforeToolSelectionHook = hookLogs.find(
|
|
(log) => log.hookCall.hook_event_name === 'BeforeToolSelection',
|
|
);
|
|
expect(beforeToolSelectionHook).toBeDefined();
|
|
expect(beforeToolSelectionHook?.hookCall.success).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('BeforeAgent Hooks - Prompt Augmentation', () => {
|
|
it('should augment prompts with BeforeAgent hooks', async () => {
|
|
await rig.setup('should augment prompts with BeforeAgent hooks', {
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.before-agent.responses',
|
|
),
|
|
});
|
|
// Create a hook script that adds context to the prompt
|
|
const hookScript = `#!/bin/bash
|
|
echo '{
|
|
"decision": "allow",
|
|
"hookSpecificOutput": {
|
|
"hookEventName": "BeforeAgent",
|
|
"additionalContext": "SYSTEM INSTRUCTION: You are in a secure environment. Always mention security compliance in your responses."
|
|
}
|
|
}'`;
|
|
|
|
const scriptPath = join(rig.testDir!, 'before_agent_hook.sh');
|
|
writeFileSync(scriptPath, hookScript);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${scriptPath}"`);
|
|
|
|
await rig.setup('should augment prompts with BeforeAgent hooks', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeAgent: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: scriptPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt = 'Hello, how are you?';
|
|
const result = await rig.run(prompt);
|
|
|
|
// The hook should have added security context, which should influence the response
|
|
expect(result).toContain('security');
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe.skip('Notification Hooks - Permission Handling', () => {
|
|
it('should handle notification hooks for tool permissions', async () => {
|
|
await rig.setup('should handle notification hooks for tool permissions');
|
|
// Create a hook script that logs notification events
|
|
const hookScript = `#!/bin/bash
|
|
echo '{
|
|
"suppressOutput": false,
|
|
"systemMessage": "Permission request logged by security hook"
|
|
}'`;
|
|
|
|
const scriptPath = join(rig.testDir!, 'notification_hook.sh');
|
|
writeFileSync(scriptPath, hookScript);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${scriptPath}"`);
|
|
|
|
await rig.setup('should handle notification hooks for tool permissions', {
|
|
settings: {
|
|
// Configure tools to enable hooks and require confirmation to trigger notifications
|
|
tools: {
|
|
enableHooks: true,
|
|
confirmationRequired: ['run_shell_command'],
|
|
},
|
|
hooks: {
|
|
Notification: [
|
|
{
|
|
matcher: 'ToolPermission',
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: scriptPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt =
|
|
'Run the command "echo test" (this should trigger a permission prompt)';
|
|
|
|
// Use stdin to automatically approve the permission
|
|
await rig.run({
|
|
prompt,
|
|
stdin: 'y\n', // Approve the permission
|
|
});
|
|
|
|
// Should find the shell command execution
|
|
const foundShellCommand = await rig.waitForToolCall('run_shell_command');
|
|
expect(foundShellCommand).toBeTruthy();
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('Sequential Hook Execution', () => {
|
|
// Note: This test checks telemetry for hook context in API requests,
|
|
// which behaves differently with mocked responses. Keeping real LLM calls.
|
|
it('should execute hooks sequentially when configured', async () => {
|
|
await rig.setup('should execute hooks sequentially when configured');
|
|
// Create two hooks that modify the input sequentially
|
|
const hook1Script = `#!/bin/bash
|
|
echo '{
|
|
"decision": "allow",
|
|
"hookSpecificOutput": {
|
|
"hookEventName": "BeforeAgent",
|
|
"additionalContext": "Step 1: Initial validation passed."
|
|
}
|
|
}'`;
|
|
|
|
const hook2Script = `#!/bin/bash
|
|
echo '{
|
|
"decision": "allow",
|
|
"hookSpecificOutput": {
|
|
"hookEventName": "BeforeAgent",
|
|
"additionalContext": "Step 2: Security check completed."
|
|
}
|
|
}'`;
|
|
|
|
const script1Path = join(rig.testDir!, 'sequential_hook1.sh');
|
|
const script2Path = join(rig.testDir!, 'sequential_hook2.sh');
|
|
|
|
writeFileSync(script1Path, hook1Script);
|
|
writeFileSync(script2Path, hook2Script);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${script1Path}"`);
|
|
execSync(`chmod +x "${script2Path}"`);
|
|
|
|
await rig.setup('should execute hooks sequentially when configured', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeAgent: [
|
|
{
|
|
sequential: true,
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: script1Path,
|
|
timeout: 5000,
|
|
},
|
|
{
|
|
type: 'command',
|
|
command: script2Path,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt = 'Hello, please help me with a task';
|
|
await rig.run(prompt);
|
|
|
|
// Should generate hook telemetry
|
|
let hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
hookTelemetryFound = await rig.waitForTelemetryEvent('api_request');
|
|
const apiRequests = rig.readAllApiRequest();
|
|
const apiRequestsTexts = apiRequests
|
|
?.filter(
|
|
(request) =>
|
|
'attributes' in request &&
|
|
typeof request['attributes'] === 'object' &&
|
|
request['attributes'] !== null &&
|
|
'request_text' in request['attributes'] &&
|
|
typeof request['attributes']['request_text'] === 'string',
|
|
)
|
|
.map((request) => request['attributes']['request_text']);
|
|
expect(apiRequestsTexts).toBeDefined();
|
|
let hasBeforeAgentHookContext = false;
|
|
let hasAfterToolHookContext = false;
|
|
for (const requestText of apiRequestsTexts) {
|
|
if (requestText.includes('Step 1: Initial validation passed')) {
|
|
hasBeforeAgentHookContext = true;
|
|
}
|
|
if (requestText.includes('Step 2: Security check completed')) {
|
|
hasAfterToolHookContext = true;
|
|
}
|
|
}
|
|
expect(hasBeforeAgentHookContext).toBeTruthy();
|
|
expect(hasAfterToolHookContext).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('Hook Input/Output Validation', () => {
|
|
it('should provide correct input format to hooks', async () => {
|
|
await rig.setup('should provide correct input format to hooks', {
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.input-validation.responses',
|
|
),
|
|
});
|
|
// Create a hook script that validates the input format
|
|
const hookScript = `#!/bin/bash
|
|
# Read JSON input from stdin
|
|
input=$(cat)
|
|
|
|
# Check for required fields
|
|
if echo "$input" | jq -e '.session_id and .cwd and .hook_event_name and .timestamp and .tool_name and .tool_input' > /dev/null 2>&1; then
|
|
echo '{"decision": "allow", "reason": "Input format is correct"}'
|
|
exit 0
|
|
else
|
|
echo '{"decision": "block", "reason": "Input format is invalid"}'
|
|
exit 0
|
|
fi`;
|
|
|
|
const scriptPath = join(rig.testDir!, 'input_validation_hook.sh');
|
|
writeFileSync(scriptPath, hookScript);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${scriptPath}"`);
|
|
|
|
await rig.setup('should provide correct input format to hooks', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeTool: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: scriptPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt = 'Create a file called input-test.txt with content "test"';
|
|
await rig.run(prompt);
|
|
|
|
// Hook should validate input format successfully
|
|
const foundWriteFile = await rig.waitForToolCall('write_file');
|
|
expect(foundWriteFile).toBeTruthy();
|
|
|
|
// Check that the file was created (hook allowed it)
|
|
const fileContent = rig.readFile('input-test.txt');
|
|
expect(fileContent).toContain('test');
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('Multiple Event Types', () => {
|
|
// Note: This test checks telemetry for hook context in API requests,
|
|
// which behaves differently with mocked responses. Keeping real LLM calls.
|
|
it('should handle hooks for all major event types', async () => {
|
|
await rig.setup('should handle hooks for all major event types');
|
|
// Create hook scripts for different events
|
|
const beforeToolScript = `#!/bin/bash
|
|
echo '{"decision": "allow", "systemMessage": "BeforeTool: File operation logged"}'`;
|
|
|
|
const afterToolScript = `#!/bin/bash
|
|
echo '{"hookSpecificOutput": {"hookEventName": "AfterTool", "additionalContext": "AfterTool: Operation completed successfully"}}'`;
|
|
|
|
const beforeAgentScript = `#!/bin/bash
|
|
echo '{"decision": "allow", "hookSpecificOutput": {"hookEventName": "BeforeAgent", "additionalContext": "BeforeAgent: User request processed"}}'`;
|
|
|
|
const beforeToolPath = join(rig.testDir!, 'before_tool.sh');
|
|
const afterToolPath = join(rig.testDir!, 'after_tool.sh');
|
|
const beforeAgentPath = join(rig.testDir!, 'before_agent.sh');
|
|
|
|
writeFileSync(beforeToolPath, beforeToolScript);
|
|
writeFileSync(afterToolPath, afterToolScript);
|
|
writeFileSync(beforeAgentPath, beforeAgentScript);
|
|
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${beforeToolPath}"`);
|
|
execSync(`chmod +x "${afterToolPath}"`);
|
|
execSync(`chmod +x "${beforeAgentPath}"`);
|
|
|
|
await rig.setup('should handle hooks for all major event types', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeAgent: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: beforeAgentPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
BeforeTool: [
|
|
{
|
|
matcher: 'write_file',
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: beforeToolPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
AfterTool: [
|
|
{
|
|
matcher: 'write_file',
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: afterToolPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt =
|
|
'Create a file called multi-event-test.txt with content ' +
|
|
'"testing multiple events", and then please reply with ' +
|
|
'everything I say just after this:"';
|
|
const result = await rig.run(prompt);
|
|
|
|
// Should execute write_file tool
|
|
const foundWriteFile = await rig.waitForToolCall('write_file');
|
|
expect(foundWriteFile).toBeTruthy();
|
|
|
|
// File should be created
|
|
const fileContent = rig.readFile('multi-event-test.txt');
|
|
expect(fileContent).toContain('testing multiple events');
|
|
|
|
// Result should contain context from all hooks
|
|
expect(result).toContain('BeforeTool: File operation logged');
|
|
|
|
// Should generate hook telemetry
|
|
let hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
hookTelemetryFound = await rig.waitForTelemetryEvent('api_request');
|
|
const apiRequests = rig.readAllApiRequest();
|
|
const apiRequestsTexts = apiRequests
|
|
?.filter(
|
|
(request) =>
|
|
'attributes' in request &&
|
|
typeof request['attributes'] === 'object' &&
|
|
request['attributes'] !== null &&
|
|
'request_text' in request['attributes'] &&
|
|
typeof request['attributes']['request_text'] === 'string',
|
|
)
|
|
.map((request) => request['attributes']['request_text']);
|
|
expect(apiRequestsTexts).toBeDefined();
|
|
let hasBeforeAgentHookContext = false;
|
|
let hasAfterToolHookContext = false;
|
|
for (const requestText of apiRequestsTexts) {
|
|
if (requestText.includes('BeforeAgent: User request processed')) {
|
|
hasBeforeAgentHookContext = true;
|
|
}
|
|
if (
|
|
requestText.includes('AfterTool: Operation completed successfully')
|
|
) {
|
|
hasAfterToolHookContext = true;
|
|
}
|
|
}
|
|
expect(hasBeforeAgentHookContext).toBeTruthy();
|
|
expect(hasAfterToolHookContext).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('Hook Error Handling', () => {
|
|
it('should handle hook failures gracefully', async () => {
|
|
await rig.setup('should handle hook failures gracefully', {
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.error-handling.responses',
|
|
),
|
|
});
|
|
// Create a hook script that fails
|
|
const failingHookScript = `#!/bin/bash
|
|
echo "Hook encountered an error" >&2
|
|
exit 1`;
|
|
|
|
const workingHookScript = `#!/bin/bash
|
|
echo '{"decision": "allow", "reason": "Working hook succeeded"}'`;
|
|
|
|
const failingPath = join(rig.testDir!, 'failing_hook.sh');
|
|
const workingPath = join(rig.testDir!, 'working_hook.sh');
|
|
|
|
writeFileSync(failingPath, failingHookScript);
|
|
writeFileSync(workingPath, workingHookScript);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${failingPath}"`);
|
|
execSync(`chmod +x "${workingPath}"`);
|
|
|
|
await rig.setup('should handle hook failures gracefully', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeTool: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: failingPath,
|
|
timeout: 5000,
|
|
},
|
|
{
|
|
type: 'command',
|
|
command: workingPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt =
|
|
'Create a file called error-test.txt with content "testing error handling"';
|
|
await rig.run(prompt);
|
|
|
|
// Despite one hook failing, the working hook should still allow the operation
|
|
const foundWriteFile = await rig.waitForToolCall('write_file');
|
|
expect(foundWriteFile).toBeTruthy();
|
|
|
|
// File should be created
|
|
const fileContent = rig.readFile('error-test.txt');
|
|
expect(fileContent).toContain('testing error handling');
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('Hook Telemetry and Observability', () => {
|
|
it('should generate telemetry events for hook executions', async () => {
|
|
await rig.setup('should generate telemetry events for hook executions', {
|
|
fakeResponsesPath: join(
|
|
import.meta.dirname,
|
|
'hooks-system.telemetry.responses',
|
|
),
|
|
});
|
|
const hookScript = `#!/bin/bash
|
|
echo '{"decision": "allow", "reason": "Telemetry test hook"}'`;
|
|
|
|
const scriptPath = join(rig.testDir!, 'telemetry_hook.sh');
|
|
writeFileSync(scriptPath, hookScript);
|
|
const { execSync } = await import('node:child_process');
|
|
execSync(`chmod +x "${scriptPath}"`);
|
|
|
|
await rig.setup('should generate telemetry events for hook executions', {
|
|
settings: {
|
|
tools: {
|
|
enableHooks: true,
|
|
},
|
|
hooks: {
|
|
BeforeTool: [
|
|
{
|
|
hooks: [
|
|
{
|
|
type: 'command',
|
|
command: scriptPath,
|
|
timeout: 5000,
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const prompt = 'Create a file called telemetry-test.txt';
|
|
await rig.run(prompt);
|
|
|
|
// Should execute the tool
|
|
const foundWriteFile = await rig.waitForToolCall('write_file');
|
|
expect(foundWriteFile).toBeTruthy();
|
|
|
|
// Should generate hook telemetry
|
|
const hookTelemetryFound = await rig.waitForTelemetryEvent('hook_call');
|
|
expect(hookTelemetryFound).toBeTruthy();
|
|
});
|
|
});
|
|
});
|