From 05cdf19123990ece1c97e9f7f8d8f223cf5df43a Mon Sep 17 00:00:00 2001 From: Christian Gunderman Date: Mon, 13 Apr 2026 09:33:05 -0700 Subject: [PATCH] Validation agent hook. --- packages/cli/src/config/config.ts | 1 + packages/cli/src/config/settingsSchema.ts | 10 ++ .../core/src/agents/deep-validation-agent.ts | 82 ++++++++++++ packages/core/src/agents/local-invocation.ts | 2 +- packages/core/src/agents/registry.test.ts | 1 + packages/core/src/agents/registry.ts | 6 +- packages/core/src/agents/types.ts | 1 + packages/core/src/config/config.ts | 62 ++++++++- .../core/src/config/deep-validation.test.ts | 121 ++++++++++++++++++ packages/core/src/core/client.ts | 4 +- packages/core/src/hooks/hookRegistry.test.ts | 2 +- packages/core/src/hooks/hookRegistry.ts | 5 +- schemas/settings.schema.json | 7 + 13 files changed, 293 insertions(+), 11 deletions(-) create mode 100644 packages/core/src/agents/deep-validation-agent.ts create mode 100644 packages/core/src/config/deep-validation.test.ts diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 25419a2d6c..f929d73daf 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -900,6 +900,7 @@ export async function loadCliConfig( memoryBoundaryMarkers: settings.context?.memoryBoundaryMarkers, importFormat: settings.context?.importFormat, debugMode, + deepValidation: settings.general?.deepValidation, question, worktreeSettings, diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index c40e87db18..60e5fafa8b 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -236,6 +236,16 @@ const SETTINGS_SCHEMA = { description: 'Enable DevTools inspector on launch.', showInDialog: false, }, + deepValidation: { + type: 'boolean', + label: 'Deep Validation', + category: 'General', + requiresRestart: false, + default: false, + description: + 'Run a subagent after the main agent finishes to perform final validation.', + showInDialog: true, + }, enableAutoUpdate: { type: 'boolean', label: 'Enable Auto Update', diff --git a/packages/core/src/agents/deep-validation-agent.ts b/packages/core/src/agents/deep-validation-agent.ts new file mode 100644 index 0000000000..6ef4cfd7ea --- /dev/null +++ b/packages/core/src/agents/deep-validation-agent.ts @@ -0,0 +1,82 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { z } from 'zod'; +import type { AgentLoopContext } from '../config/agent-loop-context.js'; +import type { LocalAgentDefinition } from './types.js'; + +const DeepValidationAgentSchema = z.object({ + report: z.string().describe('The final validation report.'), + isSatisfied: z.boolean().describe('Whether the original prompt was fully satisfied.'), +}); + +/** + * A specialized subagent that performs final validation after the main agent finishes. + * It reflects on the original prompt and determines if it was satisfied by reviewing + * the changes and running the application and tests. + */ +export const DeepValidationAgent = ( + context: AgentLoopContext, + originalPrompt: string, +): LocalAgentDefinition => ({ + kind: 'local', + name: 'deep-validation', + displayName: 'Deep Validation Agent', + internal: true, + description: + 'A specialized subagent that performs final validation. It reflects on the original prompt and determines if it was satisfied by reviewing the changes and running the application and tests, if applicable.', + inputConfig: { + inputSchema: { + type: 'object', + properties: { + originalPrompt: { + type: 'string', + description: 'The original user request or prompt that was executed.', + }, + }, + required: ['originalPrompt'], + }, + }, + outputConfig: { + outputName: 'validationResult', + description: 'The final validation report and satisfaction status.', + schema: DeepValidationAgentSchema, + }, + modelConfig: { + model: 'inherit', + }, + get toolConfig() { + const tools = context.toolRegistry?.getAllToolNames() ?? []; + return { + tools, + }; + }, + get promptConfig() { + return { + systemPrompt: `You are the Deep Validation subagent. Your job is to perform final validation after the main agent finishes. +The user's original request was: ${originalPrompt} +You must deterministically determine if the request was satisfied. +Review the changes using git, check the code, and run the application, any existing linters, formatters, and tests if applicable. +Make a comprehensive and prioritized list of any validation failures and then fix them in priority order. Take care not to leave the code worse than you found it. + +Use the following order to guide your prioritization of fixes: +- Unfulfilled aspects of the original request. +- Issues blocking correct operation of the solution, such as runtime failures, test failures, etc. +- Conventions +- Linters +- Best practices for the ecosystem. + +Try to keep your changes targeted and as minimal as possible while still resolving the validation issue. + +Return a final validation report detailing your findings and a boolean indicating if the request was fully satisfied.`, + query: `Please validate the original request: ${originalPrompt}`, + }; + }, + runConfig: { + maxTimeMinutes: 10, + maxTurns: 10, + }, +}); diff --git a/packages/core/src/agents/local-invocation.ts b/packages/core/src/agents/local-invocation.ts index 0d28dcbe64..21ea067b0d 100644 --- a/packages/core/src/agents/local-invocation.ts +++ b/packages/core/src/agents/local-invocation.ts @@ -56,7 +56,7 @@ export class LocalSubagentInvocation extends BaseToolInvocation< * @param messageBus Message bus for policy enforcement. */ constructor( - private readonly definition: LocalAgentDefinition, + private readonly definition: LocalAgentDefinition, private readonly context: AgentLoopContext, params: AgentInputs, messageBus: MessageBus, diff --git a/packages/core/src/agents/registry.test.ts b/packages/core/src/agents/registry.test.ts index 97d2c9ea09..db924a351d 100644 --- a/packages/core/src/agents/registry.test.ts +++ b/packages/core/src/agents/registry.test.ts @@ -272,6 +272,7 @@ describe('AgentRegistry', () => { codebase_investigator: { enabled: false }, cli_help: { enabled: false }, generalist: { enabled: false }, + 'deep-validation': { enabled: false }, }, }, }); diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts index 625302a6c7..a50de601b0 100644 --- a/packages/core/src/agents/registry.ts +++ b/packages/core/src/agents/registry.ts @@ -14,6 +14,7 @@ import { loadAgentsFromDirectory } from './agentLoader.js'; import { CodebaseInvestigatorAgent } from './codebase-investigator.js'; import { CliHelpAgent } from './cli-help-agent.js'; import { GeneralistAgent } from './generalist-agent.js'; +import { DeepValidationAgent } from './deep-validation-agent.js'; import { BrowserAgentDefinition } from './browser/browserAgentDefinition.js'; import { MemoryManagerAgent } from './memory-manager-agent.js'; import { A2AAuthProviderFactory } from './auth-provider/factory.js'; @@ -243,7 +244,7 @@ export class AgentRegistry { if (this.config.getDebugMode()) { debugLogger.log( - `[AgentRegistry] Loaded with ${this.agents.size} agents.`, + `[AgentRegistry] Loaded with ${this.getAllDefinitions().length} agents.`, ); } } @@ -252,6 +253,7 @@ export class AgentRegistry { this.registerLocalAgent(CodebaseInvestigatorAgent(this.config)); this.registerLocalAgent(CliHelpAgent(this.config)); this.registerLocalAgent(GeneralistAgent(this.config)); + this.registerLocalAgent(DeepValidationAgent(this.config, '${originalPrompt}')); // Register the browser agent if enabled in settings. // Tools are configured dynamically at invocation time via browserAgentFactory. @@ -669,7 +671,7 @@ export class AgentRegistry { * Returns all active agent definitions. */ getAllDefinitions(): AgentDefinition[] { - return Array.from(this.agents.values()); + return Array.from(this.agents.values()).filter((d) => !d.internal); } /** diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index 456f4cfdb3..a0ef23df55 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -193,6 +193,7 @@ export interface BaseAgentDefinition< displayName?: string; description: string; experimental?: boolean; + internal?: boolean; inputConfig: InputConfig; outputConfig?: OutputConfig; metadata?: { diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 075c5439ad..9a095a7108 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -43,7 +43,15 @@ import { EnterPlanModeTool } from '../tools/enter-plan-mode.js'; import { GeminiClient } from '../core/client.js'; import { BaseLlmClient } from '../core/baseLlmClient.js'; import { LocalLiteRtLmClient } from '../core/localLiteRtLmClient.js'; -import type { HookDefinition, HookEventName } from '../hooks/types.js'; +import { + type HookDefinition, + HookEventName, + HookType, + type AfterAgentInput, + type HookInput, +} from '../hooks/types.js'; +import { DeepValidationAgent } from '../agents/deep-validation-agent.js'; +import { LocalSubagentInvocation } from '../agents/local-invocation.js'; import { FileDiscoveryService } from '../services/fileDiscoveryService.js'; import { GitService } from '../services/gitService.js'; import { @@ -575,6 +583,7 @@ export interface ConfigParameters { toolSandboxing?: boolean; targetDir: string; debugMode: boolean; + deepValidation?: boolean; question?: string; coreTools?: string[]; @@ -740,6 +749,7 @@ export class Config implements McpContext, AgentLoopContext { private readonly targetDir: string; private workspaceContext: WorkspaceContext; private readonly debugMode: boolean; + private readonly deepValidation: boolean; private readonly question: string | undefined; private readonly worktreeSettings: WorktreeSettings | undefined; readonly enableConseca: boolean; @@ -1009,6 +1019,7 @@ export class Config implements McpContext, AgentLoopContext { this.workspaceContext = new WorkspaceContext(this.targetDir, []); this.pendingIncludeDirectories = params.includeDirectories ?? []; this.debugMode = params.debugMode; + this.deepValidation = params.deepValidation ?? false; this.question = params.question; this.worktreeSettings = params.worktreeSettings; @@ -1443,10 +1454,53 @@ export class Config implements McpContext, AgentLoopContext { } } - // Initialize hook system if enabled - if (this.getEnableHooks()) { + // Initialize hook system if enabled or if deepValidation is enabled + if (this.getEnableHooks() || this.deepValidation) { this.hookSystem = new HookSystem(this); await this.hookSystem.initialize(); + + // Register Deep Validation hook if enabled + if (this.deepValidation) { + this.hookSystem.registerHook( + { + type: HookType.Runtime, + name: 'deep-validation-hook', + action: async (input: HookInput) => { + const afterAgentInput = input as AfterAgentInput; + const invocation = new LocalSubagentInvocation( + DeepValidationAgent(this, afterAgentInput.prompt), + this, + { originalPrompt: afterAgentInput.prompt }, + this.messageBus, + ); + + const result = await invocation.execute(new AbortController().signal); + const progress = result.returnDisplay as any; + const subagentResult = progress?.result; + + let validationResult: { report: string; isSatisfied: boolean }; + try { + validationResult = JSON.parse(subagentResult); + } catch { + validationResult = { + report: subagentResult || 'Failed to parse validation report.', + isSatisfied: false, + }; + } + + const satisfactionMessage = validationResult.isSatisfied + ? '✅ Deep validation satisfied.' + : '⚠️ Deep validation NOT satisfied.'; + + return { + continue: true, + systemMessage: `${satisfactionMessage}\n\n${validationResult.report}`, + }; + }, + }, + HookEventName.AfterAgent, + ); + } } if (this.experimentalJitContext) { @@ -3508,7 +3562,7 @@ export class Config implements McpContext, AgentLoopContext { const discoveredNames = this.agentRegistry.getAllDiscoveredAgentNames(); for (const agentName of discoveredNames) { const definition = this.agentRegistry.getDiscoveredDefinition(agentName); - if (!definition) { + if (!definition || definition.internal) { continue; } try { diff --git a/packages/core/src/config/deep-validation.test.ts b/packages/core/src/config/deep-validation.test.ts new file mode 100644 index 0000000000..ae61ab74bc --- /dev/null +++ b/packages/core/src/config/deep-validation.test.ts @@ -0,0 +1,121 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { Config } from './config.js'; +import { HookEventName } from '../hooks/types.js'; +import { LocalSubagentInvocation } from '../agents/local-invocation.js'; + +vi.mock('../agents/local-invocation.js', () => ({ + LocalSubagentInvocation: vi.fn(), +})); + +vi.mock('../agents/deep-validation-agent.js', () => ({ + DeepValidationAgent: vi.fn().mockReturnValue({ + kind: 'local', + name: 'deep-validation', + displayName: 'Deep Validation Agent', + description: 'test description', + inputConfig: { inputSchema: {} }, + outputConfig: { outputName: 'validationResult', schema: { safeParse: vi.fn() } }, + promptConfig: { systemPrompt: 'test prompt', query: 'test query' }, + modelConfig: { model: 'inherit' }, + runConfig: {}, + }), +})); + +describe('Deep Validation Integration', () => { + let config: Config; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should register AfterAgent hook when deepValidation is enabled', async () => { + config = new Config({ + sessionId: 'test-session', + targetDir: '/tmp', + debugMode: false, + cwd: '/tmp', + model: 'test-model', + enableHooks: true, + deepValidation: true, + } as any); + + await config.initialize(); + + const hookSystem = config.getHookSystem(); + expect(hookSystem).toBeDefined(); + + const hooks = hookSystem?.getAllHooks(); + expect(hooks).toBeDefined(); + expect(hooks?.some(h => h.config.name === 'deep-validation-hook' && h.eventName === HookEventName.AfterAgent)).toBe(true); + }); + + it('should NOT register AfterAgent hook when deepValidation is disabled', async () => { + config = new Config({ + sessionId: 'test-session', + targetDir: '/tmp', + debugMode: false, + cwd: '/tmp', + model: 'test-model', + enableHooks: true, + deepValidation: false, + } as any); + + await config.initialize(); + + const hookSystem = config.getHookSystem(); + const hooks = hookSystem?.getAllHooks(); + expect(hooks?.some(h => h.config.name === 'deep-validation-hook')).toBeFalsy(); + }); + + it('should execute DeepValidationAgent when hook is triggered', async () => { + config = new Config({ + sessionId: 'test-session', + targetDir: '/tmp', + debugMode: false, + cwd: '/tmp', + model: 'test-model', + enableHooks: true, + deepValidation: true, + } as any); + + const mockExecute = vi.fn().mockResolvedValue({ + returnDisplay: { + result: JSON.stringify({ + report: 'All good', + isSatisfied: true, + }), + }, + }); + + (LocalSubagentInvocation as any).mockImplementation(() => ({ + execute: mockExecute, + })); + + await config.initialize(); + + const hookSystem = config.getHookSystem(); + const deepValidationEntry = hookSystem?.getAllHooks()?.find(h => h.config.name === 'deep-validation-hook'); + + expect(deepValidationEntry).toBeDefined(); + + if (deepValidationEntry && 'action' in deepValidationEntry.config) { + const result = await (deepValidationEntry.config as any).action({ prompt: 'original prompt' } as any); + + expect(LocalSubagentInvocation).toHaveBeenCalled(); + expect(mockExecute).toHaveBeenCalled(); + expect(result).toMatchObject({ + continue: true, + systemMessage: expect.stringContaining('✅ Deep validation satisfied.'), + }); + expect(result).toMatchObject({ + systemMessage: expect.stringContaining('All good'), + }); + } + }); +}); diff --git a/packages/core/src/core/client.ts b/packages/core/src/core/client.ts index 42adab3a05..880e2f6c48 100644 --- a/packages/core/src/core/client.ts +++ b/packages/core/src/core/client.ts @@ -809,7 +809,7 @@ export class GeminiClient { // Update cumulative response in hook state // We do this immediately after the stream finishes for THIS turn. - const hooksEnabled = this.config.getEnableHooks(); + const hooksEnabled = this.config.getHookSystem() !== undefined; if (hooksEnabled) { const responseText = turn.getResponseText() || ''; const hookState = this.hookStateMap.get(prompt_id); @@ -900,7 +900,7 @@ export class GeminiClient { this.config.resetTurn(); } - const hooksEnabled = this.config.getEnableHooks(); + const hooksEnabled = this.config.getHookSystem() !== undefined; const messageBus = this.context.messageBus; if (this.lastPromptId !== prompt_id) { diff --git a/packages/core/src/hooks/hookRegistry.test.ts b/packages/core/src/hooks/hookRegistry.test.ts index d8157f4ef5..051503c2ea 100644 --- a/packages/core/src/hooks/hookRegistry.test.ts +++ b/packages/core/src/hooks/hookRegistry.test.ts @@ -70,7 +70,7 @@ describe('HookRegistry', () => { mockConfig = { storage: mockStorage, getExtensions: vi.fn().mockReturnValue([]), - getHooks: vi.fn().mockReturnValue({}), + getHooks: vi.fn(), getEnableHooks: vi.fn().mockReturnValue(true).mockReturnValue({}), getProjectHooks: vi.fn().mockReturnValue({}), getDisabledHooks: vi.fn().mockReturnValue([]), isTrustedFolder: vi.fn().mockReturnValue(true), diff --git a/packages/core/src/hooks/hookRegistry.ts b/packages/core/src/hooks/hookRegistry.ts index 1dad67bad5..1190266bac 100644 --- a/packages/core/src/hooks/hookRegistry.ts +++ b/packages/core/src/hooks/hookRegistry.ts @@ -73,7 +73,10 @@ export class HookRegistry { (entry) => entry.source === ConfigSource.Runtime, ); this.entries = [...runtimeHooks]; - this.processHooksFromConfig(); + + if (this.config.getEnableHooks()) { + this.processHooksFromConfig(); + } debugLogger.debug( `Hook registry initialized with ${this.entries.length} hook entries`, diff --git a/schemas/settings.schema.json b/schemas/settings.schema.json index 52a6f1e183..6891266589 100644 --- a/schemas/settings.schema.json +++ b/schemas/settings.schema.json @@ -77,6 +77,13 @@ "default": false, "type": "boolean" }, + "deepValidation": { + "title": "Deep Validation", + "description": "Run a subagent after the main agent finishes to perform final validation.", + "markdownDescription": "Run a subagent after the main agent finishes to perform final validation. This subagent will reflect on the original prompt, review changes, and run tests to ensure satisfaction.\n\n- Category: `General`\n- Requires restart: `no`\n- Default: `false`", + "default": false, + "type": "boolean" + }, "enableAutoUpdate": { "title": "Enable Auto Update", "description": "Enable automatic updates.",