Compare commits

...

7 Commits

Author SHA1 Message Date
Christian Gunderman 0ba6bde93e Define generalist tool name. 2026-02-16 11:58:27 -08:00
Christian Gunderman fc13adaca9 Better prompt. 2026-02-15 14:32:20 -08:00
Christian Gunderman 9844066de3 Prompt changes. 2026-02-15 13:09:54 -08:00
Christian Gunderman 6d1bfa8da9 Turns for days. 2026-02-13 13:52:39 -08:00
Christian Gunderman 56d4759d41 Fix subagent break. 2026-02-13 12:03:55 -08:00
Christian Gunderman 46a0538ffd Parallelize with generalist agent. 2026-02-13 11:17:35 -08:00
Christian Gunderman 952f9e1b5e Fix subagents.eval.ts: update prompt and assertion for linter test 2026-02-13 11:17:35 -08:00
9 changed files with 199 additions and 21 deletions
+84
View File
@@ -50,4 +50,88 @@ describe('subagent eval test cases', () => {
await rig.expectToolCallSuccess(['docs-agent']);
},
});
evalTest('ALWAYS_PASSES', {
name: 'should fix linter errors in multiple projects using implicit parallelism',
prompt: 'Fix all linter errors.',
timeout: 600000,
params: {
settings: {
agents: {
overrides: {
generalist: { enabled: true },
},
},
},
},
files: {
'project-a/eslint.config.js': `
module.exports = [
{
files: ["**/*.js"],
rules: {
"no-var": "error"
}
}
];
`,
'project-a/index.js': 'var x = 1;',
'project-b/eslint.config.js': `
module.exports = [
{
files: ["**/*.js"],
rules: {
"no-console": "error"
}
}
];
`,
'project-b/main.js': 'console.log("hello");',
},
assert: async (rig) => {
const fileA = rig.readFile('project-a/index.js');
const fileB = rig.readFile('project-b/main.js');
if (fileA.includes('var x')) {
throw new Error(`project-a/index.js was not fixed. Content:\n${fileA}`);
}
// Check if console.log is present and NOT commented out or disabled.
const lines = fileB.split('\n');
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (line.includes('console.log')) {
const isCommented = line.trim().startsWith('//');
const isDisabled =
(i > 0 && lines[i - 1].includes('eslint-disable')) ||
line.includes('eslint-disable-line');
if (!isCommented && !isDisabled) {
throw new Error(
`project-b/main.js was not fixed (console.log present without disable/comment). Content:\n${fileB}`,
);
}
}
}
// Assert that the agent delegated to a subagent for each project.
const toolLogs = rig.readToolLogs();
const subagentCalls = toolLogs.filter((log) => {
if (log.toolRequest.name === 'generalist') return true;
if (log.toolRequest.name === 'delegate_to_agent') {
try {
const args = JSON.parse(log.toolRequest.args);
return args.agent_name === 'generalist';
} catch {
return false;
}
}
return false;
});
if (subagentCalls.length < 2) {
throw new Error(
`Expected at least 2 generalist calls, but found ${subagentCalls.length}`,
);
}
},
});
});
@@ -10,6 +10,8 @@ import {
GREP_TOOL_NAME,
LS_TOOL_NAME,
READ_FILE_TOOL_NAME,
SHELL_TOOL_NAME,
WEB_FETCH_TOOL_NAME,
} from '../tools/tool-names.js';
import {
DEFAULT_THINKING_MODE,
@@ -66,8 +68,8 @@ export const CodebaseInvestigatorAgent = (
name: 'codebase_investigator',
kind: 'local',
displayName: 'Codebase Investigator Agent',
description: `The specialized tool for codebase analysis, architectural mapping, and understanding system-wide dependencies.
Invoke this tool for tasks like vague requests, bug root-cause analysis, system refactoring, comprehensive feature implementation or to answer questions about the codebase that require investigation.
description: `The specialized tool for codebase analysis, architectural mapping, understanding system-wide dependencies, and VERIFYING fixes.
Invoke this tool for tasks like vague requests, bug root-cause analysis, system refactoring, comprehensive feature implementation or to answer questions about the codebase that require investigation or final verification.
It returns a structured report with key file paths, symbols, and actionable architectural insights.`,
inputConfig: {
inputSchema: {
@@ -114,12 +116,14 @@ export const CodebaseInvestigatorAgent = (
},
toolConfig: {
// Grant access only to read-only tools.
// Grant access to investigation tools.
tools: [
LS_TOOL_NAME,
READ_FILE_TOOL_NAME,
GLOB_TOOL_NAME,
GREP_TOOL_NAME,
SHELL_TOOL_NAME,
WEB_FETCH_TOOL_NAME,
],
},
@@ -144,7 +148,8 @@ You operate in a non-interactive loop and must reason based on the information p
1. **DEEP ANALYSIS, NOT JUST FILE FINDING:** Your goal is to understand the *why* behind the code. Don't just list files; explain their purpose and the role of their key components. Your final report should empower another agent to make a correct and complete fix.
2. **SYSTEMATIC & CURIOUS EXPLORATION:** Start with high-value clues (like tracebacks or ticket numbers) and broaden your search as needed. Think like a senior engineer doing a code review. An initial file contains clues (imports, function calls, puzzling logic). **If you find something you don't understand, you MUST prioritize investigating it until it is clear.** Treat confusion as a signal to dig deeper.
3. **HOLISTIC & PRECISE:** Your goal is to find the complete and minimal set of locations that need to be understood or changed. Do not stop until you are confident you have considered the side effects of a potential fix (e.g., type errors, breaking changes to callers, opportunities for code reuse).
4. **Web Search:** You are allowed to use the \`web_fetch\` tool to research libraries, language features, or concepts you don't understand (e.g., "what does gettext.translation do with localedir=None?").
4. **Tool Usage:** You are allowed to use the \`run_shell_command\` tool to run linters, tests, or other diagnostic commands to gather information or verify that issues are resolved. Do NOT use it to perform implementation changes.
5. **Web Search:** You are allowed to use the \`web_fetch\` tool to research libraries, language features, or concepts you don't understand (e.g., "what does gettext.translation do with localedir=None?").
</RULES>
---
## Scratchpad Management
@@ -9,12 +9,19 @@ import { GeneralistAgent } from './generalist-agent.js';
import { makeFakeConfig } from '../test-utils/config.js';
import type { ToolRegistry } from '../tools/tool-registry.js';
import type { AgentRegistry } from './registry.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import { GENERALIST_TOOL_NAME } from '../tools/tool-names.js';
describe('GeneralistAgent', () => {
it('should create a valid generalist agent definition', () => {
const config = makeFakeConfig();
vi.spyOn(config, 'getToolRegistry').mockReturnValue({
getAllToolNames: () => ['tool1', 'tool2', 'agent-tool'],
getAllTools: () => [
{ name: 'tool1' },
{ name: 'tool2' },
{ name: 'agent-tool' },
],
} as unknown as ToolRegistry);
vi.spyOn(config, 'getAgentRegistry').mockReturnValue({
getDirectoryContext: () => 'mock directory context',
@@ -24,7 +31,7 @@ describe('GeneralistAgent', () => {
const agent = GeneralistAgent(config);
expect(agent.name).toBe('generalist');
expect(agent.name).toBe(GENERALIST_TOOL_NAME);
expect(agent.kind).toBe('local');
expect(agent.modelConfig.model).toBe('inherit');
expect(agent.toolConfig?.tools).toBeDefined();
@@ -34,4 +41,20 @@ describe('GeneralistAgent', () => {
// Ensure it's non-interactive
expect(agent.promptConfig.systemPrompt).toContain('non-interactive');
});
it('should use fully qualified names for MCP tools', () => {
const config = makeFakeConfig();
const mockMcpTool = Object.create(DiscoveredMCPTool.prototype);
mockMcpTool.getFullyQualifiedName = () => 'server__tool';
mockMcpTool.name = 'tool';
vi.spyOn(config, 'getToolRegistry').mockReturnValue({
getAllTools: () => [{ name: 'normal-tool' }, mockMcpTool],
} as unknown as ToolRegistry);
const agent = GeneralistAgent(config);
expect(agent.toolConfig?.tools).toContain('normal-tool');
expect(agent.toolConfig?.tools).toContain('server__tool');
});
});
+15 -4
View File
@@ -8,6 +8,8 @@ import { z } from 'zod';
import type { Config } from '../config/config.js';
import { getCoreSystemPrompt } from '../core/prompts.js';
import type { LocalAgentDefinition } from './types.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import { GENERALIST_TOOL_NAME } from '../tools/tool-names.js';
const GeneralistAgentSchema = z.object({
response: z.string().describe('The final response from the agent.'),
@@ -21,10 +23,11 @@ export const GeneralistAgent = (
config: Config,
): LocalAgentDefinition<typeof GeneralistAgentSchema> => ({
kind: 'local',
name: 'generalist',
name: GENERALIST_TOOL_NAME,
displayName: 'Generalist Agent',
description:
"A general-purpose AI agent with access to all tools. Use it for complex tasks that don't fit into other specialized agents.",
description: `A general-purpose AI agent with access to all tools.
- ALWAYS use it to break up and parallelize independent pieces of a larger task, when possible.
`,
experimental: true,
inputConfig: {
inputSchema: {
@@ -47,7 +50,15 @@ export const GeneralistAgent = (
model: 'inherit',
},
get toolConfig() {
const tools = config.getToolRegistry().getAllToolNames();
const tools = config
.getToolRegistry()
.getAllTools()
.map((tool) => {
if (tool instanceof DiscoveredMCPTool) {
return tool.getFullyQualifiedName();
}
return tool.name;
});
return {
tools,
};
+6 -5
View File
@@ -27,6 +27,7 @@ import type { ToolRegistry } from '../tools/tool-registry.js';
import { ThinkingLevel } from '@google/genai';
import type { AcknowledgedAgentsService } from './acknowledgedAgents.js';
import { PolicyDecision } from '../policy/types.js';
import { GENERALIST_TOOL_NAME } from '../tools/tool-names.js';
vi.mock('./agentLoader.js', () => ({
loadAgentsFromDirectory: vi
@@ -302,14 +303,14 @@ describe('AgentRegistry', () => {
await registry.initialize();
expect(registry.getDefinition('generalist')).toBeUndefined();
expect(registry.getDefinition(GENERALIST_TOOL_NAME)).toBeUndefined();
});
it('should register generalist agent if explicitly enabled via override', async () => {
const config = makeMockedConfig({
agents: {
overrides: {
generalist: { enabled: true },
[GENERALIST_TOOL_NAME]: { enabled: true },
},
},
});
@@ -317,7 +318,7 @@ describe('AgentRegistry', () => {
await registry.initialize();
expect(registry.getDefinition('generalist')).toBeDefined();
expect(registry.getDefinition(GENERALIST_TOOL_NAME)).toBeDefined();
});
it('should NOT register a non-experimental agent if enabled is false', async () => {
@@ -340,7 +341,7 @@ describe('AgentRegistry', () => {
const config = makeMockedConfig({
agents: {
overrides: {
generalist: { enabled: false },
[GENERALIST_TOOL_NAME]: { enabled: false },
},
},
});
@@ -348,7 +349,7 @@ describe('AgentRegistry', () => {
await registry.initialize();
expect(registry.getDefinition('generalist')).toBeUndefined();
expect(registry.getDefinition(GENERALIST_TOOL_NAME)).toBeUndefined();
});
it('should load agents from active extensions', async () => {
+2 -2
View File
@@ -43,7 +43,7 @@ export const DEFAULT_QUERY_STRING = 'Get Started!';
/**
* The default maximum number of conversational turns for an agent.
*/
export const DEFAULT_MAX_TURNS = 15;
export const DEFAULT_MAX_TURNS = 50;
/**
* The default maximum execution time for an agent in minutes.
@@ -200,7 +200,7 @@ export interface RunConfig {
maxTimeMinutes?: number;
/**
* The maximum number of conversational turns.
* If not specified, defaults to DEFAULT_MAX_TURNS (15).
* If not specified, defaults to DEFAULT_MAX_TURNS (50).
*/
maxTurns?: number;
}
+9 -4
View File
@@ -11,6 +11,7 @@ import {
EDIT_TOOL_NAME,
ENTER_PLAN_MODE_TOOL_NAME,
EXIT_PLAN_MODE_TOOL_NAME,
GENERALIST_TOOL_NAME,
GLOB_TOOL_NAME,
GREP_TOOL_NAME,
MEMORY_TOOL_NAME,
@@ -497,15 +498,19 @@ function workflowStepPlan(options: PrimaryWorkflowsOptions): string {
return `2. **Plan:** An approved plan is available for this task. Use this file as a guide for your implementation. You MUST read this file before proceeding. If you discover new requirements or need to change the approach, confirm with the user and update this plan file to reflect the updated design decisions or discovered requirements.`;
}
if (options.enableCodebaseInvestigator && options.enableWriteTodosTool) {
return `2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. If 'codebase_investigator' was used, do not ignore the output of the agent, you must use it as the foundation of your plan. For complex tasks, break them down into smaller, manageable subtasks and use the \`${WRITE_TODOS_TOOL_NAME}\` tool to track your progress. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution.`;
return `2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. If 'codebase_investigator' was used, do not ignore the output of the agent, you must use it as the foundation of your plan. For complex tasks, break them down into smaller, manageable subtasks and use the \`${WRITE_TODOS_TOOL_NAME}\` tool to track your progress. When these subtasks are independent, leverage the '${GENERALIST_TOOL_NAME}' agent to execute them in parallel, increasing efficiency. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution.`;
}
if (options.enableCodebaseInvestigator) {
return `2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. If 'codebase_investigator' was used, do not ignore the output of the agent, you must use it as the foundation of your plan. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution.`;
return `2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. If 'codebase_investigator' was used, do not ignore the output of the agent, you must use it as the foundation of your plan. For tasks that can be broken down into independent sub-tasks, leverage the '${GENERALIST_TOOL_NAME}' agent to parallelize their execution. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution.`;
}
if (options.enableWriteTodosTool) {
return `2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. For complex tasks, break them down into smaller, manageable subtasks and use the \`${WRITE_TODOS_TOOL_NAME}\` tool to track your progress. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution.`;
return `2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. For complex tasks, break them down into smaller, manageable subtasks and use the \`${WRITE_TODOS_TOOL_NAME}\` tool to track your progress. When these subtasks are independent, leverage the '${GENERALIST_TOOL_NAME}' agent to execute them in parallel, increasing efficiency. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution.`;
}
return "2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution.";
return (
"2. **Plan:** Build a coherent and grounded (based on the understanding in step 1) plan for how you intend to resolve the user's task. If the user's request implies a change but does not explicitly state it, **YOU MUST ASK** for confirmation before modifying code. For tasks that can be broken down into independent sub-tasks, leverage the '" +
GENERALIST_TOOL_NAME +
"' agent to parallelize their execution. Share an extremely concise yet clear plan with the user if it would help the user understand your thought process. As part of the plan, you should use an iterative development process that includes writing unit tests to verify your changes. Use output logs or debug statements as part of this process to arrive at a solution."
);
}
function workflowVerifyStandardsSuffix(interactive: boolean): string {
+49 -1
View File
@@ -10,6 +10,7 @@ import {
EDIT_TOOL_NAME,
ENTER_PLAN_MODE_TOOL_NAME,
EXIT_PLAN_MODE_TOOL_NAME,
GENERALIST_TOOL_NAME,
GLOB_TOOL_NAME,
GREP_TOOL_NAME,
MEMORY_TOOL_NAME,
@@ -194,6 +195,12 @@ export function renderSubAgents(subAgents?: SubAgentOptions[]): string {
)
.join('\n');
const parallelismInstructions = subAgents.some(
(agent) => agent.name === GENERALIST_TOOL_NAME,
)
? renderParallelismInstructions()
: '';
return `
# Available Sub-Agents
@@ -207,7 +214,9 @@ Remember that the closest relevant sub-agent should still be used even if its ex
For example:
- A license-agent -> Should be used for a range of tasks, including reading, validating, and updating licenses and headers.
- A test-fixing-agent -> Should be used both for fixing tests as well as investigating test failures.`.trim();
- A test-fixing-agent -> Should be used both for fixing tests as well as investigating test failures.
${parallelismInstructions}`.trim();
}
export function renderAgentSkills(skills?: AgentSkillOptions[]): string {
@@ -464,6 +473,45 @@ An approved plan is available for this task at \`${approvedPlanPath}\`.
`;
}
function renderParallelismInstructions(): string {
return `
## Parallelism
You MUST ALWAYS utilize the generalist subagent to conserve context when doing repetitive tasks and parallelize independent bodies of work, even if you think you don't need to. This is very IMPORTANT to ensure you stay on track on repetitive tasks and/or complete tasks in a timely fashion.
Tasks that should be delegated include:
<should_delegate>
- Creating, refactoring, building, or updating independent projects or directories, one per generalist call.
- Refactor separate files by splitting them into batches, one batch for each generalist.
- Parallelizing finding the answer to multiple questions, one parallel generalist call per question.
- Repetitive changes across a codebase should be batched and delegated to a subagent.
</should_delegate>
<guidelines>
Try to delegate similarly sized pieces to the generalist. For example:
- Creating two projects -> each can be its own delegation.
- Repetitive code changes -> first, check the size of the work, and split it into batches of similar size.
- You can use up to 4 parallel subagents. Use as many as you can to complete the task faster.
</guidelines>
Always use parallel agents (via calls to \`generalist\`) or parallel tool calls when
able to do so safely.
<rules_for_parallelism>
Generally safe to parallelize and minimal risk of concurrency issues:
<what_you_can_parallelize>
- Edits to different files.
- Multi-step tasks that touch different folders, with generalist.
- Multi-step read-only investigations, with generalist.
- grep_search and ls to different directories.
</what_you_can_parallelize>
Not safe to parallelize. Will likely cause concurrency issues.
<what_you_cannot_parallelize>
- generalist tasks that involve making builds or edits to the same set of files.
</what_you_cannot_parallelize>
</rules_for_parallelism>`;
}
// --- Leaf Helpers (Strictly strings or simple calls) ---
function mandateConfirm(interactive: boolean): string {
+1
View File
@@ -43,6 +43,7 @@ export const ASK_USER_TOOL_NAME = 'ask_user';
export const ASK_USER_DISPLAY_NAME = 'Ask User';
export const EXIT_PLAN_MODE_TOOL_NAME = 'exit_plan_mode';
export const ENTER_PLAN_MODE_TOOL_NAME = 'enter_plan_mode';
export const GENERALIST_TOOL_NAME = 'generalist';
/**
* Mapping of legacy tool names to their current names.