Compare commits

...

5 Commits

Author SHA1 Message Date
Christian Gunderman 5b929b77af Update subagent prompt. 2026-02-03 13:29:49 -08:00
Christian Gunderman 411520a328 Inline. 2026-02-03 11:26:14 -08:00
Christian Gunderman 3dce0fe6e5 De-dupe hashing code. 2026-02-03 11:26:09 -08:00
Christian Gunderman 849b3b255c Fix the subagents eval. 2026-02-03 11:26:03 -08:00
Christian Gunderman 5be38c3a27 Add slash command for diagnosing test failures. 2026-02-03 11:25:35 -08:00
9 changed files with 91 additions and 31 deletions
+1 -2
View File
@@ -160,8 +160,7 @@ failing evaluations. It will:
`evals/logs`.
2. **Fix**: Suggest and apply targeted fixes to the prompt or tool definitions.
It prioritizes minimal changes to `prompt.ts`, tool instructions, and
modules that contribute to the prompt. It generally tries to avoid changing
the test itself.
modules that contribute to the prompt.
3. **Verify**: Re-run the test 3 times across multiple models (e.g., Gemini
3.0, Gemini 3 Flash, Gemini 2.5 Pro) to ensure stability and calculate a
success rate.
+4 -1
View File
@@ -31,7 +31,7 @@ describe('subagent eval test cases', () => {
*
* This tests the system prompt's subagent specific clauses.
*/
evalTest('USUALLY_PASSES', {
evalTest('ALWAYS_PASSES', {
name: 'should delegate to user provided agent with relevant expertise',
params: {
settings: {
@@ -46,6 +46,9 @@ describe('subagent eval test cases', () => {
'index.ts': INDEX_TS,
'README.md': 'TODO: update the README.',
},
acknowledgedAgents: {
'docs-agent': AGENT_DEFINITION,
},
assert: async (rig, _result) => {
await rig.expectToolCallSuccess(['docs-agent']);
},
+5
View File
@@ -112,6 +112,10 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
execSync('git commit --allow-empty -m "Initial commit"', execOptions);
}
if (evalCase.acknowledgedAgents) {
await rig.acknowledgeAgents(evalCase.acknowledgedAgents);
}
const result = await rig.run({
args: evalCase.prompt,
approvalMode: evalCase.approvalMode ?? 'yolo',
@@ -172,5 +176,6 @@ export interface EvalCase {
timeout?: number;
files?: Record<string, string>;
approvalMode?: 'default' | 'auto_edit' | 'yolo' | 'plan';
acknowledgedAgents?: Record<string, string>;
assert: (rig: TestRig, result: string) => Promise<void>;
}
@@ -40,25 +40,27 @@ describe('AcknowledgedAgentsService', () => {
const service = new AcknowledgedAgentsService();
const ackPath = Storage.getAcknowledgedAgentsPath();
await service.acknowledge('/project', 'AgentA', 'hash1');
await service.acknowledge('/project', 'AgentA', 'content1');
// Verify file exists and content
const content = await fs.readFile(ackPath, 'utf-8');
expect(content).toContain('"AgentA": "hash1"');
const hash1 = AcknowledgedAgentsService.computeHash('content1');
expect(content).toContain(`"AgentA": "${hash1}"`);
});
it('should return true for acknowledged agent', async () => {
const service = new AcknowledgedAgentsService();
await service.acknowledge('/project', 'AgentA', 'hash1');
await service.acknowledge('/project', 'AgentA', 'content1');
expect(await service.isAcknowledged('/project', 'AgentA', 'hash1')).toBe(
const hash1 = AcknowledgedAgentsService.computeHash('content1');
expect(await service.isAcknowledged('/project', 'AgentA', hash1)).toBe(
true,
);
expect(await service.isAcknowledged('/project', 'AgentA', 'hash2')).toBe(
false,
);
expect(await service.isAcknowledged('/project', 'AgentB', 'hash1')).toBe(
expect(await service.isAcknowledged('/project', 'AgentB', hash1)).toBe(
false,
);
});
@@ -94,4 +96,13 @@ describe('AcknowledgedAgentsService', () => {
false,
);
});
it('should compute consistent hashes', () => {
const content = 'some content';
const hash = AcknowledgedAgentsService.computeHash(content);
expect(hash).toBe(AcknowledgedAgentsService.computeHash(content));
expect(hash).not.toBe(
AcknowledgedAgentsService.computeHash('other content'),
);
});
});
@@ -6,6 +6,7 @@
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import * as crypto from 'node:crypto';
import { Storage } from '../config/storage.js';
import { debugLogger } from '../utils/debugLogger.js';
import { getErrorMessage, isNodeError } from '../utils/errors.js';
@@ -21,6 +22,10 @@ export class AcknowledgedAgentsService {
private acknowledgedAgents: AcknowledgedAgentsMap = {};
private loaded = false;
static computeHash(content: string): string {
return crypto.createHash('sha256').update(content).digest('hex');
}
async load(): Promise<void> {
if (this.loaded) return;
@@ -73,8 +78,9 @@ export class AcknowledgedAgentsService {
async acknowledge(
projectPath: string,
agentName: string,
hash: string,
content: string,
): Promise<void> {
const hash = AcknowledgedAgentsService.computeHash(content);
await this.load();
if (!this.acknowledgedAgents[projectPath]) {
this.acknowledgedAgents[projectPath] = {};
+2 -2
View File
@@ -8,8 +8,8 @@ import yaml from 'js-yaml';
import * as fs from 'node:fs/promises';
import { type Dirent } from 'node:fs';
import * as path from 'node:path';
import * as crypto from 'node:crypto';
import { z } from 'zod';
import { AcknowledgedAgentsService } from './acknowledgedAgents.js';
import type { AgentDefinition } from './types.js';
import { isValidToolName } from '../tools/tool-names.js';
import { FRONTMATTER_REGEX } from '../skills/skillLoader.js';
@@ -347,7 +347,7 @@ export async function loadAgentsFromDirectory(
const filePath = path.join(dir, entry.name);
try {
const content = await fs.readFile(filePath, 'utf-8');
const hash = crypto.createHash('sha256').update(content).digest('hex');
const hash = AcknowledgedAgentsService.computeHash(content);
const agentDefs = await parseAgentMarkdown(filePath, content);
for (const def of agentDefs) {
const agent = markdownToAgentDefinition(def, { hash, filePath });
+30 -19
View File
@@ -4,11 +4,13 @@
* SPDX-License-Identifier: Apache-2.0
*/
import * as fs from 'node:fs/promises';
import { Storage } from '../config/storage.js';
import { CoreEvent, coreEvents } from '../utils/events.js';
import type { AgentOverride, Config } from '../config/config.js';
import type { AgentDefinition, LocalAgentDefinition } from './types.js';
import { loadAgentsFromDirectory } from './agentLoader.js';
import { AcknowledgedAgentsService } from './acknowledgedAgents.js';
import { CodebaseInvestigatorAgent } from './codebase-investigator.js';
import { CliHelpAgent } from './cli-help-agent.js';
import { GeneralistAgent } from './generalist-agent.js';
@@ -81,11 +83,19 @@ export class AgentRegistry {
const ackService = this.config.getAcknowledgedAgentsService();
const projectRoot = this.config.getProjectRoot();
if (agent.metadata?.hash) {
await ackService.acknowledge(
projectRoot,
agent.name,
agent.metadata.hash,
);
let content: string;
if (agent.kind === 'remote') {
content = agent.agentCardUrl;
} else {
if (!agent.metadata.filePath) {
throw new Error(
`Cannot acknowledge local agent ${agent.name}: missing file path`,
);
}
content = await fs.readFile(agent.metadata.filePath, 'utf-8');
}
await ackService.acknowledge(projectRoot, agent.name, content);
await this.registerAgent(agent);
coreEvents.emitAgentsRefreshed();
}
@@ -146,7 +156,9 @@ export class AgentRegistry {
if (!agent.metadata) {
agent.metadata = {};
}
agent.metadata.hash = agent.agentCardUrl;
agent.metadata.hash = AcknowledgedAgentsService.computeHash(
agent.agentCardUrl,
);
}
if (!agent.metadata?.hash) {
@@ -491,26 +503,25 @@ export class AgentRegistry {
return 'No sub-agents are currently available.';
}
let context = '## Available Sub-Agents\n';
context += `Sub-agents are specialized expert agents that you can use to assist you in
the completion of all or part of a task.
let context = '# Sub-Agent Directory\n\n';
context += `Sub-agents are specialized expert models designed to handle specific domains of knowledge with greater precision and reliability than a generalist.
Each sub-agent is available as a tool of the same name.
Each sub-agent is available as a tool of the same name.
You MUST always delegate tasks to the sub-agent with the
relevant expertise, if one is available.
To ensure the highest quality results, you should prioritize delegating tasks to the specialized expert sub-agent whenever a task falls within their domain of expertise.
The following tools can be used to start sub-agents:\n\n`;
The following sub-agents are available as tools:\n\n`;
for (const [name] of this.agents) {
context += `- ${name}\n`;
for (const [name, definition] of this.agents) {
context += `- \`${name}\`: ${definition.description}\n`;
}
context += `Remember that the closest relevant sub-agent should still be used even if its expertise is broader than the given task.
context += `\nRemember that sub-agents are intended to be used for a wide range of tasks within their broader domain.
For example:
- A license-agent -> Should be used for a range of tasks, including reading, validating, and updating licenses and headers.
- A test-fixing-agent -> Should be used both for fixing tests as well as investigating test failures.`;
Examples of effective delegation:
- A \`license-agent\` should be used for tasks related to reading, validating, and updating licenses and headers.
- A \`test-fixing-agent\` should be used for both investigating test failures and implementing the actual fixes.
- A \`security-agent\` should be used for security audits, vulnerability scanning, and hardening.`;
return context;
}
+1
View File
@@ -135,6 +135,7 @@ export * from './prompts/mcp-prompts.js';
export * from './agents/types.js';
export * from './agents/agentLoader.js';
export * from './agents/local-executor.js';
export * from './agents/acknowledgedAgents.js';
// Export specific tool logic
export * from './tools/read-file.js';
+25 -1
View File
@@ -11,7 +11,11 @@ import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { env } from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
import { DEFAULT_GEMINI_MODEL, GEMINI_DIR } from '@google/gemini-cli-core';
import {
DEFAULT_GEMINI_MODEL,
GEMINI_DIR,
AcknowledgedAgentsService,
} from '@google/gemini-cli-core';
import fs from 'node:fs';
import * as pty from '@lydell/node-pty';
import stripAnsi from 'strip-ansi';
@@ -356,6 +360,26 @@ export class TestRig {
);
}
async acknowledgeAgents(agents: Record<string, string>) {
// Use realpath to ensure the path matches what the CLI sees (e.g. /var vs /private/var on macOS)
const projectRoot = fs.realpathSync(this.testDir!);
const originalHome = process.env['GEMINI_CLI_HOME'];
process.env['GEMINI_CLI_HOME'] = this.homeDir!;
try {
const service = new AcknowledgedAgentsService();
for (const [name, content] of Object.entries(agents)) {
await service.acknowledge(projectRoot, name, content);
}
} finally {
if (originalHome) {
process.env['GEMINI_CLI_HOME'] = originalHome;
} else {
delete process.env['GEMINI_CLI_HOME'];
}
}
}
createFile(fileName: string, content: string) {
const filePath = join(this.testDir!, fileName);
writeFileSync(filePath, content);