mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-07-24 08:40:58 -07:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5b929b77af | |||
| 411520a328 | |||
| 3dce0fe6e5 | |||
| 849b3b255c | |||
| 5be38c3a27 |
+1
-2
@@ -160,8 +160,7 @@ failing evaluations. It will:
|
||||
`evals/logs`.
|
||||
2. **Fix**: Suggest and apply targeted fixes to the prompt or tool definitions.
|
||||
It prioritizes minimal changes to `prompt.ts`, tool instructions, and
|
||||
modules that contribute to the prompt. It generally tries to avoid changing
|
||||
the test itself.
|
||||
modules that contribute to the prompt.
|
||||
3. **Verify**: Re-run the test 3 times across multiple models (e.g., Gemini
|
||||
3.0, Gemini 3 Flash, Gemini 2.5 Pro) to ensure stability and calculate a
|
||||
success rate.
|
||||
|
||||
@@ -31,7 +31,7 @@ describe('subagent eval test cases', () => {
|
||||
*
|
||||
* This tests the system prompt's subagent specific clauses.
|
||||
*/
|
||||
evalTest('USUALLY_PASSES', {
|
||||
evalTest('ALWAYS_PASSES', {
|
||||
name: 'should delegate to user provided agent with relevant expertise',
|
||||
params: {
|
||||
settings: {
|
||||
@@ -46,6 +46,9 @@ describe('subagent eval test cases', () => {
|
||||
'index.ts': INDEX_TS,
|
||||
'README.md': 'TODO: update the README.',
|
||||
},
|
||||
acknowledgedAgents: {
|
||||
'docs-agent': AGENT_DEFINITION,
|
||||
},
|
||||
assert: async (rig, _result) => {
|
||||
await rig.expectToolCallSuccess(['docs-agent']);
|
||||
},
|
||||
|
||||
@@ -112,6 +112,10 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
|
||||
execSync('git commit --allow-empty -m "Initial commit"', execOptions);
|
||||
}
|
||||
|
||||
if (evalCase.acknowledgedAgents) {
|
||||
await rig.acknowledgeAgents(evalCase.acknowledgedAgents);
|
||||
}
|
||||
|
||||
const result = await rig.run({
|
||||
args: evalCase.prompt,
|
||||
approvalMode: evalCase.approvalMode ?? 'yolo',
|
||||
@@ -172,5 +176,6 @@ export interface EvalCase {
|
||||
timeout?: number;
|
||||
files?: Record<string, string>;
|
||||
approvalMode?: 'default' | 'auto_edit' | 'yolo' | 'plan';
|
||||
acknowledgedAgents?: Record<string, string>;
|
||||
assert: (rig: TestRig, result: string) => Promise<void>;
|
||||
}
|
||||
|
||||
@@ -40,25 +40,27 @@ describe('AcknowledgedAgentsService', () => {
|
||||
const service = new AcknowledgedAgentsService();
|
||||
const ackPath = Storage.getAcknowledgedAgentsPath();
|
||||
|
||||
await service.acknowledge('/project', 'AgentA', 'hash1');
|
||||
await service.acknowledge('/project', 'AgentA', 'content1');
|
||||
|
||||
// Verify file exists and content
|
||||
const content = await fs.readFile(ackPath, 'utf-8');
|
||||
expect(content).toContain('"AgentA": "hash1"');
|
||||
const hash1 = AcknowledgedAgentsService.computeHash('content1');
|
||||
expect(content).toContain(`"AgentA": "${hash1}"`);
|
||||
});
|
||||
|
||||
it('should return true for acknowledged agent', async () => {
|
||||
const service = new AcknowledgedAgentsService();
|
||||
|
||||
await service.acknowledge('/project', 'AgentA', 'hash1');
|
||||
await service.acknowledge('/project', 'AgentA', 'content1');
|
||||
|
||||
expect(await service.isAcknowledged('/project', 'AgentA', 'hash1')).toBe(
|
||||
const hash1 = AcknowledgedAgentsService.computeHash('content1');
|
||||
expect(await service.isAcknowledged('/project', 'AgentA', hash1)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(await service.isAcknowledged('/project', 'AgentA', 'hash2')).toBe(
|
||||
false,
|
||||
);
|
||||
expect(await service.isAcknowledged('/project', 'AgentB', 'hash1')).toBe(
|
||||
expect(await service.isAcknowledged('/project', 'AgentB', hash1)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -94,4 +96,13 @@ describe('AcknowledgedAgentsService', () => {
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('should compute consistent hashes', () => {
|
||||
const content = 'some content';
|
||||
const hash = AcknowledgedAgentsService.computeHash(content);
|
||||
expect(hash).toBe(AcknowledgedAgentsService.computeHash(content));
|
||||
expect(hash).not.toBe(
|
||||
AcknowledgedAgentsService.computeHash('other content'),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { Storage } from '../config/storage.js';
|
||||
import { debugLogger } from '../utils/debugLogger.js';
|
||||
import { getErrorMessage, isNodeError } from '../utils/errors.js';
|
||||
@@ -21,6 +22,10 @@ export class AcknowledgedAgentsService {
|
||||
private acknowledgedAgents: AcknowledgedAgentsMap = {};
|
||||
private loaded = false;
|
||||
|
||||
static computeHash(content: string): string {
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
async load(): Promise<void> {
|
||||
if (this.loaded) return;
|
||||
|
||||
@@ -73,8 +78,9 @@ export class AcknowledgedAgentsService {
|
||||
async acknowledge(
|
||||
projectPath: string,
|
||||
agentName: string,
|
||||
hash: string,
|
||||
content: string,
|
||||
): Promise<void> {
|
||||
const hash = AcknowledgedAgentsService.computeHash(content);
|
||||
await this.load();
|
||||
if (!this.acknowledgedAgents[projectPath]) {
|
||||
this.acknowledgedAgents[projectPath] = {};
|
||||
|
||||
@@ -8,8 +8,8 @@ import yaml from 'js-yaml';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import { type Dirent } from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as crypto from 'node:crypto';
|
||||
import { z } from 'zod';
|
||||
import { AcknowledgedAgentsService } from './acknowledgedAgents.js';
|
||||
import type { AgentDefinition } from './types.js';
|
||||
import { isValidToolName } from '../tools/tool-names.js';
|
||||
import { FRONTMATTER_REGEX } from '../skills/skillLoader.js';
|
||||
@@ -347,7 +347,7 @@ export async function loadAgentsFromDirectory(
|
||||
const filePath = path.join(dir, entry.name);
|
||||
try {
|
||||
const content = await fs.readFile(filePath, 'utf-8');
|
||||
const hash = crypto.createHash('sha256').update(content).digest('hex');
|
||||
const hash = AcknowledgedAgentsService.computeHash(content);
|
||||
const agentDefs = await parseAgentMarkdown(filePath, content);
|
||||
for (const def of agentDefs) {
|
||||
const agent = markdownToAgentDefinition(def, { hash, filePath });
|
||||
|
||||
@@ -4,11 +4,13 @@
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs/promises';
|
||||
import { Storage } from '../config/storage.js';
|
||||
import { CoreEvent, coreEvents } from '../utils/events.js';
|
||||
import type { AgentOverride, Config } from '../config/config.js';
|
||||
import type { AgentDefinition, LocalAgentDefinition } from './types.js';
|
||||
import { loadAgentsFromDirectory } from './agentLoader.js';
|
||||
import { AcknowledgedAgentsService } from './acknowledgedAgents.js';
|
||||
import { CodebaseInvestigatorAgent } from './codebase-investigator.js';
|
||||
import { CliHelpAgent } from './cli-help-agent.js';
|
||||
import { GeneralistAgent } from './generalist-agent.js';
|
||||
@@ -81,11 +83,19 @@ export class AgentRegistry {
|
||||
const ackService = this.config.getAcknowledgedAgentsService();
|
||||
const projectRoot = this.config.getProjectRoot();
|
||||
if (agent.metadata?.hash) {
|
||||
await ackService.acknowledge(
|
||||
projectRoot,
|
||||
agent.name,
|
||||
agent.metadata.hash,
|
||||
);
|
||||
let content: string;
|
||||
if (agent.kind === 'remote') {
|
||||
content = agent.agentCardUrl;
|
||||
} else {
|
||||
if (!agent.metadata.filePath) {
|
||||
throw new Error(
|
||||
`Cannot acknowledge local agent ${agent.name}: missing file path`,
|
||||
);
|
||||
}
|
||||
content = await fs.readFile(agent.metadata.filePath, 'utf-8');
|
||||
}
|
||||
|
||||
await ackService.acknowledge(projectRoot, agent.name, content);
|
||||
await this.registerAgent(agent);
|
||||
coreEvents.emitAgentsRefreshed();
|
||||
}
|
||||
@@ -146,7 +156,9 @@ export class AgentRegistry {
|
||||
if (!agent.metadata) {
|
||||
agent.metadata = {};
|
||||
}
|
||||
agent.metadata.hash = agent.agentCardUrl;
|
||||
agent.metadata.hash = AcknowledgedAgentsService.computeHash(
|
||||
agent.agentCardUrl,
|
||||
);
|
||||
}
|
||||
|
||||
if (!agent.metadata?.hash) {
|
||||
@@ -491,26 +503,25 @@ export class AgentRegistry {
|
||||
return 'No sub-agents are currently available.';
|
||||
}
|
||||
|
||||
let context = '## Available Sub-Agents\n';
|
||||
context += `Sub-agents are specialized expert agents that you can use to assist you in
|
||||
the completion of all or part of a task.
|
||||
let context = '# Sub-Agent Directory\n\n';
|
||||
context += `Sub-agents are specialized expert models designed to handle specific domains of knowledge with greater precision and reliability than a generalist.
|
||||
|
||||
Each sub-agent is available as a tool of the same name.
|
||||
Each sub-agent is available as a tool of the same name.
|
||||
|
||||
You MUST always delegate tasks to the sub-agent with the
|
||||
relevant expertise, if one is available.
|
||||
To ensure the highest quality results, you should prioritize delegating tasks to the specialized expert sub-agent whenever a task falls within their domain of expertise.
|
||||
|
||||
The following tools can be used to start sub-agents:\n\n`;
|
||||
The following sub-agents are available as tools:\n\n`;
|
||||
|
||||
for (const [name] of this.agents) {
|
||||
context += `- ${name}\n`;
|
||||
for (const [name, definition] of this.agents) {
|
||||
context += `- \`${name}\`: ${definition.description}\n`;
|
||||
}
|
||||
|
||||
context += `Remember that the closest relevant sub-agent should still be used even if its expertise is broader than the given task.
|
||||
context += `\nRemember that sub-agents are intended to be used for a wide range of tasks within their broader domain.
|
||||
|
||||
For example:
|
||||
- A license-agent -> Should be used for a range of tasks, including reading, validating, and updating licenses and headers.
|
||||
- A test-fixing-agent -> Should be used both for fixing tests as well as investigating test failures.`;
|
||||
Examples of effective delegation:
|
||||
- A \`license-agent\` should be used for tasks related to reading, validating, and updating licenses and headers.
|
||||
- A \`test-fixing-agent\` should be used for both investigating test failures and implementing the actual fixes.
|
||||
- A \`security-agent\` should be used for security audits, vulnerability scanning, and hardening.`;
|
||||
|
||||
return context;
|
||||
}
|
||||
|
||||
@@ -135,6 +135,7 @@ export * from './prompts/mcp-prompts.js';
|
||||
export * from './agents/types.js';
|
||||
export * from './agents/agentLoader.js';
|
||||
export * from './agents/local-executor.js';
|
||||
export * from './agents/acknowledgedAgents.js';
|
||||
|
||||
// Export specific tool logic
|
||||
export * from './tools/read-file.js';
|
||||
|
||||
@@ -11,7 +11,11 @@ import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { env } from 'node:process';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import { DEFAULT_GEMINI_MODEL, GEMINI_DIR } from '@google/gemini-cli-core';
|
||||
import {
|
||||
DEFAULT_GEMINI_MODEL,
|
||||
GEMINI_DIR,
|
||||
AcknowledgedAgentsService,
|
||||
} from '@google/gemini-cli-core';
|
||||
import fs from 'node:fs';
|
||||
import * as pty from '@lydell/node-pty';
|
||||
import stripAnsi from 'strip-ansi';
|
||||
@@ -356,6 +360,26 @@ export class TestRig {
|
||||
);
|
||||
}
|
||||
|
||||
async acknowledgeAgents(agents: Record<string, string>) {
|
||||
// Use realpath to ensure the path matches what the CLI sees (e.g. /var vs /private/var on macOS)
|
||||
const projectRoot = fs.realpathSync(this.testDir!);
|
||||
const originalHome = process.env['GEMINI_CLI_HOME'];
|
||||
process.env['GEMINI_CLI_HOME'] = this.homeDir!;
|
||||
|
||||
try {
|
||||
const service = new AcknowledgedAgentsService();
|
||||
for (const [name, content] of Object.entries(agents)) {
|
||||
await service.acknowledge(projectRoot, name, content);
|
||||
}
|
||||
} finally {
|
||||
if (originalHome) {
|
||||
process.env['GEMINI_CLI_HOME'] = originalHome;
|
||||
} else {
|
||||
delete process.env['GEMINI_CLI_HOME'];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
createFile(fileName: string, content: string) {
|
||||
const filePath = join(this.testDir!, fileName);
|
||||
writeFileSync(filePath, content);
|
||||
|
||||
Reference in New Issue
Block a user