From 84f40768a15614f79a60b3226c1e2b953029133d Mon Sep 17 00:00:00 2001 From: Samee Zahid Date: Tue, 24 Mar 2026 12:50:48 -0700 Subject: [PATCH] feat(evals): centralize test agents into test-utils for reuse (#23616) Co-authored-by: Samee Zahid --- evals/subagents.eval.ts | 49 +++++---------- packages/test-utils/src/fixtures/agents.ts | 72 ++++++++++++++++++++++ packages/test-utils/src/index.ts | 3 +- 3 files changed, 91 insertions(+), 33 deletions(-) create mode 100644 packages/test-utils/src/fixtures/agents.ts diff --git a/evals/subagents.eval.ts b/evals/subagents.eval.ts index 3a7d8fa44f..140925964b 100644 --- a/evals/subagents.eval.ts +++ b/evals/subagents.eval.ts @@ -9,27 +9,7 @@ import path from 'node:path'; import { describe, expect } from 'vitest'; -import { evalTest } from './test-helper.js'; - -const DOCS_AGENT_DEFINITION = `--- -name: docs-agent -description: An agent with expertise in updating documentation. -tools: - - read_file - - write_file ---- -You are the docs agent. Update documentation clearly and accurately. -`; - -const TEST_AGENT_DEFINITION = `--- -name: test-agent -description: An agent with expertise in writing and updating tests. -tools: - - read_file - - write_file ---- -You are the test agent. Add or update tests. -`; +import { evalTest, TEST_AGENTS } from './test-helper.js'; const INDEX_TS = 'export const add = (a: number, b: number) => a + b;\n'; @@ -62,12 +42,12 @@ describe('subagent eval test cases', () => { }, prompt: 'Please update README.md with a description of this library.', files: { - '.gemini/agents/docs-agent.md': DOCS_AGENT_DEFINITION, + ...TEST_AGENTS.DOCS_AGENT.asFile(), 'index.ts': INDEX_TS, 'README.md': 'TODO: update the README.\n', }, assert: async (rig, _result) => { - await rig.expectToolCallSuccess(['docs-agent']); + await rig.expectToolCallSuccess([TEST_AGENTS.DOCS_AGENT.name]); }, }); @@ -92,7 +72,7 @@ describe('subagent eval test cases', () => { prompt: 'Rename the exported function in index.ts from add to sum and update the file directly.', files: { - '.gemini/agents/docs-agent.md': DOCS_AGENT_DEFINITION, + ...TEST_AGENTS.DOCS_AGENT.asFile(), 'index.ts': INDEX_TS, }, assert: async (rig, _result) => { @@ -102,9 +82,11 @@ describe('subagent eval test cases', () => { }>; expect(updatedIndex).toContain('export const sum ='); - expect(toolLogs.some((l) => l.toolRequest.name === 'docs-agent')).toBe( - false, - ); + expect( + toolLogs.some( + (l) => l.toolRequest.name === TEST_AGENTS.DOCS_AGENT.name, + ), + ).toBe(false); expect(toolLogs.some((l) => l.toolRequest.name === 'generalist')).toBe( false, ); @@ -133,7 +115,7 @@ describe('subagent eval test cases', () => { }, prompt: 'Please add a small test file that verifies add(1, 2) returns 3.', files: { - '.gemini/agents/test-agent.md': TEST_AGENT_DEFINITION, + ...TEST_AGENTS.TESTING_AGENT.asFile(), 'index.ts': INDEX_TS, 'package.json': JSON.stringify( { @@ -150,7 +132,7 @@ describe('subagent eval test cases', () => { toolRequest: { name: string }; }>; - await rig.expectToolCallSuccess(['test-agent']); + await rig.expectToolCallSuccess([TEST_AGENTS.TESTING_AGENT.name]); expect(toolLogs.some((l) => l.toolRequest.name === 'generalist')).toBe( false, ); @@ -178,8 +160,8 @@ describe('subagent eval test cases', () => { prompt: 'Add a short README description for this library and also add a test file that verifies add(1, 2) returns 3.', files: { - '.gemini/agents/docs-agent.md': DOCS_AGENT_DEFINITION, - '.gemini/agents/test-agent.md': TEST_AGENT_DEFINITION, + ...TEST_AGENTS.DOCS_AGENT.asFile(), + ...TEST_AGENTS.TESTING_AGENT.asFile(), 'index.ts': INDEX_TS, 'README.md': 'TODO: update the README.\n', 'package.json': JSON.stringify( @@ -198,7 +180,10 @@ describe('subagent eval test cases', () => { }>; const readme = readProjectFile(rig, 'README.md'); - await rig.expectToolCallSuccess(['docs-agent', 'test-agent']); + await rig.expectToolCallSuccess([ + TEST_AGENTS.DOCS_AGENT.name, + TEST_AGENTS.TESTING_AGENT.name, + ]); expect(readme).not.toContain('TODO: update the README.'); expect(toolLogs.some((l) => l.toolRequest.name === 'generalist')).toBe( false, diff --git a/packages/test-utils/src/fixtures/agents.ts b/packages/test-utils/src/fixtures/agents.ts new file mode 100644 index 0000000000..9469457227 --- /dev/null +++ b/packages/test-utils/src/fixtures/agents.ts @@ -0,0 +1,72 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Represents a test agent used in evaluations and tests. + */ +export interface TestAgent { + /** The unique name of the agent. */ + readonly name: string; + /** The full YAML/Markdown definition of the agent. */ + readonly definition: string; + /** The standard path where this agent should be saved in a test project. */ + readonly path: string; + /** A helper to spread this agent directly into a 'files' object for evalTest. */ + readonly asFile: () => Record; +} + +/** + * Helper to create a TestAgent with consistent formatting and pathing. + */ +function createAgent(options: { + name: string; + description: string; + tools: string[]; + body: string; +}): TestAgent { + const definition = `--- +name: ${options.name} +description: ${options.description} +tools: +${options.tools.map((t) => ` - ${t}`).join('\n')} +--- +${options.body} +`; + + const path = `.gemini/agents/${options.name}.md`; + + return { + name: options.name, + definition, + path, + asFile: () => ({ [path]: definition }), + }; +} + +/** + * A collection of predefined test agents for use in evaluations and tests. + */ +export const TEST_AGENTS = { + /** + * An agent with expertise in updating documentation. + */ + DOCS_AGENT: createAgent({ + name: 'docs-agent', + description: 'An agent with expertise in updating documentation.', + tools: ['read_file', 'write_file'], + body: 'You are the docs agent. Update documentation clearly and accurately.', + }), + + /** + * An agent with expertise in writing and updating tests. + */ + TESTING_AGENT: createAgent({ + name: 'testing-agent', + description: 'An agent with expertise in writing and updating tests.', + tools: ['read_file', 'write_file'], + body: 'You are the test agent. Add or update tests.', + }), +} as const; diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index 42dd12bb43..7bae818040 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -5,6 +5,7 @@ */ export * from './file-system-test-helpers.js'; -export * from './test-rig.js'; +export * from './fixtures/agents.js'; export * from './mock-utils.js'; export * from './test-mcp-server.js'; +export * from './test-rig.js';