diff --git a/evals/generalist_agent.eval.ts b/evals/generalist_agent.eval.ts new file mode 100644 index 0000000000..8e3e11e9ba --- /dev/null +++ b/evals/generalist_agent.eval.ts @@ -0,0 +1,48 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect } from 'vitest'; +import { evalTest } from './test-helper.js'; +import path from 'node:path'; +import fs from 'node:fs/promises'; + +describe('generalist_agent', () => { + evalTest('ALWAYS_PASSES', { + name: 'should be able to use generalist agent by explicitly asking the main agent to invoke it', + params: { + settings: { + agents: { + overrides: { + generalist: { enabled: true }, + }, + }, + }, + }, + prompt: + 'Please use the generalist agent to create a file called "generalist_test_file.txt" containing exactly the following text: success', + assert: async (rig) => { + // 1) Verify the generalist agent was invoked via delegate_to_agent + const foundToolCall = await rig.waitForToolCall( + 'delegate_to_agent', + undefined, + (args) => { + const parsed = JSON.parse(args); + return parsed.agent_name === 'generalist'; + }, + ); + expect( + foundToolCall, + 'Expected to find a delegate_to_agent tool call for generalist agent', + ).toBeTruthy(); + + // 2) Verify the file was created as expected + const filePath = path.join(rig.testDir!, 'generalist_test_file.txt'); + + const content = await fs.readFile(filePath, 'utf-8'); + expect(content.trim()).toBe('success'); + }, + }); +});