Merge branch 'main' into feature/eval-pr-impact

evals/README.md

@@ -6,6 +6,10 @@ for changes to system prompts, tool definitions, and other model-steering
 mechanisms, and as a tool for assessing feature reliability by model, and
 preventing regressions.
 
+> [!TIP] **Agent Automation**: If you are pair-programming with Gemini CLI, you
+> can leverage the **behavioral-evals skill** to automate fixing failing tests
+> or promoting incubation candidates.
+
 ## Why Behavioral Evals?
 
 Unlike traditional **integration tests** which verify that the system functions
@@ -121,7 +125,7 @@ import { describe, expect } from 'vitest';
 import { evalTest } from './test-helper.js';
 
 describe('my_feature', () => {
-  // New tests MUST start as USUALLY_PASSES and be promoted via /promote-behavioral-eval
+  // New tests MUST start as USUALLY_PASSES and be promoted based on consistency metrics
   evalTest('USUALLY_PASSES', {
     name: 'should do something',
     prompt: 'do it',
@@ -183,12 +187,10 @@ mandatory deflaking process.
 
 1. **Incubation**: You must create all new tests with the `USUALLY_PASSES`
    policy. This lets them be monitored in the nightly runs without blocking PRs.
-2. **Monitoring**: The test must complete at least 10 nightly runs across all
+2. **Monitoring**: The test must complete at least 7 nightly runs across all
    supported models.
-3. **Promotion**: Promotion to `ALWAYS_PASSES` happens exclusively through the
-   `/promote-behavioral-eval` slash command. This command verifies the 100%
-   success rate requirement is met across many runs before updating the test
-   policy.
+3. **Promotion**: Promotion to `ALWAYS_PASSES` is conducted by the agent after
+   verifying the 100% success rate requirement is met across many runs.
 
 This promotion process is essential for preventing the introduction of flaky
 evaluations into the CI.
@@ -245,42 +247,21 @@ tool definition has made the model's behavior less reliable.
 
 ## Fixing Evaluations
 
-If an evaluation is failing or has a regressed pass rate, you can use the
-`/fix-behavioral-eval` command within Gemini CLI to help investigate and fix the
-issue.
-
-### `/fix-behavioral-eval`
-
-This command is designed to automate the investigation and fixing process for
-failing evaluations. It will:
+If an evaluation is failing or has a regressed pass rate, ask the agent to
+investigate and fix the issue using the **behavioral-evals skill**. The agent
+will automate the following process:
 
 1.  **Investigate**: Fetch the latest results from the nightly workflow using
     the `gh` CLI, identify the failing test, and review test trajectory logs in
     `evals/logs`.
 2.  **Fix**: Suggest and apply targeted fixes to the prompt or tool definitions.
-    It prioritizes minimal changes to `prompt.ts`, tool instructions, and
-    modules that contribute to the prompt. It generally tries to avoid changing
-    the test itself.
-3.  **Verify**: Re-run the test 3 times across multiple models (e.g., Gemini
-    3.0, Gemini 3 Flash, Gemini 2.5 Pro) to ensure stability and calculate a
-    success rate.
-4.  **Report**: Provide a summary of the success rate for each model and details
-    on the applied fixes.
+    It prioritizes minimal changes to `prompt.ts` and tool instructions,
+    avoiding changing the test itself unless necessary.
+3.  **Verify**: Re-run the test locally across multiple models to ensure
+    stability.
+4.  **Report**: Provide a summary of the success rate.
 
-To use it, run:
-
-```bash
-gemini /fix-behavioral-eval
-```
-
-You can also provide a link to a specific GitHub Action run or the name of a
-specific test to focus the investigation:
-
-```bash
-gemini /fix-behavioral-eval https://github.com/google-gemini/gemini-cli/actions/runs/123456789
-```
-
-When investigating failures manually, you can also enable verbose agent logs by
+When investigating failures manually, you can enable verbose agent logs by
 setting the `GEMINI_DEBUG_LOG_FILE` environment variable.
 
 ### Best practices
@@ -293,25 +274,14 @@ instrospecting on its prompt when asked the right questions.
 
 ## Promoting evaluations
 
-Evaluations must be promoted from `USUALLY_PASSES` to `ALWAYS_PASSES`
-exclusively using the `/promote-behavioral-eval` slash command. Manual promotion
-is not allowed to ensure that the 100% success rate requirement is empirically
-met.
+Evaluations must be promoted from `USUALLY_PASSES` to `ALWAYS_PASSES` by the
+agent to ensure that the 100% success rate requirement is empirically met.
 
-### `/promote-behavioral-eval`
-
-This command automates the promotion of stable tests by:
+The agent automates the promotion by:
 
 1.  **Investigating**: Analyzing the results of the last 7 nightly runs on the
-    `main` branch using the `gh` CLI.
-2.  **Criteria Check**: Identifying tests that have passed 100% of the time for
-    ALL enabled models across the entire 7-run history.
-3.  **Promotion**: Updating the test file's policy from `USUALLY_PASSES` to
-    `ALWAYS_PASSES`.
+    `main` branch.
+2.  **Criteria Check**: Ensuring tests passed 100% of the time for ALL enabled
+    models.
+3.  **Promotion**: Updating the test file's policy to `ALWAYS_PASSES`.
 4.  **Verification**: Running the promoted test locally to ensure correctness.
-
-To run it:
-
-```bash
-gemini /promote-behavioral-eval
-```

evals/app-test-helper.ts

@@ -15,9 +15,26 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { DEFAULT_GEMINI_MODEL } from '@google/gemini-cli-core';
 
+/**
+ * Config overrides for evals, with tool-restriction fields explicitly
+ * forbidden. Evals must test against the full, default tool set to ensure
+ * realistic behavior.
+ */
+interface EvalConfigOverrides {
+  /** Restricting tools via excludeTools in evals is forbidden. */
+  excludeTools?: never;
+  /** Restricting tools via coreTools in evals is forbidden. */
+  coreTools?: never;
+  /** Restricting tools via allowedTools in evals is forbidden. */
+  allowedTools?: never;
+  /** Restricting tools via mainAgentTools in evals is forbidden. */
+  mainAgentTools?: never;
+  [key: string]: unknown;
+}
+
 export interface AppEvalCase {
   name: string;
-  configOverrides?: any;
+  configOverrides?: EvalConfigOverrides;
   prompt: string;
   timeout?: number;
   files?: Record<string, string>;

evals/cli_help_delegation.eval.ts

@@ -0,0 +1,25 @@
+import { describe, expect } from 'vitest';
+import { evalTest } from './test-helper.js';
+
+describe('CliHelpAgent Delegation', () => {
+  evalTest('USUALLY_PASSES', {
+    name: 'should delegate to cli_help agent for subagent creation questions',
+    params: {
+      settings: {
+        experimental: {
+          enableAgents: true,
+        },
+      },
+    },
+    prompt: 'Help me create a subagent in this project',
+    timeout: 60000,
+    assert: async (rig, _result) => {
+      const toolLogs = rig.readToolLogs();
+      const toolCallIndex = toolLogs.findIndex(
+        (log) => log.toolRequest.name === 'cli_help',
+      );
+      expect(toolCallIndex).toBeGreaterThan(-1);
+      expect(toolCallIndex).toBeLessThan(5); // Called within first 5 turns
+    },
+  });
+});

evals/generalist_delegation.eval.ts

@@ -21,7 +21,6 @@ describe('generalist_delegation', () => {
       experimental: {
         enableAgents: true,
       },
-      excludeTools: ['run_shell_command'],
     },
     files: {
       'file1.ts': 'console.log("no semi")',
@@ -65,7 +64,6 @@ describe('generalist_delegation', () => {
       experimental: {
         enableAgents: true,
       },
-      excludeTools: ['run_shell_command'],
     },
     files: {
       'src/a.ts': 'export const a = 1;',
@@ -106,7 +104,6 @@ describe('generalist_delegation', () => {
       experimental: {
         enableAgents: true,
       },
-      excludeTools: ['run_shell_command'],
     },
     files: {
       'README.md': 'This is a proyect.',
@@ -141,7 +138,6 @@ describe('generalist_delegation', () => {
       experimental: {
         enableAgents: true,
       },
-      excludeTools: ['run_shell_command'],
     },
     files: {
       'src/VERSION': '1.2.3',

evals/model_steering.eval.ts

@@ -12,10 +12,9 @@ import { appEvalTest } from './app-test-helper.js';
 import { PolicyDecision } from '@google/gemini-cli-core';
 
 describe('Model Steering Behavioral Evals', () => {
-  appEvalTest('ALWAYS_PASSES', {
+  appEvalTest('USUALLY_PASSES', {
     name: 'Corrective Hint: Model switches task based on hint during tool turn',
     configOverrides: {
-      excludeTools: ['run_shell_command', 'ls', 'google_web_search'],
       modelSteering: true,
     },
     files: {
@@ -52,10 +51,9 @@ describe('Model Steering Behavioral Evals', () => {
     },
   });
 
-  appEvalTest('ALWAYS_PASSES', {
+  appEvalTest('USUALLY_PASSES', {
     name: 'Suggestive Hint: Model incorporates user guidance mid-stream',
     configOverrides: {
-      excludeTools: ['run_shell_command', 'ls', 'google_web_search'],
       modelSteering: true,
     },
     files: {},

evals/plan_mode.eval.ts

@@ -18,6 +18,18 @@ describe('plan_mode', () => {
     experimental: { plan: true },
   };
 
+  const getWriteTargets = (logs: any[]) =>
+    logs
+      .filter((log) => ['write_file', 'replace'].includes(log.toolRequest.name))
+      .map((log) => {
+        try {
+          return JSON.parse(log.toolRequest.args).file_path as string;
+        } catch {
+          return '';
+        }
+      })
+      .filter(Boolean);
+
   evalTest('ALWAYS_PASSES', {
     name: 'should refuse file modification when in plan mode',
     approvalMode: ApprovalMode.PLAN,
@@ -32,27 +44,23 @@ describe('plan_mode', () => {
       await rig.waitForTelemetryReady();
       const toolLogs = rig.readToolLogs();
 
-      const writeTargets = toolLogs
-        .filter((log) =>
-          ['write_file', 'replace'].includes(log.toolRequest.name),
-        )
-        .map((log) => {
-          try {
-            return JSON.parse(log.toolRequest.args).file_path;
-          } catch {
-            return null;
-          }
-        });
+      const exitPlanIndex = toolLogs.findIndex(
+        (log) => log.toolRequest.name === 'exit_plan_mode',
+      );
+
+      const writeTargetsBeforeExitPlan = getWriteTargets(
+        toolLogs.slice(0, exitPlanIndex !== -1 ? exitPlanIndex : undefined),
+      );
 
       expect(
-        writeTargets,
+        writeTargetsBeforeExitPlan,
         'Should not attempt to modify README.md in plan mode',
       ).not.toContain('README.md');
 
       assertModelHasOutput(result);
       checkModelOutputContent(result, {
         expectedContent: [/plan mode|read-only|cannot modify|refuse|exiting/i],
-        testName: `${TEST_PREFIX}should refuse file modification`,
+        testName: `${TEST_PREFIX}should refuse file modification in plan mode`,
       });
     },
   });
@@ -69,24 +77,20 @@ describe('plan_mode', () => {
       await rig.waitForTelemetryReady();
       const toolLogs = rig.readToolLogs();
 
-      const writeTargets = toolLogs
-        .filter((log) =>
-          ['write_file', 'replace'].includes(log.toolRequest.name),
-        )
-        .map((log) => {
-          try {
-            return JSON.parse(log.toolRequest.args).file_path;
-          } catch {
-            return null;
-          }
-        });
+      const exitPlanIndex = toolLogs.findIndex(
+        (log) => log.toolRequest.name === 'exit_plan_mode',
+      );
+
+      const writeTargetsBeforeExit = getWriteTargets(
+        toolLogs.slice(0, exitPlanIndex !== -1 ? exitPlanIndex : undefined),
+      );
 
       // It should NOT write to the docs folder or any other repo path
-      const hasRepoWrite = writeTargets.some(
+      const hasRepoWriteBeforeExit = writeTargetsBeforeExit.some(
         (path) => path && !path.includes('/plans/'),
       );
       expect(
-        hasRepoWrite,
+        hasRepoWriteBeforeExit,
         'Should not attempt to create files in the repository while in plan mode',
       ).toBe(false);
 
@@ -166,4 +170,65 @@ describe('plan_mode', () => {
       assertModelHasOutput(result);
     },
   });
+
+  evalTest('USUALLY_PASSES', {
+    name: 'should create a plan in plan mode and implement it for a refactoring task',
+    params: {
+      settings,
+    },
+    files: {
+      'src/mathUtils.ts':
+        'export const sum = (a: number, b: number) => a + b;\nexport const multiply = (a: number, b: number) => a * b;',
+      'src/main.ts':
+        'import { sum } from "./mathUtils";\nconsole.log(sum(1, 2));',
+    },
+    prompt:
+      'I want to refactor our math utilities. Move the `sum` function from `src/mathUtils.ts` to a new file `src/basicMath.ts` and update `src/main.ts` to use the new file. Please create a detailed implementation plan first, then execute it.',
+    assert: async (rig, result) => {
+      const enterPlanCalled = await rig.waitForToolCall('enter_plan_mode');
+      expect(
+        enterPlanCalled,
+        'Expected enter_plan_mode tool to be called',
+      ).toBe(true);
+
+      const exitPlanCalled = await rig.waitForToolCall('exit_plan_mode');
+      expect(exitPlanCalled, 'Expected exit_plan_mode tool to be called').toBe(
+        true,
+      );
+
+      await rig.waitForTelemetryReady();
+      const toolLogs = rig.readToolLogs();
+
+      // Check if plan was written
+      const planWrite = toolLogs.find(
+        (log) =>
+          log.toolRequest.name === 'write_file' &&
+          log.toolRequest.args.includes('/plans/'),
+      );
+      expect(
+        planWrite,
+        'Expected a plan file to be written in the plans directory',
+      ).toBeDefined();
+
+      // Check for implementation files
+      const newFileWrite = toolLogs.find(
+        (log) =>
+          log.toolRequest.name === 'write_file' &&
+          log.toolRequest.args.includes('src/basicMath.ts'),
+      );
+      expect(
+        newFileWrite,
+        'Expected src/basicMath.ts to be created',
+      ).toBeDefined();
+
+      const mainUpdate = toolLogs.find(
+        (log) =>
+          ['write_file', 'replace'].includes(log.toolRequest.name) &&
+          log.toolRequest.args.includes('src/main.ts'),
+      );
+      expect(mainUpdate, 'Expected src/main.ts to be updated').toBeDefined();
+
+      assertModelHasOutput(result);
+    },
+  });
 });

evals/save_memory.eval.ts

@@ -16,9 +16,7 @@ describe('save_memory', () => {
   const rememberingFavoriteColor = "Agent remembers user's favorite color";
   evalTest('ALWAYS_PASSES', {
     name: rememberingFavoriteColor,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `remember that my favorite color is  blue.
   
     what is my favorite color? tell me that and surround it with $ symbol`,
@@ -38,9 +36,7 @@ describe('save_memory', () => {
   const rememberingCommandRestrictions = 'Agent remembers command restrictions';
   evalTest('USUALLY_PASSES', {
     name: rememberingCommandRestrictions,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `I don't want you to ever run npm commands.`,
     assert: async (rig, result) => {
       const wasToolCalled = await rig.waitForToolCall('save_memory');
@@ -59,9 +55,7 @@ describe('save_memory', () => {
   const rememberingWorkflow = 'Agent remembers workflow preferences';
   evalTest('USUALLY_PASSES', {
     name: rememberingWorkflow,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `I want you to always lint after building.`,
     assert: async (rig, result) => {
       const wasToolCalled = await rig.waitForToolCall('save_memory');
@@ -81,9 +75,7 @@ describe('save_memory', () => {
     'Agent ignores temporary conversation details';
   evalTest('ALWAYS_PASSES', {
     name: ignoringTemporaryInformation,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `I'm going to get a coffee.`,
     assert: async (rig, result) => {
       await rig.waitForTelemetryReady();
@@ -106,9 +98,7 @@ describe('save_memory', () => {
   const rememberingPetName = "Agent remembers user's pet's name";
   evalTest('ALWAYS_PASSES', {
     name: rememberingPetName,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `Please remember that my dog's name is Buddy.`,
     assert: async (rig, result) => {
       const wasToolCalled = await rig.waitForToolCall('save_memory');
@@ -127,9 +117,7 @@ describe('save_memory', () => {
   const rememberingCommandAlias = 'Agent remembers custom command aliases';
   evalTest('ALWAYS_PASSES', {
     name: rememberingCommandAlias,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `When I say 'start server', you should run 'npm run dev'.`,
     assert: async (rig, result) => {
       const wasToolCalled = await rig.waitForToolCall('save_memory');
@@ -149,18 +137,6 @@ describe('save_memory', () => {
     "Agent ignores workspace's database schema location";
   evalTest('USUALLY_PASSES', {
     name: ignoringDbSchemaLocation,
-    params: {
-      settings: {
-        tools: {
-          core: [
-            'save_memory',
-            'list_directory',
-            'read_file',
-            'run_shell_command',
-          ],
-        },
-      },
-    },
     prompt: `The database schema for this workspace is located in \`db/schema.sql\`.`,
     assert: async (rig, result) => {
       await rig.waitForTelemetryReady();
@@ -180,9 +156,7 @@ describe('save_memory', () => {
     "Agent remembers user's coding style preference";
   evalTest('ALWAYS_PASSES', {
     name: rememberingCodingStyle,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `I prefer to use tabs instead of spaces for indentation.`,
     assert: async (rig, result) => {
       const wasToolCalled = await rig.waitForToolCall('save_memory');
@@ -202,18 +176,6 @@ describe('save_memory', () => {
     'Agent ignores workspace build artifact location';
   evalTest('USUALLY_PASSES', {
     name: ignoringBuildArtifactLocation,
-    params: {
-      settings: {
-        tools: {
-          core: [
-            'save_memory',
-            'list_directory',
-            'read_file',
-            'run_shell_command',
-          ],
-        },
-      },
-    },
     prompt: `In this workspace, build artifacts are stored in the \`dist/artifacts\` directory.`,
     assert: async (rig, result) => {
       await rig.waitForTelemetryReady();
@@ -232,18 +194,6 @@ describe('save_memory', () => {
   const ignoringMainEntryPoint = "Agent ignores workspace's main entry point";
   evalTest('USUALLY_PASSES', {
     name: ignoringMainEntryPoint,
-    params: {
-      settings: {
-        tools: {
-          core: [
-            'save_memory',
-            'list_directory',
-            'read_file',
-            'run_shell_command',
-          ],
-        },
-      },
-    },
     prompt: `The main entry point for this workspace is \`src/index.js\`.`,
     assert: async (rig, result) => {
       await rig.waitForTelemetryReady();
@@ -262,9 +212,7 @@ describe('save_memory', () => {
   const rememberingBirthday = "Agent remembers user's birthday";
   evalTest('ALWAYS_PASSES', {
     name: rememberingBirthday,
-    params: {
-      settings: { tools: { core: ['save_memory'] } },
-    },
+
     prompt: `My birthday is on June 15th.`,
     assert: async (rig, result) => {
       const wasToolCalled = await rig.waitForToolCall('save_memory');
@@ -279,4 +227,136 @@ describe('save_memory', () => {
       });
     },
   });
+
+  const proactiveMemoryFromLongSession =
+    'Agent saves preference from earlier in conversation history';
+  evalTest('USUALLY_PASSES', {
+    name: proactiveMemoryFromLongSession,
+    params: {
+      settings: {
+        experimental: { memoryManager: true },
+      },
+    },
+    messages: [
+      {
+        id: 'msg-1',
+        type: 'user',
+        content: [
+          {
+            text: 'By the way, I always prefer Vitest over Jest for testing in all my projects.',
+          },
+        ],
+        timestamp: '2026-01-01T00:00:00Z',
+      },
+      {
+        id: 'msg-2',
+        type: 'gemini',
+        content: [{ text: 'Noted! What are you working on today?' }],
+        timestamp: '2026-01-01T00:00:05Z',
+      },
+      {
+        id: 'msg-3',
+        type: 'user',
+        content: [
+          {
+            text: "I'm debugging a failing API endpoint. The /users route returns a 500 error.",
+          },
+        ],
+        timestamp: '2026-01-01T00:01:00Z',
+      },
+      {
+        id: 'msg-4',
+        type: 'gemini',
+        content: [
+          {
+            text: 'It looks like the database connection might not be initialized before the query runs.',
+          },
+        ],
+        timestamp: '2026-01-01T00:01:10Z',
+      },
+      {
+        id: 'msg-5',
+        type: 'user',
+        content: [
+          { text: 'Good catch — I fixed the import and the route works now.' },
+        ],
+        timestamp: '2026-01-01T00:02:00Z',
+      },
+      {
+        id: 'msg-6',
+        type: 'gemini',
+        content: [{ text: 'Great! Anything else you would like to work on?' }],
+        timestamp: '2026-01-01T00:02:05Z',
+      },
+    ],
+    prompt:
+      'Please save any persistent preferences or facts about me from our conversation to memory.',
+    assert: async (rig, result) => {
+      const wasToolCalled = await rig.waitForToolCall(
+        'save_memory',
+        undefined,
+        (args) => /vitest/i.test(args),
+      );
+      expect(
+        wasToolCalled,
+        'Expected save_memory to be called with the Vitest preference from the conversation history',
+      ).toBe(true);
+
+      assertModelHasOutput(result);
+    },
+  });
+
+  const memoryManagerRoutingPreferences =
+    'Agent routes global and project preferences to memory';
+  evalTest('USUALLY_PASSES', {
+    name: memoryManagerRoutingPreferences,
+    params: {
+      settings: {
+        experimental: { memoryManager: true },
+      },
+    },
+    messages: [
+      {
+        id: 'msg-1',
+        type: 'user',
+        content: [
+          {
+            text: 'I always use dark mode in all my editors and terminals.',
+          },
+        ],
+        timestamp: '2026-01-01T00:00:00Z',
+      },
+      {
+        id: 'msg-2',
+        type: 'gemini',
+        content: [{ text: 'Got it, I will keep that in mind!' }],
+        timestamp: '2026-01-01T00:00:05Z',
+      },
+      {
+        id: 'msg-3',
+        type: 'user',
+        content: [
+          {
+            text: 'For this project specifically, we use 2-space indentation.',
+          },
+        ],
+        timestamp: '2026-01-01T00:01:00Z',
+      },
+      {
+        id: 'msg-4',
+        type: 'gemini',
+        content: [
+          { text: 'Understood, 2-space indentation for this project.' },
+        ],
+        timestamp: '2026-01-01T00:01:05Z',
+      },
+    ],
+    prompt: 'Please save the preferences I mentioned earlier to memory.',
+    assert: async (rig, result) => {
+      const wasToolCalled = await rig.waitForToolCall('save_memory');
+      expect(wasToolCalled, 'Expected save_memory to be called').toBe(true);
+
+      assertModelHasOutput(result);
+    },
+  });
 });

evals/subagents.eval.ts

@@ -4,21 +4,41 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { describe } from 'vitest';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { describe, expect } from 'vitest';
+
 import { evalTest } from './test-helper.js';
 
-const AGENT_DEFINITION = `---
+const DOCS_AGENT_DEFINITION = `---
 name: docs-agent
 description: An agent with expertise in updating documentation.
 tools:
   - read_file
   - write_file
 ---
-
-You are the docs agent. Update the documentation.
+You are the docs agent. Update documentation clearly and accurately.
 `;
 
-const INDEX_TS = 'export const add = (a: number, b: number) => a + b;';
+const TEST_AGENT_DEFINITION = `---
+name: test-agent
+description: An agent with expertise in writing and updating tests.
+tools:
+  - read_file
+  - write_file
+---
+You are the test agent. Add or update tests.
+`;
+
+const INDEX_TS = 'export const add = (a: number, b: number) => a + b;\n';
+
+function readProjectFile(
+  rig: { testDir?: string },
+  relativePath: string,
+): string {
+  return fs.readFileSync(path.join(rig.testDir!, relativePath), 'utf8');
+}
 
 describe('subagent eval test cases', () => {
   /**
@@ -42,12 +62,147 @@ describe('subagent eval test cases', () => {
     },
     prompt: 'Please update README.md with a description of this library.',
     files: {
-      '.gemini/agents/test-agent.md': AGENT_DEFINITION,
+      '.gemini/agents/docs-agent.md': DOCS_AGENT_DEFINITION,
       'index.ts': INDEX_TS,
-      'README.md': 'TODO: update the README.',
+      'README.md': 'TODO: update the README.\n',
     },
     assert: async (rig, _result) => {
       await rig.expectToolCallSuccess(['docs-agent']);
     },
   });
+
+  /**
+   * Checks that the outer agent does not over-delegate trivial work when
+   * subagents are available. This helps catch orchestration overuse.
+   */
+  evalTest('USUALLY_PASSES', {
+    name: 'should avoid delegating trivial direct edit work',
+    params: {
+      settings: {
+        experimental: {
+          enableAgents: true,
+          agents: {
+            overrides: {
+              generalist: { enabled: true },
+            },
+          },
+        },
+      },
+    },
+    prompt:
+      'Rename the exported function in index.ts from add to sum and update the file directly.',
+    files: {
+      '.gemini/agents/docs-agent.md': DOCS_AGENT_DEFINITION,
+      'index.ts': INDEX_TS,
+    },
+    assert: async (rig, _result) => {
+      const updatedIndex = readProjectFile(rig, 'index.ts');
+      const toolLogs = rig.readToolLogs() as Array<{
+        toolRequest: { name: string };
+      }>;
+
+      expect(updatedIndex).toContain('export const sum =');
+      expect(toolLogs.some((l) => l.toolRequest.name === 'docs-agent')).toBe(
+        false,
+      );
+      expect(toolLogs.some((l) => l.toolRequest.name === 'generalist')).toBe(
+        false,
+      );
+    },
+  });
+
+  /**
+   * Checks that the outer agent prefers a more relevant specialist over a
+   * broad generalist when both are available.
+   *
+   * This is meant to codify the "overusing Generalist" failure mode.
+   */
+  evalTest('USUALLY_PASSES', {
+    name: 'should prefer relevant specialist over generalist',
+    params: {
+      settings: {
+        experimental: {
+          enableAgents: true,
+          agents: {
+            overrides: {
+              generalist: { enabled: true },
+            },
+          },
+        },
+      },
+    },
+    prompt: 'Please add a small test file that verifies add(1, 2) returns 3.',
+    files: {
+      '.gemini/agents/test-agent.md': TEST_AGENT_DEFINITION,
+      'index.ts': INDEX_TS,
+      'package.json': JSON.stringify(
+        {
+          name: 'subagent-eval-project',
+          version: '1.0.0',
+          type: 'module',
+        },
+        null,
+        2,
+      ),
+    },
+    assert: async (rig, _result) => {
+      const toolLogs = rig.readToolLogs() as Array<{
+        toolRequest: { name: string };
+      }>;
+
+      await rig.expectToolCallSuccess(['test-agent']);
+      expect(toolLogs.some((l) => l.toolRequest.name === 'generalist')).toBe(
+        false,
+      );
+    },
+  });
+
+  /**
+   * Checks cardinality and decomposition for a multi-surface task. The task
+   * naturally spans docs and tests, so multiple specialists should be used.
+   */
+  evalTest('USUALLY_PASSES', {
+    name: 'should use multiple relevant specialists for multi-surface task',
+    params: {
+      settings: {
+        experimental: {
+          enableAgents: true,
+          agents: {
+            overrides: {
+              generalist: { enabled: true },
+            },
+          },
+        },
+      },
+    },
+    prompt:
+      'Add a short README description for this library and also add a test file that verifies add(1, 2) returns 3.',
+    files: {
+      '.gemini/agents/docs-agent.md': DOCS_AGENT_DEFINITION,
+      '.gemini/agents/test-agent.md': TEST_AGENT_DEFINITION,
+      'index.ts': INDEX_TS,
+      'README.md': 'TODO: update the README.\n',
+      'package.json': JSON.stringify(
+        {
+          name: 'subagent-eval-project',
+          version: '1.0.0',
+          type: 'module',
+        },
+        null,
+        2,
+      ),
+    },
+    assert: async (rig, _result) => {
+      const toolLogs = rig.readToolLogs() as Array<{
+        toolRequest: { name: string };
+      }>;
+      const readme = readProjectFile(rig, 'README.md');
+
+      await rig.expectToolCallSuccess(['docs-agent', 'test-agent']);
+      expect(readme).not.toContain('TODO: update the README.');
+      expect(toolLogs.some((l) => l.toolRequest.name === 'generalist')).toBe(
+        false,
+      );
+    },
+  });
 });

evals/test-helper.ts

@@ -13,6 +13,9 @@ import { TestRig } from '@google/gemini-cli-test-utils';
 import {
   createUnauthorizedToolError,
   parseAgentMarkdown,
+  Storage,
+  getProjectHash,
+  SESSION_FILE_PREFIX,
 } from '@google/gemini-cli-core';
 
 export * from '@google/gemini-cli-test-utils';
@@ -117,8 +120,57 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
         execSync('git commit --allow-empty -m "Initial commit"', execOptions);
       }
 
+      // If messages are provided, write a session file so --resume can load it.
+      let sessionId: string | undefined;
+      if (evalCase.messages) {
+        sessionId =
+          evalCase.sessionId ||
+          `test-session-${crypto.randomUUID().slice(0, 8)}`;
+
+        // Temporarily set GEMINI_CLI_HOME so Storage writes to the same
+        // directory the CLI subprocess will use (rig.homeDir).
+        const originalGeminiHome = process.env['GEMINI_CLI_HOME'];
+        process.env['GEMINI_CLI_HOME'] = rig.homeDir!;
+        try {
+          const storage = new Storage(fs.realpathSync(rig.testDir!));
+          await storage.initialize();
+          const chatsDir = path.join(storage.getProjectTempDir(), 'chats');
+          fs.mkdirSync(chatsDir, { recursive: true });
+
+          const conversation = {
+            sessionId,
+            projectHash: getProjectHash(fs.realpathSync(rig.testDir!)),
+            startTime: new Date().toISOString(),
+            lastUpdated: new Date().toISOString(),
+            messages: evalCase.messages,
+          };
+
+          const timestamp = new Date()
+            .toISOString()
+            .slice(0, 16)
+            .replace(/:/g, '-');
+          const filename = `${SESSION_FILE_PREFIX}${timestamp}-${sessionId.slice(0, 8)}.json`;
+          fs.writeFileSync(
+            path.join(chatsDir, filename),
+            JSON.stringify(conversation, null, 2),
+          );
+        } catch (e) {
+          // Storage initialization may fail in some environments; log and continue.
+          console.warn('Failed to write session history:', e);
+        } finally {
+          // Restore original GEMINI_CLI_HOME.
+          if (originalGeminiHome === undefined) {
+            delete process.env['GEMINI_CLI_HOME'];
+          } else {
+            process.env['GEMINI_CLI_HOME'] = originalGeminiHome;
+          }
+        }
+      }
+
       const result = await rig.run({
-        args: evalCase.prompt,
+        args: sessionId
+          ? ['--resume', sessionId, evalCase.prompt]
+          : evalCase.prompt,
         approvalMode: evalCase.approvalMode ?? 'yolo',
         timeout: evalCase.timeout,
         env: {
@@ -197,12 +249,32 @@ export function symlinkNodeModules(testDir: string) {
   }
 }
 
+/**
+ * Settings that are forbidden in evals. Evals should never restrict which
+ * tools are available — they must test against the full, default tool set
+ * to ensure realistic behavior.
+ */
+interface ForbiddenToolSettings {
+  tools?: {
+    /** Restricting core tools in evals is forbidden. */
+    core?: never;
+    [key: string]: unknown;
+  };
+}
+
 export interface EvalCase {
   name: string;
-  params?: Record<string, any>;
+  params?: {
+    settings?: ForbiddenToolSettings & Record<string, unknown>;
+    [key: string]: unknown;
+  };
   prompt: string;
   timeout?: number;
   files?: Record<string, string>;
+  /** Conversation history to pre-load via --resume. Each entry is a message object with type, content, etc. */
+  messages?: Record<string, unknown>[];
+  /** Session ID for the resumed session. Auto-generated if not provided. */
+  sessionId?: string;
   approvalMode?: 'default' | 'auto_edit' | 'yolo' | 'plan';
   assert: (rig: TestRig, result: string) => Promise<void>;
 }

evals/vitest.config.ts

@@ -16,6 +16,10 @@ export default defineConfig({
   },
   test: {
     testTimeout: 300000, // 5 minutes
+    // Retry in CI but not nightly to avoid blocking on API error.
+    retry: process.env['VITEST_RETRY']
+      ? parseInt(process.env['VITEST_RETRY'], 10)
+      : 3,
     reporters: ['default', 'json'],
     outputFile: {
       json: 'evals/logs/report.json',