feat(evals): add tool call formatter and integrate failure summaries (#28305)

Co-authored-by: David Pierce <davidapierce@google.com>

evals/hierarchical_memory.eval.ts

@@ -8,12 +8,10 @@ import { describe, expect } from 'vitest';
 import { evalTest, assertModelHasOutput } from './test-helper.js';
 
 describe('Hierarchical Memory', () => {
-  const conflictResolutionTest =
-    'Agent follows hierarchy for contradictory instructions';
   evalTest('ALWAYS_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: conflictResolutionTest,
+    name: 'Agent follows hierarchy for contradictory instructions',
     params: {
       settings: {
         security: {
@@ -47,11 +45,10 @@ What is my favorite fruit? Tell me just the name of the fruit.`,
     },
   });
 
-  const provenanceAwarenessTest = 'Agent is aware of memory provenance';
   evalTest('USUALLY_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: provenanceAwarenessTest,
+    name: 'Agent is aware of memory provenance',
     params: {
       settings: {
         security: {
@@ -88,11 +85,10 @@ Provide the answer as an XML block like this:
     },
   });
 
-  const extensionVsGlobalTest = 'Extension memory wins over Global memory';
   evalTest('ALWAYS_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: extensionVsGlobalTest,
+    name: 'Extension memory wins over Global memory',
     params: {
       settings: {
         security: {

evals/memory_persistence.eval.ts

@@ -74,12 +74,10 @@ async function waitForSessionScratchpad(
 }
 
 describe('memory persistence', () => {
-  const proactiveMemoryFromLongSession =
-    'Agent saves preference from earlier in conversation history';
   evalTest('USUALLY_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: proactiveMemoryFromLongSession,
+    name: 'Agent saves preference from earlier in conversation history',
     messages: [
       {
         id: 'msg-1',
@@ -195,12 +193,10 @@ describe('memory persistence', () => {
     },
   });
 
-  const memoryRoutesTeamConventionsToProjectGemini =
-    'Agent routes team-shared project conventions to ./GEMINI.md';
   evalTest('USUALLY_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: memoryRoutesTeamConventionsToProjectGemini,
+    name: 'Agent routes team-shared project conventions to ./GEMINI.md',
     messages: [
       {
         id: 'msg-1',
@@ -303,12 +299,10 @@ describe('memory persistence', () => {
     },
   });
 
-  const memorySessionScratchpad =
-    'Session summary persists memory scratchpad for memory-saving sessions';
   evalTest('USUALLY_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: memorySessionScratchpad,
+    name: 'Session summary persists memory scratchpad for memory-saving sessions',
     sessionId: 'memory-scratchpad-eval',
     messages: [
       {
@@ -395,12 +389,10 @@ describe('memory persistence', () => {
     },
   });
 
-  const memoryRoutesUserProject =
-    'Agent routes personal-to-user project notes to user-project memory';
   evalTest('USUALLY_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: memoryRoutesUserProject,
+    name: 'Agent routes personal-to-user project notes to user-project memory',
     prompt: `Please remember my personal local dev setup for THIS project's Postgres database. This is private to my machine — do NOT commit it to the repo.
 
 Connection details:
@@ -486,12 +478,10 @@ Quirks to remember:
     },
   });
 
-  const memoryRoutesCrossProjectToGlobal =
-    'Agent routes cross-project personal preferences to ~/.gemini/GEMINI.md';
   evalTest('USUALLY_PASSES', {
     suiteName: 'default',
     suiteType: 'behavioral',
-    name: memoryRoutesCrossProjectToGlobal,
+    name: 'Agent routes cross-project personal preferences to ~/.gemini/GEMINI.md',
     prompt:
       'Please remember this about me in general: across all my projects I always prefer Prettier with single quotes and trailing commas, and I always prefer tabs over spaces for indentation. These are my personal coding-style defaults that follow me into every workspace.',
     assert: async (rig, result) => {

evals/snapshot_fidelity.eval.ts

@@ -21,7 +21,7 @@ function snapshotEvalTest(policy: EvalPolicy, evalCase: ComponentEvalCase) {
 describe('snapshot_fidelity', () => {
   snapshotEvalTest('ALWAYS_PASSES', {
     suiteName: 'default',
-    suiteType: 'behavioral',
+    suiteType: 'component-level',
     name: 'SnapshotGenerator strictly retains specific empirical facts',
     assert: async (config) => {
       // 1. Construct a highly specific mock transcript containing 3 empirical facts we can test for:

evals/test-helper.test.ts

@@ -216,4 +216,109 @@ describe('evalTest reliability logic', () => {
       }
     }
   });
+
+  it('should append tool call chain to assertion failure error messages', async () => {
+    const mockRig = {
+      setup: vi.fn(),
+      run: vi.fn(),
+      cleanup: vi.fn(),
+      readToolLogs: vi.fn().mockReturnValue([]),
+      _lastRunStderr: '',
+    } as any;
+    (TestRig as any).mockReturnValue(mockRig);
+
+    mockRig.run.mockResolvedValue('Success');
+    mockRig.readToolLogs.mockReturnValue([
+      {
+        toolRequest: {
+          name: 'grep_search',
+          args: '{"query":"TODO"}',
+          success: true,
+          duration_ms: 42,
+        },
+      },
+      {
+        toolRequest: {
+          name: 'read_file',
+          args: '{"path":"/src/foo.ts"}',
+          success: false,
+          duration_ms: 15,
+          error: 'File not found',
+          error_type: 'ENOENT',
+        },
+      },
+    ]);
+
+    const assertionError = new Error('Expected tool to be called');
+
+    try {
+      await internalEvalTest({
+        suiteName: 'test',
+        suiteType: 'behavioral',
+        name: 'test-tool-chain',
+        prompt: 'do something',
+        assert: async () => {
+          throw assertionError;
+        },
+      });
+      expect.unreachable('Expected internalEvalTest to throw');
+    } catch (error: unknown) {
+      expect(error).toBeInstanceOf(Error);
+      const msg = (error as Error).message;
+      expect(msg).toContain('Expected tool to be called');
+      expect(msg).toContain('Tool Call Chain (2 calls)');
+      expect(msg).toContain('grep_search');
+      expect(msg).toContain('read_file');
+      expect(msg).toContain('[ENOENT] File not found');
+    }
+  });
+
+  it('should not crash when error.message is read-only (frozen error)', async () => {
+    const mockRig = {
+      setup: vi.fn(),
+      run: vi.fn(),
+      cleanup: vi.fn(),
+      readToolLogs: vi.fn(),
+      _lastRunStderr: '',
+    } as any;
+    (TestRig as any).mockReturnValue(mockRig);
+
+    mockRig.run.mockResolvedValue('Success');
+    mockRig.readToolLogs.mockReturnValue([
+      {
+        toolRequest: {
+          name: 'read_file',
+          args: '{"path":"/foo.ts"}',
+          success: true,
+          duration_ms: 10,
+        },
+      },
+    ]);
+
+    // Simulate a frozen error whose message property cannot be mutated
+    const frozenError = Object.freeze(new Error('Frozen assertion error'));
+
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    try {
+      await expect(
+        internalEvalTest({
+          suiteName: 'test',
+          suiteType: 'behavioral',
+          name: 'test-frozen-error',
+          prompt: 'do something',
+          assert: async () => {
+            throw frozenError;
+          },
+        }),
+      ).rejects.toThrow('Frozen assertion error');
+
+      // Should have warned that the message could not be mutated
+      expect(warnSpy).toHaveBeenCalledWith(
+        expect.stringContaining('Could not append tool call chain'),
+      );
+    } finally {
+      warnSpy.mockRestore();
+    }
+  });
 });

evals/test-helper.ts

@@ -10,6 +10,7 @@ import path from 'node:path';
 import crypto from 'node:crypto';
 import { execSync } from 'node:child_process';
 import { TestRig } from '@google/gemini-cli-test-utils';
+import { formatToolLogChain } from '../scripts/utils/tool-log-formatter.js';
 import {
   createUnauthorizedToolError,
   parseAgentMarkdown,
@@ -186,6 +187,23 @@ export async function internalEvalTest(evalCase: EvalCase) {
 
       await evalCase.assert(rig, result);
       isSuccess = true;
+    } catch (error: unknown) {
+      const toolLogs = rig.readToolLogs();
+      if (toolLogs && toolLogs.length > 0) {
+        const summary = formatToolLogChain(toolLogs);
+        if (error instanceof Error) {
+          try {
+            error.message = `${error.message}\n\nTool Call Chain (${toolLogs.length} calls):\n${summary}`;
+          } catch {
+            // Error object may be frozen or have a read-only message property.
+            // The original error is still re-thrown, so no failure is hidden.
+            console.warn(
+              `[eval] Could not append tool call chain to error message (${toolLogs.length} calls)`,
+            );
+          }
+        }
+      }
+      throw error;
     } finally {
       if (isSuccess) {
         await fs.promises.unlink(activityLogFile).catch((err) => {

evals/tsconfig.json

@@ -7,7 +7,7 @@
       "@google/gemini-cli": ["../packages/cli/index.ts"]
     }
   },
-  "include": ["**/*.ts"],
+  "include": ["**/*.ts", "../scripts/utils/tool-log-formatter.ts"],
   "exclude": ["logs"],
   "references": [{ "path": "../packages/core" }, { "path": "../packages/cli" }]
 }

package.json

@@ -32,6 +32,7 @@
     "schema:settings": "tsx ./scripts/generate-settings-schema.ts",
     "docs:settings": "tsx ./scripts/generate-settings-doc.ts",
     "docs:keybindings": "tsx ./scripts/generate-keybindings-doc.ts",
+    "eval:validate": "tsx ./scripts/eval-validate-cli.ts",
     "eval:inventory": "tsx ./scripts/eval-inventory-cli.ts",
     "eval:inventory:json": "tsx ./scripts/eval-inventory-cli.ts --json",
     "eval:report": "tsx ./scripts/eval-report-cli.ts",

packages/core/src/ide/ide-connection-utils.test.ts

@@ -31,6 +31,21 @@ vi.mock('node:fs', async (importOriginal) => {
       ...actual.promises,
       readFile: vi.fn(),
       readdir: vi.fn(),
+      realpath: vi.fn((p) => Promise.resolve(p)),
+      stat: vi.fn(() =>
+        Promise.resolve({ uid: process.getuid ? process.getuid() : 1000 }),
+      ),
+      open: vi.fn((filePath: string) =>
+        Promise.resolve({
+          stat: () => fs.promises.stat(filePath),
+          readFile: (options?: string | { encoding?: string }) =>
+            fs.promises.readFile(
+              filePath,
+              options as unknown as BufferEncoding | undefined,
+            ),
+          close: () => Promise.resolve(),
+        } as unknown as fs.promises.FileHandle),
+      ),
     },
     realpathSync: (p: string) => p,
     existsSync: vi.fn(() => false),
@@ -430,6 +445,141 @@ describe('ide-connection-utils', () => {
 
       expect(result).toEqual(config2);
     });
+
+    it('should NOT filter out config if all found config files are mismatched/invalid workspaces, returning the best sorted match so that the correct Directory Mismatch error is raised downstream', async () => {
+      const invalidConfig1 = {
+        port: '1111',
+        workspacePath: '/invalid/workspace1',
+      };
+      const invalidConfig2 = {
+        port: '2222',
+        workspacePath: '/invalid/workspace2',
+      };
+      vi.mocked(fs.promises.readFile).mockRejectedValueOnce(
+        new Error('not found'),
+      );
+      (
+        vi.mocked(fs.promises.readdir) as Mock<
+          (path: fs.PathLike) => Promise<string[]>
+        >
+      ).mockResolvedValue([
+        'gemini-ide-server-12345-111.json',
+        'gemini-ide-server-12345-222.json',
+      ]);
+      vi.mocked(fs.promises.readFile)
+        .mockResolvedValueOnce(JSON.stringify(invalidConfig1))
+        .mockResolvedValueOnce(JSON.stringify(invalidConfig2));
+
+      const result = await getConnectionConfigFromFile(12345);
+
+      expect(result).toEqual(invalidConfig1);
+    });
+
+    it('should prioritize the config matching the port from the environment variable when all found config files are mismatched/invalid workspaces', async () => {
+      vi.stubEnv('GEMINI_CLI_IDE_SERVER_PORT', '2222');
+      const invalidConfig1 = {
+        port: '1111',
+        workspacePath: '/invalid/workspace1',
+      };
+      const invalidConfig2 = {
+        port: '2222',
+        workspacePath: '/invalid/workspace2',
+      };
+      vi.mocked(fs.promises.readFile).mockRejectedValueOnce(
+        new Error('not found'),
+      );
+      (
+        vi.mocked(fs.promises.readdir) as Mock<
+          (path: fs.PathLike) => Promise<string[]>
+        >
+      ).mockResolvedValue([
+        'gemini-ide-server-12345-111.json',
+        'gemini-ide-server-12345-222.json',
+      ]);
+      vi.mocked(fs.promises.readFile)
+        .mockResolvedValueOnce(JSON.stringify(invalidConfig1))
+        .mockResolvedValueOnce(JSON.stringify(invalidConfig2));
+
+      const result = await getConnectionConfigFromFile(12345);
+
+      expect(result).toEqual(invalidConfig2);
+    });
+
+    it.runIf(process.getuid !== undefined)(
+      'should reject and ignore config files owned by a different user UID to prevent hijacking/information disclosure',
+      async () => {
+        const config1 = {
+          port: '1111',
+          workspacePath: '/test/workspace',
+        };
+        vi.mocked(fs.promises.readFile).mockRejectedValueOnce(
+          new Error('not found'),
+        );
+        (
+          vi.mocked(fs.promises.readdir) as Mock<
+            (path: fs.PathLike) => Promise<string[]>
+          >
+        ).mockResolvedValue(['gemini-ide-server-12345-111.json']);
+        vi.mocked(fs.promises.readFile).mockResolvedValueOnce(
+          JSON.stringify(config1),
+        );
+
+        const otherUid = (process.getuid ? process.getuid() : 1000) + 1;
+        vi.mocked(fs.promises.stat).mockResolvedValueOnce({
+          uid: otherUid,
+        } as unknown as fs.Stats);
+
+        const result = await getConnectionConfigFromFile(12345);
+
+        expect(result).toBeUndefined();
+      },
+    );
+
+    it('should accept and parse config files owned by the current user UID', async () => {
+      const config1 = {
+        port: '1111',
+        workspacePath: '/test/workspace',
+      };
+      vi.mocked(fs.promises.readFile).mockRejectedValueOnce(
+        new Error('not found'),
+      );
+      (
+        vi.mocked(fs.promises.readdir) as Mock<
+          (path: fs.PathLike) => Promise<string[]>
+        >
+      ).mockResolvedValue(['gemini-ide-server-12345-111.json']);
+      vi.mocked(fs.promises.readFile).mockResolvedValueOnce(
+        JSON.stringify(config1),
+      );
+
+      const currentUid = process.getuid ? process.getuid() : 1000;
+      vi.mocked(fs.promises.stat).mockResolvedValueOnce({
+        uid: currentUid,
+      } as unknown as fs.Stats);
+
+      const result = await getConnectionConfigFromFile(12345);
+
+      expect(result).toEqual(config1);
+    });
+
+    it('should reject and ignore config files if fs.promises.open throws an error', async () => {
+      vi.mocked(fs.promises.readFile).mockRejectedValueOnce(
+        new Error('not found'),
+      );
+      (
+        vi.mocked(fs.promises.readdir) as Mock<
+          (path: fs.PathLike) => Promise<string[]>
+        >
+      ).mockResolvedValue(['gemini-ide-server-12345-111.json']);
+
+      vi.mocked(fs.promises.open).mockRejectedValueOnce(
+        new Error('symlink loop / permission denied'),
+      );
+
+      const result = await getConnectionConfigFromFile(12345);
+
+      expect(result).toBeUndefined();
+    });
   });
 
   describe('validateWorkspacePath', () => {

packages/core/src/ide/ide-connection-utils.ts

@@ -109,6 +109,26 @@ export function getStdioConfigFromEnv(): StdioConfig | undefined {
 
 const IDE_SERVER_FILE_REGEX = /^gemini-ide-server-(\d+)-\d+\.json$/;
 
+async function verifyAndReadFile(
+  filePath: string,
+): Promise<string | undefined> {
+  let handle: fs.promises.FileHandle | undefined;
+  try {
+    handle = await fs.promises.open(filePath, 'r');
+    const stat = await handle.stat();
+    if (process.getuid && stat.uid !== process.getuid()) {
+      return undefined;
+    }
+    return await handle.readFile('utf8');
+  } catch {
+    return undefined;
+  } finally {
+    if (handle) {
+      await handle.close();
+    }
+  }
+}
+
 export async function getConnectionConfigFromFile(
   pid: number,
 ): Promise<
@@ -122,7 +142,10 @@ export async function getConnectionConfigFromFile(
       'ide',
       `gemini-ide-server-${pid}.json`,
     );
-    const portFileContents = await fs.promises.readFile(portFile, 'utf8');
+    const portFileContents = await verifyAndReadFile(portFile);
+    if (!portFileContents) {
+      throw new Error('Verification failed or file not found');
+    }
     const parsed: unknown = JSON.parse(portFileContents);
     type ConfigType = ConnectionConfig & {
       workspacePath?: string;
@@ -164,23 +187,21 @@ export async function getConnectionConfigFromFile(
 
   sortConnectionFiles(matchingFiles, pid);
 
-  let fileContents: string[];
-  try {
-    fileContents = await Promise.all(
-      matchingFiles.map((file) =>
-        fs.promises.readFile(path.join(portFileDir, file), 'utf8'),
-      ),
-    );
-  } catch (e) {
-    logger.debug('Failed to read IDE connection config file(s):', e);
-    return undefined;
-  }
+  const fileContents = await Promise.all(
+    matchingFiles.map((file) =>
+      verifyAndReadFile(path.join(portFileDir, file)),
+    ),
+  );
+
   const parsedContents = fileContents.map(
     (
       content,
     ):
       | (ConnectionConfig & { workspacePath?: string; ideInfo?: IdeInfo })
       | undefined => {
+      if (!content) {
+        return undefined;
+      }
       try {
         const parsed: unknown = JSON.parse(content);
         type ConfigType = ConnectionConfig & {
@@ -219,6 +240,31 @@ export async function getConnectionConfigFromFile(
   );
 
   if (validWorkspaces.length === 0) {
+    // If no workspace matches the current CWD, but we found and parsed
+    // valid connection config file(s), return the best-sorted config.
+    // This lets downstream connection logic raise a helpful, detailed
+    // "Directory mismatch" warning instead of a generic connection error.
+    let fileIndex = -1;
+    const portFromEnv = getPortFromEnv();
+    if (portFromEnv) {
+      fileIndex = parsedContents.findIndex(
+        (content) =>
+          !!content &&
+          content.port !== undefined &&
+          String(content.port) === portFromEnv,
+      );
+    }
+    if (fileIndex === -1) {
+      fileIndex = parsedContents.findIndex((content) => !!content);
+    }
+
+    if (fileIndex !== -1) {
+      const selected = parsedContents[fileIndex]!;
+      logger.debug(
+        `Selected best mismatched IDE connection file: ${matchingFiles[fileIndex]}`,
+      );
+      return selected;
+    }
     return undefined;
   }
 
@@ -234,7 +280,8 @@ export async function getConnectionConfigFromFile(
   const portFromEnv = getPortFromEnv();
   if (portFromEnv) {
     const matchingPortIndex = validWorkspaces.findIndex(
-      (content) => String(content.port) === portFromEnv,
+      (content) =>
+        content.port !== undefined && String(content.port) === portFromEnv,
     );
     if (matchingPortIndex !== -1) {
       const selected = validWorkspaces[matchingPortIndex];

scripts/eval-validate-cli.ts

@@ -0,0 +1,94 @@
+#!/usr/bin/env tsx
+
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @fileoverview CLI entry point for the eval validate command.
+ *
+ * Usage:
+ *   npm run eval:validate
+ *   npm run eval:validate -- --json
+ *   npm run eval:validate -- --root /path/to/repo
+ *   npm run eval:validate -- evals/some-file.eval.ts [--json]
+ */
+
+import path from 'node:path';
+import { collectInventory } from './utils/eval-inventory.js';
+import { buildToolRegistry } from './utils/tool-registry.js';
+import {
+  validateInventory,
+  formatValidationReport,
+  formatValidationJson,
+} from './utils/eval-validate.js';
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  const rootFlagIndex = args.indexOf('--root');
+  let repoRoot: string | undefined;
+  if (rootFlagIndex !== -1) {
+    repoRoot = args[rootFlagIndex + 1];
+    if (repoRoot === undefined) {
+      console.error(
+        'Error: --root requires a directory path argument but none was provided.',
+      );
+      process.exit(1);
+    }
+    if (repoRoot.startsWith('--')) {
+      console.error(
+        `Error: --root value "${repoRoot}" looks like a flag. Provide a valid directory path.`,
+      );
+      process.exit(1);
+    }
+    args.splice(rootFlagIndex, 2);
+  }
+
+  const resolvedRoot = repoRoot ? path.resolve(repoRoot) : process.cwd();
+
+  const jsonFlagIndex = args.indexOf('--json');
+  const jsonMode = jsonFlagIndex !== -1;
+  if (jsonMode) args.splice(jsonFlagIndex, 1);
+
+  const filePaths = args.filter((a) => !a.startsWith('--'));
+
+  const inventory = await collectInventory(resolvedRoot);
+
+  if (inventory.totalFiles === 0) {
+    console.error('No eval files found under evals/.');
+    process.exit(1);
+  }
+
+  const registry = buildToolRegistry();
+  const result = validateInventory(inventory, registry, {
+    filePaths: filePaths.length > 0 ? filePaths : undefined,
+  });
+
+  if (result.unmatchedFilePaths && result.unmatchedFilePaths.length > 0) {
+    console.error(
+      'Error: The following requested file(s) were not found or did not contain any eval cases:',
+    );
+    for (const f of result.unmatchedFilePaths) {
+      console.error(`  - ${f}`);
+    }
+    process.exit(1);
+  }
+
+  if (jsonMode) {
+    console.log(formatValidationJson(result, resolvedRoot));
+  } else {
+    console.log(formatValidationReport(result, resolvedRoot));
+  }
+
+  if (result.totalViolations > 0) {
+    process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error('Fatal error:', error);
+  process.exit(1);
+});

scripts/tests/eval-validate.test.ts

@@ -0,0 +1,745 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import path from 'node:path';
+import { execSync } from 'node:child_process';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('node:child_process', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('node:child_process')>();
+  return {
+    ...actual,
+    execSync: vi.fn().mockReturnValue(''),
+  };
+});
+import {
+  validateInventory,
+  formatValidationReport,
+  formatValidationJson,
+  VALIDATION_RULES,
+  type ValidationJsonOutput,
+} from '../utils/eval-validate.js';
+import { buildToolRegistry } from '../utils/tool-registry.js';
+import { type InventoryResult } from '../utils/eval-inventory.js';
+import type {
+  EvalCaseRecord,
+  EvalFileAnalysis,
+  EvalAnalysisDiagnostic,
+} from '../utils/eval-analysis.js';
+
+function makeCase(overrides: Partial<EvalCaseRecord> = {}): EvalCaseRecord {
+  return {
+    filePath: '/repo/evals/test.eval.ts',
+    relativePath: 'evals/test.eval.ts',
+    helperName: 'evalTest',
+    baseHelperName: 'evalTest',
+    policy: 'USUALLY_PASSES',
+    name: 'test case',
+    suiteName: 'default',
+    suiteType: 'behavioral',
+    hasFiles: false,
+    hasSetup: false,
+    hasPrompt: true,
+    hasAssertBody: false,
+    prompt: 'Describe how the function works.',
+    toolReferences: ['grep_search'],
+    location: { line: 10, column: 3 },
+    ...overrides,
+  };
+}
+
+function makeFile(
+  cases: EvalCaseRecord[],
+  overrides: Partial<EvalFileAnalysis> = {},
+): EvalFileAnalysis {
+  const firstCase = cases[0];
+  return {
+    filePath: firstCase?.filePath ?? '/repo/evals/test.eval.ts',
+    relativePath: firstCase?.relativePath ?? 'evals/test.eval.ts',
+    helpers: { evalTest: 'evalTest' },
+    cases,
+    toolReferences: [],
+    diagnostics: [],
+    ...overrides,
+  };
+}
+
+function makeInventory(
+  files: EvalFileAnalysis[],
+  overrides: Partial<InventoryResult> = {},
+): InventoryResult {
+  const allCases = files.flatMap((f) => f.cases);
+  const allDiagnostics = files.flatMap((f) => [...f.diagnostics]);
+  return {
+    totalFiles: files.length,
+    totalCases: allCases.length,
+    repoRoot: '/repo',
+    files,
+    cases: allCases,
+    diagnostics: allDiagnostics,
+    ...overrides,
+  };
+}
+
+const FIXED_DATE = new Date('2026-07-05T00:00:00.000Z');
+
+describe('eval-validate', () => {
+  const registry = buildToolRegistry();
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+  });
+
+  describe('VALIDATION_RULES', () => {
+    it('exports the expected nine rules', () => {
+      expect(VALIDATION_RULES).toHaveLength(9);
+      const ids = VALIDATION_RULES.map((r) => r.id);
+      expect(ids).toEqual([
+        'file-naming',
+        'valid-policy',
+        'suite-metadata',
+        'prompt-presence',
+        'case-name-static',
+        'invalid-tool-refs',
+        'positive-assertion',
+        'workspace-setup',
+        'new-evals-policy',
+      ]);
+    });
+
+    it('every rule has a non-empty description', () => {
+      for (const rule of VALIDATION_RULES) {
+        expect(rule.description.length).toBeGreaterThan(0);
+      }
+    });
+  });
+
+  describe('validateInventory', () => {
+    it('returns zero violations for a well-formed eval', () => {
+      const inventory = makeInventory([makeFile([makeCase()])]);
+      const result = validateInventory(inventory, registry);
+
+      expect(result.totalFiles).toBe(1);
+      expect(result.totalCases).toBe(1);
+      expect(result.totalViolations).toBe(0);
+      expect(result.validFiles).toBe(1);
+      expect(result.invalidFiles).toBe(0);
+      expect(result.violations).toEqual([]);
+    });
+
+    it('returns correct file summaries for clean inventory', () => {
+      const inventory = makeInventory([
+        makeFile([
+          makeCase({ name: 'case one' }),
+          makeCase({ name: 'case two' }),
+        ]),
+      ]);
+      const result = validateInventory(inventory, registry);
+
+      expect(result.fileSummaries).toEqual([
+        {
+          relativePath: 'evals/test.eval.ts',
+          totalCases: 2,
+          violationCount: 0,
+        },
+      ]);
+    });
+
+    it('flags file-naming violation for non-.eval.ts files', () => {
+      const c = makeCase({
+        filePath: '/repo/evals/bad-name.ts',
+        relativePath: 'evals/bad-name.ts',
+      });
+      const f = makeFile([c], {
+        filePath: '/repo/evals/bad-name.ts',
+        relativePath: 'evals/bad-name.ts',
+      });
+      const result = validateInventory(makeInventory([f]), registry);
+
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'file-naming',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain('bad-name.ts');
+    });
+
+    it('checks relativePath for file-naming when filePath differs', () => {
+      const c = makeCase({
+        filePath: '/repo/evals/good.eval.ts',
+        relativePath: 'evals/bad-name.ts',
+      });
+      const f = makeFile([c], {
+        filePath: '/repo/evals/good.eval.ts',
+        relativePath: 'evals/bad-name.ts',
+      });
+      const result = validateInventory(makeInventory([f]), registry);
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'file-naming',
+      );
+      expect(violations).toHaveLength(1);
+    });
+
+    it('accepts .eval.tsx files', () => {
+      const c = makeCase({
+        filePath: '/repo/evals/component.eval.tsx',
+        relativePath: 'evals/component.eval.tsx',
+      });
+      const f = makeFile([c], {
+        filePath: '/repo/evals/component.eval.tsx',
+        relativePath: 'evals/component.eval.tsx',
+      });
+      const result = validateInventory(makeInventory([f]), registry);
+      expect(
+        result.violations.filter((v) => v.ruleId === 'file-naming'),
+      ).toHaveLength(0);
+    });
+
+    it('flags valid-policy violation for unknown policy', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase({ policy: 'unknown' })])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'valid-policy',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain('"unknown"');
+    });
+
+    it('accepts all three valid policies', () => {
+      for (const policy of [
+        'ALWAYS_PASSES',
+        'USUALLY_PASSES',
+        'USUALLY_FAILS',
+      ] as const) {
+        const result = validateInventory(
+          makeInventory([makeFile([makeCase({ policy })])]),
+          registry,
+        );
+        expect(
+          result.violations.filter((v) => v.ruleId === 'valid-policy'),
+        ).toHaveLength(0);
+      }
+    });
+
+    it('flags missing suiteName', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase({ suiteName: undefined })])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'suite-metadata',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain('suiteName');
+    });
+
+    it('flags missing suiteType', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase({ suiteType: undefined })])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'suite-metadata',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain('suiteType');
+    });
+
+    it('flags both missing suiteName and suiteType', () => {
+      const result = validateInventory(
+        makeInventory([
+          makeFile([makeCase({ suiteName: undefined, suiteType: undefined })]),
+        ]),
+        registry,
+      );
+      expect(
+        result.violations.filter((v) => v.ruleId === 'suite-metadata'),
+      ).toHaveLength(2);
+    });
+
+    it('flags missing prompt', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase({ hasPrompt: false })])]),
+        registry,
+      );
+      expect(
+        result.violations.filter((v) => v.ruleId === 'prompt-presence'),
+      ).toHaveLength(1);
+    });
+
+    it('flags case name that could not be statically resolved', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase({ name: '<unknown>' })])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'case-name-static',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain('static string literal');
+    });
+
+    it('does not flag a case literally named "unknown"', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase({ name: 'unknown' })])]),
+        registry,
+      );
+      expect(
+        result.violations.filter((v) => v.ruleId === 'case-name-static'),
+      ).toHaveLength(0);
+    });
+
+    it('collects multiple violations from a single case', () => {
+      const result = validateInventory(
+        makeInventory([
+          makeFile([
+            makeCase({
+              policy: 'unknown',
+              suiteName: undefined,
+              suiteType: undefined,
+              hasPrompt: false,
+              name: '<unknown>',
+            }),
+          ]),
+        ]),
+        registry,
+      );
+      expect(result.totalViolations).toBe(5);
+      expect(result.invalidFiles).toBe(1);
+      expect(result.validFiles).toBe(0);
+    });
+
+    it('filters by relative file paths', () => {
+      const f1 = makeFile(
+        [
+          makeCase({
+            filePath: '/repo/evals/a.eval.ts',
+            relativePath: 'evals/a.eval.ts',
+            name: 'case a',
+          }),
+        ],
+        {
+          filePath: '/repo/evals/a.eval.ts',
+          relativePath: 'evals/a.eval.ts',
+        },
+      );
+      const f2 = makeFile(
+        [
+          makeCase({
+            filePath: '/repo/evals/b.eval.ts',
+            relativePath: 'evals/b.eval.ts',
+            name: 'case b',
+            policy: 'unknown',
+          }),
+        ],
+        {
+          filePath: '/repo/evals/b.eval.ts',
+          relativePath: 'evals/b.eval.ts',
+        },
+      );
+      const result = validateInventory(makeInventory([f1, f2]), registry, {
+        filePaths: ['evals/a.eval.ts'],
+      });
+      expect(result.totalFiles).toBe(1);
+      expect(result.totalViolations).toBe(0);
+    });
+
+    it('filters by absolute file paths', () => {
+      const f1 = makeFile(
+        [
+          makeCase({
+            filePath: '/repo/evals/a.eval.ts',
+            relativePath: 'evals/a.eval.ts',
+            name: 'case a',
+          }),
+        ],
+        {
+          filePath: '/repo/evals/a.eval.ts',
+          relativePath: 'evals/a.eval.ts',
+        },
+      );
+      const f2 = makeFile(
+        [
+          makeCase({
+            filePath: '/repo/evals/b.eval.ts',
+            relativePath: 'evals/b.eval.ts',
+            name: 'case b',
+            policy: 'unknown',
+          }),
+        ],
+        {
+          filePath: '/repo/evals/b.eval.ts',
+          relativePath: 'evals/b.eval.ts',
+        },
+      );
+      const result = validateInventory(makeInventory([f1, f2]), registry, {
+        filePaths: ['/repo/evals/b.eval.ts'],
+      });
+      expect(result.totalFiles).toBe(1);
+      expect(result.totalViolations).toBeGreaterThanOrEqual(1);
+    });
+
+    it('filters by relative file paths with dot-slash prefix (./)', () => {
+      const f1 = makeFile(
+        [
+          makeCase({
+            filePath: '/repo/evals/a.eval.ts',
+            relativePath: 'evals/a.eval.ts',
+            name: 'case a',
+          }),
+        ],
+        {
+          filePath: '/repo/evals/a.eval.ts',
+          relativePath: 'evals/a.eval.ts',
+        },
+      );
+      const result = validateInventory(makeInventory([f1]), registry, {
+        filePaths: ['./evals/a.eval.ts'],
+      });
+      expect(result.totalFiles).toBe(1);
+      expect(result.unmatchedFilePaths).toEqual([]);
+    });
+
+    it('tracks and returns unmatched file paths', () => {
+      const f1 = makeFile(
+        [
+          makeCase({
+            filePath: '/repo/evals/a.eval.ts',
+            relativePath: 'evals/a.eval.ts',
+            name: 'case a',
+          }),
+        ],
+        {
+          filePath: '/repo/evals/a.eval.ts',
+          relativePath: 'evals/a.eval.ts',
+        },
+      );
+      const result = validateInventory(makeInventory([f1]), registry, {
+        filePaths: ['./evals/a.eval.ts', 'evals/missing.eval.ts'],
+      });
+      expect(result.totalFiles).toBe(1);
+      expect(result.unmatchedFilePaths).toEqual(['evals/missing.eval.ts']);
+    });
+
+    it('forwards analyzer diagnostics in the result', () => {
+      const diag: EvalAnalysisDiagnostic = {
+        severity: 'warning',
+        message: 'Could not resolve wrapper helper',
+        filePath: '/repo/evals/test.eval.ts',
+        location: { line: 5, column: 1 },
+      };
+      const f = makeFile([makeCase()], { diagnostics: [diag] });
+      const result = validateInventory(makeInventory([f]), registry);
+      expect(result.analyzerDiagnostics).toHaveLength(1);
+      expect(result.analyzerDiagnostics[0].message).toBe(
+        'Could not resolve wrapper helper',
+      );
+    });
+
+    it('flags invalid-tool-refs when unrecognized tools are reported in diagnostics', () => {
+      const diag: EvalAnalysisDiagnostic = {
+        severity: 'warning',
+        message: 'Unrecognized tool name extracted: "bad_tool"',
+        filePath: '/repo/evals/test.eval.ts',
+        location: { line: 5, column: 1 },
+      };
+      const f = makeFile([makeCase()], { diagnostics: [diag] });
+      const result = validateInventory(makeInventory([f]), registry);
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'invalid-tool-refs',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain('bad_tool');
+    });
+
+    it('flags positive-assertion when standard test has no tool references', () => {
+      const c = makeCase({ toolReferences: [] });
+      const result = validateInventory(
+        makeInventory([makeFile([c])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'positive-assertion',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain(
+        'assert function does not track any tool references',
+      );
+    });
+
+    it('does not flag positive-assertion for component-level tests', () => {
+      const c1 = makeCase({
+        baseHelperName: 'componentEvalTest',
+        toolReferences: [],
+      });
+      const c2 = makeCase({
+        suiteType: 'component-level',
+        toolReferences: [],
+      });
+      const result = validateInventory(
+        makeInventory([makeFile([c1, c2])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'positive-assertion',
+      );
+      expect(violations).toHaveLength(0);
+    });
+
+    it('flags workspace-setup when workspace prompt has no files or setup config', () => {
+      const c = makeCase({
+        prompt: 'Please edit app.ts and fix the typo.',
+        suiteType: 'workspace',
+        hasFiles: false,
+        hasSetup: false,
+      });
+      const result = validateInventory(
+        makeInventory([makeFile([c])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'workspace-setup',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain('suggests workspace interaction');
+    });
+
+    it('accepts workspace prompt if files or setup are provided', () => {
+      const c1 = makeCase({
+        prompt: 'Please edit app.ts and fix the typo.',
+        hasFiles: true,
+        hasSetup: false,
+      });
+      const c2 = makeCase({
+        prompt: 'Please edit app.ts and fix the typo.',
+        hasFiles: false,
+        hasSetup: true,
+      });
+      const result = validateInventory(
+        makeInventory([makeFile([c1, c2])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'workspace-setup',
+      );
+      expect(violations).toHaveLength(0);
+    });
+
+    it('flags new-evals-policy when a new file has ALWAYS_PASSES policy', () => {
+      const c = makeCase({
+        filePath: '/repo/evals/untracked-test.eval.ts',
+        relativePath: 'evals/untracked-test.eval.ts',
+        policy: 'ALWAYS_PASSES',
+      });
+
+      // First call: git status --porcelain (working-tree additions)
+      vi.mocked(execSync).mockReturnValueOnce(
+        '?? evals/untracked-test.eval.ts\n',
+      );
+      // Subsequent calls for git merge-base can throw — addFromOutput won't add anything
+      vi.mocked(execSync).mockImplementationOnce(() => {
+        throw new Error('no merge base');
+      });
+
+      const result = validateInventory(
+        makeInventory([makeFile([c])]),
+        registry,
+      );
+      const violations = result.violations.filter(
+        (v) => v.ruleId === 'new-evals-policy',
+      );
+      expect(violations).toHaveLength(1);
+      expect(violations[0].message).toContain(
+        'not use ALWAYS_PASSES policy initially',
+      );
+    });
+
+    it('validates the real evals directory without any rule violations', async () => {
+      const { collectInventory } = await import('../utils/eval-inventory.js');
+      const repoRoot = path.resolve(import.meta.dirname, '../../');
+      const inventory = await collectInventory(repoRoot);
+      const result = validateInventory(inventory, registry);
+      expect(result.totalFiles).toBeGreaterThanOrEqual(1);
+      expect(result.totalViolations).toBe(0);
+      expect(result.violations).toEqual([]);
+    });
+  });
+
+  describe('formatValidationReport', () => {
+    it('shows success message when there are no violations', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase()])]),
+        registry,
+      );
+      const report = formatValidationReport(result, '/repo');
+
+      expect(report).toContain('Eval Validation Report');
+      expect(report).toContain('1 files');
+      expect(report).toContain('0 violations');
+      expect(report).toContain('✓ All eval cases pass validation.');
+    });
+
+    it('shows violations grouped by file with rule IDs', () => {
+      const result = validateInventory(
+        makeInventory([
+          makeFile([makeCase({ policy: 'unknown', suiteName: undefined })]),
+        ]),
+        registry,
+      );
+      const report = formatValidationReport(result, '/repo');
+      expect(report).toContain('[valid-policy]');
+      expect(report).toContain('[suite-metadata]');
+      expect(report).toContain('Summary');
+      expect(report).toContain('violation(s) found');
+    });
+
+    it('converts absolute paths to relative in report', () => {
+      const result = validateInventory(
+        makeInventory([
+          makeFile([
+            makeCase({
+              filePath: '/repo/evals/test.eval.ts',
+              policy: 'unknown',
+            }),
+          ]),
+        ]),
+        registry,
+      );
+      const report = formatValidationReport(result, '/repo');
+      expect(report).toContain('evals/test.eval.ts');
+      expect(report).not.toContain('/repo/evals/test.eval.ts');
+    });
+
+    it('includes analyzer diagnostics section when present', () => {
+      const diag: EvalAnalysisDiagnostic = {
+        severity: 'warning',
+        message: 'Could not resolve wrapper',
+        filePath: '/repo/evals/test.eval.ts',
+        location: { line: 5, column: 1 },
+      };
+      const f = makeFile([makeCase({ policy: 'unknown' })], {
+        diagnostics: [diag],
+      });
+      const result = validateInventory(makeInventory([f]), registry);
+      const report = formatValidationReport(result, '/repo');
+      expect(report).toContain('Analyzer Diagnostics (1)');
+      expect(report).toContain('Could not resolve wrapper');
+    });
+  });
+
+  describe('formatValidationJson', () => {
+    it('returns valid JSON with the expected schema', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase()])]),
+        registry,
+      );
+      const parsed: ValidationJsonOutput = JSON.parse(
+        formatValidationJson(result, '/repo', FIXED_DATE),
+      );
+
+      expect(parsed.version).toBe(1);
+      expect(parsed.generated).toBe('2026-07-05T00:00:00.000Z');
+      expect(parsed.summary.totalFiles).toBe(1);
+      expect(parsed.summary.totalCases).toBe(1);
+      expect(parsed.summary.totalViolations).toBe(0);
+      expect(parsed.summary.validFiles).toBe(1);
+      expect(parsed.summary.invalidFiles).toBe(0);
+      expect(parsed.violations).toEqual([]);
+    });
+
+    it('includes violations in the JSON output', () => {
+      const result = validateInventory(
+        makeInventory([
+          makeFile([makeCase({ policy: 'unknown', suiteName: undefined })]),
+        ]),
+        registry,
+      );
+      const parsed: ValidationJsonOutput = JSON.parse(
+        formatValidationJson(result, '/repo', FIXED_DATE),
+      );
+
+      expect(parsed.summary.totalViolations).toBeGreaterThanOrEqual(2);
+      expect(parsed.violations.length).toBe(parsed.summary.totalViolations);
+      for (const v of parsed.violations) {
+        expect(v).toHaveProperty('ruleId');
+        expect(v).toHaveProperty('message');
+        expect(v).toHaveProperty('filePath');
+        expect(v).toHaveProperty('location');
+        expect(typeof v.location.line).toBe('number');
+        expect(typeof v.location.column).toBe('number');
+      }
+    });
+
+    it('uses relative paths in JSON violations', () => {
+      const result = validateInventory(
+        makeInventory([
+          makeFile([
+            makeCase({
+              filePath: '/repo/evals/test.eval.ts',
+              policy: 'unknown',
+            }),
+          ]),
+        ]),
+        registry,
+      );
+      const parsed: ValidationJsonOutput = JSON.parse(
+        formatValidationJson(result, '/repo', FIXED_DATE),
+      );
+      for (const v of parsed.violations) {
+        expect(v.filePath).not.toMatch(/^\//);
+        expect(v.filePath).toContain('evals/test.eval.ts');
+      }
+    });
+
+    it('respects SOURCE_DATE_EPOCH for reproducible output', () => {
+      vi.stubEnv('SOURCE_DATE_EPOCH', '1000000000');
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase()])]),
+        registry,
+      );
+      const parsed: ValidationJsonOutput = JSON.parse(
+        formatValidationJson(result, '/repo'),
+      );
+      expect(parsed.generated).toBe('2001-09-09T01:46:40.000Z');
+    });
+
+    it('respects EVAL_VALIDATE_STABLE_DATE for deterministic output', () => {
+      vi.stubEnv('EVAL_VALIDATE_STABLE_DATE', '1');
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase()])]),
+        registry,
+      );
+      const parsed: ValidationJsonOutput = JSON.parse(
+        formatValidationJson(result, '/repo'),
+      );
+      expect(parsed.generated).toBe('1970-01-01T00:00:00.000Z');
+    });
+
+    it('JSON output schema is stable', () => {
+      const result = validateInventory(
+        makeInventory([makeFile([makeCase({ policy: 'unknown' })])]),
+        registry,
+      );
+      const parsed: ValidationJsonOutput = JSON.parse(
+        formatValidationJson(result, '/repo', FIXED_DATE),
+      );
+      expect(Object.keys(parsed).sort()).toEqual([
+        'generated',
+        'summary',
+        'version',
+        'violations',
+      ]);
+      expect(Object.keys(parsed.summary).sort()).toEqual([
+        'invalidFiles',
+        'totalCases',
+        'totalFiles',
+        'totalViolations',
+        'validFiles',
+      ]);
+    });
+  });
+});

scripts/tests/tool-log-formatter.test.ts

@@ -0,0 +1,194 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+  formatToolLogChain,
+  type ToolLogEntry,
+} from '../utils/tool-log-formatter.js';
+
+function makeEntry(
+  overrides: Partial<ToolLogEntry['toolRequest']> = {},
+): ToolLogEntry {
+  return {
+    toolRequest: {
+      name: 'test_tool',
+      args: '{}',
+      success: true,
+      duration_ms: 100,
+      ...overrides,
+    },
+  };
+}
+
+describe('formatToolLogChain', () => {
+  it('returns empty string for empty log array', () => {
+    expect(formatToolLogChain([])).toBe('');
+  });
+
+  it('returns empty string for null/undefined input', () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    expect(formatToolLogChain(null as any)).toBe('');
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    expect(formatToolLogChain(undefined as any)).toBe('');
+  });
+
+  it('formats a single successful tool call', () => {
+    const logs = [makeEntry({ name: 'grep_search', duration_ms: 42 })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('1.');
+    expect(result).toContain('grep_search()');
+    expect(result).toContain('✓');
+    expect(result).toContain('42ms');
+  });
+
+  it('formats a single failed tool call with error details', () => {
+    const logs = [
+      makeEntry({
+        name: 'read_file',
+        args: '{"path":"/src/foo.ts"}',
+        success: false,
+        duration_ms: 80,
+        error: 'File not found',
+        error_type: 'ENOENT',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('read_file(');
+    expect(result).toContain('path="/src/foo.ts"');
+    expect(result).toContain('✗');
+    expect(result).toContain('80ms');
+    expect(result).toContain('↳ Error: [ENOENT] File not found');
+  });
+
+  it('formats arguments as key=value pairs', () => {
+    const logs = [
+      makeEntry({
+        name: 'grep_search',
+        args: '{"query":"TODO","path":"/src"}',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('query="TODO"');
+    expect(result).toContain('path="/src"');
+  });
+
+  it('truncates long argument values', () => {
+    const longValue = 'a'.repeat(100);
+    const logs = [
+      makeEntry({
+        name: 'write_file',
+        args: JSON.stringify({ content: longValue }),
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('…');
+    expect(result).not.toContain(longValue);
+  });
+
+  it('handles invalid JSON in args gracefully', () => {
+    const logs = [
+      makeEntry({
+        name: 'shell',
+        args: 'not-json {{{',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('shell(');
+    expect(result).toContain('not-json');
+  });
+
+  it('handles JSON null args without crashing', () => {
+    // JSON.parse('null') returns null; Object.entries(null) would throw TypeError
+    const logs = [makeEntry({ name: 'some_tool', args: 'null' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('some_tool(');
+    expect(result).toContain('null');
+  });
+
+  it('handles JSON primitive string args without producing garbage output', () => {
+    // JSON.parse('"hello"') returns a string; Object.entries("hello") would produce char pairs
+    const logs = [makeEntry({ name: 'some_tool', args: '"hello"' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('some_tool(');
+    expect(result).toContain('hello');
+    // Should NOT produce character-index pairs like 0="h"
+    expect(result).not.toMatch(/0="h"/);
+  });
+
+  it('handles JSON array args without producing indexed output', () => {
+    // JSON.parse('[1,2,3]') returns an array; Object.entries([1,2,3]) would produce index pairs
+    const logs = [makeEntry({ name: 'some_tool', args: '[1, 2, 3]' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('some_tool(');
+    // Should NOT produce array-index pairs like 0="1"
+    expect(result).not.toMatch(/0="1"/);
+  });
+
+  it('formats multiple tool calls with correct numbering', () => {
+    const logs = [
+      makeEntry({ name: 'grep_search', duration_ms: 10 }),
+      makeEntry({ name: 'read_file', duration_ms: 20 }),
+      makeEntry({
+        name: 'write_file',
+        success: false,
+        duration_ms: 30,
+        error: 'Permission denied',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    const lines = result.split('\n');
+    expect(lines[0]).toContain('1.');
+    expect(lines[0]).toContain('grep_search');
+    expect(lines[1]).toContain('2.');
+    expect(lines[1]).toContain('read_file');
+    expect(lines[2]).toContain('3.');
+    expect(lines[2]).toContain('write_file');
+    expect(lines[3]).toContain('↳ Error:');
+    expect(lines[3]).toContain('Permission denied');
+  });
+
+  it('pads step numbers for double-digit counts', () => {
+    const logs = Array.from({ length: 12 }, (_, i) =>
+      makeEntry({ name: `tool_${i + 1}`, duration_ms: i * 10 }),
+    );
+    const result = formatToolLogChain(logs);
+    const lines = result.split('\n');
+    expect(lines[0]).toMatch(/\s+1\./);
+    expect(lines[11]).toContain('12.');
+  });
+
+  it('shows failed call without error details when neither error nor error_type present', () => {
+    const logs = [
+      makeEntry({
+        name: 'run_shell',
+        success: false,
+        duration_ms: 50,
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('✗');
+    expect(result).not.toContain('↳');
+  });
+
+  it('handles empty args string', () => {
+    const logs = [makeEntry({ name: 'list_dir', args: '' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('list_dir()');
+  });
+
+  it('formats non-string argument values correctly', () => {
+    const logs = [
+      makeEntry({
+        name: 'some_tool',
+        args: '{"count":42,"nested":{"a":1},"flag":true}',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('count="42"');
+    expect(result).toContain('flag="true"');
+  });
+});

scripts/utils/eval-analysis.ts

@@ -49,7 +49,10 @@ export interface EvalCaseRecord {
   suiteType?: string;
   timeout?: number;
   hasFiles: boolean;
+  hasSetup: boolean;
   hasPrompt: boolean;
+  hasAssertBody: boolean;
+  prompt?: string;
   toolReferences: readonly string[];
   location: EvalSourceLocation;
 }
@@ -161,7 +164,10 @@ export function analyzeEvalSource(
       suiteType: getStaticStringProperty(evalCase, 'suiteType'),
       timeout: getStaticNumberProperty(evalCase, 'timeout'),
       hasFiles: hasProperty(evalCase, 'files'),
+      hasSetup: hasProperty(evalCase, 'setup'),
       hasPrompt: hasProperty(evalCase, 'prompt'),
+      hasAssertBody: Boolean(assertBody),
+      prompt: getStaticStringProperty(evalCase, 'prompt'),
       toolReferences: Object.freeze([...new Set(toolRefs)].sort()),
       location: getLocation(sourceFile, callExpression),
     });
@@ -592,19 +598,32 @@ function extractFromWaitForToolCall(
   refs: { name: string; node: ts.Node }[],
 ) {
   const expr = call.expression;
+  let methodName: string | undefined;
+
+  if (ts.isPropertyAccessExpression(expr)) {
+    methodName = expr.name.text;
+  } else if (ts.isIdentifier(expr)) {
+    methodName = expr.text;
+  }
+
+  if (!methodName) return;
+
   if (
-    !ts.isPropertyAccessExpression(expr) ||
-    expr.name.text !== 'waitForToolCall'
+    methodName === 'waitForToolCall' ||
+    methodName === 'waitForPendingConfirmation'
   ) {
-    return;
-  }
-  const firstArg = call.arguments[0];
-  if (!firstArg) {
-    return;
-  }
-  const resolved = resolveStringValue(firstArg, importedConstants);
-  if (resolved) {
-    refs.push({ name: resolved, node: firstArg });
+    const firstArg = call.arguments[0];
+    if (firstArg) {
+      const resolved = resolveStringValue(firstArg, importedConstants);
+      if (resolved) {
+        refs.push({ name: resolved, node: firstArg });
+      }
+    }
+  } else if (
+    methodName === 'expectSubagentCall' ||
+    methodName === 'expectSubagent'
+  ) {
+    refs.push({ name: 'invoke_agent', node: call });
   }
 }
 

scripts/utils/eval-validate.ts

@@ -0,0 +1,642 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import path from 'node:path';
+import { execSync } from 'node:child_process';
+import type {
+  EvalCaseRecord,
+  EvalAnalysisDiagnostic,
+} from './eval-analysis.js';
+import type { InventoryResult } from './eval-inventory.js';
+import type { ToolRegistry } from './tool-registry.js';
+
+export interface ValidationRule {
+  id: string;
+  description: string;
+}
+
+export interface ValidationViolation {
+  ruleId: string;
+  message: string;
+  filePath: string;
+  location: { line: number; column: number };
+}
+
+export interface FileValidationSummary {
+  relativePath: string;
+  totalCases: number;
+  violationCount: number;
+}
+
+export interface ValidationResult {
+  totalFiles: number;
+  totalCases: number;
+  totalViolations: number;
+  invalidFiles: number;
+  validFiles: number;
+  violations: ValidationViolation[];
+  fileSummaries: FileValidationSummary[];
+  analyzerDiagnostics: readonly EvalAnalysisDiagnostic[];
+  unmatchedFilePaths?: string[];
+}
+
+export const VALIDATION_RULES: readonly ValidationRule[] = [
+  {
+    id: 'file-naming',
+    description:
+      'Eval file relativePath must match the *.eval.ts or *.eval.tsx naming convention.',
+  },
+  {
+    id: 'valid-policy',
+    description:
+      'Policy must be one of ALWAYS_PASSES, USUALLY_PASSES, or USUALLY_FAILS.',
+  },
+  {
+    id: 'suite-metadata',
+    description:
+      'Both suiteName and suiteType must be present as static string literals.',
+  },
+  {
+    id: 'prompt-presence',
+    description: 'The prompt property must be present in the eval case object.',
+  },
+  {
+    id: 'case-name-static',
+    description:
+      'The case name must be a static string literal, not a computed value.',
+  },
+  {
+    id: 'invalid-tool-refs',
+    description:
+      'All tools referenced in assertions must be valid, existing tools in the registry.',
+  },
+  {
+    id: 'positive-assertion',
+    description:
+      'Behavioral evaluation cases must assert on at least one tool call.',
+  },
+  {
+    id: 'workspace-setup',
+    description:
+      'Workspace behavior evaluations must provide files or a setup function.',
+  },
+  {
+    id: 'new-evals-policy',
+    description: 'New evaluations must not use ALWAYS_PASSES policy initially.',
+  },
+];
+
+const VALID_FILE_SUFFIXES = ['.eval.ts', '.eval.tsx'] as const;
+const VALID_POLICIES = new Set([
+  'ALWAYS_PASSES',
+  'USUALLY_PASSES',
+  'USUALLY_FAILS',
+]);
+
+function checkFileNaming(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+): ValidationViolation | undefined {
+  // Check the relativePath because collectInventory's glob already filters by
+  // *.eval.{ts,tsx}. This rule catches cases from manually constructed
+  // inventories or future non-glob discovery paths.
+  const checkPath = evalCase.relativePath || filePath;
+  const base = path.basename(checkPath);
+  if (!VALID_FILE_SUFFIXES.some((suffix) => base.endsWith(suffix))) {
+    return {
+      ruleId: 'file-naming',
+      message: `File "${base}" does not match the required *.eval.ts or *.eval.tsx naming convention.`,
+      filePath,
+      location: evalCase.location,
+    };
+  }
+  return undefined;
+}
+
+function checkValidPolicy(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+): ValidationViolation | undefined {
+  if (!VALID_POLICIES.has(evalCase.policy)) {
+    return {
+      ruleId: 'valid-policy',
+      message: `Policy "${evalCase.policy}" is not valid. Must be one of: ${[...VALID_POLICIES].join(', ')}.`,
+      filePath,
+      location: evalCase.location,
+    };
+  }
+  return undefined;
+}
+
+function checkSuiteMetadata(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+): ValidationViolation[] {
+  const violations: ValidationViolation[] = [];
+  if (!evalCase.suiteName) {
+    violations.push({
+      ruleId: 'suite-metadata',
+      message:
+        'Missing suiteName. Add a static suiteName string to the eval case object.',
+      filePath,
+      location: evalCase.location,
+    });
+  }
+  if (!evalCase.suiteType) {
+    violations.push({
+      ruleId: 'suite-metadata',
+      message:
+        'Missing suiteType. Add a static suiteType string to the eval case object.',
+      filePath,
+      location: evalCase.location,
+    });
+  }
+  return violations;
+}
+
+function checkPromptPresence(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+): ValidationViolation | undefined {
+  if (
+    evalCase.baseHelperName === 'componentEvalTest' ||
+    evalCase.suiteType === 'component-level'
+  ) {
+    return undefined;
+  }
+  if (!evalCase.hasPrompt) {
+    return {
+      ruleId: 'prompt-presence',
+      message:
+        'Eval case is missing a prompt property. Every case must include a prompt.',
+      filePath,
+      location: evalCase.location,
+    };
+  }
+  return undefined;
+}
+
+function checkCaseNameStatic(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+): ValidationViolation | undefined {
+  // The analyzer stores unresolved names as '<unknown>' (eval-analysis.ts L159).
+  if (evalCase.name === '<unknown>') {
+    return {
+      ruleId: 'case-name-static',
+      message:
+        'Case name could not be resolved to a static string literal. Use a plain string, not a variable or template expression.',
+      filePath,
+      location: evalCase.location,
+    };
+  }
+  return undefined;
+}
+
+const EXEMPT_SUITE_TYPES = new Set([
+  'component-level',
+  'text',
+  'prose',
+  'steering',
+  'memory',
+]);
+
+function checkPositiveAssertion(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+): ValidationViolation | undefined {
+  if (
+    evalCase.baseHelperName === 'componentEvalTest' ||
+    (evalCase.suiteType && EXEMPT_SUITE_TYPES.has(evalCase.suiteType))
+  ) {
+    return undefined;
+  }
+
+  // If case has tool references or has an assertion body (e.g. negative or custom assertions), pass
+  if (evalCase.toolReferences.length > 0 || evalCase.hasAssertBody) {
+    return undefined;
+  }
+
+  return {
+    ruleId: 'positive-assertion',
+    message:
+      'Eval case assert function does not track any tool references. Use at least one positive tool assertion.',
+    filePath,
+    location: evalCase.location,
+  };
+}
+
+function checkWorkspaceSetup(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+): ValidationViolation | undefined {
+  if (
+    evalCase.baseHelperName === 'componentEvalTest' ||
+    (evalCase.suiteType && EXEMPT_SUITE_TYPES.has(evalCase.suiteType)) ||
+    evalCase.hasFiles ||
+    evalCase.hasSetup
+  ) {
+    return undefined;
+  }
+
+  if (
+    evalCase.suiteType === 'workspace' ||
+    evalCase.suiteType === 'file-system'
+  ) {
+    return {
+      ruleId: 'workspace-setup',
+      message:
+        'Eval case suggests workspace interaction (files/git), but neither "files" nor a "setup" hook is specified.',
+      filePath,
+      location: evalCase.location,
+    };
+  }
+
+  return undefined;
+}
+
+function getNewEvalFiles(repoRoot?: string): Set<string> {
+  const newFiles = new Set<string>();
+  const cwd = repoRoot || process.cwd();
+
+  function addFromOutput(output: string): void {
+    for (const line of output.split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      // git status --porcelain: "A  path" or "?? path"
+      // git diff --name-only: bare path
+      let filePath: string;
+      if (trimmed.startsWith('A') || trimmed.startsWith('??')) {
+        filePath = trimmed.slice(2).trim();
+      } else {
+        filePath = trimmed;
+      }
+      if (filePath.startsWith('"') && filePath.endsWith('"')) {
+        filePath = filePath.slice(1, -1);
+      }
+      if (filePath) {
+        newFiles.add(path.resolve(cwd, filePath).replace(/\\/g, '/'));
+      }
+    }
+  }
+
+  try {
+    // Working-tree additions (useful locally)
+    const status = execSync('git status --porcelain', {
+      cwd,
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    addFromOutput(status);
+  } catch {
+    // Ignore
+  }
+
+  try {
+    // PR-level additions against the merge base (works in CI where working
+    // tree is clean). Try origin/main, fall back to main, then HEAD~1.
+    const bases = ['origin/main', 'main', 'HEAD~1'];
+    for (const base of bases) {
+      try {
+        const mergeBase = execSync(`git merge-base HEAD ${base}`, {
+          cwd,
+          encoding: 'utf8',
+          stdio: ['ignore', 'pipe', 'ignore'],
+        }).trim();
+        if (mergeBase) {
+          const diff = execSync(
+            `git diff --diff-filter=A --name-only ${mergeBase}`,
+            {
+              cwd,
+              encoding: 'utf8',
+              stdio: ['ignore', 'pipe', 'ignore'],
+            },
+          );
+          addFromOutput(diff);
+          break; // succeeded, no need to try next base
+        }
+      } catch {
+        // Try next base
+      }
+    }
+  } catch {
+    // Ignore
+  }
+
+  return newFiles;
+}
+
+function checkNewEvalPolicy(
+  evalCase: EvalCaseRecord,
+  filePath: string,
+  newFiles: Set<string>,
+): ValidationViolation | undefined {
+  const normalizedPath = path.resolve(filePath).replace(/\\/g, '/');
+  if (newFiles.has(normalizedPath) && evalCase.policy === 'ALWAYS_PASSES') {
+    return {
+      ruleId: 'new-evals-policy',
+      message:
+        'New evaluations must not use ALWAYS_PASSES policy initially. Use USUALLY_PASSES.',
+      filePath,
+      location: evalCase.location,
+    };
+  }
+  return undefined;
+}
+
+/**
+ * Validates every eval case in the given inventory and returns a
+ * ValidationResult. Pass `options.filePaths` to restrict to a subset of
+ * files (absolute or relative to `inventory.repoRoot`).
+ *
+ * The `_registry` parameter is reserved for future tool-name validation rules.
+ */
+export function validateInventory(
+  inventory: InventoryResult,
+  _registry: ToolRegistry,
+  options: { filePaths?: string[] } = {},
+): ValidationResult {
+  const { filePaths: filterPaths } = options;
+
+  const matchedFilterSet = new Set<string>();
+  const filterMap = new Map<string, string>(); // maps normalized relative path -> original filter path
+
+  const filterSet: Set<string> | undefined = filterPaths
+    ? new Set(
+        filterPaths.map((p) => {
+          let abs: string;
+          if (path.isAbsolute(p)) {
+            abs = p;
+          } else {
+            abs = path.resolve(inventory.repoRoot || process.cwd(), p);
+          }
+          const rel = inventory.repoRoot
+            ? path.relative(inventory.repoRoot, abs)
+            : abs;
+          const normalized = rel.replace(/\\/g, '/');
+          filterMap.set(normalized, p);
+          return normalized;
+        }),
+      )
+    : undefined;
+
+  const fileSummaryMap = new Map<
+    string,
+    { totalCases: number; violationCount: number }
+  >();
+  const allViolations: ValidationViolation[] = [];
+  const newFiles = getNewEvalFiles(inventory.repoRoot);
+
+  for (const fileAnalysis of inventory.files) {
+    const relativePath = fileAnalysis.relativePath;
+    if (filterSet) {
+      if (!filterSet.has(relativePath)) {
+        continue;
+      }
+      matchedFilterSet.add(relativePath);
+    }
+
+    if (!fileSummaryMap.has(relativePath)) {
+      fileSummaryMap.set(relativePath, { totalCases: 0, violationCount: 0 });
+    }
+
+    for (const evalCase of fileAnalysis.cases) {
+      const summary = fileSummaryMap.get(relativePath)!;
+      summary.totalCases += 1;
+      const fp = evalCase.filePath;
+      const caseViolations: ValidationViolation[] = [];
+
+      const fn = checkFileNaming(evalCase, fp);
+      if (fn) caseViolations.push(fn);
+
+      const pol = checkValidPolicy(evalCase, fp);
+      if (pol) caseViolations.push(pol);
+
+      caseViolations.push(...checkSuiteMetadata(evalCase, fp));
+
+      const pr = checkPromptPresence(evalCase, fp);
+      if (pr) caseViolations.push(pr);
+
+      const ns = checkCaseNameStatic(evalCase, fp);
+      if (ns) caseViolations.push(ns);
+
+      const pos = checkPositiveAssertion(evalCase, fp);
+      if (pos) caseViolations.push(pos);
+
+      const ws = checkWorkspaceSetup(evalCase, fp);
+      if (ws) caseViolations.push(ws);
+
+      const nep = checkNewEvalPolicy(evalCase, fp, newFiles);
+      if (nep) caseViolations.push(nep);
+
+      allViolations.push(...caseViolations);
+      summary.violationCount += caseViolations.length;
+    }
+  }
+
+  // Elevate unrecognized tool warning diagnostics to validation violations
+  for (const d of inventory.diagnostics) {
+    if (d.message.startsWith('Unrecognized tool name extracted:')) {
+      const displayPath = resolveDisplayPath(d.filePath, inventory.repoRoot);
+      if (filterSet && !filterSet.has(displayPath)) {
+        continue;
+      }
+      allViolations.push({
+        ruleId: 'invalid-tool-refs',
+        message: d.message,
+        filePath: d.filePath,
+        location: d.location,
+      });
+
+      const summary = fileSummaryMap.get(displayPath);
+      if (summary) {
+        summary.violationCount += 1;
+      }
+    }
+  }
+
+  const fileSummaries: FileValidationSummary[] = [...fileSummaryMap.entries()]
+    .map(([relativePath, s]) => ({
+      relativePath,
+      totalCases: s.totalCases,
+      violationCount: s.violationCount,
+    }))
+    .sort((a, b) => a.relativePath.localeCompare(b.relativePath, 'en'));
+
+  const totalFiles = fileSummaries.length;
+  const totalCases = fileSummaries.reduce((sum, f) => sum + f.totalCases, 0);
+  const totalViolations = allViolations.length;
+  const invalidFiles = fileSummaries.filter((f) => f.violationCount > 0).length;
+
+  const unmatchedFilePaths: string[] = [];
+  if (filterSet) {
+    for (const f of filterSet) {
+      if (!matchedFilterSet.has(f)) {
+        const original = filterMap.get(f);
+        if (original !== undefined) {
+          unmatchedFilePaths.push(original);
+        }
+      }
+    }
+  }
+
+  return {
+    totalFiles,
+    totalCases,
+    totalViolations,
+    invalidFiles,
+    validFiles: totalFiles - invalidFiles,
+    violations: allViolations,
+    fileSummaries,
+    analyzerDiagnostics: inventory.diagnostics,
+    unmatchedFilePaths,
+  };
+}
+
+function resolveDisplayPath(filePath: string, repoRoot?: string): string {
+  if (filePath === '<inline>') return filePath;
+  if (repoRoot && path.isAbsolute(filePath)) {
+    return path.relative(repoRoot, filePath).replace(/\\/g, '/');
+  }
+  return filePath;
+}
+
+function appendAnalyzerDiagnostics(
+  lines: string[],
+  diagnostics: readonly EvalAnalysisDiagnostic[],
+  repoRoot?: string,
+): void {
+  lines.push(`Analyzer Diagnostics (${diagnostics.length})`);
+  lines.push('────────────────────────');
+  for (const d of diagnostics) {
+    const displayPath = resolveDisplayPath(d.filePath, repoRoot);
+    lines.push(
+      `⚠ ${displayPath}:${d.location.line}:${d.location.column} — ${d.message}`,
+    );
+  }
+  lines.push('');
+}
+
+export function formatValidationReport(
+  result: ValidationResult,
+  repoRoot?: string,
+): string {
+  const lines: string[] = [];
+
+  lines.push('Eval Validation Report');
+  lines.push('══════════════════════');
+  lines.push('');
+  lines.push(
+    `${result.totalFiles} files · ${result.totalCases} cases · ${result.totalViolations} violations`,
+  );
+
+  if (result.totalViolations === 0) {
+    lines.push('');
+    lines.push('✓ All eval cases pass validation.');
+    if (result.analyzerDiagnostics.length > 0) {
+      lines.push('');
+      appendAnalyzerDiagnostics(lines, result.analyzerDiagnostics, repoRoot);
+    }
+    return lines.join('\n');
+  }
+
+  lines.push('');
+
+  const byFile = new Map<string, ValidationViolation[]>();
+  for (const v of result.violations) {
+    const displayPath = resolveDisplayPath(v.filePath, repoRoot);
+    const existing = byFile.get(displayPath);
+    if (existing) {
+      existing.push(v);
+    } else {
+      byFile.set(displayPath, [v]);
+    }
+  }
+
+  for (const [displayPath, violations] of [...byFile.entries()].sort(
+    ([a], [b]) => a.localeCompare(b, 'en'),
+  )) {
+    lines.push(displayPath);
+    for (const v of violations) {
+      lines.push(
+        `  ✗ [${v.ruleId}] ${v.location.line}:${v.location.column} — ${v.message}`,
+      );
+    }
+    lines.push('');
+  }
+
+  lines.push('Summary');
+  lines.push('───────');
+  lines.push(`  ${result.validFiles} / ${result.totalFiles} files pass`);
+  lines.push(`  ${result.totalViolations} violation(s) found`);
+  lines.push('');
+
+  if (result.analyzerDiagnostics.length > 0) {
+    appendAnalyzerDiagnostics(lines, result.analyzerDiagnostics, repoRoot);
+  }
+
+  return lines.join('\n');
+}
+
+export interface ValidationJsonViolation {
+  ruleId: string;
+  message: string;
+  filePath: string;
+  location: { line: number; column: number };
+}
+
+export interface ValidationJsonOutput {
+  version: 1;
+  generated: string;
+  summary: {
+    totalFiles: number;
+    totalCases: number;
+    totalViolations: number;
+    validFiles: number;
+    invalidFiles: number;
+  };
+  violations: ValidationJsonViolation[];
+}
+
+export function formatValidationJson(
+  result: ValidationResult,
+  repoRoot?: string,
+  now?: Date,
+): string {
+  let generatedDate = now;
+  if (!generatedDate && process.env.SOURCE_DATE_EPOCH) {
+    const epoch = parseInt(process.env.SOURCE_DATE_EPOCH, 10);
+    if (!isNaN(epoch)) generatedDate = new Date(epoch * 1000);
+  }
+  if (
+    !generatedDate &&
+    (process.env.EVAL_VALIDATE_STABLE_DATE ||
+      process.env.EVAL_INVENTORY_DETERMINISTIC)
+  ) {
+    generatedDate = new Date(0);
+  }
+  if (!generatedDate) generatedDate = new Date();
+
+  const output: ValidationJsonOutput = {
+    version: 1,
+    generated: generatedDate.toISOString(),
+    summary: {
+      totalFiles: result.totalFiles,
+      totalCases: result.totalCases,
+      totalViolations: result.totalViolations,
+      validFiles: result.validFiles,
+      invalidFiles: result.invalidFiles,
+    },
+    violations: result.violations.map((v) => ({
+      ruleId: v.ruleId,
+      message: v.message,
+      filePath: resolveDisplayPath(v.filePath, repoRoot),
+      location: { line: v.location.line, column: v.location.column },
+    })),
+  };
+
+  return JSON.stringify(output, null, 2);
+}

scripts/utils/tool-log-formatter.ts

@@ -0,0 +1,84 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface ToolLogEntry {
+  toolRequest: {
+    name: string;
+    args: string;
+    success: boolean;
+    duration_ms: number;
+    prompt_id?: string;
+    error?: string;
+    error_type?: string;
+  };
+}
+
+const MAX_ARG_VALUE_LENGTH = 60;
+
+function formatArgs(argsJson: string): string {
+  if (!argsJson || argsJson === '{}') {
+    return '';
+  }
+
+  let parsed: Record<string, unknown>;
+  try {
+    const val = JSON.parse(argsJson);
+    if (val === null || typeof val !== 'object' || Array.isArray(val)) {
+      return truncate(argsJson, MAX_ARG_VALUE_LENGTH);
+    }
+    parsed = val as Record<string, unknown>;
+  } catch {
+    return truncate(argsJson, MAX_ARG_VALUE_LENGTH);
+  }
+
+  const pairs: string[] = [];
+  for (const [key, value] of Object.entries(parsed)) {
+    const strValue = typeof value === 'string' ? value : JSON.stringify(value);
+    pairs.push(
+      `${key}=${JSON.stringify(truncate(String(strValue), MAX_ARG_VALUE_LENGTH))}`,
+    );
+  }
+
+  return pairs.join(', ');
+}
+
+function truncate(str: string, max: number): string {
+  if (str.length <= max) {
+    return str;
+  }
+  return str.slice(0, max - 1) + '…';
+}
+
+export function formatToolLogChain(logs: ToolLogEntry[]): string {
+  if (!logs || logs.length === 0) {
+    return '';
+  }
+
+  const lines: string[] = [];
+  const padWidth = String(logs.length).length;
+
+  for (let i = 0; i < logs.length; i++) {
+    const { toolRequest: t } = logs[i];
+    const idx = String(i + 1).padStart(padWidth, ' ');
+    const argsStr = formatArgs(t.args);
+    const call = argsStr ? `${t.name}(${argsStr})` : `${t.name}()`;
+
+    const status = t.success ? '✓' : '✗';
+    const duration = `${t.duration_ms}ms`;
+
+    lines.push(`  ${idx}. ${call} ── ${status} ${duration}`);
+
+    if (!t.success && (t.error || t.error_type)) {
+      const errorType = t.error_type ? `[${t.error_type}] ` : '';
+      const errorMsg = t.error ? truncate(t.error, 120) : 'Unknown error';
+      lines.push(
+        `  ${' '.repeat(padWidth)}   ↳ Error: ${errorType}${errorMsg}`,
+      );
+    }
+  }
+
+  return lines.join('\n');
+}

scripts/utils/tool-registry.ts

@@ -108,6 +108,32 @@ export function buildToolRegistry(): ToolRegistry {
     }
   }
 
+  const customTools: Array<[string, ToolCategory]> = [
+    ['read_background_output', 'shell'],
+    ['list_background_processes', 'shell'],
+    ['mutation-agent', 'agent'],
+    ['generalist', 'agent'],
+    ['generalist-agent', 'agent'],
+    ['coder-agent', 'agent'],
+    ['task-tracker-agent', 'agent'],
+  ];
+
+  for (const [name, category] of customTools) {
+    aliasLookup.set(name, name);
+    const entry: ToolRegistryEntry = {
+      name,
+      category,
+      aliases: Object.freeze([]),
+    };
+    tools.set(name, entry);
+    const group = categoryGroups.get(category);
+    if (group) {
+      group.push(entry);
+    } else {
+      categoryGroups.set(category, [entry]);
+    }
+  }
+
   const frozenCategories = new Map<
     ToolCategory,
     readonly ToolRegistryEntry[]