chore(evals): add tool-log-formatter to tsconfig includes

evals/test-helper.test.ts

@@ -216,4 +216,109 @@ describe('evalTest reliability logic', () => {
       }
     }
   });
+
+  it('should append tool call chain to assertion failure error messages', async () => {
+    const mockRig = {
+      setup: vi.fn(),
+      run: vi.fn(),
+      cleanup: vi.fn(),
+      readToolLogs: vi.fn().mockReturnValue([]),
+      _lastRunStderr: '',
+    } as any;
+    (TestRig as any).mockReturnValue(mockRig);
+
+    mockRig.run.mockResolvedValue('Success');
+    mockRig.readToolLogs.mockReturnValue([
+      {
+        toolRequest: {
+          name: 'grep_search',
+          args: '{"query":"TODO"}',
+          success: true,
+          duration_ms: 42,
+        },
+      },
+      {
+        toolRequest: {
+          name: 'read_file',
+          args: '{"path":"/src/foo.ts"}',
+          success: false,
+          duration_ms: 15,
+          error: 'File not found',
+          error_type: 'ENOENT',
+        },
+      },
+    ]);
+
+    const assertionError = new Error('Expected tool to be called');
+
+    try {
+      await internalEvalTest({
+        suiteName: 'test',
+        suiteType: 'behavioral',
+        name: 'test-tool-chain',
+        prompt: 'do something',
+        assert: async () => {
+          throw assertionError;
+        },
+      });
+      expect.unreachable('Expected internalEvalTest to throw');
+    } catch (error: unknown) {
+      expect(error).toBeInstanceOf(Error);
+      const msg = (error as Error).message;
+      expect(msg).toContain('Expected tool to be called');
+      expect(msg).toContain('Tool Call Chain (2 calls)');
+      expect(msg).toContain('grep_search');
+      expect(msg).toContain('read_file');
+      expect(msg).toContain('[ENOENT] File not found');
+    }
+  });
+
+  it('should not crash when error.message is read-only (frozen error)', async () => {
+    const mockRig = {
+      setup: vi.fn(),
+      run: vi.fn(),
+      cleanup: vi.fn(),
+      readToolLogs: vi.fn(),
+      _lastRunStderr: '',
+    } as any;
+    (TestRig as any).mockReturnValue(mockRig);
+
+    mockRig.run.mockResolvedValue('Success');
+    mockRig.readToolLogs.mockReturnValue([
+      {
+        toolRequest: {
+          name: 'read_file',
+          args: '{"path":"/foo.ts"}',
+          success: true,
+          duration_ms: 10,
+        },
+      },
+    ]);
+
+    // Simulate a frozen error whose message property cannot be mutated
+    const frozenError = Object.freeze(new Error('Frozen assertion error'));
+
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    try {
+      await expect(
+        internalEvalTest({
+          suiteName: 'test',
+          suiteType: 'behavioral',
+          name: 'test-frozen-error',
+          prompt: 'do something',
+          assert: async () => {
+            throw frozenError;
+          },
+        }),
+      ).rejects.toThrow('Frozen assertion error');
+
+      // Should have warned that the message could not be mutated
+      expect(warnSpy).toHaveBeenCalledWith(
+        expect.stringContaining('Could not append tool call chain'),
+      );
+    } finally {
+      warnSpy.mockRestore();
+    }
+  });
 });

evals/test-helper.ts

@@ -10,6 +10,7 @@ import path from 'node:path';
 import crypto from 'node:crypto';
 import { execSync } from 'node:child_process';
 import { TestRig } from '@google/gemini-cli-test-utils';
+import { formatToolLogChain } from '../scripts/utils/tool-log-formatter.js';
 import {
   createUnauthorizedToolError,
   parseAgentMarkdown,
@@ -186,6 +187,23 @@ export async function internalEvalTest(evalCase: EvalCase) {
 
       await evalCase.assert(rig, result);
       isSuccess = true;
+    } catch (error: unknown) {
+      const toolLogs = rig.readToolLogs();
+      if (toolLogs && toolLogs.length > 0) {
+        const summary = formatToolLogChain(toolLogs);
+        if (error instanceof Error) {
+          try {
+            error.message = `${error.message}\n\nTool Call Chain (${toolLogs.length} calls):\n${summary}`;
+          } catch {
+            // Error object may be frozen or have a read-only message property.
+            // The original error is still re-thrown, so no failure is hidden.
+            console.warn(
+              `[eval] Could not append tool call chain to error message (${toolLogs.length} calls)`,
+            );
+          }
+        }
+      }
+      throw error;
     } finally {
       if (isSuccess) {
         await fs.promises.unlink(activityLogFile).catch((err) => {

evals/tsconfig.json

@@ -7,7 +7,7 @@
       "@google/gemini-cli": ["../packages/cli/index.ts"]
     }
   },
-  "include": ["**/*.ts"],
+  "include": ["**/*.ts", "../scripts/utils/tool-log-formatter.ts"],
   "exclude": ["logs"],
   "references": [{ "path": "../packages/core" }, { "path": "../packages/cli" }]
 }

scripts/tests/tool-log-formatter.test.ts

@@ -0,0 +1,194 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+  formatToolLogChain,
+  type ToolLogEntry,
+} from '../utils/tool-log-formatter.js';
+
+function makeEntry(
+  overrides: Partial<ToolLogEntry['toolRequest']> = {},
+): ToolLogEntry {
+  return {
+    toolRequest: {
+      name: 'test_tool',
+      args: '{}',
+      success: true,
+      duration_ms: 100,
+      ...overrides,
+    },
+  };
+}
+
+describe('formatToolLogChain', () => {
+  it('returns empty string for empty log array', () => {
+    expect(formatToolLogChain([])).toBe('');
+  });
+
+  it('returns empty string for null/undefined input', () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    expect(formatToolLogChain(null as any)).toBe('');
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    expect(formatToolLogChain(undefined as any)).toBe('');
+  });
+
+  it('formats a single successful tool call', () => {
+    const logs = [makeEntry({ name: 'grep_search', duration_ms: 42 })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('1.');
+    expect(result).toContain('grep_search()');
+    expect(result).toContain('✓');
+    expect(result).toContain('42ms');
+  });
+
+  it('formats a single failed tool call with error details', () => {
+    const logs = [
+      makeEntry({
+        name: 'read_file',
+        args: '{"path":"/src/foo.ts"}',
+        success: false,
+        duration_ms: 80,
+        error: 'File not found',
+        error_type: 'ENOENT',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('read_file(');
+    expect(result).toContain('path="/src/foo.ts"');
+    expect(result).toContain('✗');
+    expect(result).toContain('80ms');
+    expect(result).toContain('↳ Error: [ENOENT] File not found');
+  });
+
+  it('formats arguments as key=value pairs', () => {
+    const logs = [
+      makeEntry({
+        name: 'grep_search',
+        args: '{"query":"TODO","path":"/src"}',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('query="TODO"');
+    expect(result).toContain('path="/src"');
+  });
+
+  it('truncates long argument values', () => {
+    const longValue = 'a'.repeat(100);
+    const logs = [
+      makeEntry({
+        name: 'write_file',
+        args: JSON.stringify({ content: longValue }),
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('…');
+    expect(result).not.toContain(longValue);
+  });
+
+  it('handles invalid JSON in args gracefully', () => {
+    const logs = [
+      makeEntry({
+        name: 'shell',
+        args: 'not-json {{{',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('shell(');
+    expect(result).toContain('not-json');
+  });
+
+  it('handles JSON null args without crashing', () => {
+    // JSON.parse('null') returns null; Object.entries(null) would throw TypeError
+    const logs = [makeEntry({ name: 'some_tool', args: 'null' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('some_tool(');
+    expect(result).toContain('null');
+  });
+
+  it('handles JSON primitive string args without producing garbage output', () => {
+    // JSON.parse('"hello"') returns a string; Object.entries("hello") would produce char pairs
+    const logs = [makeEntry({ name: 'some_tool', args: '"hello"' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('some_tool(');
+    expect(result).toContain('hello');
+    // Should NOT produce character-index pairs like 0="h"
+    expect(result).not.toMatch(/0="h"/);
+  });
+
+  it('handles JSON array args without producing indexed output', () => {
+    // JSON.parse('[1,2,3]') returns an array; Object.entries([1,2,3]) would produce index pairs
+    const logs = [makeEntry({ name: 'some_tool', args: '[1, 2, 3]' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('some_tool(');
+    // Should NOT produce array-index pairs like 0="1"
+    expect(result).not.toMatch(/0="1"/);
+  });
+
+  it('formats multiple tool calls with correct numbering', () => {
+    const logs = [
+      makeEntry({ name: 'grep_search', duration_ms: 10 }),
+      makeEntry({ name: 'read_file', duration_ms: 20 }),
+      makeEntry({
+        name: 'write_file',
+        success: false,
+        duration_ms: 30,
+        error: 'Permission denied',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    const lines = result.split('\n');
+    expect(lines[0]).toContain('1.');
+    expect(lines[0]).toContain('grep_search');
+    expect(lines[1]).toContain('2.');
+    expect(lines[1]).toContain('read_file');
+    expect(lines[2]).toContain('3.');
+    expect(lines[2]).toContain('write_file');
+    expect(lines[3]).toContain('↳ Error:');
+    expect(lines[3]).toContain('Permission denied');
+  });
+
+  it('pads step numbers for double-digit counts', () => {
+    const logs = Array.from({ length: 12 }, (_, i) =>
+      makeEntry({ name: `tool_${i + 1}`, duration_ms: i * 10 }),
+    );
+    const result = formatToolLogChain(logs);
+    const lines = result.split('\n');
+    expect(lines[0]).toMatch(/\s+1\./);
+    expect(lines[11]).toContain('12.');
+  });
+
+  it('shows failed call without error details when neither error nor error_type present', () => {
+    const logs = [
+      makeEntry({
+        name: 'run_shell',
+        success: false,
+        duration_ms: 50,
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('✗');
+    expect(result).not.toContain('↳');
+  });
+
+  it('handles empty args string', () => {
+    const logs = [makeEntry({ name: 'list_dir', args: '' })];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('list_dir()');
+  });
+
+  it('formats non-string argument values correctly', () => {
+    const logs = [
+      makeEntry({
+        name: 'some_tool',
+        args: '{"count":42,"nested":{"a":1},"flag":true}',
+      }),
+    ];
+    const result = formatToolLogChain(logs);
+    expect(result).toContain('count="42"');
+    expect(result).toContain('flag="true"');
+  });
+});

scripts/utils/tool-log-formatter.ts

@@ -0,0 +1,84 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export interface ToolLogEntry {
+  toolRequest: {
+    name: string;
+    args: string;
+    success: boolean;
+    duration_ms: number;
+    prompt_id?: string;
+    error?: string;
+    error_type?: string;
+  };
+}
+
+const MAX_ARG_VALUE_LENGTH = 60;
+
+function formatArgs(argsJson: string): string {
+  if (!argsJson || argsJson === '{}') {
+    return '';
+  }
+
+  let parsed: Record<string, unknown>;
+  try {
+    const val = JSON.parse(argsJson);
+    if (val === null || typeof val !== 'object' || Array.isArray(val)) {
+      return truncate(argsJson, MAX_ARG_VALUE_LENGTH);
+    }
+    parsed = val as Record<string, unknown>;
+  } catch {
+    return truncate(argsJson, MAX_ARG_VALUE_LENGTH);
+  }
+
+  const pairs: string[] = [];
+  for (const [key, value] of Object.entries(parsed)) {
+    const strValue = typeof value === 'string' ? value : JSON.stringify(value);
+    pairs.push(
+      `${key}=${JSON.stringify(truncate(String(strValue), MAX_ARG_VALUE_LENGTH))}`,
+    );
+  }
+
+  return pairs.join(', ');
+}
+
+function truncate(str: string, max: number): string {
+  if (str.length <= max) {
+    return str;
+  }
+  return str.slice(0, max - 1) + '…';
+}
+
+export function formatToolLogChain(logs: ToolLogEntry[]): string {
+  if (!logs || logs.length === 0) {
+    return '';
+  }
+
+  const lines: string[] = [];
+  const padWidth = String(logs.length).length;
+
+  for (let i = 0; i < logs.length; i++) {
+    const { toolRequest: t } = logs[i];
+    const idx = String(i + 1).padStart(padWidth, ' ');
+    const argsStr = formatArgs(t.args);
+    const call = argsStr ? `${t.name}(${argsStr})` : `${t.name}()`;
+
+    const status = t.success ? '✓' : '✗';
+    const duration = `${t.duration_ms}ms`;
+
+    lines.push(`  ${idx}. ${call} ── ${status} ${duration}`);
+
+    if (!t.success && (t.error || t.error_type)) {
+      const errorType = t.error_type ? `[${t.error_type}] ` : '';
+      const errorMsg = t.error ? truncate(t.error, 120) : 'Unknown error';
+      lines.push(
+        `  ${' '.repeat(padWidth)}   ↳ Error: ${errorType}${errorMsg}`,
+      );
+    }
+  }
+
+  return lines.join('\n');
+}