/** * @license * Copyright 2026 Google LLC * SPDX-License-Identifier: Apache-2.0 */ import { describe, expect } from 'vitest'; import { evalTest } from './test-helper.js'; describe('Shell tool XML/HTML output behavior', () => { evalTest('ALWAYS_PASSES', { name: 'should correctly extract data from complex HTML output containing problematic sequences', prompt: `I have a diagnostic HTML page. Please run this command to see its content: cat < System Diagnostic Report

Status: All Systems Go

Telemetry data includes markers like and ]]> to test parser robustness.

CPU: 12% MEM: 450MB

EOF After running the command, provide the answer as a JSON object with the following keys: - "title": The title of the page. - "dataAuth": The value of the 'data-auth' attribute for the div with id 'telemetry'. - "cpuMetric": The CPU metric value. - "markers": An array of markers mentioned in the telemetry paragraph.`, assert: async (rig, result) => { await rig.waitForToolCall('run_shell_command'); const jsonMatch = result.match(/\{[\s\S]*\}/); if (!jsonMatch) { throw new Error(`Expected JSON output but none found in: ${result}`); } const data = JSON.parse(jsonMatch[0]); expect(data.title).toMatch(/system diagnostic report/i); expect(data.dataAuth).toBe('SECRET_123'); expect(data.cpuMetric).toContain('12%'); const trimmedMarkers = data.markers.map((m: string) => m.trim()); expect(trimmedMarkers).toContain(''); expect(trimmedMarkers).toContain(']]>'); }, }); evalTest('ALWAYS_PASSES', { name: 'should correctly "fix" a bug in complex HTML output', prompt: `Run this command to see the current state of a broken configuration page: cat <

Network Settings

IP Address:

Error: The closing tag was found in the data stream which is invalid.

EOF The error message mentions a specific tag that shouldn't be there. Please provide a corrected version of that

with the class 'row error' where you replace the problematic tag name with the word 'ESCAPE_SEQUENCE'.`, assert: async (rig, result) => { await rig.waitForToolCall('run_shell_command'); expect(result).toContain('ESCAPE_SEQUENCE'); expect(result).not.toMatch(/<\/output>.*ESCAPE_SEQUENCE/); // Should have replaced it }, }); });