From 28e667bd97820859b9f28bbd535fd35ec661cd41 Mon Sep 17 00:00:00 2001
From: Jacob MacDonald <jakemac@google.com>
Date: Mon, 13 Oct 2025 09:35:00 -0700
Subject: [PATCH] Give explicit instructions for failure text in
 json-output.test.ts (#11029)

---
 integration-tests/json-output.test.ts | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/integration-tests/json-output.test.ts b/integration-tests/json-output.test.ts
index ddda72fb40..1376a4b9e9 100644
--- a/integration-tests/json-output.test.ts
+++ b/integration-tests/json-output.test.ts
@@ -87,7 +87,8 @@ describe('JSON output', () => {
 
   it('should not exit on tool errors and allow model to self-correct in JSON mode', async () => {
     const result = await rig.run(
-      'Read the contents of /path/to/nonexistent/file.txt and tell me what it says',
+      `Read the contents of ${rig.testDir}/path/to/nonexistent/file.txt and tell me what it says. ` +
+        'On error, respond to the user with exactly the text "File not found".',
       '--output-format',
       'json',
     );
@@ -99,14 +100,22 @@ describe('JSON output', () => {
     expect(parsed).toHaveProperty('response');
     expect(typeof parsed.response).toBe('string');
 
-    // The model should acknowledge the error in its response
+    // The model should acknowledge the error in its response with exactly the
+    // text "File not found" based on the instruction above, but we also match
+    // some other forms. If you get flakes for this test please file an issue to
+    // come up with a more robust solution.
     expect(parsed.response.toLowerCase()).toMatch(
       /cannot|does not exist|doesn't exist|not found|unable to|error|couldn't/,
     );
 
-    // Stats should be present, indicating the session completed normally
+    // Stats should be present, indicating the session completed normally.
     expect(parsed).toHaveProperty('stats');
+
+    // Should see one failed tool call in the stats.
     expect(parsed.stats).toHaveProperty('tools');
+    expect(parsed.stats.tools.totalCalls).toBe(1);
+    expect(parsed.stats.tools.totalFail).toBe(1);
+    expect(parsed.stats.tools.totalSuccess).toBe(0);
 
     // Should NOT have an error field at the top level
     expect(parsed.error).toBeUndefined();