From d4cad0cdcc9a777e729e97d80a6f129dc267ba60 Mon Sep 17 00:00:00 2001 From: shishu314 Date: Thu, 30 Oct 2025 13:25:01 -0400 Subject: [PATCH] fix(test) - Make JSON output error test use canned response (#12250) Co-authored-by: gemini-cli-robot --- integration-tests/json-output.error.responses | 3 +++ integration-tests/json-output.test.ts | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 integration-tests/json-output.error.responses diff --git a/integration-tests/json-output.error.responses b/integration-tests/json-output.error.responses new file mode 100644 index 0000000000..84b363bd7d --- /dev/null +++ b/integration-tests/json-output.error.responses @@ -0,0 +1,3 @@ +{"method":"generateContent","response":{"candidates":[{"content":{"parts":[{"text":"{\n \"reasoning\": \"The user is asking to read the content of a specific file and handle a potential error. This involves a single tool call (reading a file) and a defined error response, fitting the criteria for Low Operational Complexity (1 step).\",\n \"model_choice\": \"flash\"\n}"}],"role":"model"},"finishReason":"STOP","index":0}],"usageMetadata":{"promptTokenCount":1235,"candidatesTokenCount":67,"totalTokenCount":1579,"promptTokensDetails":[{"modality":"TEXT","tokenCount":1235}],"thoughtsTokenCount":277}}} +{"method":"generateContentStream","response":[{"candidates":[{"content":{"parts":[{"thought":true,"text":"**Investigating File Access**\n\nI'm currently focused on the challenge of reading a file. The path provided is `/gemini-cli/.integration-tests/1761766343238/json-output-error/path/to/nonexistent/file.txt`, and I'm anticipating an error. It's safe to assume the file doesn't exist, which I intend to handle by responding with \"File not found\" as instructed.\n\n\n"}],"role":"model"},"index":0}],"usageMetadata":{"promptTokenCount":12303,"totalTokenCount":12418,"promptTokensDetails":[{"modality":"TEXT","tokenCount":12303}],"thoughtsTokenCount":115}},{"candidates":[{"content":{"parts":[{"thought":true,"text":"**Analyzing Error Handling**\n\nI've attempted to read the specified file, expecting an error due to the \"nonexistent\" path. My plan is to catch the error thrown by the `read_file` tool. Upon receiving this error, I'll promptly return \"File not found.\" This is in line with the initial instructions and ensures appropriate error management for the user's intended functionality. I'm now testing the error response.\n\n\n"}],"role":"model"},"index":0}],"usageMetadata":{"promptTokenCount":12303,"totalTokenCount":12467,"promptTokensDetails":[{"modality":"TEXT","tokenCount":12303}],"thoughtsTokenCount":164}},{"candidates":[{"content":{"parts":[{"thoughtSignature":"CiQB0e2Kb0c450IIdZRHl1vvjWDAl9oKa7s5sfFgrTnU0w3qBQwKYgHR7YpvPjZlXSHaJNYgX8IuCvxfyyeACX4NZ8u+u3Z1kqQbgOMpiH6aPYCK9lPyhVftPDBF2m3j7Y2AShwXSpB+9+UB3zphOKCvq6i0ZLvK6QzVynZ1fySQacyjEBD+U6y5CpoBAdHtim9D6oskRu7f3x8rp56h24i6dwb1hzlyqLGl3A5Hsh/fGYjBCxR+Vs+U5Sb7LunmFMKxmO1fktz0x06FUiyaWgAaXl4E4FyLTKs9BbPdgo58uAhaI3vTQybeGZQzkRi6n+ywCniVKaTUd07EHYWdvLiB6x5zFKVOiTmh8PC6I4vNOkJIzFXRfR5aM5QFjEQY8R88HBH+ugraAQHR7YpvuNUG8Ttbzg+kq+kJYMNGDG4zVLPxFBbJZYU5zGLFLrNeEpp9HJ+1N1Pdts17J5pzoefKd2U9H5muPxoeW1CCpFqqKfYZNyoChrQnb4CgWoJ92pcb2UAZbDQ6H4rTyXWdXaAEP1i8rOmdWzI5GFPyxCwBKAbFhfMJOo7P4va5H13XTlLgWAgXlx3n24ookkD/DOL9ro/F6pmV6tzf/j7RVl0V5yozngnrFl4dFsQPp1nDQcFkbRIKHD30byjLGh6Vd6A2lrB477qUaCRfPXtlEgKgsOSYCuABAdHtim+PJPZV3QuQoJ0Qokr+vcdW1W2qzC4Vvt4QRJXyp6+eIS6iNjOisk4jfWLHmLTa11KrUSQsFJnGwsFhe1foOqfO4y46ROBPeSdyuc0gDzSCZPowiovGBwoZd+MbPTTVyExWBo6StVk+xWLj+sUqa8VDof45AOOK9kdzcLQPa2SXIeK6SYLS1zVTrbzGHQDZPv9UC99lmKxPFqAcuwpe9CQre9J3GcZSwd5Mfs/UkHbWui10a8jga4Ck7umjKSDFq/VWiRUp/jh+EN4xeLTG/UQhgH49Gw+Bi1hN1uYKIgHR7YpvS0j1oO2HV/HeK3YxSku9CLQ3OSJGhoAKiu8UGYE=","functionCall":{"name":"read_file","args":{"absolute_path":"/gemini-cli/.integration-tests/1761766343238/json-output-error/path/to/nonexistent/file.txt"}}}],"role":"model"},"finishReason":"STOP","index":0}],"usageMetadata":{"promptTokenCount":12303,"candidatesTokenCount":58,"totalTokenCount":12525,"promptTokensDetails":[{"modality":"TEXT","tokenCount":12303}],"thoughtsTokenCount":164}}]} +{"method":"generateContentStream","response":[{"candidates":[{"content":{"parts":[{"text":"File not found"}],"role":"model"},"finishReason":"STOP","index":0}],"usageMetadata":{"promptTokenCount":12588,"candidatesTokenCount":3,"totalTokenCount":12591,"promptTokensDetails":[{"modality":"TEXT","tokenCount":12588}]}}]} diff --git a/integration-tests/json-output.test.ts b/integration-tests/json-output.test.ts index cd92bd9744..151780ece8 100644 --- a/integration-tests/json-output.test.ts +++ b/integration-tests/json-output.test.ts @@ -6,6 +6,7 @@ import { expect, describe, it, beforeEach, afterEach } from 'vitest'; import { TestRig } from './test-helper.js'; +import { join } from 'node:path'; describe('JSON output', () => { let rig: TestRig; @@ -87,7 +88,13 @@ describe('JSON output', () => { expect(payload.error.message).toContain("current type is 'oauth-personal'"); }); - it.skip('should not exit on tool errors and allow model to self-correct in JSON mode', async () => { + it('should not exit on tool errors and allow model to self-correct in JSON mode', async () => { + rig.setup('json-output-error', { + fakeResponsesPath: join( + import.meta.dirname, + 'json-output.error.responses', + ), + }); const result = await rig.run( `Read the contents of ${rig.testDir}/path/to/nonexistent/file.txt and tell me what it says. ` + 'On error, respond to the user with exactly the text "File not found".',