diff --git a/evals/auto_distillation.eval.ts b/evals/auto_distillation.eval.ts new file mode 100644 index 0000000000..49116e0663 --- /dev/null +++ b/evals/auto_distillation.eval.ts @@ -0,0 +1,84 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect } from 'vitest'; +import path from 'node:path'; +import fs from 'node:fs'; +import { appEvalTest } from './app-test-helper.js'; +import { + userText, + mockGenerateContentStreamText, +} from '@google/gemini-cli-core'; + +describe('Auto-Distillation Behavioral Evals', () => { + appEvalTest('USUALLY_PASSES', { + name: 'Agent successfully navigates truncated output using the structural map to extract a secret', + timeout: 180000, + configOverrides: { autoDistillation: true }, + setup: async (rig) => { + const testDir = rig.getTestDir(); + + const mockData: any = { + system_info: { + version: '1.0.0', + uptime: 999999, + environment: 'production', + }, + active_sessions: [], + quarantined_payloads: [ + { id: 'Subject-01', status: 'cleared' }, + { + id: 'Subject-89', + secret_token: 'the_cake_is_a_lie', + status: 'held_for_review', + }, + { id: 'Subject-99', status: 'cleared' }, + ], + archived_metrics: [], + }; + + for (let i = 0; i < 300; i++) { + mockData.active_sessions.push({ + session_id: `sess_${i.toString().padStart(4, '0')}`, + duration_ms: Math.floor(Math.random() * 10000), + bytes_transferred: Math.floor(Math.random() * 50000), + }); + } + + for (let i = 0; i < 2000; i++) { + mockData.archived_metrics.push({ + timestamp: Date.now() - i * 1000, + cpu_load: parseFloat(Math.random().toFixed(4)), + mem_usage: parseFloat(Math.random().toFixed(4)), + }); + } + + const massiveString = JSON.stringify(mockData, null, 2); + + fs.writeFileSync( + path.join(testDir, 'server_state_dump.json'), + massiveString, + ); + }, + script: [ + userText( + 'We have a critical error in production. Are you ready to help?', + ), + mockGenerateContentStreamText( + 'I am ready. Please provide the details of the error.', + ), + ], + prompt: `My application crashed with: "FATAL: Subject-89 held for review in quarantine". \n\nPlease run \`cat server_state_dump.json\` to investigate. The file is massive, so your tool output will be automatically truncated and you will receive a structural map instead. Use that structural map to determine the right command to extract the \`secret_token\` for Subject-89. Please state the exact secret token when you find it.`, + assert: async (rig) => { + await rig.drainBreakpointsUntilIdle(undefined, 60000); + + const finalOutput = rig.getStaticOutput(); + + // Ensure the agent correctly extracted the secret token after navigating the distilled output + expect(finalOutput).toContain('the_cake_is_a_lie'); + }, + }); +}); diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 777950c0ca..03445f04fa 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -840,6 +840,7 @@ export async function loadCliConfig( skillsSupport: settings.skills?.enabled ?? true, disabledSkills: settings.skills?.disabled, experimentalJitContext: settings.experimental?.jitContext, + autoDistillation: settings.experimental?.autoDistillation, experimentalMemoryManager: settings.experimental?.memoryManager, modelSteering: settings.experimental?.modelSteering, topicUpdateNarration: settings.experimental?.topicUpdateNarration, diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index ea6b9f9239..d688b85699 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -1870,6 +1870,15 @@ const SETTINGS_SCHEMA = { description: 'Enable local and remote subagents.', showInDialog: false, }, + autoDistillation: { + type: 'boolean', + label: 'Auto Distillation', + category: 'Experimental', + requiresRestart: true, + default: false, + description: 'Enable automatic distillation for large tool outputs.', + showInDialog: false, + }, extensionManagement: { type: 'boolean', label: 'Extension Management', diff --git a/packages/cli/src/integration-tests/__snapshots__/autoDistillation.test.tsx.snap b/packages/cli/src/integration-tests/__snapshots__/autoDistillation.test.tsx.snap new file mode 100644 index 0000000000..d73c3d92a8 --- /dev/null +++ b/packages/cli/src/integration-tests/__snapshots__/autoDistillation.test.tsx.snap @@ -0,0 +1,73 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Auto-distillation Integration > should truncate and summarize massive tool outputs, and we should golden the chat history 1`] = ` +[ + { + "parts": [ + { + "text": "", + }, + ], + "role": "user", + }, + { + "parts": [ + { + "text": "Fetch the massive file.", + }, + ], + "role": "user", + }, + { + "parts": [ + { + "text": "I will now fetch the data.", + }, + { + "functionCall": { + "args": { + "command": "cat large.txt", + }, + "id": "", + "name": "run_shell_command", + }, + }, + ], + "role": "model", + }, + { + "parts": [ + { + "functionResponse": { + "id": "", + "name": "run_shell_command", + "response": { + "output": "Output too large. Showing first 10 and last 40 characters. For full output see: /.gemini/tmp//tool-outputs/session-/run_shell_command__.txt +Output: ca + +... [39 characters omitted] ... + + +Exit Code: 1 +Process Group PGID: + +--- Structural Map of Truncated Content --- +- Line 1: Header +- Lines 2-5000: User data +- Line 5001: Footer", + }, + }, + }, + ], + "role": "user", + }, + { + "parts": [ + { + "text": "I got the summarized output. Task complete.", + }, + ], + "role": "model", + }, +] +`; diff --git a/packages/cli/src/integration-tests/autoDistillation.test.tsx b/packages/cli/src/integration-tests/autoDistillation.test.tsx new file mode 100644 index 0000000000..b5ddacccd5 --- /dev/null +++ b/packages/cli/src/integration-tests/autoDistillation.test.tsx @@ -0,0 +1,68 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { AppRig } from '../test-utils/AppRig.js'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { FakeContentGenerator } from '@google/gemini-cli-core'; +import { PolicyDecision } from '@google/gemini-cli-core'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +describe('Auto-distillation Integration', () => { + let rig: AppRig | undefined; + + afterEach(async () => { + if (rig) { + await rig.unmount(); + } + vi.restoreAllMocks(); + }); + + it('should truncate and summarize massive tool outputs, and we should golden the chat history', async () => { + const fakeResponsesPath = path.join( + __dirname, + '../test-utils/fixtures/auto-distillation.responses', + ); + const contentGenerator = + await FakeContentGenerator.fromFile(fakeResponsesPath); + rig = new AppRig({ + contentGenerator, + configOverrides: { autoDistillation: true }, + }); + + await rig.initialize(); + + const config = rig.getConfig(); + // 50 chars threshold. > 75 chars triggers summarization + vi.spyOn(config, 'getTruncateToolOutputThreshold').mockReturnValue(50); + + rig.setToolPolicy('run_shell_command', PolicyDecision.ASK_USER); + + rig.setMockCommands([ + { + command: /cat large.txt/, + result: { + output: 'A'.repeat(100), + exitCode: 0, + }, + }, + ]); + + await rig.render(); + await rig.waitForIdle(); + + await rig.sendMessage('Fetch the massive file.'); + + await rig.waitForOutput('Shell'); + await rig.resolveTool('Shell'); + + await rig.waitForOutput('Task complete.'); + + expect(rig.getCuratedHistory()).toMatchSnapshot(); + }); +}); diff --git a/packages/cli/src/test-utils/fixtures/auto-distillation.responses b/packages/cli/src/test-utils/fixtures/auto-distillation.responses new file mode 100644 index 0000000000..68ece6f858 --- /dev/null +++ b/packages/cli/src/test-utils/fixtures/auto-distillation.responses @@ -0,0 +1,3 @@ +{"method":"generateContentStream","response":[{"candidates":[{"content":{"role":"model","parts":[{"text":"I will now fetch the data."},{"functionCall":{"name":"run_shell_command","args":{"command":"cat large.txt"}}}]},"finishReason":"STOP"}]}]} +{"method":"generateContent","response":{"candidates":[{"content":{"role":"model","parts":[{"text":"- Line 1: Header\n- Lines 2-5000: User data\n- Line 5001: Footer"}]},"finishReason":"STOP"}]}} +{"method":"generateContentStream","response":[{"candidates":[{"content":{"role":"model","parts":[{"text":"I got the summarized output. Task complete."}]},"finishReason":"STOP"}]}]} diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 93473d776e..0e4be38a87 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -637,6 +637,7 @@ export interface ConfigParameters { disabledSkills?: string[]; adminSkillsEnabled?: boolean; experimentalJitContext?: boolean; + autoDistillation?: boolean; experimentalMemoryManager?: boolean; topicUpdateNarration?: boolean; toolOutputMasking?: Partial; @@ -863,6 +864,7 @@ export class Config implements McpContext, AgentLoopContext { private readonly adminSkillsEnabled: boolean; private readonly experimentalJitContext: boolean; + private readonly autoDistillation: boolean; private readonly experimentalMemoryManager: boolean; private readonly topicUpdateNarration: boolean; private readonly disableLLMCorrection: boolean; @@ -1028,6 +1030,7 @@ export class Config implements McpContext, AgentLoopContext { ); this.experimentalJitContext = params.experimentalJitContext ?? true; + this.autoDistillation = params.autoDistillation ?? false; this.experimentalMemoryManager = params.experimentalMemoryManager ?? false; this.topicUpdateNarration = params.topicUpdateNarration ?? false; this.modelSteering = params.modelSteering ?? false; @@ -2186,6 +2189,10 @@ export class Config implements McpContext, AgentLoopContext { return this.experimentalJitContext; } + isAutoDistillationEnabled(): boolean { + return this.autoDistillation; + } + isMemoryManagerEnabled(): boolean { return this.experimentalMemoryManager; } diff --git a/packages/core/src/scheduler/tool-executor.test.ts b/packages/core/src/scheduler/tool-executor.test.ts index ff9edd83f3..ece9a9c0c6 100644 --- a/packages/core/src/scheduler/tool-executor.test.ts +++ b/packages/core/src/scheduler/tool-executor.test.ts @@ -335,8 +335,10 @@ describe('ToolExecutor', () => { it('should truncate large shell output', async () => { // 1. Setup Config for Truncation + vi.spyOn(config, 'getTruncateToolOutputThreshold').mockReturnValue(10); vi.spyOn(config.storage, 'getProjectTempDir').mockReturnValue('/tmp'); + vi.spyOn(config.storage, 'getProjectTempDir').mockReturnValue('/tmp'); const mockTool = new MockTool({ name: SHELL_TOOL_NAME }); const invocation = mockTool.build({}); @@ -396,8 +398,10 @@ describe('ToolExecutor', () => { it('should truncate large MCP tool output with single text Part', async () => { // 1. Setup Config for Truncation + vi.spyOn(config, 'getTruncateToolOutputThreshold').mockReturnValue(10); vi.spyOn(config.storage, 'getProjectTempDir').mockReturnValue('/tmp'); + vi.spyOn(config.storage, 'getProjectTempDir').mockReturnValue('/tmp'); const mcpToolName = 'get_big_text'; const messageBus = createMockMessageBus(); @@ -440,8 +444,9 @@ describe('ToolExecutor', () => { }); // 4. Verify Truncation Logic + const stringifiedLongText = JSON.stringify([{ text: longText }], null, 2); expect(fileUtils.saveTruncatedToolOutput).toHaveBeenCalledWith( - longText, + stringifiedLongText, mcpToolName, 'call-mcp-trunc', expect.any(String), @@ -449,7 +454,7 @@ describe('ToolExecutor', () => { ); expect(fileUtils.formatTruncatedToolOutput).toHaveBeenCalledWith( - longText, + stringifiedLongText, '/tmp/truncated_output.txt', 10, ); @@ -460,8 +465,9 @@ describe('ToolExecutor', () => { } }); - it('should not truncate MCP tool output with multiple Parts', async () => { + it('should truncate MCP tool output with multiple Parts', async () => { vi.spyOn(config, 'getTruncateToolOutputThreshold').mockReturnValue(10); + vi.spyOn(config.storage, 'getProjectTempDir').mockReturnValue('/tmp'); const messageBus = createMockMessageBus(); const mcpTool = new DiscoveredMCPTool( @@ -501,9 +507,26 @@ describe('ToolExecutor', () => { onUpdateToolCall: vi.fn(), }); - // Should NOT have been truncated - expect(fileUtils.saveTruncatedToolOutput).not.toHaveBeenCalled(); - expect(fileUtils.formatTruncatedToolOutput).not.toHaveBeenCalled(); + const longText1 = 'This is long text that exceeds the threshold.'; + const stringifiedLongText = JSON.stringify( + [{ text: longText1 }, { text: 'second part' }], + null, + 2, + ); + + // Should HAVE been truncated now + expect(fileUtils.saveTruncatedToolOutput).toHaveBeenCalledWith( + stringifiedLongText, + 'get_big_text', + 'call-mcp-multi', + expect.any(String), + 'test-session-id', + ); + expect(fileUtils.formatTruncatedToolOutput).toHaveBeenCalledWith( + stringifiedLongText, + '/tmp/truncated_output.txt', + 10, + ); expect(result.status).toBe(CoreToolCallStatus.Success); }); @@ -712,8 +735,10 @@ describe('ToolExecutor', () => { it('should truncate large shell output even on cancellation', async () => { // 1. Setup Config for Truncation + vi.spyOn(config, 'getTruncateToolOutputThreshold').mockReturnValue(10); vi.spyOn(config.storage, 'getProjectTempDir').mockReturnValue('/tmp'); + vi.spyOn(config.storage, 'getProjectTempDir').mockReturnValue('/tmp'); const mockTool = new MockTool({ name: SHELL_TOOL_NAME }); const invocation = mockTool.build({}); diff --git a/packages/core/src/scheduler/tool-executor.ts b/packages/core/src/scheduler/tool-executor.ts index 81232d39d9..fcfb9776ab 100644 --- a/packages/core/src/scheduler/tool-executor.ts +++ b/packages/core/src/scheduler/tool-executor.ts @@ -6,9 +6,9 @@ import { ToolErrorType, + runInDevTraceSpan, ToolOutputTruncatedEvent, logToolOutputTruncated, - runInDevTraceSpan, type ToolCallRequestInfo, type ToolCallResponseInfo, type ToolResult, @@ -19,11 +19,13 @@ import { import { isAbortError } from '../utils/errors.js'; import { SHELL_TOOL_NAME } from '../tools/tool-names.js'; import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; +import { ToolOutputDistillationService } from '../services/toolDistillationService.js'; import { executeToolWithHooks } from '../core/coreToolHookTriggers.js'; import { saveTruncatedToolOutput, formatTruncatedToolOutput, } from '../utils/fileUtils.js'; + import { convertToFunctionResponse } from '../utils/generateContentResponseUtilities.js'; import { CoreToolCallStatus, @@ -180,6 +182,15 @@ export class ToolExecutor { call: ToolCall, content: PartListUnion, ): Promise<{ truncatedContent: PartListUnion; outputFile?: string }> { + if (this.config.isAutoDistillationEnabled()) { + const distiller = new ToolOutputDistillationService( + this.config, + this.context.geminiClient, + this.context.promptId, + ); + return distiller.distill(call.request.name, call.request.callId, content); + } + const toolName = call.request.name; const callId = call.request.callId; let outputFile: string | undefined; @@ -242,8 +253,6 @@ export class ToolExecutor { threshold, ); - // We need to return a NEW array to avoid mutating the original toolResult if it matters, - // though here we are creating the response so it's probably fine to mutate or return new. const truncatedContent: Part[] = [ { ...firstPart, text: truncatedText }, ]; diff --git a/packages/core/src/services/toolDistillationService.ts b/packages/core/src/services/toolDistillationService.ts new file mode 100644 index 0000000000..5fe430fa33 --- /dev/null +++ b/packages/core/src/services/toolDistillationService.ts @@ -0,0 +1,203 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + LlmRole, + ToolOutputTruncatedEvent, + logToolOutputTruncated, + debugLogger, + type Config, +} from '../index.js'; +import type { PartListUnion } from '@google/genai'; +import { type GeminiClient } from '../core/client.js'; +import { DEFAULT_GEMINI_FLASH_LITE_MODEL } from '../config/models.js'; +import { + saveTruncatedToolOutput, + formatTruncatedToolOutput, +} from '../utils/fileUtils.js'; +import { + READ_FILE_TOOL_NAME, + READ_MANY_FILES_TOOL_NAME, +} from '../tools/tool-names.js'; + +export interface DistilledToolOutput { + truncatedContent: PartListUnion; + outputFile?: string; +} + +export class ToolOutputDistillationService { + constructor( + private readonly config: Config, + private readonly geminiClient: GeminiClient, + private readonly promptId: string, + ) {} + + /** + * Distills a tool's output if it exceeds configured length thresholds, preserving + * the agent's context window. This includes saving the raw output to disk, replacing + * the output with a truncated placeholder, and optionally summarizing the output + * via a secondary LLM call if the output is massively oversized. + */ + async distill( + toolName: string, + callId: string, + content: PartListUnion, + ): Promise { + // Explicitly bypass escape hatches that natively handle large outputs + if (this.isExemptFromDistillation(toolName)) { + return { truncatedContent: content }; + } + + const threshold = this.config.getTruncateToolOutputThreshold(); + if (threshold <= 0) { + return { truncatedContent: content }; + } + + const originalContentLength = this.calculateContentLength(content); + + if (originalContentLength > threshold) { + return this.performDistillation( + toolName, + callId, + content, + originalContentLength, + threshold, + ); + } + + return { truncatedContent: content }; + } + + private isExemptFromDistillation(toolName: string): boolean { + return ( + toolName === READ_FILE_TOOL_NAME || toolName === READ_MANY_FILES_TOOL_NAME + ); + } + + private calculateContentLength(content: PartListUnion): number { + if (typeof content === 'string') { + return content.length; + } + + if (Array.isArray(content)) { + return content.reduce((acc, part) => { + if ( + typeof part === 'object' && + part !== null && + 'text' in part && + typeof part.text === 'string' + ) { + return acc + part.text.length; + } + return acc; + }, 0); + } + + return 0; + } + + private stringifyContent(content: PartListUnion): string { + return typeof content === 'string' + ? content + : JSON.stringify(content, null, 2); + } + + private async performDistillation( + toolName: string, + callId: string, + content: PartListUnion, + originalContentLength: number, + threshold: number, + ): Promise { + const stringifiedContent = this.stringifyContent(content); + + // Save the raw, untruncated string to disk for human review + const { outputFile: savedPath } = await saveTruncatedToolOutput( + stringifiedContent, + toolName, + callId, + this.config.storage.getProjectTempDir(), + this.promptId, + ); + + let truncatedText = formatTruncatedToolOutput( + stringifiedContent, + savedPath, + threshold, + ); + + // If the output is massively oversized, attempt to generate a structural map + const summarizationThreshold = threshold * 1.5; + if (originalContentLength > summarizationThreshold) { + const summaryText = await this.generateStructuralMap( + toolName, + stringifiedContent, + Math.floor(summarizationThreshold), + ); + + if (summaryText) { + truncatedText += `\n\n--- Structural Map of Truncated Content ---\n${summaryText}`; + } + } + + logToolOutputTruncated( + this.config, + new ToolOutputTruncatedEvent(this.promptId, { + toolName, + originalContentLength, + truncatedContentLength: truncatedText.length, + threshold, + }), + ); + + return { + truncatedContent: + typeof content === 'string' ? truncatedText : [{ text: truncatedText }], + outputFile: savedPath, + }; + } + + /** + * Calls a fast, internal model (Flash-Lite) to provide a high-level summary + * of the truncated content's structure. + */ + private async generateStructuralMap( + toolName: string, + stringifiedContent: string, + maxPreviewLen: number, + ): Promise { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 15000); // 15s timeout + + const promptText = `The following output from the tool '${toolName}' is extremely large and has been truncated. Please provide a very brief, high-level structural map of its contents (e.g., key sections, JSON schema outline, or line number ranges for major components). Keep the summary under 10 lines. Do not attempt to summarize the specific data values, just the structure so another agent knows what is inside. + +Output to summarize: +${stringifiedContent.slice(0, maxPreviewLen)}...`; + + const summaryResponse = await this.geminiClient.generateContent( + { + model: DEFAULT_GEMINI_FLASH_LITE_MODEL, + overrideScope: 'internal-summarizer', + }, + [{ parts: [{ text: promptText }] }], + controller.signal, + LlmRole.MAIN, + ); + + clearTimeout(timeoutId); + + return summaryResponse.candidates?.[0]?.content?.parts?.[0]?.text; + } catch (e) { + // Fail gracefully, summarization is a progressive enhancement + debugLogger.debug( + 'Failed to generate structural map for truncated output:', + e, + ); + return undefined; + } + } +} diff --git a/packages/core/src/tools/web-fetch.ts b/packages/core/src/tools/web-fetch.ts index 27a60c4259..674b3c75c2 100644 --- a/packages/core/src/tools/web-fetch.ts +++ b/packages/core/src/tools/web-fetch.ts @@ -21,8 +21,8 @@ import { getErrorMessage } from '../utils/errors.js'; import { ApprovalMode } from '../policy/types.js'; import { getResponseText } from '../utils/partUtils.js'; import { fetchWithTimeout, isPrivateIp } from '../utils/fetch.js'; -import { truncateString } from '../utils/textUtils.js'; import { convert } from 'html-to-text'; +import { truncateString } from '../utils/textUtils.js'; import { logWebFetchFallbackAttempt, WebFetchFallbackAttemptEvent, @@ -40,11 +40,12 @@ import { LRUCache } from 'mnemonist'; import type { AgentLoopContext } from '../config/agent-loop-context.js'; const URL_FETCH_TIMEOUT_MS = 10000; -const MAX_CONTENT_LENGTH = 250000; + const MAX_EXPERIMENTAL_FETCH_SIZE = 10 * 1024 * 1024; // 10MB +const MAX_CONTENT_LENGTH = 100000; +const TRUNCATION_WARNING = '\n\n... [Content truncated due to size limit] ...'; const USER_AGENT = 'Mozilla/5.0 (compatible; Google-Gemini-CLI/1.0; +https://github.com/google-gemini/gemini-cli)'; -const TRUNCATION_WARNING = '\n\n... [Content truncated due to size limit] ...'; // Rate limiting configuration const RATE_LIMIT_WINDOW_MS = 60000; // 1 minute @@ -331,9 +332,15 @@ class WebFetchToolInvocation extends BaseToolInvocation< textContent = rawContent; } - // Cap at MAX_CONTENT_LENGTH initially to avoid excessive memory usage - // before the global budget allocation. - return truncateString(textContent, MAX_CONTENT_LENGTH, ''); + if (!this.context.config.isAutoDistillationEnabled()) { + return truncateString( + textContent, + MAX_CONTENT_LENGTH, + TRUNCATION_WARNING, + ); + } + + return textContent; } private filterAndValidateUrls(urls: string[]): { @@ -399,28 +406,10 @@ class WebFetchToolInvocation extends BaseToolInvocation< }; } - // Smart Budget Allocation (Water-filling algorithm) for successes - const sortedSuccesses = [...successes].sort( - (a, b) => a.content.length - b.content.length, - ); - - let remainingBudget = MAX_CONTENT_LENGTH; - let remainingUrls = sortedSuccesses.length; const finalContentsByUrl = new Map(); - for (const success of sortedSuccesses) { - const fairShare = Math.floor(remainingBudget / remainingUrls); - const allocated = Math.min(success.content.length, fairShare); - - const truncated = truncateString( - success.content, - allocated, - TRUNCATION_WARNING, - ); - - finalContentsByUrl.set(success.url, truncated); - remainingBudget -= truncated.length; - remainingUrls--; + for (const success of successes) { + finalContentsByUrl.set(success.url, success.content); } const aggregatedContent = uniqueUrls @@ -656,14 +645,21 @@ ${aggregatedContent} ); if (status >= 400) { - const rawResponseText = bodyBuffer.toString('utf8'); + let rawResponseText = bodyBuffer.toString('utf8'); + if (!this.context.config.isAutoDistillationEnabled()) { + rawResponseText = truncateString( + rawResponseText, + 10000, + '\n\n... [Error response truncated] ...', + ); + } const headers: Record = {}; response.headers.forEach((value, key) => { headers[key] = value; }); const errorContent = `Request failed with status ${status} Headers: ${JSON.stringify(headers, null, 2)} -Response: ${truncateString(rawResponseText, 10000, '\n\n... [Error response truncated] ...')}`; +Response: ${rawResponseText}`; debugLogger.error( `[WebFetchTool] Experimental fetch failed with status ${status} for ${url}`, ); @@ -679,11 +675,10 @@ Response: ${truncateString(rawResponseText, 10000, '\n\n... [Error response trun lowContentType.includes('text/plain') || lowContentType.includes('application/json') ) { - const text = truncateString( - bodyBuffer.toString('utf8'), - MAX_CONTENT_LENGTH, - TRUNCATION_WARNING, - ); + let text = bodyBuffer.toString('utf8'); + if (!this.context.config.isAutoDistillationEnabled()) { + text = truncateString(text, MAX_CONTENT_LENGTH, TRUNCATION_WARNING); + } return { llmContent: text, returnDisplay: `Fetched ${contentType} content from ${url}`, @@ -692,16 +687,19 @@ Response: ${truncateString(rawResponseText, 10000, '\n\n... [Error response trun if (lowContentType.includes('text/html')) { const html = bodyBuffer.toString('utf8'); - const textContent = truncateString( - convert(html, { - wordwrap: false, - selectors: [ - { selector: 'a', options: { ignoreHref: false, baseUrl: url } }, - ], - }), - MAX_CONTENT_LENGTH, - TRUNCATION_WARNING, - ); + let textContent = convert(html, { + wordwrap: false, + selectors: [ + { selector: 'a', options: { ignoreHref: false, baseUrl: url } }, + ], + }); + if (!this.context.config.isAutoDistillationEnabled()) { + textContent = truncateString( + textContent, + MAX_CONTENT_LENGTH, + TRUNCATION_WARNING, + ); + } return { llmContent: textContent, returnDisplay: `Fetched and converted HTML content from ${url}`, @@ -726,11 +724,10 @@ Response: ${truncateString(rawResponseText, 10000, '\n\n... [Error response trun } // Fallback for unknown types - try as text - const text = truncateString( - bodyBuffer.toString('utf8'), - MAX_CONTENT_LENGTH, - TRUNCATION_WARNING, - ); + let text = bodyBuffer.toString('utf8'); + if (!this.context.config.isAutoDistillationEnabled()) { + text = truncateString(text, MAX_CONTENT_LENGTH, TRUNCATION_WARNING); + } return { llmContent: text, returnDisplay: `Fetched ${contentType || 'unknown'} content from ${url}`,