feat(core): centralize line truncation and apply to all tools

This commit is contained in:
Christian Gunderman
2026-03-02 18:53:36 -08:00
parent 9850f01894
commit ef38202dbd
19 changed files with 636 additions and 260 deletions
+8 -64
View File
@@ -18,9 +18,7 @@ import type {
ModelSelectionConfig, ModelSelectionConfig,
GenerateContentResponsePromptFeedback, GenerateContentResponsePromptFeedback,
GenerateContentResponseUsageMetadata, GenerateContentResponseUsageMetadata,
Part,
SafetySetting, SafetySetting,
PartUnion,
SpeechConfigUnion, SpeechConfigUnion,
ThinkingConfig, ThinkingConfig,
ToolListUnion, ToolListUnion,
@@ -28,6 +26,11 @@ import type {
} from '@google/genai'; } from '@google/genai';
import { GenerateContentResponse } from '@google/genai'; import { GenerateContentResponse } from '@google/genai';
import { debugLogger } from '../utils/debugLogger.js'; import { debugLogger } from '../utils/debugLogger.js';
import {
isPart,
toParts,
toPartWithThoughtAsText,
} from '../utils/partUtils.js';
import type { Credits } from './types.js'; import type { Credits } from './types.js';
export interface CAGenerateContentRequest { export interface CAGenerateContentRequest {
@@ -193,22 +196,12 @@ function maybeToContent(content?: ContentUnion): Content | undefined {
return toContent(content); return toContent(content);
} }
function isPart(c: ContentUnion): c is PartUnion {
return (
typeof c === 'object' &&
c !== null &&
!Array.isArray(c) &&
!('parts' in c) &&
!('role' in c)
);
}
function toContent(content: ContentUnion): Content { function toContent(content: ContentUnion): Content {
if (Array.isArray(content)) { if (Array.isArray(content)) {
// it's a PartsUnion[] // it's a PartsUnion[]
return { return {
role: 'user', role: 'user',
parts: toParts(content), parts: toParts(content).map(toPartWithThoughtAsText),
}; };
} }
if (typeof content === 'string') { if (typeof content === 'string') {
@@ -222,65 +215,16 @@ function toContent(content: ContentUnion): Content {
// it's a Content - process parts to handle thought filtering // it's a Content - process parts to handle thought filtering
return { return {
...content, ...content,
parts: content.parts parts: toParts(content.parts).map(toPartWithThoughtAsText),
? toParts(content.parts.filter((p) => p != null))
: [],
}; };
} }
// it's a Part // it's a Part
return { return {
role: 'user', role: 'user',
parts: [toPart(content)], parts: [toPartWithThoughtAsText(content)],
}; };
} }
export function toParts(parts: PartUnion[]): Part[] {
return parts.map(toPart);
}
function toPart(part: PartUnion): Part {
if (typeof part === 'string') {
// it's a string
return { text: part };
}
// Handle thought parts for CountToken API compatibility
// The CountToken API expects parts to have certain required "oneof" fields initialized,
// but thought parts don't conform to this schema and cause API failures
if ('thought' in part && part.thought) {
const thoughtText = `[Thought: ${part.thought}]`;
const newPart = { ...part };
delete (newPart as Record<string, unknown>)['thought'];
const hasApiContent =
'functionCall' in newPart ||
'functionResponse' in newPart ||
'inlineData' in newPart ||
'fileData' in newPart;
if (hasApiContent) {
// It's a functionCall or other non-text part. Just strip the thought.
return newPart;
}
// If no other valid API content, this must be a text part.
// Combine existing text (if any) with the thought, preserving other properties.
const text = (newPart as { text?: unknown }).text;
const existingText = text ? String(text) : '';
const combinedText = existingText
? `${existingText}\n${thoughtText}`
: thoughtText;
return {
...newPart,
text: combinedText,
};
}
return part;
}
function toVertexGenerationConfig( function toVertexGenerationConfig(
config?: GenerateContentConfig, config?: GenerateContentConfig,
): VertexGenerationConfig | undefined { ): VertexGenerationConfig | undefined {
+1 -1
View File
@@ -16,7 +16,7 @@ import type {
GenerateContentConfig, GenerateContentConfig,
GenerateContentParameters, GenerateContentParameters,
} from '@google/genai'; } from '@google/genai';
import { toParts } from '../code_assist/converter.js'; import { toParts } from '../utils/partUtils.js';
import { createUserContent, FinishReason } from '@google/genai'; import { createUserContent, FinishReason } from '@google/genai';
import { retryWithBackoff, isRetryableError } from '../utils/retry.js'; import { retryWithBackoff, isRetryableError } from '../utils/retry.js';
import type { ValidationRequiredError } from '../utils/googleQuotaErrors.js'; import type { ValidationRequiredError } from '../utils/googleQuotaErrors.js';
+10 -33
View File
@@ -12,7 +12,12 @@ import type {
FunctionCallingConfig, FunctionCallingConfig,
} from '@google/genai'; } from '@google/genai';
import { DEFAULT_GEMINI_FLASH_MODEL } from '../config/models.js'; import { DEFAULT_GEMINI_FLASH_MODEL } from '../config/models.js';
import { getResponseText } from '../utils/partUtils.js'; import {
getResponseText,
isContent,
isTextPart,
toParts,
} from '../utils/partUtils.js';
/** /**
* Decoupled LLM request format - stable across Gemini CLI versions * Decoupled LLM request format - stable across Gemini CLI versions
@@ -87,32 +92,6 @@ export abstract class HookTranslator {
abstract fromHookToolConfig(hookToolConfig: HookToolConfig): ToolConfig; abstract fromHookToolConfig(hookToolConfig: HookToolConfig): ToolConfig;
} }
/**
* Type guard to check if a value has a text property
*/
function hasTextProperty(value: unknown): value is { text: string } {
return (
typeof value === 'object' &&
value !== null &&
'text' in value &&
typeof (value as { text: unknown }).text === 'string'
);
}
/**
* Type guard to check if content has role and parts properties
*/
function isContentWithParts(
content: unknown,
): content is { role: string; parts: unknown } {
return (
typeof content === 'object' &&
content !== null &&
'role' in content &&
'parts' in content
);
}
/** /**
* Helper to safely extract generation config from SDK request * Helper to safely extract generation config from SDK request
* The SDK uses a config field that contains generation parameters * The SDK uses a config field that contains generation parameters
@@ -174,7 +153,7 @@ export class HookTranslatorGenAIv1 extends HookTranslator {
role: 'user', role: 'user',
content, content,
}); });
} else if (isContentWithParts(content)) { } else if (isContent(content)) {
const role = const role =
content.role === 'model' content.role === 'model'
? ('model' as const) ? ('model' as const)
@@ -182,13 +161,11 @@ export class HookTranslatorGenAIv1 extends HookTranslator {
? ('system' as const) ? ('system' as const)
: ('user' as const); : ('user' as const);
const parts = Array.isArray(content.parts) const parts = toParts(content.parts);
? content.parts
: [content.parts];
// Extract only text parts - intentionally filtering out non-text content // Extract only text parts - intentionally filtering out non-text content
const textContent = parts const textContent = parts
.filter(hasTextProperty) .filter(isTextPart)
.map((part) => part.text) .map((part) => part.text)
.join(''); .join('');
@@ -273,7 +250,7 @@ export class HookTranslatorGenAIv1 extends HookTranslator {
// Extract text parts from the candidate // Extract text parts from the candidate
const textParts = const textParts =
candidate.content?.parts candidate.content?.parts
?.filter(hasTextProperty) ?.filter(isTextPart)
.map((part) => part.text) || []; .map((part) => part.text) || [];
return { return {
@@ -16,6 +16,7 @@ import { resolveClassifierModel, isGemini3Model } from '../../config/models.js';
import { createUserContent, Type } from '@google/genai'; import { createUserContent, Type } from '@google/genai';
import type { Config } from '../../config/config.js'; import type { Config } from '../../config/config.js';
import { debugLogger } from '../../utils/debugLogger.js'; import { debugLogger } from '../../utils/debugLogger.js';
import { toParts } from '../../utils/partUtils.js';
import type { LocalLiteRtLmClient } from '../../core/localLiteRtLmClient.js'; import type { LocalLiteRtLmClient } from '../../core/localLiteRtLmClient.js';
import { LlmRole } from '../../telemetry/types.js'; import { LlmRole } from '../../telemetry/types.js';
import { AuthType } from '../../core/contentGenerator.js'; import { AuthType } from '../../core/contentGenerator.js';
@@ -152,14 +153,9 @@ export class NumericalClassifierStrategy implements RoutingStrategy {
const finalHistory = context.history.slice(-HISTORY_TURNS_FOR_CONTEXT); const finalHistory = context.history.slice(-HISTORY_TURNS_FOR_CONTEXT);
// Wrap the user's request in tags to prevent prompt injection // Wrap the user's request in tags to prevent prompt injection
const requestParts = Array.isArray(context.request) const requestParts = toParts(context.request);
? context.request
: [context.request];
const sanitizedRequest = requestParts.map((part) => { const sanitizedRequest = requestParts.map((part) => {
if (typeof part === 'string') {
return { text: part };
}
if (part.text) { if (part.text) {
return { text: part.text }; return { text: part.text };
} }
@@ -10,10 +10,15 @@ import {
type Config, type Config,
type ToolResult, type ToolResult,
type AnyToolInvocation, type AnyToolInvocation,
type AnyDeclarativeTool,
} from '../index.js'; } from '../index.js';
import { makeFakeConfig } from '../test-utils/config.js'; import { makeFakeConfig } from '../test-utils/config.js';
import { MockTool } from '../test-utils/mock-tool.js'; import { MockTool } from '../test-utils/mock-tool.js';
import type { ScheduledToolCall } from './types.js'; import type {
ScheduledToolCall,
SuccessfulToolCall,
ToolCall,
} from './types.js';
import { CoreToolCallStatus } from './types.js'; import { CoreToolCallStatus } from './types.js';
import { SHELL_TOOL_NAME } from '../tools/tool-names.js'; import { SHELL_TOOL_NAME } from '../tools/tool-names.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
@@ -28,6 +33,7 @@ import {
GEN_AI_TOOL_DESCRIPTION, GEN_AI_TOOL_DESCRIPTION,
GEN_AI_TOOL_NAME, GEN_AI_TOOL_NAME,
} from '../telemetry/constants.js'; } from '../telemetry/constants.js';
import { DEFAULT_MAX_LINE_LENGTH } from '../utils/constants.js';
// Mock file utils // Mock file utils
vi.mock('../utils/fileUtils.js', () => ({ vi.mock('../utils/fileUtils.js', () => ({
@@ -136,7 +142,7 @@ describe('ToolExecutor', () => {
const spanArgs = vi.mocked(runInDevTraceSpan).mock.calls[0]; const spanArgs = vi.mocked(runInDevTraceSpan).mock.calls[0];
const fn = spanArgs[1]; const fn = spanArgs[1];
const metadata = { attributes: {} }; const metadata = { name: 'test-span', attributes: {} };
await fn({ metadata, endSpan: vi.fn() }); await fn({ metadata, endSpan: vi.fn() });
expect(metadata).toMatchObject({ expect(metadata).toMatchObject({
input: scheduledCall.request, input: scheduledCall.request,
@@ -199,7 +205,7 @@ describe('ToolExecutor', () => {
const spanArgs = vi.mocked(runInDevTraceSpan).mock.calls[0]; const spanArgs = vi.mocked(runInDevTraceSpan).mock.calls[0];
const fn = spanArgs[1]; const fn = spanArgs[1];
const metadata = { attributes: {} }; const metadata = { name: 'test-span', attributes: {} };
await fn({ metadata, endSpan: vi.fn() }); await fn({ metadata, endSpan: vi.fn() });
expect(metadata).toMatchObject({ expect(metadata).toMatchObject({
error: new Error('Tool Failed'), error: new Error('Tool Failed'),
@@ -528,4 +534,56 @@ describe('ToolExecutor', () => {
}), }),
); );
}); });
it('should truncate excessively long lines in multi-part tool results (subagent/MCP style)', async () => {
const longLine = 'a'.repeat(DEFAULT_MAX_LINE_LENGTH + 100);
const multiPartCall: ToolCall = {
status: CoreToolCallStatus.Scheduled,
request: {
callId: 'call-multipart',
name: 'testTool',
args: {},
isClientInitiated: false,
prompt_id: 'p1',
},
tool: {
name: 'testTool',
description: 'desc',
parametersJsonSchema: {},
isReadOnly: true,
} as unknown as AnyDeclarativeTool,
invocation: {
execute: vi.fn(),
} as unknown as AnyToolInvocation,
};
vi.mocked(coreToolHookTriggers.executeToolWithHooks).mockResolvedValueOnce({
llmContent: [
{ text: 'short line' },
{ text: `prefix ${longLine} suffix` },
],
returnDisplay: 'done',
});
const result = await executor.execute({
call: multiPartCall,
signal: new AbortController().signal,
onUpdateToolCall: vi.fn(),
});
expect(result.status).toBe(CoreToolCallStatus.Success);
const response = (result as SuccessfulToolCall).response;
expect(response.responseParts).toHaveLength(1);
const firstPart = response.responseParts[0];
const functionResponse =
'functionResponse' in firstPart ? firstPart.functionResponse : undefined;
if (!functionResponse) {
throw new Error('Expected functionResponse part');
}
const outputText = (functionResponse.response as { output: string }).output;
expect(outputText).toContain('short line');
expect(outputText).toContain('[Truncated');
expect(outputText.length).toBeLessThan(longLine.length + 100);
});
}); });
+41 -8
View File
@@ -16,7 +16,6 @@ import { ToolOutputTruncatedEvent } from '../telemetry/types.js';
import { runInDevTraceSpan } from '../telemetry/trace.js'; import { runInDevTraceSpan } from '../telemetry/trace.js';
import { truncateLongLines } from '../utils/textUtils.js'; import { truncateLongLines } from '../utils/textUtils.js';
import { DEFAULT_MAX_LINE_LENGTH } from '../utils/constants.js'; import { DEFAULT_MAX_LINE_LENGTH } from '../utils/constants.js';
import { SHELL_TOOL_NAME } from '../tools/tool-names.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import { ShellToolInvocation } from '../tools/shell.js'; import { ShellToolInvocation } from '../tools/shell.js';
import { executeToolWithHooks } from '../core/coreToolHookTriggers.js'; import { executeToolWithHooks } from '../core/coreToolHookTriggers.js';
@@ -24,6 +23,7 @@ import {
saveTruncatedToolOutput, saveTruncatedToolOutput,
formatTruncatedToolOutput, formatTruncatedToolOutput,
} from '../utils/fileUtils.js'; } from '../utils/fileUtils.js';
import { isTextPart } from '../utils/partUtils.js';
import { convertToFunctionResponse } from '../utils/generateContentResponseUtilities.js'; import { convertToFunctionResponse } from '../utils/generateContentResponseUtilities.js';
import type { import type {
CompletedToolCall, CompletedToolCall,
@@ -232,16 +232,12 @@ export class ToolExecutor {
call: ToolCall, call: ToolCall,
toolResult: ToolResult, toolResult: ToolResult,
): Promise<SuccessfulToolCall> { ): Promise<SuccessfulToolCall> {
let content = toolResult.llmContent;
let outputFile: string | undefined;
const toolName = call.request.originalRequestName || call.request.name; const toolName = call.request.originalRequestName || call.request.name;
const callId = call.request.callId; const callId = call.request.callId;
let content = toolResult.llmContent;
let outputFile: string | undefined;
if (typeof content === 'string') { if (typeof content === 'string') {
content = truncateLongLines(content, DEFAULT_MAX_LINE_LENGTH);
}
if (typeof content === 'string' && toolName === SHELL_TOOL_NAME) {
const threshold = this.config.getTruncateToolOutputThreshold(); const threshold = this.config.getTruncateToolOutputThreshold();
if (threshold > 0 && content.length > threshold) { if (threshold > 0 && content.length > threshold) {
@@ -273,7 +269,7 @@ export class ToolExecutor {
call.tool instanceof DiscoveredMCPTool call.tool instanceof DiscoveredMCPTool
) { ) {
const firstPart = content[0]; const firstPart = content[0];
if (typeof firstPart === 'object' && typeof firstPart.text === 'string') { if (isTextPart(firstPart)) {
const textContent = firstPart.text; const textContent = firstPart.text;
const threshold = this.config.getTruncateToolOutputThreshold(); const threshold = this.config.getTruncateToolOutputThreshold();
@@ -307,6 +303,43 @@ export class ToolExecutor {
} }
} }
// Final safety pass: truncate excessively long lines in every tool result (including subagents and MCP tools).
// This acts as a universal guardrail to prevent token-limit errors.
/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-return */
if (typeof content === 'string') {
content = truncateLongLines(content, DEFAULT_MAX_LINE_LENGTH);
} else if (Array.isArray(content)) {
content = Array.from(content as unknown[]).map((part: any) => {
if (typeof part?.text === 'string') {
return {
...part,
text: truncateLongLines(part.text, DEFAULT_MAX_LINE_LENGTH),
};
}
if (typeof part?.thought === 'string') {
return {
...part,
thought: truncateLongLines(part.thought, DEFAULT_MAX_LINE_LENGTH),
};
}
return part;
});
} else {
const p: any = content;
if (typeof p?.text === 'string') {
content = {
...p,
text: truncateLongLines(p.text, DEFAULT_MAX_LINE_LENGTH),
};
} else if (typeof p?.thought === 'string') {
content = {
...p,
thought: truncateLongLines(p.thought, DEFAULT_MAX_LINE_LENGTH),
};
}
}
/* eslint-enable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-return */
const response = convertToFunctionResponse( const response = convertToFunctionResponse(
toolName, toolName,
callId, callId,
+1 -9
View File
@@ -20,6 +20,7 @@ import type {
PartUnion, PartUnion,
} from '@google/genai'; } from '@google/genai';
import { truncateString } from '../utils/textUtils.js'; import { truncateString } from '../utils/textUtils.js';
import { isPart } from '../utils/partUtils.js';
// 160KB limit for the total size of string content in a log entry. // 160KB limit for the total size of string content in a log entry.
// The total log entry size limit is 256KB. We leave ~96KB (approx 37%) for JSON overhead (escaping, structure) and other fields. // The total log entry size limit is 256KB. We leave ~96KB (approx 37%) for JSON overhead (escaping, structure) and other fields.
@@ -131,15 +132,6 @@ export function toInputMessages(contents: Content[]): InputMessages {
return messages; return messages;
} }
function isPart(value: unknown): value is Part {
return (
typeof value === 'object' &&
value !== null &&
!Array.isArray(value) &&
!('parts' in value)
);
}
function toPart(part: PartUnion): Part { function toPart(part: PartUnion): Part {
if (typeof part === 'string') { if (typeof part === 'string') {
return { text: part }; return { text: part };
@@ -81,7 +81,7 @@ import {
} from '../dynamic-declaration-helpers.js'; } from '../dynamic-declaration-helpers.js';
import { import {
DEFAULT_MAX_LINES_TEXT_FILE, DEFAULT_MAX_LINES_TEXT_FILE,
MAX_LINE_LENGTH_TEXT_FILE, DEFAULT_MAX_LINE_LENGTH,
MAX_FILE_SIZE_MB, MAX_FILE_SIZE_MB,
} from '../../../utils/constants.js'; } from '../../../utils/constants.js';
@@ -91,7 +91,7 @@ import {
export const GEMINI_3_SET: CoreToolSet = { export const GEMINI_3_SET: CoreToolSet = {
read_file: { read_file: {
name: READ_FILE_TOOL_NAME, name: READ_FILE_TOOL_NAME,
description: `Reads and returns the content of a specified file. To maintain context efficiency, you MUST use 'start_line' and 'end_line' for targeted, surgical reads of specific sections. For your safety, the tool will automatically truncate output exceeding ${DEFAULT_MAX_LINES_TEXT_FILE} lines, ${MAX_LINE_LENGTH_TEXT_FILE} characters per line, or ${MAX_FILE_SIZE_MB}MB in size; however, triggering these limits is considered token-inefficient. Always retrieve only the minimum content necessary for your next step. Handles text, images (PNG, JPG, GIF, WEBP, SVG, BMP), audio files (MP3, WAV, AIFF, AAC, OGG, FLAC), and PDF files.`, description: `Reads and returns the content of a specified file. To maintain context efficiency, you MUST use 'start_line' and 'end_line' for targeted, surgical reads of specific sections. For your safety, the tool will automatically truncate output exceeding ${DEFAULT_MAX_LINES_TEXT_FILE} lines, ${DEFAULT_MAX_LINE_LENGTH} characters per line, or ${MAX_FILE_SIZE_MB}MB in size; however, triggering these limits is considered token-inefficient. Always retrieve only the minimum content necessary for your next step. Handles text, images (PNG, JPG, GIF, WEBP, SVG, BMP), audio files (MP3, WAV, AIFF, AAC, OGG, FLAC), and PDF files.`,
parametersJsonSchema: { parametersJsonSchema: {
type: 'object', type: 'object',
properties: { properties: {
+3 -8
View File
@@ -290,21 +290,16 @@ describe('ReadFileTool', () => {
'File size exceeds the 20MB limit', 'File size exceeds the 20MB limit',
); );
}); });
it('should return full lines regardless of length, relying on central truncation in ToolExecutor', async () => {
it('should handle text file with lines exceeding maximum length', async () => {
const filePath = path.join(tempRootDir, 'longlines.txt'); const filePath = path.join(tempRootDir, 'longlines.txt');
const longLine = 'a'.repeat(2500); // Exceeds MAX_LINE_LENGTH_TEXT_FILE (2000) const longLine = 'a'.repeat(2500); // Exceeds DEFAULT_MAX_LINE_LENGTH (2000)
const fileContent = `Short line\n${longLine}\nAnother short line`; const fileContent = `Short line\n${longLine}\nAnother short line`;
await fsp.writeFile(filePath, fileContent, 'utf-8'); await fsp.writeFile(filePath, fileContent, 'utf-8');
const params: ReadFileToolParams = { file_path: filePath }; const params: ReadFileToolParams = { file_path: filePath };
const invocation = tool.build(params); const invocation = tool.build(params);
const result = await invocation.execute(abortSignal); const result = await invocation.execute(abortSignal);
expect(result.llmContent).toContain( expect(result.llmContent).toContain(longLine);
'IMPORTANT: The file content has been truncated',
);
expect(result.llmContent).toContain('--- FILE CONTENT (truncated) ---');
expect(result.returnDisplay).toContain('some lines were shortened');
}); });
it('should handle image file and return appropriate content', async () => { it('should handle image file and return appropriate content', async () => {
+37
View File
@@ -740,6 +740,43 @@ describe('RipGrepTool', () => {
expect(result.returnDisplay).toContain('Found 1 match'); expect(result.returnDisplay).toContain('Found 1 match');
}); });
it('should truncate extremely long lines to prevent token limit errors', async () => {
const filePath = path.join(tempRootDir, 'longFile.txt');
await fs.writeFile(filePath, 'dummy content');
// Create a 20,000 character long line
const longPrefix = 'A'.repeat(10000);
const longSuffix = 'B'.repeat(10000);
const longLine = `${longPrefix} match ${longSuffix}\n`;
mockSpawn.mockImplementation(
createMockSpawn({
outputData:
JSON.stringify({
type: 'match',
data: {
path: { text: 'longFile.txt' },
line_number: 1,
lines: { text: longLine },
submatches: [
{ match: { text: 'match' }, start: 10001, end: 10006 },
],
},
}) + '\n',
exitCode: 0,
}),
);
const params: RipGrepToolParams = { pattern: 'match', no_ignore: true };
const invocation = grepTool.build(params);
const result = await invocation.execute(abortSignal);
expect(result.llmContent).toContain('longFile.txt');
expect(result.llmContent).toContain('[Truncated: showing characters');
expect(result.llmContent).toContain('match');
expect(result.llmContent).toContain('of 20007');
});
it('should handle regex special characters correctly', async () => { it('should handle regex special characters correctly', async () => {
// Setup specific mock for this test - regex pattern 'foo.*bar' should match 'const foo = "bar";' // Setup specific mock for this test - regex pattern 'foo.*bar' should match 'const foo = "bar";'
mockSpawn.mockImplementation( mockSpawn.mockImplementation(
+74 -27
View File
@@ -25,6 +25,8 @@ import {
} from '../utils/ignorePatterns.js'; } from '../utils/ignorePatterns.js';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js'; import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
import { execStreaming } from '../utils/shell-utils.js'; import { execStreaming } from '../utils/shell-utils.js';
import { truncateLine } from '../utils/textUtils.js';
import { DEFAULT_MAX_LINE_LENGTH } from '../utils/constants.js';
import { import {
DEFAULT_TOTAL_MAX_MATCHES, DEFAULT_TOTAL_MAX_MATCHES,
DEFAULT_SEARCH_TIMEOUT_MS, DEFAULT_SEARCH_TIMEOUT_MS,
@@ -33,6 +35,34 @@ import { RIP_GREP_DEFINITION } from './definitions/coreTools.js';
import { resolveToolDeclaration } from './definitions/resolver.js'; import { resolveToolDeclaration } from './definitions/resolver.js';
import { type GrepMatch, formatGrepResults } from './grep-utils.js'; import { type GrepMatch, formatGrepResults } from './grep-utils.js';
interface RipGrepJson {
type: string;
data: RipGrepData;
}
interface RipGrepData {
path: { text: string };
line_number: number;
lines: { text: string };
submatches?: Array<{
match: { text: string };
start: number;
end: number;
}>;
}
function isRipGrepJson(value: unknown): value is RipGrepJson {
return (
typeof value === 'object' &&
value !== null &&
'type' in value &&
typeof (value as { type: unknown }).type === 'string' &&
'data' in value &&
typeof (value as { data: unknown }).data === 'object' &&
(value as { data: unknown }).data !== null
);
}
function getRgCandidateFilenames(): readonly string[] { function getRgCandidateFilenames(): readonly string[] {
return process.platform === 'win32' ? ['rg.exe', 'rg'] : ['rg']; return process.platform === 'win32' ? ['rg.exe', 'rg'] : ['rg'];
} }
@@ -494,34 +524,51 @@ class GrepToolInvocation extends BaseToolInvocation<
basePath: string, basePath: string,
): GrepMatch | null { ): GrepMatch | null {
try { try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const json: unknown = JSON.parse(line);
const json = JSON.parse(line); if (isRipGrepJson(json)) {
if (json.type === 'match' || json.type === 'context') { if (json.type === 'match' || json.type === 'context') {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const data = json.data;
const data = json.data; // Defensive check: ensure text properties exist (skips binary/invalid encoding)
// Defensive check: ensure text properties exist (skips binary/invalid encoding) if (data.path?.text && data.lines?.text) {
if (data.path?.text && data.lines?.text) { const absoluteFilePath = path.resolve(basePath, data.path.text);
const absoluteFilePath = path.resolve(basePath, data.path.text); const relativeCheck = path.relative(basePath, absoluteFilePath);
const relativeCheck = path.relative(basePath, absoluteFilePath); if (
if ( relativeCheck === '..' ||
relativeCheck === '..' || relativeCheck.startsWith(`..${path.sep}`) ||
relativeCheck.startsWith(`..${path.sep}`) || path.isAbsolute(relativeCheck)
path.isAbsolute(relativeCheck) ) {
) { return null;
return null; }
const relativeFilePath = path.relative(basePath, absoluteFilePath);
let lineText = data.lines.text.trimEnd();
if (lineText.length > DEFAULT_MAX_LINE_LENGTH) {
const centerIndex =
data.submatches &&
Array.isArray(data.submatches) &&
data.submatches.length > 0
? Math.floor(
(data.submatches[0].start + data.submatches[0].end) / 2,
)
: undefined;
lineText = truncateLine(lineText, {
maxLength: DEFAULT_MAX_LINE_LENGTH,
centerIndex,
includeStats: true,
});
}
return {
absolutePath: absoluteFilePath,
filePath: relativeFilePath || path.basename(absoluteFilePath),
lineNumber: data.line_number,
line: lineText,
isContext: json.type === 'context',
};
} }
const relativeFilePath = path.relative(basePath, absoluteFilePath);
return {
absolutePath: absoluteFilePath,
filePath: relativeFilePath || path.basename(absoluteFilePath),
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
lineNumber: data.line_number,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
line: data.lines.text.trimEnd(),
isContext: json.type === 'context',
};
} }
} }
} catch (error) { } catch (error) {
+1 -1
View File
@@ -6,7 +6,7 @@
export const REFERENCE_CONTENT_START = '--- Content from referenced files ---'; export const REFERENCE_CONTENT_START = '--- Content from referenced files ---';
export const REFERENCE_CONTENT_END = '--- End of content ---'; export const REFERENCE_CONTENT_END = '--- End of content ---';
export const DEFAULT_MAX_LINE_LENGTH = 2000;
export const DEFAULT_MAX_LINES_TEXT_FILE = 2000; export const DEFAULT_MAX_LINES_TEXT_FILE = 2000;
export const MAX_LINE_LENGTH_TEXT_FILE = 2000;
export const MAX_FILE_SIZE_MB = 20; export const MAX_FILE_SIZE_MB = 20;
+3 -26
View File
@@ -15,11 +15,7 @@ import { ToolErrorType } from '../tools/tool-error.js';
import { BINARY_EXTENSIONS } from './ignorePatterns.js'; import { BINARY_EXTENSIONS } from './ignorePatterns.js';
import { createRequire as createModuleRequire } from 'node:module'; import { createRequire as createModuleRequire } from 'node:module';
import { debugLogger } from './debugLogger.js'; import { debugLogger } from './debugLogger.js';
import { import { DEFAULT_MAX_LINES_TEXT_FILE, MAX_FILE_SIZE_MB } from './constants.js';
DEFAULT_MAX_LINES_TEXT_FILE,
MAX_LINE_LENGTH_TEXT_FILE,
MAX_FILE_SIZE_MB,
} from './constants.js';
const requireModule = createModuleRequire(import.meta.url); const requireModule = createModuleRequire(import.meta.url);
@@ -495,22 +491,8 @@ export async function processSingleFileContent(
const actualStart = Math.min(sliceStart, originalLineCount); const actualStart = Math.min(sliceStart, originalLineCount);
const selectedLines = lines.slice(actualStart, sliceEnd); const selectedLines = lines.slice(actualStart, sliceEnd);
let linesWereTruncatedInLength = false; const isTruncated = actualStart > 0 || sliceEnd < originalLineCount;
const formattedLines = selectedLines.map((line) => { const llmContent = selectedLines.join('\n');
if (line.length > MAX_LINE_LENGTH_TEXT_FILE) {
linesWereTruncatedInLength = true;
return (
line.substring(0, MAX_LINE_LENGTH_TEXT_FILE) + '... [truncated]'
);
}
return line;
});
const isTruncated =
actualStart > 0 ||
sliceEnd < originalLineCount ||
linesWereTruncatedInLength;
const llmContent = formattedLines.join('\n');
// By default, return nothing to streamline the common case of a successful read_file. // By default, return nothing to streamline the common case of a successful read_file.
let returnDisplay = ''; let returnDisplay = '';
@@ -518,11 +500,6 @@ export async function processSingleFileContent(
returnDisplay = `Read lines ${ returnDisplay = `Read lines ${
actualStart + 1 actualStart + 1
}-${sliceEnd} of ${originalLineCount} from ${relativePathForDisplay}`; }-${sliceEnd} of ${originalLineCount} from ${relativePathForDisplay}`;
if (linesWereTruncatedInLength) {
returnDisplay += ' (some lines were shortened)';
}
} else if (linesWereTruncatedInLength) {
returnDisplay = `Read all ${originalLineCount} lines from ${relativePathForDisplay} (some lines were shortened)`;
} }
return { return {
@@ -10,7 +10,14 @@ import type {
FunctionCall, FunctionCall,
PartListUnion, PartListUnion,
} from '@google/genai'; } from '@google/genai';
import { getResponseText } from './partUtils.js'; import {
isTextPart,
isInlineDataPart,
isFileDataPart,
isFunctionResponsePart,
getResponseText,
toParts,
} from './partUtils.js';
import { supportsMultimodalFunctionResponse } from '../config/models.js'; import { supportsMultimodalFunctionResponse } from '../config/models.js';
import { debugLogger } from './debugLogger.js'; import { debugLogger } from './debugLogger.js';
@@ -31,18 +38,6 @@ function createFunctionResponsePart(
}; };
} }
function toParts(input: PartListUnion): Part[] {
const parts: Part[] = [];
for (const part of Array.isArray(input) ? input : [input]) {
if (typeof part === 'string') {
parts.push({ text: part });
} else if (part) {
parts.push(part);
}
}
return parts;
}
export function convertToFunctionResponse( export function convertToFunctionResponse(
toolName: string, toolName: string,
callId: string, callId: string,
@@ -61,13 +56,13 @@ export function convertToFunctionResponse(
const fileDataParts: Part[] = []; const fileDataParts: Part[] = [];
for (const part of parts) { for (const part of parts) {
if (part.text !== undefined) { if (isTextPart(part)) {
textParts.push(part.text); textParts.push(part.text);
} else if (part.inlineData) { } else if (isInlineDataPart(part)) {
inlineDataParts.push(part); inlineDataParts.push(part);
} else if (part.fileData) { } else if (isFileDataPart(part)) {
fileDataParts.push(part); fileDataParts.push(part);
} else if (part.functionResponse) { } else if (isFunctionResponsePart(part)) {
if (parts.length > 1) { if (parts.length > 1) {
debugLogger.warn( debugLogger.warn(
'convertToFunctionResponse received multiple parts with a functionResponse. Only the functionResponse will be used, other parts will be ignored', 'convertToFunctionResponse received multiple parts with a functionResponse. Only the functionResponse will be used, other parts will be ignored',
+76 -8
View File
@@ -10,6 +10,13 @@ import {
getResponseText, getResponseText,
flatMapTextParts, flatMapTextParts,
appendToLastTextPart, appendToLastTextPart,
isPart,
isTextPart,
isContent,
isFunctionCallPart,
isFunctionResponsePart,
isInlineDataPart,
isFileDataPart,
} from './partUtils.js'; } from './partUtils.js';
import type { GenerateContentResponse, Part, PartUnion } from '@google/genai'; import type { GenerateContentResponse, Part, PartUnion } from '@google/genai';
@@ -28,6 +35,61 @@ const mockResponse = (
}); });
describe('partUtils', () => { describe('partUtils', () => {
describe('type guards', () => {
it('isPart', () => {
expect(isPart({ text: 'hi' })).toBe(true);
expect(isPart({ functionCall: { name: 'f' } })).toBe(true);
expect(isPart('string')).toBe(false);
expect(isPart(['array'])).toBe(false);
expect(isPart({ parts: [] })).toBe(false);
expect(isPart(null)).toBe(false);
});
it('isTextPart', () => {
expect(isTextPart({ text: 'hi' })).toBe(true);
expect(isTextPart({ text: '' })).toBe(true);
expect(isTextPart({ functionCall: { name: 'f' } })).toBe(false);
expect(isTextPart({ text: 123 })).toBe(false);
});
it('isContent', () => {
expect(isContent({ parts: [] })).toBe(true);
expect(isContent({ role: 'user', parts: [{ text: 'hi' }] })).toBe(true);
expect(isContent({ text: 'hi' })).toBe(false);
});
it('isFunctionCallPart', () => {
expect(isFunctionCallPart({ functionCall: { name: 'f' } })).toBe(true);
expect(
isFunctionCallPart({ functionCall: { name: 'f', args: {} } }),
).toBe(true);
expect(isFunctionCallPart({ text: 'hi' })).toBe(false);
});
it('isFunctionResponsePart', () => {
expect(
isFunctionResponsePart({
functionResponse: { name: 'f', response: {} },
}),
).toBe(true);
expect(isFunctionResponsePart({ text: 'hi' })).toBe(false);
});
it('isInlineDataPart', () => {
expect(
isInlineDataPart({ inlineData: { mimeType: 't', data: 'd' } }),
).toBe(true);
expect(isInlineDataPart({ text: 'hi' })).toBe(false);
});
it('isFileDataPart', () => {
expect(
isFileDataPart({ fileData: { mimeType: 't', fileUri: 'u' } }),
).toBe(true);
expect(isFileDataPart({ text: 'hi' })).toBe(false);
});
});
describe('partToString (default behavior)', () => { describe('partToString (default behavior)', () => {
it('should return empty string for undefined or null', () => { it('should return empty string for undefined or null', () => {
// @ts-expect-error Testing invalid input // @ts-expect-error Testing invalid input
@@ -81,7 +143,7 @@ describe('partUtils', () => {
}); });
it('should return descriptive string for videoMetadata part', () => { it('should return descriptive string for videoMetadata part', () => {
const part = { videoMetadata: {} } as Part; const part: Part = { videoMetadata: {} };
expect(partToString(part, verboseOptions)).toBe('[Video Metadata]'); expect(partToString(part, verboseOptions)).toBe('[Video Metadata]');
}); });
@@ -91,38 +153,44 @@ describe('partUtils', () => {
}); });
it('should return descriptive string for codeExecutionResult part', () => { it('should return descriptive string for codeExecutionResult part', () => {
const part = { codeExecutionResult: {} } as Part; const part: Part = { codeExecutionResult: {} };
expect(partToString(part, verboseOptions)).toBe( expect(partToString(part, verboseOptions)).toBe(
'[Code Execution Result]', '[Code Execution Result]',
); );
}); });
it('should return descriptive string for executableCode part', () => { it('should return descriptive string for executableCode part', () => {
const part = { executableCode: {} } as Part; const part: Part = { executableCode: {} };
expect(partToString(part, verboseOptions)).toBe('[Executable Code]'); expect(partToString(part, verboseOptions)).toBe('[Executable Code]');
}); });
it('should return descriptive string for fileData part', () => { it('should return descriptive string for fileData part', () => {
const part = { fileData: {} } as Part; const part: Part = {
fileData: { mimeType: 'image/png', fileUri: 'u' },
};
expect(partToString(part, verboseOptions)).toBe('[File Data]'); expect(partToString(part, verboseOptions)).toBe('[File Data]');
}); });
it('should return descriptive string for functionCall part', () => { it('should return descriptive string for functionCall }', () => {
const part = { functionCall: { name: 'myFunction' } } as Part; const part: Part = { functionCall: { name: 'myFunction' } };
expect(partToString(part, verboseOptions)).toBe( expect(partToString(part, verboseOptions)).toBe(
'[Function Call: myFunction]', '[Function Call: myFunction]',
); );
}); });
it('should return descriptive string for functionResponse part', () => { it('should return descriptive string for functionResponse part', () => {
const part = { functionResponse: { name: 'myFunction' } } as Part; const part: Part = {
functionResponse: { name: 'myFunction', response: {} },
};
expect(partToString(part, verboseOptions)).toBe( expect(partToString(part, verboseOptions)).toBe(
'[Function Response: myFunction]', '[Function Response: myFunction]',
); );
}); });
it('should return descriptive string for inlineData part', () => { it('should return descriptive string for inlineData part', () => {
const part = { inlineData: { mimeType: 'image/png', data: '' } } as Part; const part: Part = {
inlineData: { mimeType: 'image/png', data: '' },
};
expect(partToString(part, verboseOptions)).toBe('<image/png>'); expect(partToString(part, verboseOptions)).toBe('<image/png>');
}); });
+225 -40
View File
@@ -3,14 +3,209 @@
* Copyright 2025 Google LLC * Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
*/ */
import type { import type {
GenerateContentResponse, GenerateContentResponse,
PartListUnion, PartListUnion,
Part, Part,
PartUnion, PartUnion,
Content,
FunctionCall,
FunctionResponse,
Blob as InlineData,
FileData,
} from '@google/genai'; } from '@google/genai';
/**
* Type guard to check if a value is a Gemini Part object.
*/
export function isPart(value: unknown): value is Part {
return (
typeof value === 'object' &&
value !== null &&
!Array.isArray(value) &&
!('parts' in value)
);
}
/**
* Type guard to check if a value is a text-based Part object.
*/
export function isTextPart(value: unknown): value is { text: string } {
return (
typeof value === 'object' &&
value !== null &&
'text' in value &&
typeof (value as Record<string, unknown>)['text'] === 'string'
);
}
/**
* Type guard to check if a value is a Gemini Content object.
*/
export function isContent(value: unknown): value is Content {
return typeof value === 'object' && value !== null && 'parts' in value;
}
/**
* Normalizes a PartListUnion or Content parts into a standard Part array.
* Filters out null or undefined items.
*/
export function toParts(value: PartListUnion | undefined | null): Part[] {
if (value == null) {
return [];
}
const array = Array.isArray(value) ? value : [value];
return array
.filter((p): p is PartUnion => p != null)
.map((p) => (typeof p === 'string' ? { text: p } : p));
}
/**
* Handles thought parts for API compatibility (e.g. CountToken API).
* The CountToken API expects parts to have certain required "oneof" fields initialized,
* but thought parts don't always conform to this schema and can cause API failures.
* This function merges thoughts into text parts.
*/
export function toPartWithThoughtAsText(part: PartUnion): Part {
if (typeof part === 'string') {
return { text: part };
}
const maybeThoughtPart: unknown = part;
if (isThoughtPart(maybeThoughtPart)) {
const thoughtText = `[Thought: ${maybeThoughtPart.thought}]`;
const newPart: Record<string, unknown> = { ...part };
delete newPart['thought'];
const hasApiContent =
'functionCall' in newPart ||
'functionResponse' in newPart ||
'inlineData' in newPart ||
'fileData' in newPart;
if (hasApiContent) {
// It's a functionCall or other non-text part. Just strip the thought.
return newPart as Part;
}
// If no other valid API content, this must be a text part.
// Combine existing text (if any) with the thought, preserving other properties.
const text = newPart['text'] ?? '';
const existingText = typeof text === 'string' ? text : String(text);
const combinedText = existingText
? `${existingText}\n${thoughtText}`
: thoughtText;
return {
...newPart,
text: combinedText,
} as Part;
}
return part;
}
/**
* Type guard to check if a value is a function call Part object.
*/
export function isFunctionCallPart(
value: unknown,
): value is { functionCall: FunctionCall } {
if (
typeof value !== 'object' ||
value === null ||
!('functionCall' in value)
) {
return false;
}
const functionCall = (value as Record<string, unknown>)['functionCall'];
return (
typeof functionCall === 'object' &&
functionCall !== null &&
'name' in functionCall &&
typeof (functionCall as Record<string, unknown>)['name'] === 'string'
);
}
/**
* Type guard to check if a value is a function response Part object.
*/
export function isFunctionResponsePart(
value: unknown,
): value is { functionResponse: FunctionResponse } {
if (
typeof value !== 'object' ||
value === null ||
!('functionResponse' in value)
) {
return false;
}
const functionResponse = (value as Record<string, unknown>)[
'functionResponse'
];
return (
typeof functionResponse === 'object' &&
functionResponse !== null &&
'name' in functionResponse &&
typeof (functionResponse as Record<string, unknown>)['name'] === 'string'
);
}
/**
* Type guard to check if a value is an inline data Part object.
*/
export function isInlineDataPart(
value: unknown,
): value is { inlineData: InlineData } {
if (typeof value !== 'object' || value === null || !('inlineData' in value)) {
return false;
}
const inlineData = (value as Record<string, unknown>)['inlineData'];
return (
typeof inlineData === 'object' &&
inlineData !== null &&
'mimeType' in inlineData &&
typeof (inlineData as Record<string, unknown>)['mimeType'] === 'string'
);
}
/**
* Type guard to check if a value is a file data Part object.
*/
export function isFileDataPart(
value: unknown,
): value is { fileData: FileData } {
if (typeof value !== 'object' || value === null || !('fileData' in value)) {
return false;
}
const fileData = (value as Record<string, unknown>)['fileData'];
return (
typeof fileData === 'object' &&
fileData !== null &&
'mimeType' in fileData &&
typeof (fileData as Record<string, unknown>)['mimeType'] === 'string'
);
}
/**
* A Gemini Part object that contains a 'thought' string.
* This extends the base Part interface but overrides the 'thought' property type
* from boolean to string to support thinking models.
*/
export type ThoughtPart = Omit<Part, 'thought'> & { thought: string };
/**
* Type guard to check if a value is a thought Part object.
*/
export function isThoughtPart(value: unknown): value is ThoughtPart {
return (
typeof value === 'object' &&
value !== null &&
'thought' in value &&
typeof (value as Record<string, unknown>)['thought'] === 'string'
);
}
/** /**
* Converts a PartListUnion into a string. * Converts a PartListUnion into a string.
* If verbose is true, includes summary representations of non-text parts. * If verbose is true, includes summary representations of non-text parts.
@@ -29,63 +224,57 @@ export function partToString(
return value.map((part) => partToString(part, options)).join(''); return value.map((part) => partToString(part, options)).join('');
} }
// Cast to Part, assuming it might contain project-specific fields
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
const part = value as Part & {
videoMetadata?: unknown;
thought?: string;
codeExecutionResult?: unknown;
executableCode?: unknown;
};
if (options?.verbose) { if (options?.verbose) {
if (part.videoMetadata !== undefined) { if ('videoMetadata' in value && value.videoMetadata !== undefined) {
return `[Video Metadata]`; return `[Video Metadata]`;
} }
if (part.thought !== undefined) {
return `[Thought: ${part.thought}]`; const maybeThoughtPart: unknown = value;
if (isThoughtPart(maybeThoughtPart)) {
return `[Thought: ${maybeThoughtPart.thought}]`;
} }
if (part.codeExecutionResult !== undefined) {
if (
'codeExecutionResult' in value &&
value.codeExecutionResult !== undefined
) {
return `[Code Execution Result]`; return `[Code Execution Result]`;
} }
if (part.executableCode !== undefined) { if ('executableCode' in value && value.executableCode !== undefined) {
return `[Executable Code]`; return `[Executable Code]`;
} }
// Standard Part fields // Standard Part fields
if (part.fileData !== undefined) { if (isFileDataPart(value)) {
return `[File Data]`; return `[File Data]`;
} }
if (part.functionCall !== undefined) { if (isFunctionCallPart(value)) {
return `[Function Call: ${part.functionCall.name}]`; return `[Function Call: ${value.functionCall.name}]`;
} }
if (part.functionResponse !== undefined) { if (isFunctionResponsePart(value)) {
return `[Function Response: ${part.functionResponse.name}]`; return `[Function Response: ${value.functionResponse.name}]`;
} }
if (part.inlineData !== undefined) { if (isInlineDataPart(value)) {
return `<${part.inlineData.mimeType}>`; return `<${value.inlineData.mimeType}>`;
} }
} }
return part.text ?? ''; return isTextPart(value) ? value.text : '';
} }
export function getResponseText( export function getResponseText(
response: GenerateContentResponse, response: GenerateContentResponse,
): string | null { ): string | null {
if (response.candidates && response.candidates.length > 0) { const candidate = response.candidates?.[0];
const candidate = response.candidates[0];
if ( if (candidate?.content?.parts && candidate.content.parts.length > 0) {
candidate.content && return candidate.content.parts
candidate.content.parts && .filter(
candidate.content.parts.length > 0 (part): part is { text: string } =>
) { isTextPart(part) && !isThoughtPart(part),
return candidate.content.parts )
.filter((part) => part.text && !part.thought) .map((part) => part.text)
.map((part) => part.text) .join('');
.join('');
}
} }
return null; return null;
} }
@@ -105,11 +294,7 @@ export async function flatMapTextParts(
transform: (text: string) => Promise<PartUnion[]>, transform: (text: string) => Promise<PartUnion[]>,
): Promise<PartUnion[]> { ): Promise<PartUnion[]> {
const result: PartUnion[] = []; const result: PartUnion[] = [];
const partArray = Array.isArray(parts) const partArray = toParts(parts);
? parts
: typeof parts === 'string'
? [{ text: parts }]
: [parts];
for (const part of partArray) { for (const part of partArray) {
let textToProcess: string | undefined; let textToProcess: string | undefined;
+73
View File
@@ -9,8 +9,81 @@ import {
safeLiteralReplace, safeLiteralReplace,
truncateString, truncateString,
safeTemplateReplace, safeTemplateReplace,
truncateLine,
truncateLongLines,
} from './textUtils.js'; } from './textUtils.js';
describe('truncateLine', () => {
it('should not truncate when within maxLength', () => {
expect(truncateLine('hello', { maxLength: 10 })).toBe('hello');
});
it('should truncate and add ellipsis when exceeding maxLength', () => {
expect(truncateLine('hello world', { maxLength: 5 })).toBe('hello ...');
});
it('should include stats when requested', () => {
const result = truncateLine('hello world', {
maxLength: 5,
includeStats: true,
});
expect(result).toBe('hello [Truncated to 5 characters (total length: 11)]');
});
it('should center truncation around centerIndex', () => {
const text = 'abcdefghijklmnopqrstuvwxyz'; // 26 chars
const result = truncateLine(text, { maxLength: 10, centerIndex: 13 });
// Center is 13. Half length is 5.
// Start = 13 - 5 = 8.
// End = 8 + 10 = 18.
// substring(8, 18) = 'ijklmnopqr'
expect(result).toBe('... ijklmnopqr ...');
});
it('should handle centerIndex near start', () => {
const text = 'abcdefghijklmnopqrstuvwxyz';
const result = truncateLine(text, { maxLength: 10, centerIndex: 2 });
// Center is 2. Half length 5. Start = max(0, 2-5) = 0. End = 10.
expect(result).toBe('abcdefghij ...');
});
it('should handle centerIndex near end', () => {
const text = 'abcdefghijklmnopqrstuvwxyz';
const result = truncateLine(text, { maxLength: 10, centerIndex: 24 });
// Center is 24. Half length 5. Start = 24 - 5 = 19. End = 19 + 10 = 29.
// End > 26, so End = 26. Start = max(0, 26 - 10) = 16.
expect(result).toBe('... qrstuvwxyz');
});
});
describe('truncateLongLines', () => {
it('should truncate only long lines', () => {
const text = 'short\nthis is a very long line indeed\nanother short';
const result = truncateLongLines(text, 15, false);
expect(result).toBe('short\nthis is a very ...\nanother short');
});
it('should preserve LF line endings', () => {
const text = 'line 1 is long\nline 2';
const result = truncateLongLines(text, 10, false);
expect(result).toBe('line 1 is ...\nline 2');
expect(result).not.toContain('\r');
});
it('should preserve CRLF line endings', () => {
const text = 'line 1 is long\r\nline 2';
const result = truncateLongLines(text, 10, false);
expect(result).toBe('line 1 is ...\r\nline 2');
expect(result).toContain('\r\n');
});
it('should handle empty or null input', () => {
expect(truncateLongLines('', 10)).toBe('');
// @ts-expect-error testing null
expect(truncateLongLines(null, 10)).toBe(null);
});
});
describe('safeLiteralReplace', () => { describe('safeLiteralReplace', () => {
it('returns original string when oldString empty or not found', () => { it('returns original string when oldString empty or not found', () => {
expect(safeLiteralReplace('abc', '', 'X')).toBe('abc'); expect(safeLiteralReplace('abc', '', 'X')).toBe('abc');
+4 -2
View File
@@ -155,7 +155,9 @@ export function truncateLongLines(
includeStats = true, includeStats = true,
): string { ): string {
if (!text) return text; if (!text) return text;
const lines = text.split('\n');
const lineEnding = detectLineEnding(text);
const lines = text.split(/\r?\n/);
let modified = false; let modified = false;
const processed = lines.map((line) => { const processed = lines.map((line) => {
@@ -166,7 +168,7 @@ export function truncateLongLines(
return line; return line;
}); });
return modified ? processed.join('\n') : text; return modified ? processed.join(lineEnding) : text;
} }
/** /**
+2 -5
View File
@@ -7,6 +7,7 @@
import type { PartListUnion, Part } from '@google/genai'; import type { PartListUnion, Part } from '@google/genai';
import type { ContentGenerator } from '../core/contentGenerator.js'; import type { ContentGenerator } from '../core/contentGenerator.js';
import { debugLogger } from './debugLogger.js'; import { debugLogger } from './debugLogger.js';
import { toParts } from './partUtils.js';
// Token estimation constants // Token estimation constants
// ASCII characters (0-127) are roughly 4 chars per token // ASCII characters (0-127) are roughly 4 chars per token
@@ -140,11 +141,7 @@ export async function calculateRequestTokenCount(
contentGenerator: ContentGenerator, contentGenerator: ContentGenerator,
model: string, model: string,
): Promise<number> { ): Promise<number> {
const parts: Part[] = Array.isArray(request) const parts = toParts(request);
? request.map((p) => (typeof p === 'string' ? { text: p } : p))
: typeof request === 'string'
? [{ text: request }]
: [request];
// Use countTokens API only for heavy media parts that are hard to estimate. // Use countTokens API only for heavy media parts that are hard to estimate.
const hasMedia = parts.some((p) => { const hasMedia = parts.some((p) => {