diff --git a/packages/cli/src/zed-integration/schema.ts b/packages/cli/src/zed-integration/schema.ts index 4c96213100..b35cc47d5c 100644 --- a/packages/cli/src/zed-integration/schema.ts +++ b/packages/cli/src/zed-integration/schema.ts @@ -84,6 +84,8 @@ export type AgentCapabilities = z.infer; export type AuthMethod = z.infer; +export type PromptCapabilities = z.infer; + export type ClientResponse = z.infer; export type ClientNotification = z.infer; @@ -270,8 +272,15 @@ export const mcpServerSchema = z.object({ name: z.string(), }); +export const promptCapabilitiesSchema = z.object({ + audio: z.boolean().optional(), + embeddedContext: z.boolean().optional(), + image: z.boolean().optional(), +}); + export const agentCapabilitiesSchema = z.object({ - loadSession: z.boolean(), + loadSession: z.boolean().optional(), + promptCapabilities: promptCapabilitiesSchema.optional(), }); export const authMethodSchema = z.object({ diff --git a/packages/cli/src/zed-integration/zedIntegration.ts b/packages/cli/src/zed-integration/zedIntegration.ts index 9a10e1a838..f9a1bcf3ac 100644 --- a/packages/cli/src/zed-integration/zedIntegration.ts +++ b/packages/cli/src/zed-integration/zedIntegration.ts @@ -99,6 +99,11 @@ class GeminiAgent { authMethods, agentCapabilities: { loadSession: false, + promptCapabilities: { + image: true, + audio: true, + embeddedContext: true, + }, }, }; } @@ -374,74 +379,75 @@ class Session { ); } - const invocation = tool.build(args); - const confirmationDetails = - await invocation.shouldConfirmExecute(abortSignal); + try { + const invocation = tool.build(args); - if (confirmationDetails) { - const content: acp.ToolCallContent[] = []; + const confirmationDetails = + await invocation.shouldConfirmExecute(abortSignal); - if (confirmationDetails.type === 'edit') { - content.push({ - type: 'diff', - path: confirmationDetails.fileName, - oldText: confirmationDetails.originalContent, - newText: confirmationDetails.newContent, + if (confirmationDetails) { + const content: acp.ToolCallContent[] = []; + + if (confirmationDetails.type === 'edit') { + content.push({ + type: 'diff', + path: confirmationDetails.fileName, + oldText: confirmationDetails.originalContent, + newText: confirmationDetails.newContent, + }); + } + + const params: acp.RequestPermissionRequest = { + sessionId: this.id, + options: toPermissionOptions(confirmationDetails), + toolCall: { + toolCallId: callId, + status: 'pending', + title: invocation.getDescription(), + content, + locations: invocation.toolLocations(), + kind: tool.kind, + }, + }; + + const output = await this.client.requestPermission(params); + const outcome = + output.outcome.outcome === 'cancelled' + ? ToolConfirmationOutcome.Cancel + : z + .nativeEnum(ToolConfirmationOutcome) + .parse(output.outcome.optionId); + + await confirmationDetails.onConfirm(outcome); + + switch (outcome) { + case ToolConfirmationOutcome.Cancel: + return errorResponse( + new Error(`Tool "${fc.name}" was canceled by the user.`), + ); + case ToolConfirmationOutcome.ProceedOnce: + case ToolConfirmationOutcome.ProceedAlways: + case ToolConfirmationOutcome.ProceedAlwaysServer: + case ToolConfirmationOutcome.ProceedAlwaysTool: + case ToolConfirmationOutcome.ModifyWithEditor: + break; + default: { + const resultOutcome: never = outcome; + throw new Error(`Unexpected: ${resultOutcome}`); + } + } + } else { + await this.sendUpdate({ + sessionUpdate: 'tool_call', + toolCallId: callId, + status: 'in_progress', + title: invocation.getDescription(), + content: [], + locations: invocation.toolLocations(), + kind: tool.kind, }); } - const params: acp.RequestPermissionRequest = { - sessionId: this.id, - options: toPermissionOptions(confirmationDetails), - toolCall: { - toolCallId: callId, - status: 'pending', - title: invocation.getDescription(), - content, - locations: invocation.toolLocations(), - kind: tool.kind, - }, - }; - - const output = await this.client.requestPermission(params); - const outcome = - output.outcome.outcome === 'cancelled' - ? ToolConfirmationOutcome.Cancel - : z - .nativeEnum(ToolConfirmationOutcome) - .parse(output.outcome.optionId); - - await confirmationDetails.onConfirm(outcome); - - switch (outcome) { - case ToolConfirmationOutcome.Cancel: - return errorResponse( - new Error(`Tool "${fc.name}" was canceled by the user.`), - ); - case ToolConfirmationOutcome.ProceedOnce: - case ToolConfirmationOutcome.ProceedAlways: - case ToolConfirmationOutcome.ProceedAlwaysServer: - case ToolConfirmationOutcome.ProceedAlwaysTool: - case ToolConfirmationOutcome.ModifyWithEditor: - break; - default: { - const resultOutcome: never = outcome; - throw new Error(`Unexpected: ${resultOutcome}`); - } - } - } else { - await this.sendUpdate({ - sessionUpdate: 'tool_call', - toolCallId: callId, - status: 'in_progress', - title: invocation.getDescription(), - content: [], - locations: invocation.toolLocations(), - kind: tool.kind, - }); - } - - try { const toolResult: ToolResult = await invocation.execute(abortSignal); const content = toToolCallContent(toolResult); @@ -488,45 +494,59 @@ class Session { message: acp.ContentBlock[], abortSignal: AbortSignal, ): Promise { + const FILE_URI_SCHEME = 'file://'; + + const embeddedContext: acp.EmbeddedResourceResource[] = []; + const parts = message.map((part) => { switch (part.type) { case 'text': return { text: part.text }; - case 'resource_link': + case 'image': + case 'audio': return { - fileData: { - mimeData: part.mimeType, - name: part.name, - fileUri: part.uri, + inlineData: { + mimeType: part.mimeType, + data: part.data, }, }; + case 'resource_link': { + if (part.uri.startsWith(FILE_URI_SCHEME)) { + return { + fileData: { + mimeData: part.mimeType, + name: part.name, + fileUri: part.uri.slice(FILE_URI_SCHEME.length), + }, + }; + } else { + return { text: `@${part.uri}` }; + } + } case 'resource': { - return { - fileData: { - mimeData: part.resource.mimeType, - name: part.resource.uri, - fileUri: part.resource.uri, - }, - }; + embeddedContext.push(part.resource); + return { text: `@${part.resource.uri}` }; } default: { - throw new Error(`Unexpected chunk type: '${part.type}'`); + const unreachable: never = part; + throw new Error(`Unexpected chunk type: '${unreachable}'`); } } }); const atPathCommandParts = parts.filter((part) => 'fileData' in part); - if (atPathCommandParts.length === 0) { + if (atPathCommandParts.length === 0 && embeddedContext.length === 0) { return parts; } + const atPathToResolvedSpecMap = new Map(); + // Get centralized file discovery service const fileDiscovery = this.config.getFileService(); const respectGitIgnore = this.config.getFileFilteringRespectGitIgnore(); const pathSpecsToRead: string[] = []; - const atPathToResolvedSpecMap = new Map(); const contentLabelsForDisplay: string[] = []; const ignoredPaths: string[] = []; @@ -634,6 +654,7 @@ class Session { contentLabelsForDisplay.push(pathName); } } + // Construct the initial part of the query for the LLM let initialQueryText = ''; for (let i = 0; i < parts.length; i++) { @@ -687,94 +708,123 @@ class Session { `Ignored ${ignoredPaths.length} ${ignoreType} files: ${ignoredPaths.join(', ')}`, ); } - // Fallback for lone "@" or completely invalid @-commands resulting in empty initialQueryText - if (pathSpecsToRead.length === 0) { + + const processedQueryParts: Part[] = [{ text: initialQueryText }]; + + if (pathSpecsToRead.length === 0 && embeddedContext.length === 0) { + // Fallback for lone "@" or completely invalid @-commands resulting in empty initialQueryText console.warn('No valid file paths found in @ commands to read.'); return [{ text: initialQueryText }]; } - const processedQueryParts: Part[] = [{ text: initialQueryText }]; - const toolArgs = { - paths: pathSpecsToRead, - respectGitIgnore, // Use configuration setting - }; - const callId = `${readManyFilesTool.name}-${Date.now()}`; - - try { - const invocation = readManyFilesTool.build(toolArgs); - - await this.sendUpdate({ - sessionUpdate: 'tool_call', - toolCallId: callId, - status: 'in_progress', - title: invocation.getDescription(), - content: [], - locations: invocation.toolLocations(), - kind: readManyFilesTool.kind, - }); - - const result = await invocation.execute(abortSignal); - const content = toToolCallContent(result) || { - type: 'content', - content: { - type: 'text', - text: `Successfully read: ${contentLabelsForDisplay.join(', ')}`, - }, + if (pathSpecsToRead.length > 0) { + const toolArgs = { + paths: pathSpecsToRead, + respectGitIgnore, // Use configuration setting }; - await this.sendUpdate({ - sessionUpdate: 'tool_call_update', - toolCallId: callId, - status: 'completed', - content: content ? [content] : [], - }); - if (Array.isArray(result.llmContent)) { - const fileContentRegex = /^--- (.*?) ---\n\n([\s\S]*?)\n\n$/; - processedQueryParts.push({ - text: '\n--- Content from referenced files ---', + + const callId = `${readManyFilesTool.name}-${Date.now()}`; + + try { + const invocation = readManyFilesTool.build(toolArgs); + + await this.sendUpdate({ + sessionUpdate: 'tool_call', + toolCallId: callId, + status: 'in_progress', + title: invocation.getDescription(), + content: [], + locations: invocation.toolLocations(), + kind: readManyFilesTool.kind, }); - for (const part of result.llmContent) { - if (typeof part === 'string') { - const match = fileContentRegex.exec(part); - if (match) { - const filePathSpecInContent = match[1]; // This is a resolved pathSpec - const fileActualContent = match[2].trim(); - processedQueryParts.push({ - text: `\nContent from @${filePathSpecInContent}:\n`, - }); - processedQueryParts.push({ text: fileActualContent }); - } else { - processedQueryParts.push({ text: part }); - } - } else { - // part is a Part object. - processedQueryParts.push(part); - } - } - processedQueryParts.push({ text: '\n--- End of content ---' }); - } else { - console.warn( - 'read_many_files tool returned no content or empty content.', - ); - } - return processedQueryParts; - } catch (error: unknown) { - await this.sendUpdate({ - sessionUpdate: 'tool_call_update', - toolCallId: callId, - status: 'failed', - content: [ - { - type: 'content', - content: { - type: 'text', - text: `Error reading files (${contentLabelsForDisplay.join(', ')}): ${getErrorMessage(error)}`, - }, + + const result = await invocation.execute(abortSignal); + const content = toToolCallContent(result) || { + type: 'content', + content: { + type: 'text', + text: `Successfully read: ${contentLabelsForDisplay.join(', ')}`, }, - ], + }; + await this.sendUpdate({ + sessionUpdate: 'tool_call_update', + toolCallId: callId, + status: 'completed', + content: content ? [content] : [], + }); + if (Array.isArray(result.llmContent)) { + const fileContentRegex = /^--- (.*?) ---\n\n([\s\S]*?)\n\n$/; + processedQueryParts.push({ + text: '\n--- Content from referenced files ---', + }); + for (const part of result.llmContent) { + if (typeof part === 'string') { + const match = fileContentRegex.exec(part); + if (match) { + const filePathSpecInContent = match[1]; // This is a resolved pathSpec + const fileActualContent = match[2].trim(); + processedQueryParts.push({ + text: `\nContent from @${filePathSpecInContent}:\n`, + }); + processedQueryParts.push({ text: fileActualContent }); + } else { + processedQueryParts.push({ text: part }); + } + } else { + // part is a Part object. + processedQueryParts.push(part); + } + } + } else { + console.warn( + 'read_many_files tool returned no content or empty content.', + ); + } + } catch (error: unknown) { + await this.sendUpdate({ + sessionUpdate: 'tool_call_update', + toolCallId: callId, + status: 'failed', + content: [ + { + type: 'content', + content: { + type: 'text', + text: `Error reading files (${contentLabelsForDisplay.join(', ')}): ${getErrorMessage(error)}`, + }, + }, + ], + }); + + throw error; + } + } + + if (embeddedContext.length > 0) { + processedQueryParts.push({ + text: '\n--- Content from referenced context ---', }); - throw error; + for (const contextPart of embeddedContext) { + processedQueryParts.push({ + text: `\nContent from @${contextPart.uri}:\n`, + }); + if ('text' in contextPart) { + processedQueryParts.push({ + text: contextPart.text, + }); + } else { + processedQueryParts.push({ + inlineData: { + mimeType: contextPart.mimeType ?? 'application/octet-stream', + data: contextPart.blob, + }, + }); + } + } } + + return processedQueryParts; } debug(msg: string) { @@ -785,6 +835,10 @@ class Session { } function toToolCallContent(toolResult: ToolResult): acp.ToolCallContent | null { + if (toolResult.error?.message) { + throw new Error(toolResult.error.message); + } + if (toolResult.returnDisplay) { if (typeof toolResult.returnDisplay === 'string') { return {