Add experimental built-in visualization tooling

This commit is contained in:
Dmitry Lyalin
2026-02-10 20:37:42 -05:00
parent ef02cec2cd
commit e907822dd5
16 changed files with 3325 additions and 7 deletions

View File

@@ -106,10 +106,6 @@ export const profiler = {
}
if (idleInPastSecond >= 5) {
if (this.openedDebugConsole === false) {
this.openedDebugConsole = true;
appEvents.emit(AppEvent.OpenDebugConsole);
}
debugLogger.error(
`${idleInPastSecond} frames rendered while the app was ` +
`idle in the past second. This likely indicates severe infinite loop ` +

View File

@@ -8,6 +8,7 @@ import { renderWithProviders } from '../../../test-utils/render.js';
import { ToolResultDisplay } from './ToolResultDisplay.js';
import { describe, it, expect, vi, beforeEach } from 'vitest';
import type { AnsiOutput } from '@google/gemini-cli-core';
import type { VisualizationResult as VisualizationDisplay } from './VisualizationDisplay.js';
// Mock UIStateContext partially
const mockUseUIState = vi.fn();
@@ -190,6 +191,77 @@ describe('ToolResultDisplay', () => {
expect(output).toMatchSnapshot();
});
it('renders visualization result', () => {
const visualization: VisualizationDisplay = {
type: 'visualization',
kind: 'bar',
title: 'Fastest BMW 0-60',
unit: 's',
data: {
series: [
{
name: 'BMW',
points: [
{ label: 'M5 CS', value: 2.9 },
{ label: 'M8 Competition', value: 3.0 },
],
},
],
},
meta: {
truncated: false,
originalItemCount: 2,
},
};
const { lastFrame } = render(
<ToolResultDisplay
resultDisplay={visualization}
terminalWidth={80}
availableTerminalHeight={20}
/>,
);
const output = lastFrame();
expect(output).toContain('Fastest BMW 0-60');
expect(output).toContain('M5 CS');
expect(output).toContain('2.90s');
});
it('renders diagram visualization result', () => {
const visualization: VisualizationDisplay = {
type: 'visualization',
kind: 'diagram',
title: 'Service Graph',
data: {
diagramKind: 'architecture',
nodes: [
{ id: 'ui', label: 'Web UI', type: 'frontend' },
{ id: 'api', label: 'API', type: 'service' },
],
edges: [{ from: 'ui', to: 'api', label: 'HTTPS' }],
},
meta: {
truncated: false,
originalItemCount: 2,
},
};
const { lastFrame } = render(
<ToolResultDisplay
resultDisplay={visualization}
terminalWidth={80}
availableTerminalHeight={20}
/>,
);
const output = lastFrame();
expect(output).toContain('Service Graph');
expect(output).toContain('Web UI');
expect(output).toContain('API');
expect(output).toContain('Notes:');
expect(output).toContain('Web UI -> API: HTTPS');
});
it('does not fall back to plain text if availableHeight is set and not in alternate buffer', () => {
mockUseAlternateBuffer.mockReturnValue(false);
// availableHeight calculation: 20 - 1 - 5 = 14 > 3

View File

@@ -19,6 +19,10 @@ import { Scrollable } from '../shared/Scrollable.js';
import { ScrollableList } from '../shared/ScrollableList.js';
import { SCROLL_TO_ITEM_END } from '../shared/VirtualizedList.js';
import { ACTIVE_SHELL_MAX_LINES } from '../../constants.js';
import {
VisualizationResultDisplay,
type VisualizationResult,
} from './VisualizationDisplay.js';
const STATIC_HEIGHT = 1;
const RESERVED_LINE_COUNT = 6; // for tool name, status, padding, and 'ShowMoreLines' hint
@@ -180,6 +184,19 @@ export const ToolResultDisplay: React.FC<ToolResultDisplayProps> = ({
{truncatedResultDisplay}
</Text>
);
} else if (
typeof truncatedResultDisplay === 'object' &&
'type' in truncatedResultDisplay &&
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
(truncatedResultDisplay as VisualizationResult).type === 'visualization'
) {
content = (
<VisualizationResultDisplay
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
visualization={truncatedResultDisplay as VisualizationResult}
width={childWidth}
/>
);
} else if (
typeof truncatedResultDisplay === 'object' &&
'fileDiff' in truncatedResultDisplay

View File

@@ -0,0 +1,162 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, expect, it } from 'vitest';
import { renderWithProviders } from '../../../test-utils/render.js';
import {
VisualizationResultDisplay,
type VisualizationResult as VisualizationDisplay,
} from './VisualizationDisplay.js';
describe('VisualizationResultDisplay', () => {
const render = (ui: React.ReactElement) => renderWithProviders(ui);
it('renders bar visualization', () => {
const visualization: VisualizationDisplay = {
type: 'visualization',
kind: 'bar',
title: 'BMW 0-60',
unit: 's',
data: {
series: [
{
name: 'BMW',
points: [
{ label: 'M5 CS', value: 2.9 },
{ label: 'M8', value: 3.0 },
],
},
],
},
meta: {
truncated: false,
originalItemCount: 2,
},
};
const { lastFrame } = render(
<VisualizationResultDisplay visualization={visualization} width={80} />,
);
expect(lastFrame()).toContain('BMW 0-60');
expect(lastFrame()).toContain('M5 CS');
expect(lastFrame()).toContain('2.90s');
});
it('renders line and pie visualizations', () => {
const line: VisualizationDisplay = {
type: 'visualization',
kind: 'line',
title: 'BMW models by year',
xLabel: 'Year',
yLabel: 'Models',
data: {
series: [
{
name: 'Total',
points: [
{ label: '2021', value: 15 },
{ label: '2022', value: 16 },
{ label: '2023', value: 17 },
],
},
],
},
meta: {
truncated: false,
originalItemCount: 3,
},
};
const pie: VisualizationDisplay = {
type: 'visualization',
kind: 'pie',
data: {
slices: [
{ label: 'M', value: 40 },
{ label: 'X', value: 60 },
],
},
meta: {
truncated: false,
originalItemCount: 2,
},
};
const lineFrame = render(
<VisualizationResultDisplay visualization={line} width={70} />,
).lastFrame();
const pieFrame = render(
<VisualizationResultDisplay visualization={pie} width={70} />,
).lastFrame();
expect(lineFrame).toContain('Total:');
expect(lineFrame).toContain('x: Year | y: Models');
expect(pieFrame).toContain('Slice');
expect(pieFrame).toContain('Share');
});
it('renders rich table visualization with metric bars', () => {
const visualization: VisualizationDisplay = {
type: 'visualization',
kind: 'table',
title: 'Risk Table',
data: {
columns: ['Path', 'Score', 'Lines'],
rows: [
['src/core.ts', 95, 210],
['src/ui.tsx', 45, 80],
],
metricColumns: [1],
},
meta: {
truncated: true,
originalItemCount: 8,
},
};
const { lastFrame } = render(
<VisualizationResultDisplay visualization={visualization} width={90} />,
);
const frame = lastFrame();
expect(frame).toContain('Risk Table');
expect(frame).toContain('Path');
expect(frame).toContain('Showing truncated data (8 original items)');
});
it('renders diagram visualization with UML-like nodes', () => {
const visualization: VisualizationDisplay = {
type: 'visualization',
kind: 'diagram',
title: 'Service Architecture',
data: {
diagramKind: 'architecture',
direction: 'LR',
nodes: [
{ id: 'ui', label: 'Web UI', type: 'frontend' },
{ id: 'api', label: 'API', type: 'service' },
],
edges: [{ from: 'ui', to: 'api', label: 'HTTPS' }],
},
meta: {
truncated: false,
originalItemCount: 2,
},
};
const { lastFrame } = render(
<VisualizationResultDisplay visualization={visualization} width={90} />,
);
const frame = lastFrame();
expect(frame).toContain('Service Architecture');
expect(frame).toContain('┌');
expect(frame).toContain('>');
expect(frame).toContain('Notes:');
expect(frame).toContain('Web UI -> API: HTTPS');
});
});

File diff suppressed because it is too large Load Diff

View File

@@ -36,6 +36,7 @@ import { WebSearchTool } from '../tools/web-search.js';
import { AskUserTool } from '../tools/ask-user.js';
import { ExitPlanModeTool } from '../tools/exit-plan-mode.js';
import { EnterPlanModeTool } from '../tools/enter-plan-mode.js';
import { RenderVisualizationTool } from '../tools/render-visualization.js';
import { GeminiClient } from '../core/client.js';
import { BaseLlmClient } from '../core/baseLlmClient.js';
import type { HookDefinition, HookEventName } from '../hooks/types.js';
@@ -2422,6 +2423,9 @@ export class Config {
maybeRegister(AskUserTool, () =>
registry.registerTool(new AskUserTool(this.messageBus)),
);
maybeRegister(RenderVisualizationTool, () =>
registry.registerTool(new RenderVisualizationTool(this.messageBus)),
);
if (this.getUseWriteTodos()) {
maybeRegister(WriteTodosTool, () =>
registry.registerTool(new WriteTodosTool(this.messageBus)),

View File

@@ -157,6 +157,7 @@ export * from './tools/read-many-files.js';
export * from './tools/mcp-client.js';
export * from './tools/mcp-tool.js';
export * from './tools/write-todos.js';
export * from './tools/render-visualization.js';
// MCP OAuth
export { MCPOAuthProvider } from './mcp/oauth-provider.js';

View File

@@ -28,6 +28,7 @@ import {
ENTER_PLAN_MODE_TOOL_NAME,
GLOB_TOOL_NAME,
GREP_TOOL_NAME,
RENDER_VISUALIZATION_TOOL_NAME,
} from '../tools/tool-names.js';
import { resolveModel, isPreviewModel } from '../config/models.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
@@ -185,6 +186,9 @@ export class PromptProvider {
isGemini3,
enableShellEfficiency: config.getEnableShellOutputEfficiency(),
interactiveShellEnabled: config.isInteractiveShellEnabled(),
enableVisualizationTool: enabledToolNames.has(
RENDER_VISUALIZATION_TOOL_NAME,
),
}),
),
sandbox: this.withSection('sandbox', () => getSandboxMode()),

View File

@@ -15,6 +15,7 @@ import {
GREP_TOOL_NAME,
MEMORY_TOOL_NAME,
READ_FILE_TOOL_NAME,
RENDER_VISUALIZATION_TOOL_NAME,
SHELL_TOOL_NAME,
WRITE_FILE_TOOL_NAME,
WRITE_TODOS_TOOL_NAME,
@@ -61,6 +62,7 @@ export interface OperationalGuidelinesOptions {
isGemini3: boolean;
enableShellEfficiency: boolean;
interactiveShellEnabled: boolean;
enableVisualizationTool?: boolean;
}
export type SandboxMode = 'macos-seatbelt' | 'generic' | 'outside';
@@ -270,7 +272,7 @@ ${shellEfficiencyGuidelines(options.enableShellEfficiency)}
- **Command Execution:** Use the '${SHELL_TOOL_NAME}' tool for running shell commands, remembering the safety rule to explain modifying commands first.${toolUsageInteractive(
options.interactive,
options.interactiveShellEnabled,
)}${toolUsageRememberingFacts(options)}
)}${toolUsageVisualization(options.enableVisualizationTool)}${toolUsageRememberingFacts(options)}
- **Respect User Confirmations:** Most tool calls (also denoted as 'function calls') will first require confirmation from the user, where they will either approve or cancel the function call. If a user cancels a function call, respect their choice and do _not_ try to make the function call again. It is okay to request the tool call again _only_ if the user requests that same tool call on a subsequent prompt. When a user cancels a function call, assume best intentions from the user and consider inquiring if they prefer any alternative paths forward.
## Interaction Details
@@ -621,6 +623,18 @@ function toolUsageRememberingFacts(
return base + suffix;
}
function toolUsageVisualization(enabled?: boolean): string {
if (!enabled) {
return '';
}
return `
- **Visualization Tool:** Use '${RENDER_VISUALIZATION_TOOL_NAME}' for compact visual output with these kinds only: \`bar\`, \`line\`, \`pie\`, \`table\`, \`diagram\`.
- **Canonical data shapes:** bar/line -> \`data.series=[{name,points:[{label,value}]}]\`; pie -> \`data.slices=[{label,value}]\`; table -> \`data.columns + data.rows\`; diagram -> \`data.nodes + data.edges\` with optional \`direction: "LR"|"TB"\`.
- **Shorthand accepted:** bar/line and pie also accept key/value maps; table accepts \`headers\` alias; diagram accepts \`links\`/\`connections\` and edge keys \`source/target\`.
- **Selection rule:** For engineering dashboards (tests/build/risk/trace/coverage/cost), consolidate into a single rich \`table\`; for UML-like architecture and flow, use \`diagram\`.`;
}
function gitRepoKeepUserInformed(interactive: boolean): string {
return interactive
? `

View File

@@ -14,6 +14,7 @@ import {
GREP_TOOL_NAME,
MEMORY_TOOL_NAME,
READ_FILE_TOOL_NAME,
RENDER_VISUALIZATION_TOOL_NAME,
SHELL_TOOL_NAME,
WRITE_FILE_TOOL_NAME,
WRITE_TODOS_TOOL_NAME,
@@ -63,6 +64,7 @@ export interface OperationalGuidelinesOptions {
interactive: boolean;
isGemini3: boolean;
interactiveShellEnabled: boolean;
enableVisualizationTool?: boolean;
}
export type SandboxMode = 'macos-seatbelt' | 'generic' | 'outside';
@@ -292,7 +294,7 @@ export function renderOperationalGuidelines(
- **Command Execution:** Use the ${formatToolName(SHELL_TOOL_NAME)} tool for running shell commands, remembering the safety rule to explain modifying commands first.${toolUsageInteractive(
options.interactive,
options.interactiveShellEnabled,
)}${toolUsageRememberingFacts(options)}
)}${toolUsageVisualization(options.enableVisualizationTool)}${toolUsageRememberingFacts(options)}
- **Confirmation Protocol:** If a tool call is declined or cancelled, respect the decision immediately. Do not re-attempt the action or "negotiate" for the same tool call unless the user explicitly directs you to. Offer an alternative technical path if possible.
## Interaction Details
@@ -640,6 +642,18 @@ function toolUsageRememberingFacts(
return base + suffix;
}
function toolUsageVisualization(enabled?: boolean): string {
if (!enabled) {
return '';
}
return `
- **Visualization Tool:** Use ${formatToolName(RENDER_VISUALIZATION_TOOL_NAME)} for compact visual output with these kinds only: \`bar\`, \`line\`, \`pie\`, \`table\`, \`diagram\`.
- **Canonical data shapes:** bar/line -> \`data.series=[{name,points:[{label,value}]}]\`; pie -> \`data.slices=[{label,value}]\`; table -> \`data.columns + data.rows\`; diagram -> \`data.nodes + data.edges\` with optional \`direction: "LR"|"TB"\`.
- **Shorthand accepted:** bar/line and pie also accept key/value maps; table accepts \`headers\` alias; diagram accepts \`links\`/\`connections\` and edge keys \`source/target\`.
- **Selection rule:** For engineering dashboards (tests/build/risk/trace/coverage/cost), consolidate into a single rich \`table\`; for UML-like architecture and flow, use \`diagram\`.`;
}
function gitRepoKeepUserInformed(interactive: boolean): string {
return interactive
? `

View File

@@ -14,6 +14,7 @@ export const LS_TOOL_NAME = 'list_directory';
export const READ_FILE_TOOL_NAME = 'read_file';
export const SHELL_TOOL_NAME = 'run_shell_command';
export const WRITE_FILE_TOOL_NAME = 'write_file';
export const RENDER_VISUALIZATION_TOOL_NAME = 'render_visualization';
// ============================================================================
// READ_FILE TOOL

View File

@@ -0,0 +1,339 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, expect, it } from 'vitest';
import {
RenderVisualizationTool,
type RenderVisualizationToolParams,
renderVisualizationTestUtils,
} from './render-visualization.js';
import { createMockMessageBus } from '../test-utils/mock-message-bus.js';
const signal = new AbortController().signal;
describe('RenderVisualizationTool', () => {
const tool = new RenderVisualizationTool(createMockMessageBus());
it('renders bar visualization', async () => {
const params: RenderVisualizationToolParams = {
visualizationKind: 'bar',
title: 'BMW 0-60',
unit: 's',
sort: 'asc',
data: {
series: [
{
name: 'BMW',
points: [
{ label: 'M5 CS', value: 2.9 },
{ label: 'M8 Competition', value: 3.0 },
{ label: 'XM Label Red', value: 3.7 },
],
},
],
},
};
const result = await tool.buildAndExecute(params, signal);
expect(result.llmContent).toContain('Visualization rendered: bar');
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'bar',
title: 'BMW 0-60',
});
});
it('renders line visualization with multiple series', async () => {
const params: RenderVisualizationToolParams = {
visualizationKind: 'line',
data: {
series: [
{
name: 'Sedan',
points: [
{ label: '2021', value: 5 },
{ label: '2022', value: 6 },
{ label: '2023', value: 7 },
],
},
{
name: 'SUV',
points: [
{ label: '2021', value: 4 },
{ label: '2022', value: 5 },
{ label: '2023', value: 6 },
],
},
],
},
};
const result = await tool.buildAndExecute(params, signal);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'line',
});
});
it('accepts shorthand bar payloads from root key/value maps', async () => {
const result = await tool.buildAndExecute(
{
visualizationKind: 'bar',
data: {
North: 320,
South: 580,
East: 450,
West: 490,
},
},
signal,
);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'bar',
});
const display = result.returnDisplay as {
data: {
series: Array<{ points: Array<{ label: string; value: number }> }>;
};
};
expect(display.data.series[0]?.points).toEqual(
expect.arrayContaining([
{ label: 'North', value: 320 },
{ label: 'South', value: 580 },
{ label: 'East', value: 450 },
{ label: 'West', value: 490 },
]),
);
});
it('renders pie visualization', async () => {
const params: RenderVisualizationToolParams = {
visualizationKind: 'pie',
data: {
slices: [
{ label: 'M', value: 40 },
{ label: 'X', value: 35 },
{ label: 'i', value: 25 },
],
},
};
const result = await tool.buildAndExecute(params, signal);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'pie',
});
});
it('accepts shorthand pie payloads from series points', async () => {
const result = await tool.buildAndExecute(
{
visualizationKind: 'pie',
data: {
series: [
{
name: 'Browsers',
points: [
{ label: 'Chrome', value: 65 },
{ label: 'Safari', value: 18 },
],
},
],
},
},
signal,
);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'pie',
data: {
slices: [
{ label: 'Chrome', value: 65 },
{ label: 'Safari', value: 18 },
],
},
});
});
it('renders rich table visualization', async () => {
const params: RenderVisualizationToolParams = {
visualizationKind: 'table',
data: {
columns: ['Path', 'Score', 'Lines'],
rows: [
['src/core.ts', 90, 220],
['src/ui.tsx', 45, 80],
],
metricColumns: [1, 2],
},
};
const result = await tool.buildAndExecute(params, signal);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'table',
});
});
it('accepts table headers alias', async () => {
const result = await tool.buildAndExecute(
{
visualizationKind: 'table',
data: {
headers: ['Name', 'Score'],
rows: [
['alpha', 90],
['beta', 75],
],
},
},
signal,
);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'table',
data: {
columns: ['Name', 'Score'],
},
});
});
it('renders diagram visualization', async () => {
const params: RenderVisualizationToolParams = {
visualizationKind: 'diagram',
data: {
diagramKind: 'architecture',
direction: 'LR',
nodes: [
{ id: 'ui', label: 'Web UI', type: 'frontend' },
{ id: 'api', label: 'API', type: 'service' },
{ id: 'db', label: 'Postgres', type: 'database' },
],
edges: [
{ from: 'ui', to: 'api', label: 'HTTPS' },
{ from: 'api', to: 'db', label: 'SQL' },
],
},
};
const result = await tool.buildAndExecute(params, signal);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'diagram',
});
});
it('accepts shorthand diagram payload aliases', async () => {
const result = await tool.buildAndExecute(
{
visualizationKind: 'diagram',
data: {
diagramKind: 'flowchart',
direction: 'top-bottom',
nodes: ['Start', 'Validate', 'Done'],
links: [
{ source: 'start', target: 'validate', label: 'ok' },
{ source: 'validate', target: 'done' },
],
},
},
signal,
);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'diagram',
data: {
direction: 'TB',
},
});
});
it('converts legacy dashboard payloads into table', async () => {
const params: RenderVisualizationToolParams = {
visualizationKind: 'table',
data: {
failures: [
{
testName: 'should compile',
file: 'src/a.test.ts',
durationMs: 120,
status: 'failed',
isNew: true,
},
],
},
};
const result = await tool.buildAndExecute(params, signal);
expect(result.returnDisplay).toMatchObject({
type: 'visualization',
kind: 'table',
data: {
columns: ['Status', 'Test', 'DurationMs', 'File', 'IsNew'],
},
});
});
it('rejects invalid payloads', async () => {
await expect(
tool.buildAndExecute(
{
visualizationKind: 'bar',
data: {
series: [
{
name: 'BMW',
points: [{ label: 'M5', value: -1 }],
},
],
},
},
signal,
),
).rejects.toThrow('bar does not support negative values');
await expect(
tool.buildAndExecute(
{
visualizationKind: 'diagram',
data: {
diagramKind: 'flowchart',
nodes: [{ id: 'start', label: 'Start' }],
edges: [],
},
},
signal,
),
).rejects.toThrow('flowchart requires at least one edge');
});
it('exposes normalization helper for tests', () => {
const normalized = renderVisualizationTestUtils.normalizeByKind(
'table',
{
columns: ['Name', 'Score'],
rows: [
['A', 10],
['B', 20],
],
},
'none',
10,
);
expect(normalized.originalItemCount).toBe(2);
expect(normalized.truncated).toBe(false);
expect(normalized.data).toMatchObject({
columns: ['Name', 'Score'],
});
});
});

File diff suppressed because it is too large Load Diff

View File

@@ -9,6 +9,7 @@ import {
GREP_TOOL_NAME,
LS_TOOL_NAME,
READ_FILE_TOOL_NAME,
RENDER_VISUALIZATION_TOOL_NAME,
SHELL_TOOL_NAME,
WRITE_FILE_TOOL_NAME,
} from './definitions/coreTools.js';
@@ -22,6 +23,7 @@ export {
GREP_TOOL_NAME,
LS_TOOL_NAME,
READ_FILE_TOOL_NAME,
RENDER_VISUALIZATION_TOOL_NAME,
SHELL_TOOL_NAME,
WRITE_FILE_TOOL_NAME,
};
@@ -94,6 +96,7 @@ export const ALL_BUILTIN_TOOL_NAMES = [
MEMORY_TOOL_NAME,
ACTIVATE_SKILL_TOOL_NAME,
ASK_USER_TOOL_NAME,
RENDER_VISUALIZATION_TOOL_NAME,
] as const;
/**

View File

@@ -664,7 +664,86 @@ export interface TodoList {
todos: Todo[];
}
export type ToolResultDisplay = string | FileDiff | AnsiOutput | TodoList;
export const VISUALIZATION_KINDS = [
'bar',
'line',
'pie',
'table',
'diagram',
] as const;
export type VisualizationKind = (typeof VISUALIZATION_KINDS)[number];
export const DIAGRAM_KINDS = ['architecture', 'flowchart'] as const;
export type DiagramKind = (typeof DIAGRAM_KINDS)[number];
export interface DiagramNode {
id: string;
label: string;
type?: string;
}
export interface DiagramEdge {
from: string;
to: string;
label?: string;
}
export interface VisualizationPoint {
label: string;
value: number;
}
export interface VisualizationSeries {
name: string;
points: VisualizationPoint[];
}
export interface VisualizationPieSlice {
label: string;
value: number;
}
export interface VisualizationTableData {
columns: string[];
rows: Array<Array<string | number | boolean>>;
metricColumns?: number[];
}
export interface VisualizationDiagramData {
diagramKind: DiagramKind;
direction?: 'LR' | 'TB';
nodes: DiagramNode[];
edges: DiagramEdge[];
}
export type VisualizationData =
| { series: VisualizationSeries[] }
| { slices: VisualizationPieSlice[] }
| VisualizationTableData
| VisualizationDiagramData;
export interface VisualizationDisplay {
type: 'visualization';
kind: VisualizationKind;
title?: string;
subtitle?: string;
xLabel?: string;
yLabel?: string;
unit?: string;
data: VisualizationData;
meta?: {
truncated: boolean;
originalItemCount: number;
validationWarnings?: string[];
};
}
export type ToolResultDisplay =
| string
| FileDiff
| AnsiOutput
| TodoList
| VisualizationDisplay;
export type TodoStatus = 'pending' | 'in_progress' | 'completed' | 'cancelled';

245
plan/visualer.md Normal file
View File

@@ -0,0 +1,245 @@
# Built-in Native Visualization Tool for Gemini CLI (V1)
## Summary
Implement a first-class built-in tool `render_visualization` (no MCP) that
supports `bar`, `line`, and `table` outputs in the terminal using Ink-native
rendering. The model will be guided to use this tool during natural-language
reasoning loops (research -> normalize -> visualize), so users do not need to
format input data manually.
This design directly supports these scenarios:
- "Fastest 3 BMW + comparative 0-60 chart"
- Follow-up in same session: "BMW models per year for last 5 years" with dynamic
data volumes and consistent cross-platform rendering.
## Goals and Success Criteria
- User asks a natural-language question requiring comparison/trend display.
- Model can:
1. Gather data via existing tools (web/file/etc),
2. Transform raw findings into visualization schema,
3. Call `render_visualization` correctly,
4. Render readable inline chart/table in CLI.
- Works across modern terminals on macOS, Linux, Windows.
- Follow-up turns in same session continue to use tool correctly without user
schema knowledge.
## Product Decisions (Locked)
- V1 scope: one tool with 3 visualization types: `bar | line | table`.
- UI surface: inline tool result panel only.
- Rendering stack: pure Ink + custom renderers (no chart library dependency in
v1).
- Invocation behavior: explicit user visualization intent
(chart/show/compare/trend/table).
- Prompting: tool description + explicit system-prompt guidance snippet for
visualization decisioning.
- Data entry: primary structured schema; optional text parsing fallback for
convenience.
## Public API / Interface Changes
### New built-in tool
- Name: `render_visualization`
- Display: `Render Visualization`
- Category: built-in core tool (`Kind.Other`)
### Tool input schema
- `chartType` (required): `"bar" | "line" | "table"`
- `title` (optional): string
- `subtitle` (optional): string
- `xLabel` (optional): string
- `yLabel` (optional): string
- `series` (required): array of series
- `series[].name` (required): string
- `series[].points` (required): array of `{ label: string, value: number }`
- `sort` (optional): `"none" | "asc" | "desc"` (applies to single-series
bar/table)
- `maxPoints` (optional): number (default 30, cap 200)
- `inputText` (optional fallback): string (JSON/table/CSV-like text to parse
when `series` omitted)
- `unit` (optional): string (e.g., `"s"`)
### Tool output display type
Add new `ToolResultDisplay` variant:
- `VisualizationDisplay`:
- `type: "visualization"`
- `chartType: "bar" | "line" | "table"`
- `title?`, `subtitle?`, `xLabel?`, `yLabel?`, `unit?`
- `series`
- `meta`:
`{ truncated: boolean, originalPointCount: number, fallbackMode?: "unicode"|"ascii" }`
## Architecture and Data Flow
## 1. Tool implementation (core)
- New file: `packages/core/src/tools/render-visualization.ts`
- Implements validation + normalization pipeline:
1. Resolve input source:
- use `series` if provided,
- else parse `inputText`.
2. Validate numeric values (finite only), normalize labels as strings.
3. Apply chart-specific constraints:
- `line`: preserve chronological order unless explicit sort disabled.
- `bar/table`: optional sort.
4. Apply volume controls (`maxPoints`, truncation metadata).
5. Return:
- `llmContent`: concise factual summary + normalized data preview.
- `returnDisplay`: typed `VisualizationDisplay`.
## 2. Built-in registration
- Register in `packages/core/src/config/config.ts` (`createToolRegistry()`).
- Add tool-name constants and built-in lists:
- `packages/core/src/tools/tool-names.ts`
- `packages/core/src/tools/definitions/coreTools.ts`
## 3. Prompt guidance (critical for this scenario)
- Update prompt snippets so model reliably chooses this tool:
- In tool-usage guidance section: "When user asks to compare, chart, trend, or
show tabular metrics, gather/compute data first, then call
`render_visualization` with structured series."
- Keep short and deterministic; do not over-prescribe style.
- Include 1 canonical example in tool description: "BMW 0-60 comparison."
## 4. CLI rendering (Ink)
- Extend `packages/cli/src/ui/components/messages/ToolResultDisplay.tsx` to
handle `VisualizationDisplay`.
- Add renderer components:
- `VisualizationDisplay.tsx` dispatcher
- `BarChartDisplay.tsx`
- `LineChartDisplay.tsx`
- `TableVizDisplay.tsx`
### Rendering behavior
- Inline panel, width-aware, no alternate screen required.
- Unicode first (`█`, box chars), ASCII fallback when needed.
- Label truncation + right-aligned values.
- Height caps to preserve conversational viewport.
- Multi-series behavior:
- V1: `line` supports multi-series.
- `bar/table`: single-series required in v1 (validation error if more than one).
## 5. Natural-language to schema reliability strategy
- Primary expectation: model transforms researched data into `series`.
- Fallback parser for `inputText` supports:
- JSON object map
- JSON array records
- Markdown table
- CSV-like 2-column text
- Ambiguous prose parsing intentionally rejected with actionable error +
accepted examples.
- This avoids silent bad charts and improves reasoning-loop consistency.
## 6. Dynamic volume strategy
- Defaults:
- `maxPoints = 30`
- Hard cap `200`
- For larger sets:
- `bar/table`: keep top N by absolute value (or chronological when user asks
"last N years").
- `line`: downsample uniformly while preserving first/last points.
- Always annotate truncation in `meta` and human-readable footer.
## Testing Plan
## Core tests
- New: `packages/core/src/tools/render-visualization.test.ts`
- Cases:
- Valid single-series bar (BMW 0-60 style).
- Valid line trend (yearly counts).
- Valid table rendering payload.
- Multi-series line accepted.
- Multi-series bar rejected.
- `inputText` parse success for JSON/table/CSV.
- Ambiguous prose rejected with guidance.
- Sort behavior correctness.
- Volume truncation/downsampling correctness.
- Unit handling and numeric validation.
## Registration and schema tests
- Update `packages/core/src/config/config.test.ts`:
- tool registers by default
- respects `tools.core` allowlist and `tools.exclude`
- Update tool definition snapshots for model function declarations.
## Prompt tests
- Update prompt snapshot tests to assert presence of visualization guidance line
and tool name substitution.
## UI tests
- New:
- `packages/cli/src/ui/components/messages/VisualizationDisplay.test.tsx`
- `BarChartDisplay.test.tsx`
- `LineChartDisplay.test.tsx`
- `TableVizDisplay.test.tsx`
- Validate:
- width adaptation (narrow/normal)
- unicode/ascii fallback
- long labels
- truncation indicators
- no overflow crashes
## Integration tests
- Add scenario tests for:
1. Research + bar chart call pattern.
2. Same-session follow-up question with different metric and chart type.
- Validate tool call args shape and final rendered output branch selection.
## Rollout Plan
- Feature flag: `experimental.visualizationToolV1` default `false`.
- Dogfood + internal beta.
- Telemetry:
- invocation rate
- schema validation failure rate
- parse fallback usage
- render fallback (unicode->ascii) rate
- per-chart-type success.
- Flip default to `true` after stability threshold.
## Risks and Mitigations
- Risk: model under-calls tool.
- Mitigation: explicit prompt guidance + strong tool description examples.
- Risk: terminal incompatibilities.
- Mitigation: deterministic ASCII fallback and conservative layout.
- Risk: oversized datasets.
- Mitigation: hard caps + truncation/downsampling metadata.
- Risk: noisy prose parsing.
- Mitigation: strict parser and explicit rejection path.
## Assumptions and Defaults
- Modern terminal baseline supports ANSI color and common Unicode; fallback
always available.
- V1 interactivity (keyboard-driven selections/buttons) is out of scope.
- Browser/WebView rendering is out of scope.
- V1 excludes negative-value bars.