mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-07-28 02:31:05 -07:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1e52da87ce | |||
| 554d52369d | |||
| 21348834d7 | |||
| 14596de440 |
@@ -98,6 +98,7 @@ export interface CliArgs {
|
||||
acceptRawOutputRisk: boolean | undefined;
|
||||
isCommand: boolean | undefined;
|
||||
simulateUser: boolean | undefined;
|
||||
knowledgeSources: string[] | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -304,6 +305,14 @@ export async function parseArguments(
|
||||
type: 'boolean',
|
||||
description:
|
||||
'Run the user simulation agent in the background for evaluation purposes.',
|
||||
})
|
||||
.option('knowledge-sources', {
|
||||
type: 'array',
|
||||
string: true,
|
||||
nargs: 1,
|
||||
description:
|
||||
'List of files or folders to load into the user simulator context (comma-separated or multiple --knowledge-sources)',
|
||||
coerce: coerceCommaSeparated,
|
||||
}),
|
||||
)
|
||||
// Register MCP subcommands
|
||||
@@ -806,6 +815,7 @@ export async function loadCliConfig(
|
||||
disableYoloMode:
|
||||
settings.security?.disableYoloMode || settings.admin?.secureModeEnabled,
|
||||
simulateUser: !!argv.simulateUser,
|
||||
knowledgeSources: argv.knowledgeSources,
|
||||
disableAlwaysAllow:
|
||||
settings.security?.disableAlwaysAllow ||
|
||||
settings.admin?.secureModeEnabled,
|
||||
|
||||
@@ -514,6 +514,7 @@ describe('gemini.tsx main function kitty protocol', () => {
|
||||
acceptRawOutputRisk: undefined,
|
||||
isCommand: undefined,
|
||||
simulateUser: undefined,
|
||||
knowledgeSources: undefined,
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
} from '@google/gemini-cli-core';
|
||||
import type { Writable } from 'node:stream';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export class UserSimulator {
|
||||
private isRunning = false;
|
||||
@@ -19,6 +20,8 @@ export class UserSimulator {
|
||||
private lastScreenContent = '';
|
||||
private isProcessing = false;
|
||||
private interactionsFile: string | null = null;
|
||||
|
||||
private knowledgeBase = '';
|
||||
private actionHistory: string[] = [];
|
||||
|
||||
constructor(
|
||||
@@ -31,6 +34,10 @@ export class UserSimulator {
|
||||
if (!this.config.getSimulateUser()) {
|
||||
return;
|
||||
}
|
||||
const sources = (this.config as any).getKnowledgeSources?.() || [];
|
||||
if (sources.length > 0) {
|
||||
this.loadKnowledge(sources);
|
||||
}
|
||||
this.interactionsFile = `interactions_${Date.now()}.txt`;
|
||||
this.isRunning = true;
|
||||
this.timer = setInterval(() => this.tick(), 3000);
|
||||
@@ -45,6 +52,30 @@ export class UserSimulator {
|
||||
debugLogger.log('User simulator stopped');
|
||||
}
|
||||
|
||||
private loadKnowledge(paths: string[]) {
|
||||
const loadFromPath = (p: string) => {
|
||||
try {
|
||||
if (!fs.existsSync(p)) return;
|
||||
const stats = fs.statSync(p);
|
||||
if (stats.isFile()) {
|
||||
const content = fs.readFileSync(p, 'utf-8');
|
||||
this.knowledgeBase += `\nFile: ${p}\nContent:\n${content}\n---\n`;
|
||||
} else if (stats.isDirectory()) {
|
||||
const files = fs.readdirSync(p);
|
||||
for (const file of files) {
|
||||
loadFromPath(path.join(p, file));
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
debugLogger.error(`Failed to load knowledge from ${p}`, e);
|
||||
}
|
||||
};
|
||||
|
||||
for (const p of paths) {
|
||||
loadFromPath(p);
|
||||
}
|
||||
}
|
||||
|
||||
private async tick() {
|
||||
if (!this.isRunning || this.isProcessing) return;
|
||||
|
||||
@@ -86,10 +117,15 @@ export class UserSimulator {
|
||||
? `\nThe original goal was: "${originalGoal}"\n`
|
||||
: '';
|
||||
|
||||
const historyInstruction =
|
||||
this.actionHistory.length > 0
|
||||
? `\nYou have previously taken the following actions (in order):\n${this.actionHistory.map((a, i) => `${i + 1}. ${JSON.stringify(a)}`).join('\n')}\nPay close attention to whether you have already asked for the original goal. If you have already submitted the original goal, DO NOT repeat it verbatim. If the UI shows your typed text but hasn't submitted it yet, just output \\r to press Enter.\n`
|
||||
: '';
|
||||
|
||||
|
||||
const knowledgeInstruction = this.knowledgeBase
|
||||
? `\nUser Knowledge Base:\nUse this information to answer questions if applicable. If the answer is not here, respond as you normally would.\n${this.knowledgeBase}\n`
|
||||
: '';
|
||||
|
||||
const historyInstruction = this.actionHistory.length > 0
|
||||
? `\nRecent Simulator Actions (last 10):\n${this.actionHistory.slice(-10).map((a, i) => `${i + 1}. ${JSON.stringify(a)}`).join('\n')}\n`
|
||||
: '';
|
||||
|
||||
const prompt = `You are evaluating a CLI agent by simulating a user sitting at the terminal.
|
||||
Look carefully at the screen and determine the CLI's current state:
|
||||
@@ -114,7 +150,7 @@ CRITICAL RULES:
|
||||
- RULE 2: If there is an "Action Required" or confirmation prompt on the screen, YOU MUST HANDLE IT (State 2). This takes precedence over everything else.
|
||||
- RULE 3: Output ONLY the raw characters to send, <WAIT>, or <DONE>.
|
||||
- RULE 4: Do NOT output markdown, explanations of your thought process, or quotes.
|
||||
${goalInstruction}${historyInstruction}
|
||||
${goalInstruction}${knowledgeInstruction}${historyInstruction}
|
||||
|
||||
Here is the current terminal screen output:
|
||||
|
||||
@@ -186,22 +222,45 @@ ${strippedScreen}
|
||||
debugLogger.log(
|
||||
'[SIMULATOR] Skipping action (model decided to <WAIT>)',
|
||||
);
|
||||
this.actionHistory.push('<WAIT>');
|
||||
if (this.interactionsFile) {
|
||||
fs.appendFileSync(
|
||||
this.interactionsFile,
|
||||
`[LOG] [SIMULATOR] Action History updated with: "<WAIT>"\n\n`,
|
||||
);
|
||||
}
|
||||
this.lastScreenContent = normalizedScreen;
|
||||
return;
|
||||
}
|
||||
|
||||
if (responseText) {
|
||||
const keys = responseText.replace(/\\n/g, '\r').replace(/\\r/g, '\r');
|
||||
const readableAction = trimmedResponse.replace(/\\r/g, '[ENTER]');
|
||||
this.actionHistory.push(readableAction);
|
||||
|
||||
debugLogger.log(
|
||||
`[SIMULATOR] Sending to stdin: ${JSON.stringify(keys)}`,
|
||||
);
|
||||
|
||||
this.actionHistory.push(keys);
|
||||
if (this.interactionsFile) {
|
||||
fs.appendFileSync(
|
||||
this.interactionsFile,
|
||||
`[LOG] [SIMULATOR] Action History updated with: ${JSON.stringify(keys)}\n\n`,
|
||||
);
|
||||
}
|
||||
|
||||
this.stdinBuffer.write(keys);
|
||||
this.lastScreenContent = normalizedScreen;
|
||||
} else {
|
||||
debugLogger.log('[SIMULATOR] Skipping (empty response)');
|
||||
|
||||
this.actionHistory.push('<EMPTY>');
|
||||
if (this.interactionsFile) {
|
||||
fs.appendFileSync(
|
||||
this.interactionsFile,
|
||||
`[LOG] [SIMULATOR] Action History updated with: "<EMPTY>"\n\n`,
|
||||
);
|
||||
}
|
||||
|
||||
this.lastScreenContent = normalizedScreen;
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
|
||||
@@ -853,7 +853,7 @@ const ChoiceQuestionView: React.FC<ChoiceQuestionViewProps> = ({
|
||||
listHeight && !isAlternateBuffer
|
||||
? question.unconstrainedHeight
|
||||
? Math.max(1, listHeight - selectionItems.length * 2)
|
||||
: Math.min(15, Math.max(1, listHeight - DIALOG_PADDING))
|
||||
: Math.min(15, Math.max(1, listHeight - Math.min(selectionItems.length, 5) * 2))
|
||||
: undefined;
|
||||
|
||||
const maxItemsToShow =
|
||||
|
||||
@@ -30,6 +30,8 @@ exports[`AskUserDialog > Scroll Arrows (useAlternateBuffer: false) > shows scrol
|
||||
Description 1
|
||||
2. Option 2
|
||||
Description 2
|
||||
3. Option 3
|
||||
Description 3
|
||||
▼
|
||||
|
||||
Enter to select · ↑/↓ to navigate · Esc to cancel
|
||||
|
||||
@@ -650,6 +650,7 @@ export interface ConfigParameters {
|
||||
overageStrategy?: OverageStrategy;
|
||||
};
|
||||
simulateUser?: boolean;
|
||||
knowledgeSources?: string[];
|
||||
}
|
||||
|
||||
export class Config implements McpContext, AgentLoopContext {
|
||||
@@ -868,6 +869,7 @@ export class Config implements McpContext, AgentLoopContext {
|
||||
readonly injectionService: InjectionService;
|
||||
private approvedPlanPath: string | undefined;
|
||||
private readonly simulateUser: boolean;
|
||||
private readonly knowledgeSources: string[];
|
||||
|
||||
constructor(params: ConfigParameters) {
|
||||
this._sessionId = params.sessionId;
|
||||
@@ -1098,6 +1100,7 @@ export class Config implements McpContext, AgentLoopContext {
|
||||
this.eventEmitter = params.eventEmitter;
|
||||
this.enableConseca = params.enableConseca ?? false;
|
||||
this.simulateUser = params.simulateUser ?? false;
|
||||
this.knowledgeSources = params.knowledgeSources ?? [];
|
||||
|
||||
// Initialize Safety Infrastructure
|
||||
const contextBuilder = new ContextBuilder(this);
|
||||
@@ -2515,6 +2518,10 @@ export class Config implements McpContext, AgentLoopContext {
|
||||
getSimulateUser(): boolean {
|
||||
return this.simulateUser;
|
||||
}
|
||||
|
||||
getKnowledgeSources(): string[] {
|
||||
return this.knowledgeSources;
|
||||
}
|
||||
|
||||
getAcpMode(): boolean {
|
||||
return this.acpMode;
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# User Simulation Feature - Design Document
|
||||
|
||||
*Branch: `user-simulation`*
|
||||
|
||||
## 1. Overview
|
||||
The User Simulation feature enables an autonomous LLM-driven agent to act as a realistic user in interactive terminal sessions. It aims to support integration testing, evaluation, and background evaluation of system capabilities by effectively observing the terminal interface and supplying contextual standard input (stdin).
|
||||
|
||||
This is accomplished by adding a `--simulate-user` option to the CLI, intercepting terminal updates via a simulated stdout reader, computing appropriate responses via an LLM, and piping these directly into a `PassThrough` stream masquerading as `process.stdin`.
|
||||
|
||||
## 2. Core Architecture
|
||||
|
||||
### 2.1 Configurations and Triggers
|
||||
**Affected Files:**
|
||||
- `packages/cli/src/config/config.ts`
|
||||
- `packages/core/src/config/config.ts`
|
||||
- `packages/cli/src/interactiveCli.tsx`
|
||||
|
||||
**Implementation:**
|
||||
The feature is controlled via the explicit CLI argument `--simulate-user`. When supplied, the argument flows through the `Config` interface (`config.getSimulateUser()`), modifying how the interactive CLI bootstraps.
|
||||
- Instead of attaching the raw `process.stdin` directly to the `ink` render environment, the UI engine binds to a Node.js `PassThrough` stream if simulation is enabled.
|
||||
- The Interactive CLI session begins by initiating the `UserSimulator`, providing it access to the `Config`, the `simulatedStdin` buffer, and an observable snapshot of the `lastFrame` (the current state of the UI render).
|
||||
|
||||
### 2.2 The `UserSimulator` Service
|
||||
**Affected Files:**
|
||||
- `packages/cli/src/services/UserSimulator.ts`
|
||||
|
||||
**Implementation:**
|
||||
The `UserSimulator` class governs the core observation and input loops. It handles interactions asynchronously outside of the primary command execution flow.
|
||||
|
||||
#### Loop Execution
|
||||
The simulator runs via a repeating timer (3-second intervals `setInterval`), checking `tick()`. It ensures only one evaluation happens sequentially using a mutex lock (`this.isProcessing`).
|
||||
|
||||
#### Screen Processing and Environment Context
|
||||
The engine receives the rendered terminal snapshot (`lastFrame`) and applies sanitizations:
|
||||
- Removes ANSI terminal colors and cursor properties using regex filtering.
|
||||
- Strips variable UI animation symbols (Spinners like ⠋, ⠙) and time indicators (e.g., `[ 7s ]`).
|
||||
These visual stability checks allow the simulator to detect meaningful frame changes instead of being repeatedly triggered by minor clock updates.
|
||||
|
||||
#### Simulation Brain & LLM Integration
|
||||
The simulator utilizes `PREVIEW_GEMINI_FLASH_MODEL`, leveraging the newly configured `thinking: true` capacity (`packages/core/src/config/defaultModelConfigs.ts`) for better reasoning, operating under the distinct telemetry role `LlmRole.UTILITY_SIMULATOR`.
|
||||
|
||||
The LLM is provided an extensive context schema encapsulating:
|
||||
- **Terminal output:** Current stripped terminal representation.
|
||||
- **Original task objective:** The intention of the CLI session.
|
||||
- **Action Memory:** A tracked sequence (`actionHistory`) of previously executed actions to avoid getting stuck in inquiry loops.
|
||||
|
||||
#### State Machine Interpretation
|
||||
The model translates the screen context into specific behavioral states:
|
||||
- **STATE 1 (WAIT):** Triggered if any indication of internal "thinking" or processing exists (active spinners/timers). Returns `<WAIT>`.
|
||||
- **STATE 2 (ACTION/AUTHORIZATION):** Prompt requests an approval (e.g., "Allow execution"). Supplies strict raw choice input.
|
||||
- **STATE 3 (DONE):** Fully idle string detection paired with task completion fulfillment. Gracefully types `Thank you\r` and eventually exits indicating `<DONE>`.
|
||||
- **STATE 4 (INPUT):** Idle state where continuation text instruction is needed, supplying direct commands injected to stdin.
|
||||
|
||||
#### Safeguards and Debugging
|
||||
- **Anti-Stall Mechanism:** The simulator tracks repetitive actions. If the LLM repeats the exact same non-empty interaction `MAX_REPEATS` (3) times, it terminates the process abruptly to prevent infinite loops.
|
||||
- **Interaction Transcripts:** Binds an audit log, dumping each parsed `[SIMULATOR] Screen Content Seen` block, the constructed `Prompt Used`, and `Raw model response` to discrete contextual `interactions_<timestamp>.txt` files, making it traceable.
|
||||
|
||||
### 2.3 Required Ecosystem Changes
|
||||
- `packages/core/src/telemetry/llmRole.ts`: A new internal trace role `UTILITY_SIMULATOR` has been appended to separate evaluation events from main-thread generations.
|
||||
- `packages/core/src/config/defaultModelConfigs.ts`: Added dynamic thought logic parameter (`thinking: true`) for `gemini-3-flash-preview` to enhance the simulation depth reasoning.
|
||||
- Unit Testing Mocks: Updated dummy interface `mockConfig.ts` to ensure compatibility by safely mocking `getSimulateUser`.
|
||||
|
||||
## 3. Workflow Summary
|
||||
1. The user launches `gemini --simulate-user`.
|
||||
2. Interactive UI bindings swap `process.stdin` for a readable `PassThrough` stream.
|
||||
3. Every 3 seconds, `UserSimulator.tick()` accesses the newest render frame.
|
||||
4. The frame is stripped of unstable metrics/spinners. If the clean frame is unchanged since the last tick, it waits.
|
||||
5. If changed, the simulator invokes Gemini Flash Preview (Thinking).
|
||||
6. Gemini observes the state—emitting `<WAIT>` if busy, action keys for selections, `<DONE>` if completed, or commands to be resolved.
|
||||
7. Return text is piped cleanly back to `simulatedStdin`—advancing the ink render state.
|
||||
|
||||
*This design effectively encapsulates the user interface flow without imposing side-effects on standard operations.*
|
||||
Reference in New Issue
Block a user