mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-08-08 16:11:58 -07:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7bb9d8bd92 | |||
| 457793c279 | |||
| 058fb6c64e | |||
| 43eb0a9df2 | |||
| 67dc645b10 | |||
| 96aa6004fb | |||
| 8fb2f1e7f8 | |||
| 941a479855 | |||
| a4b3229513 |
+21
-1
@@ -17,8 +17,22 @@ You can specify the output format using the `--output-format` flag.
|
||||
Returns a single JSON object containing the response and usage statistics.
|
||||
|
||||
- **Schema:**
|
||||
- `session_id`: (string, optional) Session ID.
|
||||
- `auth_method`: (string, optional) Authentication method. Emitted with
|
||||
`--debug`.
|
||||
- `user_tier`: (string, optional) User tier name. Emitted with `--debug`.
|
||||
- `response`: (string) The model's final answer.
|
||||
- `stats`: (object) Token usage and API latency metrics.
|
||||
- `stats.api_requests`: (number, optional) Total API requests. Emitted with
|
||||
`--debug`.
|
||||
- `stats.api_errors`: (number, optional) Total API errors. Emitted with
|
||||
`--debug`.
|
||||
- `stats.retry_count`: (number, optional) Total retries. Emitted with
|
||||
`--debug`.
|
||||
- `stats.loop_detected`: (boolean, optional) Whether a loop was detected.
|
||||
Emitted with `--debug`.
|
||||
- `stats.loop_type`: (string, optional) Loop classification. Emitted with
|
||||
`--debug`.
|
||||
- `error`: (object, optional) Error details if the request failed.
|
||||
|
||||
#### Streaming JSON output
|
||||
@@ -26,13 +40,19 @@ Returns a single JSON object containing the response and usage statistics.
|
||||
Returns a stream of newline-delimited JSON (JSONL) events.
|
||||
|
||||
- **Event types:**
|
||||
- `init`: Session metadata (session ID, model).
|
||||
- `init`: Session metadata (session ID, model). Includes `auth_method` and
|
||||
`user_tier` with `--debug`.
|
||||
- `message`: User and assistant message chunks.
|
||||
- `tool_use`: Tool call requests with arguments.
|
||||
- `tool_result`: Output from executed tools.
|
||||
- `error`: Non-fatal warnings and system errors.
|
||||
- `retry`: Retry attempt diagnostics. Emitted with `--debug`.
|
||||
- `loop_detected`: Loop detection diagnostics. Emitted with `--debug`.
|
||||
- `result`: Final outcome with aggregated statistics.
|
||||
|
||||
In debug mode (`--debug`), `result.stats` also includes `api_requests`,
|
||||
`api_errors`, and `retry_count`.
|
||||
|
||||
## Exit codes
|
||||
|
||||
The CLI returns standard exit codes to indicate the result of the headless
|
||||
|
||||
@@ -14,6 +14,7 @@ import type {
|
||||
UserFeedbackPayload,
|
||||
} from '@google/gemini-cli-core';
|
||||
import {
|
||||
AuthType,
|
||||
ToolErrorType,
|
||||
GeminiEventType,
|
||||
OutputFormat,
|
||||
@@ -183,6 +184,7 @@ describe('runNonInteractive', () => {
|
||||
getIdeMode: vi.fn().mockReturnValue(false),
|
||||
|
||||
getContentGeneratorConfig: vi.fn().mockReturnValue({}),
|
||||
getUserTierName: vi.fn().mockReturnValue(undefined),
|
||||
getDebugMode: vi.fn().mockReturnValue(false),
|
||||
getOutputFormat: vi.fn().mockReturnValue('text'),
|
||||
getModel: vi.fn().mockReturnValue('test-model'),
|
||||
@@ -733,6 +735,79 @@ describe('runNonInteractive', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('should include debug diagnostics in JSON output when debug mode is enabled', async () => {
|
||||
const events: ServerGeminiStreamEvent[] = [
|
||||
{ type: GeminiEventType.Content, value: 'Hello World' },
|
||||
{
|
||||
type: GeminiEventType.Finished,
|
||||
value: { reason: undefined, usageMetadata: { totalTokenCount: 10 } },
|
||||
},
|
||||
];
|
||||
const debugMetrics: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: {
|
||||
totalRequests: 2,
|
||||
totalErrors: 1,
|
||||
totalLatencyMs: 1234,
|
||||
},
|
||||
tokens: {
|
||||
input: 10,
|
||||
prompt: 10,
|
||||
candidates: 5,
|
||||
total: 15,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
roles: {},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: {
|
||||
accept: 0,
|
||||
reject: 0,
|
||||
modify: 0,
|
||||
auto_accept: 0,
|
||||
},
|
||||
byName: {},
|
||||
},
|
||||
files: {
|
||||
totalLinesAdded: 0,
|
||||
totalLinesRemoved: 0,
|
||||
},
|
||||
};
|
||||
|
||||
mockGeminiClient.sendMessageStream.mockReturnValue(
|
||||
createStreamFromEvents(events),
|
||||
);
|
||||
vi.mocked(mockConfig.getOutputFormat).mockReturnValue(OutputFormat.JSON);
|
||||
vi.mocked(mockConfig.getDebugMode).mockReturnValue(true);
|
||||
vi.mocked(mockConfig.getContentGeneratorConfig).mockReturnValue({
|
||||
authType: AuthType.USE_GEMINI,
|
||||
});
|
||||
vi.mocked(mockConfig.getUserTierName).mockReturnValue('free');
|
||||
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(debugMetrics);
|
||||
|
||||
await runNonInteractive({
|
||||
config: mockConfig,
|
||||
settings: mockSettings,
|
||||
input: 'Test input',
|
||||
prompt_id: 'prompt-id-json-debug',
|
||||
});
|
||||
|
||||
const parsed = JSON.parse(getWrittenOutput());
|
||||
expect(parsed.auth_method).toBe('gemini-api-key');
|
||||
expect(parsed.user_tier).toBe('free');
|
||||
expect(parsed.stats.api_requests).toBe(2);
|
||||
expect(parsed.stats.api_errors).toBe(1);
|
||||
expect(parsed.stats.retry_count).toBe(0);
|
||||
});
|
||||
|
||||
it('should write JSON output with stats for tool-only commands (no text response)', async () => {
|
||||
// Test the scenario where a command completes successfully with only tool calls
|
||||
// but no text response - this would have caught the original bug
|
||||
@@ -1415,6 +1490,10 @@ describe('runNonInteractive', () => {
|
||||
CoreEvent.UserFeedback,
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(mockCoreEvents.on).toHaveBeenCalledWith(
|
||||
CoreEvent.RetryAttempt,
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(mockCoreEvents.drainBacklogs).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -1440,6 +1519,10 @@ describe('runNonInteractive', () => {
|
||||
CoreEvent.UserFeedback,
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(mockCoreEvents.off).toHaveBeenCalledWith(
|
||||
CoreEvent.RetryAttempt,
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it('logs to process.stderr when UserFeedback event is received', async () => {
|
||||
@@ -1723,6 +1806,75 @@ describe('runNonInteractive', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it('should emit loop_detected and legacy warning error events in debug stream-json mode', async () => {
|
||||
vi.mocked(mockConfig.getOutputFormat).mockReturnValue(
|
||||
OutputFormat.STREAM_JSON,
|
||||
);
|
||||
vi.mocked(mockConfig.getDebugMode).mockReturnValue(true);
|
||||
vi.mocked(mockConfig.getContentGeneratorConfig).mockReturnValue({
|
||||
authType: AuthType.LOGIN_WITH_GOOGLE,
|
||||
});
|
||||
vi.mocked(mockConfig.getUserTierName).mockReturnValue('pro');
|
||||
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(
|
||||
MOCK_SESSION_METRICS,
|
||||
);
|
||||
|
||||
const events: ServerGeminiStreamEvent[] = [
|
||||
{
|
||||
type: GeminiEventType.LoopDetected,
|
||||
value: { loopType: 'llm_detected_loop' },
|
||||
},
|
||||
{
|
||||
type: GeminiEventType.Finished,
|
||||
value: { reason: undefined, usageMetadata: { totalTokenCount: 0 } },
|
||||
},
|
||||
];
|
||||
mockGeminiClient.sendMessageStream.mockReturnValue(
|
||||
createStreamFromEvents(events),
|
||||
);
|
||||
|
||||
await runNonInteractive({
|
||||
config: mockConfig,
|
||||
settings: mockSettings,
|
||||
input: 'Loop debug test',
|
||||
prompt_id: 'prompt-id-loop-debug',
|
||||
});
|
||||
|
||||
const outputLines = getWrittenOutput()
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => JSON.parse(line));
|
||||
|
||||
expect(outputLines[0]).toMatchObject({
|
||||
type: 'init',
|
||||
auth_method: 'oauth-personal',
|
||||
user_tier: 'pro',
|
||||
});
|
||||
expect(outputLines).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: 'loop_detected',
|
||||
loop_type: 'llm_detected_loop',
|
||||
}),
|
||||
);
|
||||
expect(outputLines).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: 'error',
|
||||
severity: 'warning',
|
||||
message: 'Loop detected, stopping execution',
|
||||
}),
|
||||
);
|
||||
expect(outputLines).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: 'result',
|
||||
stats: expect.objectContaining({
|
||||
api_requests: 0,
|
||||
api_errors: 0,
|
||||
retry_count: 0,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should log error when tool recording fails', async () => {
|
||||
const toolCallEvent: ServerGeminiStreamEvent = {
|
||||
type: GeminiEventType.ToolCallRequest,
|
||||
|
||||
@@ -9,6 +9,7 @@ import type {
|
||||
ToolCallRequestInfo,
|
||||
ResumedSessionData,
|
||||
UserFeedbackPayload,
|
||||
RetryAttemptPayload,
|
||||
} from '@google/gemini-cli-core';
|
||||
import { isSlashCommand } from './ui/utils/commandUtils.js';
|
||||
import type { LoadedSettings } from './config/settings.js';
|
||||
@@ -94,6 +95,10 @@ export async function runNonInteractive({
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
let retryCount = 0;
|
||||
let loopDetected = false;
|
||||
let detectedLoopType: string | undefined;
|
||||
const debugMode = config.getDebugMode();
|
||||
const streamFormatter =
|
||||
config.getOutputFormat() === OutputFormat.STREAM_JSON
|
||||
? new StreamJsonFormatter()
|
||||
@@ -181,6 +186,27 @@ export async function runNonInteractive({
|
||||
}
|
||||
};
|
||||
|
||||
const handleRetryAttempt = (payload: RetryAttemptPayload) => {
|
||||
retryCount++;
|
||||
if (!debugMode) return;
|
||||
if (streamFormatter) {
|
||||
streamFormatter.emitEvent({
|
||||
type: JsonStreamEventType.RETRY,
|
||||
timestamp: new Date().toISOString(),
|
||||
attempt: payload.attempt,
|
||||
max_attempts: payload.maxAttempts,
|
||||
delay_ms: payload.delayMs,
|
||||
error: payload.error,
|
||||
model: payload.model,
|
||||
});
|
||||
} else if (config.getOutputFormat() === OutputFormat.TEXT) {
|
||||
const errorSuffix = payload.error ? `: ${payload.error}` : '';
|
||||
process.stderr.write(
|
||||
`[RETRY] Attempt ${payload.attempt}/${payload.maxAttempts} for model ${payload.model} (delay: ${payload.delayMs}ms)${errorSuffix}\n`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
let errorToHandle: unknown | undefined;
|
||||
try {
|
||||
consolePatcher.patch();
|
||||
@@ -199,6 +225,8 @@ export async function runNonInteractive({
|
||||
setupStdinCancellation();
|
||||
|
||||
coreEvents.on(CoreEvent.UserFeedback, handleUserFeedback);
|
||||
coreEvents.on(CoreEvent.RetryAttempt, handleRetryAttempt);
|
||||
|
||||
coreEvents.drainBacklogs();
|
||||
|
||||
// Handle EPIPE errors when the output is piped to a command that closes early.
|
||||
@@ -228,12 +256,18 @@ export async function runNonInteractive({
|
||||
}
|
||||
|
||||
// Emit init event for streaming JSON
|
||||
const authMethod = debugMode
|
||||
? config.getContentGeneratorConfig()?.authType
|
||||
: undefined;
|
||||
const userTier = debugMode ? config.getUserTierName() : undefined;
|
||||
if (streamFormatter) {
|
||||
streamFormatter.emitEvent({
|
||||
type: JsonStreamEventType.INIT,
|
||||
timestamp: new Date().toISOString(),
|
||||
session_id: config.getSessionId(),
|
||||
model: config.getModel(),
|
||||
...(authMethod && { auth_method: authMethod }),
|
||||
...(userTier && { user_tier: userTier }),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -345,7 +379,33 @@ export async function runNonInteractive({
|
||||
}
|
||||
toolCallRequests.push(event.value);
|
||||
} else if (event.type === GeminiEventType.LoopDetected) {
|
||||
if (streamFormatter) {
|
||||
const loopType = event.value?.loopType;
|
||||
loopDetected = true;
|
||||
if (loopType) {
|
||||
detectedLoopType = loopType;
|
||||
}
|
||||
|
||||
if (debugMode) {
|
||||
if (streamFormatter) {
|
||||
streamFormatter.emitEvent({
|
||||
type: JsonStreamEventType.LOOP_DETECTED,
|
||||
timestamp: new Date().toISOString(),
|
||||
...(loopType && { loop_type: loopType }),
|
||||
});
|
||||
// Keep emitting the legacy warning event for existing parsers.
|
||||
streamFormatter.emitEvent({
|
||||
type: JsonStreamEventType.ERROR,
|
||||
timestamp: new Date().toISOString(),
|
||||
severity: 'warning',
|
||||
message: 'Loop detected, stopping execution',
|
||||
});
|
||||
} else if (config.getOutputFormat() === OutputFormat.TEXT) {
|
||||
const loopTypeStr = loopType ? ` (${loopType})` : '';
|
||||
process.stderr.write(
|
||||
`[WARNING] Loop detected${loopTypeStr}, stopping execution\n`,
|
||||
);
|
||||
}
|
||||
} else if (streamFormatter) {
|
||||
streamFormatter.emitEvent({
|
||||
type: JsonStreamEventType.ERROR,
|
||||
timestamp: new Date().toISOString(),
|
||||
@@ -380,6 +440,7 @@ export async function runNonInteractive({
|
||||
stats: streamFormatter.convertToStreamStats(
|
||||
metrics,
|
||||
durationMs,
|
||||
{ retryCount, includeDiagnostics: debugMode },
|
||||
),
|
||||
});
|
||||
}
|
||||
@@ -479,13 +540,27 @@ export async function runNonInteractive({
|
||||
stats: streamFormatter.convertToStreamStats(
|
||||
metrics,
|
||||
durationMs,
|
||||
{ retryCount, includeDiagnostics: debugMode },
|
||||
),
|
||||
});
|
||||
} else if (config.getOutputFormat() === OutputFormat.JSON) {
|
||||
const formatter = new JsonFormatter();
|
||||
const stats = uiTelemetryService.getMetrics();
|
||||
textOutput.write(
|
||||
formatter.format(config.getSessionId(), responseText, stats),
|
||||
formatter.format(
|
||||
config.getSessionId(),
|
||||
responseText,
|
||||
stats,
|
||||
undefined,
|
||||
authMethod,
|
||||
userTier,
|
||||
{
|
||||
includeDiagnostics: debugMode,
|
||||
retryCount,
|
||||
loopDetected,
|
||||
loopType: detectedLoopType,
|
||||
},
|
||||
),
|
||||
);
|
||||
} else {
|
||||
textOutput.ensureTrailingNewline(); // Ensure a final newline
|
||||
@@ -503,13 +578,29 @@ export async function runNonInteractive({
|
||||
type: JsonStreamEventType.RESULT,
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'success',
|
||||
stats: streamFormatter.convertToStreamStats(metrics, durationMs),
|
||||
stats: streamFormatter.convertToStreamStats(metrics, durationMs, {
|
||||
retryCount,
|
||||
includeDiagnostics: debugMode,
|
||||
}),
|
||||
});
|
||||
} else if (config.getOutputFormat() === OutputFormat.JSON) {
|
||||
const formatter = new JsonFormatter();
|
||||
const stats = uiTelemetryService.getMetrics();
|
||||
textOutput.write(
|
||||
formatter.format(config.getSessionId(), responseText, stats),
|
||||
formatter.format(
|
||||
config.getSessionId(),
|
||||
responseText,
|
||||
stats,
|
||||
undefined,
|
||||
authMethod,
|
||||
userTier,
|
||||
{
|
||||
includeDiagnostics: debugMode,
|
||||
retryCount,
|
||||
loopDetected,
|
||||
loopType: detectedLoopType,
|
||||
},
|
||||
),
|
||||
);
|
||||
} else {
|
||||
textOutput.ensureTrailingNewline(); // Ensure a final newline
|
||||
@@ -524,6 +615,7 @@ export async function runNonInteractive({
|
||||
cleanupStdinCancellation();
|
||||
|
||||
consolePatcher.cleanup();
|
||||
coreEvents.off(CoreEvent.RetryAttempt, handleRetryAttempt);
|
||||
coreEvents.off(CoreEvent.UserFeedback, handleUserFeedback);
|
||||
}
|
||||
|
||||
|
||||
@@ -636,7 +636,11 @@ export class GeminiClient {
|
||||
|
||||
const loopDetected = await this.loopDetector.turnStarted(signal);
|
||||
if (loopDetected) {
|
||||
yield { type: GeminiEventType.LoopDetected };
|
||||
const loopType = this.loopDetector.getDetectedLoopType();
|
||||
yield {
|
||||
type: GeminiEventType.LoopDetected,
|
||||
value: loopType ? { loopType } : undefined,
|
||||
};
|
||||
return turn;
|
||||
}
|
||||
|
||||
@@ -689,7 +693,11 @@ export class GeminiClient {
|
||||
|
||||
for await (const event of resultStream) {
|
||||
if (this.loopDetector.addAndCheck(event)) {
|
||||
yield { type: GeminiEventType.LoopDetected };
|
||||
const loopType = this.loopDetector.getDetectedLoopType();
|
||||
yield {
|
||||
type: GeminiEventType.LoopDetected,
|
||||
value: loopType ? { loopType } : undefined,
|
||||
};
|
||||
controller.abort();
|
||||
return turn;
|
||||
}
|
||||
|
||||
@@ -207,6 +207,7 @@ export type ServerGeminiFinishedEvent = {
|
||||
|
||||
export type ServerGeminiLoopDetectedEvent = {
|
||||
type: GeminiEventType.LoopDetected;
|
||||
value?: { loopType: string };
|
||||
};
|
||||
|
||||
export type ServerGeminiCitationEvent = {
|
||||
|
||||
@@ -41,6 +41,25 @@ describe('JsonFormatter', () => {
|
||||
expect(parsed.response).toBe('Red text and Green text');
|
||||
});
|
||||
|
||||
it('should include auth method and user tier when provided', () => {
|
||||
const formatter = new JsonFormatter();
|
||||
const formatted = formatter.format(
|
||||
'test-session-id',
|
||||
'hello',
|
||||
undefined,
|
||||
undefined,
|
||||
'gemini-api-key',
|
||||
'free',
|
||||
);
|
||||
|
||||
expect(JSON.parse(formatted)).toEqual({
|
||||
session_id: 'test-session-id',
|
||||
auth_method: 'gemini-api-key',
|
||||
user_tier: 'free',
|
||||
response: 'hello',
|
||||
});
|
||||
});
|
||||
|
||||
it('should strip control characters from response text', () => {
|
||||
const formatter = new JsonFormatter();
|
||||
const responseWithControlChars =
|
||||
@@ -138,6 +157,87 @@ describe('JsonFormatter', () => {
|
||||
expect(JSON.parse(formatted)).toEqual(expected);
|
||||
});
|
||||
|
||||
it('should include debug diagnostic stats when enabled', () => {
|
||||
const formatter = new JsonFormatter();
|
||||
const stats: SessionMetrics = {
|
||||
models: {
|
||||
'gemini-2.5-pro': {
|
||||
api: {
|
||||
totalRequests: 2,
|
||||
totalErrors: 1,
|
||||
totalLatencyMs: 1234,
|
||||
},
|
||||
tokens: {
|
||||
input: 10,
|
||||
prompt: 10,
|
||||
candidates: 5,
|
||||
total: 15,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
roles: {},
|
||||
},
|
||||
'gemini-2.5-flash': {
|
||||
api: {
|
||||
totalRequests: 3,
|
||||
totalErrors: 0,
|
||||
totalLatencyMs: 2345,
|
||||
},
|
||||
tokens: {
|
||||
input: 10,
|
||||
prompt: 10,
|
||||
candidates: 5,
|
||||
total: 15,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
roles: {},
|
||||
},
|
||||
},
|
||||
tools: {
|
||||
totalCalls: 0,
|
||||
totalSuccess: 0,
|
||||
totalFail: 0,
|
||||
totalDurationMs: 0,
|
||||
totalDecisions: {
|
||||
accept: 0,
|
||||
reject: 0,
|
||||
modify: 0,
|
||||
auto_accept: 0,
|
||||
},
|
||||
byName: {},
|
||||
},
|
||||
files: {
|
||||
totalLinesAdded: 0,
|
||||
totalLinesRemoved: 0,
|
||||
},
|
||||
};
|
||||
|
||||
const formatted = formatter.format(
|
||||
'test-session-id',
|
||||
'hello',
|
||||
stats,
|
||||
undefined,
|
||||
'oauth-personal',
|
||||
'pro',
|
||||
{
|
||||
includeDiagnostics: true,
|
||||
retryCount: 0,
|
||||
loopDetected: true,
|
||||
loopType: 'llm_detected_loop',
|
||||
},
|
||||
);
|
||||
|
||||
const parsed = JSON.parse(formatted);
|
||||
expect(parsed.stats.api_requests).toBe(5);
|
||||
expect(parsed.stats.api_errors).toBe(1);
|
||||
expect(parsed.stats.retry_count).toBe(0);
|
||||
expect(parsed.stats.loop_detected).toBe(true);
|
||||
expect(parsed.stats.loop_type).toBe('llm_detected_loop');
|
||||
});
|
||||
|
||||
it('should format error as JSON', () => {
|
||||
const formatter = new JsonFormatter();
|
||||
const error: JsonError = {
|
||||
|
||||
@@ -6,7 +6,14 @@
|
||||
|
||||
import stripAnsi from 'strip-ansi';
|
||||
import type { SessionMetrics } from '../telemetry/uiTelemetry.js';
|
||||
import type { JsonError, JsonOutput } from './types.js';
|
||||
import type { JsonError, JsonOutput, JsonOutputStats } from './types.js';
|
||||
|
||||
type JsonFormatDiagnostics = {
|
||||
includeDiagnostics?: boolean;
|
||||
retryCount?: number;
|
||||
loopDetected?: boolean;
|
||||
loopType?: string;
|
||||
};
|
||||
|
||||
export class JsonFormatter {
|
||||
format(
|
||||
@@ -14,6 +21,9 @@ export class JsonFormatter {
|
||||
response?: string,
|
||||
stats?: SessionMetrics,
|
||||
error?: JsonError,
|
||||
authMethod?: string,
|
||||
userTier?: string,
|
||||
diagnostics?: JsonFormatDiagnostics,
|
||||
): string {
|
||||
const output: JsonOutput = {};
|
||||
|
||||
@@ -21,12 +31,41 @@ export class JsonFormatter {
|
||||
output.session_id = sessionId;
|
||||
}
|
||||
|
||||
if (authMethod) {
|
||||
output.auth_method = authMethod;
|
||||
}
|
||||
|
||||
if (userTier) {
|
||||
output.user_tier = userTier;
|
||||
}
|
||||
|
||||
if (response !== undefined) {
|
||||
output.response = stripAnsi(response);
|
||||
}
|
||||
|
||||
if (stats) {
|
||||
output.stats = stats;
|
||||
const outputStats: JsonOutputStats = { ...stats };
|
||||
if (diagnostics?.includeDiagnostics) {
|
||||
let apiRequests = 0;
|
||||
let apiErrors = 0;
|
||||
for (const modelMetrics of Object.values(stats.models)) {
|
||||
apiRequests += modelMetrics.api.totalRequests;
|
||||
apiErrors += modelMetrics.api.totalErrors;
|
||||
}
|
||||
|
||||
outputStats.api_requests = apiRequests;
|
||||
outputStats.api_errors = apiErrors;
|
||||
outputStats.retry_count = diagnostics.retryCount ?? 0;
|
||||
|
||||
if (diagnostics.loopDetected) {
|
||||
outputStats.loop_detected = true;
|
||||
}
|
||||
|
||||
if (diagnostics.loopType) {
|
||||
outputStats.loop_type = diagnostics.loopType;
|
||||
}
|
||||
}
|
||||
output.stats = outputStats;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
|
||||
@@ -473,6 +473,68 @@ describe('StreamJsonFormatter', () => {
|
||||
|
||||
expect(result.duration_ms).toBe(5000);
|
||||
});
|
||||
|
||||
it('should include diagnostic stats when enabled', () => {
|
||||
const metrics = createMockMetrics();
|
||||
metrics.models['gemini-pro'] = {
|
||||
api: { totalRequests: 2, totalErrors: 1, totalLatencyMs: 1000 },
|
||||
tokens: {
|
||||
input: 10,
|
||||
prompt: 10,
|
||||
candidates: 5,
|
||||
total: 15,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
roles: {},
|
||||
};
|
||||
metrics.models['gemini-flash'] = {
|
||||
api: { totalRequests: 3, totalErrors: 0, totalLatencyMs: 2000 },
|
||||
tokens: {
|
||||
input: 20,
|
||||
prompt: 20,
|
||||
candidates: 10,
|
||||
total: 30,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
roles: {},
|
||||
};
|
||||
|
||||
const result = formatter.convertToStreamStats(metrics, 750, {
|
||||
includeDiagnostics: true,
|
||||
retryCount: 0,
|
||||
});
|
||||
|
||||
expect(result.api_requests).toBe(5);
|
||||
expect(result.api_errors).toBe(1);
|
||||
expect(result.retry_count).toBe(0);
|
||||
});
|
||||
|
||||
it('should not include diagnostic stats when disabled', () => {
|
||||
const metrics = createMockMetrics();
|
||||
metrics.models['gemini-pro'] = {
|
||||
api: { totalRequests: 2, totalErrors: 1, totalLatencyMs: 1000 },
|
||||
tokens: {
|
||||
input: 10,
|
||||
prompt: 10,
|
||||
candidates: 5,
|
||||
total: 15,
|
||||
cached: 0,
|
||||
thoughts: 0,
|
||||
tool: 0,
|
||||
},
|
||||
roles: {},
|
||||
};
|
||||
|
||||
const result = formatter.convertToStreamStats(metrics, 750);
|
||||
|
||||
expect(result.api_requests).toBeUndefined();
|
||||
expect(result.api_errors).toBeUndefined();
|
||||
expect(result.retry_count).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('JSON validity', () => {
|
||||
|
||||
@@ -39,6 +39,7 @@ export class StreamJsonFormatter {
|
||||
convertToStreamStats(
|
||||
metrics: SessionMetrics,
|
||||
durationMs: number,
|
||||
options?: { retryCount?: number; includeDiagnostics?: boolean },
|
||||
): StreamStats {
|
||||
let totalTokens = 0;
|
||||
let inputTokens = 0;
|
||||
@@ -55,7 +56,7 @@ export class StreamJsonFormatter {
|
||||
input += modelMetrics.tokens.input;
|
||||
}
|
||||
|
||||
return {
|
||||
const stats: StreamStats = {
|
||||
total_tokens: totalTokens,
|
||||
input_tokens: inputTokens,
|
||||
output_tokens: outputTokens,
|
||||
@@ -64,5 +65,19 @@ export class StreamJsonFormatter {
|
||||
duration_ms: durationMs,
|
||||
tool_calls: metrics.tools.totalCalls,
|
||||
};
|
||||
|
||||
if (options?.includeDiagnostics) {
|
||||
let apiRequests = 0;
|
||||
let apiErrors = 0;
|
||||
for (const modelMetrics of Object.values(metrics.models)) {
|
||||
apiRequests += modelMetrics.api.totalRequests;
|
||||
apiErrors += modelMetrics.api.totalErrors;
|
||||
}
|
||||
stats.api_requests = apiRequests;
|
||||
stats.api_errors = apiErrors;
|
||||
stats.retry_count = options.retryCount ?? 0;
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,10 +18,20 @@ export interface JsonError {
|
||||
code?: string | number;
|
||||
}
|
||||
|
||||
export interface JsonOutputStats extends SessionMetrics {
|
||||
api_requests?: number;
|
||||
api_errors?: number;
|
||||
retry_count?: number;
|
||||
loop_detected?: boolean;
|
||||
loop_type?: string;
|
||||
}
|
||||
|
||||
export interface JsonOutput {
|
||||
session_id?: string;
|
||||
auth_method?: string;
|
||||
user_tier?: string;
|
||||
response?: string;
|
||||
stats?: SessionMetrics;
|
||||
stats?: JsonOutputStats;
|
||||
error?: JsonError;
|
||||
}
|
||||
|
||||
@@ -33,6 +43,8 @@ export enum JsonStreamEventType {
|
||||
TOOL_RESULT = 'tool_result',
|
||||
ERROR = 'error',
|
||||
RESULT = 'result',
|
||||
RETRY = 'retry',
|
||||
LOOP_DETECTED = 'loop_detected',
|
||||
}
|
||||
|
||||
export interface BaseJsonStreamEvent {
|
||||
@@ -44,6 +56,8 @@ export interface InitEvent extends BaseJsonStreamEvent {
|
||||
type: JsonStreamEventType.INIT;
|
||||
session_id: string;
|
||||
model: string;
|
||||
auth_method?: string;
|
||||
user_tier?: string;
|
||||
}
|
||||
|
||||
export interface MessageEvent extends BaseJsonStreamEvent {
|
||||
@@ -86,6 +100,9 @@ export interface StreamStats {
|
||||
input: number;
|
||||
duration_ms: number;
|
||||
tool_calls: number;
|
||||
api_requests?: number;
|
||||
api_errors?: number;
|
||||
retry_count?: number;
|
||||
}
|
||||
|
||||
export interface ResultEvent extends BaseJsonStreamEvent {
|
||||
@@ -98,10 +115,26 @@ export interface ResultEvent extends BaseJsonStreamEvent {
|
||||
stats?: StreamStats;
|
||||
}
|
||||
|
||||
export interface RetryEvent extends BaseJsonStreamEvent {
|
||||
type: JsonStreamEventType.RETRY;
|
||||
attempt: number;
|
||||
max_attempts: number;
|
||||
delay_ms: number;
|
||||
error?: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface LoopDetectedStreamEvent extends BaseJsonStreamEvent {
|
||||
type: JsonStreamEventType.LOOP_DETECTED;
|
||||
loop_type?: string;
|
||||
}
|
||||
|
||||
export type JsonStreamEvent =
|
||||
| InitEvent
|
||||
| MessageEvent
|
||||
| ToolUseEvent
|
||||
| ToolResultEvent
|
||||
| ErrorEvent
|
||||
| ResultEvent;
|
||||
| ResultEvent
|
||||
| RetryEvent
|
||||
| LoopDetectedStreamEvent;
|
||||
|
||||
@@ -118,6 +118,9 @@ export class LoopDetectionService {
|
||||
private llmCheckInterval = DEFAULT_LLM_CHECK_INTERVAL;
|
||||
private lastCheckTurn = 0;
|
||||
|
||||
// Detected loop type tracking
|
||||
private detectedLoopType: string | null = null;
|
||||
|
||||
// Session-level disable flag
|
||||
private disabledForSession = false;
|
||||
|
||||
@@ -208,6 +211,7 @@ export class LoopDetectionService {
|
||||
this.toolCallRepetitionCount = 1;
|
||||
}
|
||||
if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) {
|
||||
this.detectedLoopType = 'consecutive_identical_tool_calls';
|
||||
logLoopDetected(
|
||||
this.config,
|
||||
new LoopDetectedEvent(
|
||||
@@ -321,6 +325,7 @@ export class LoopDetectionService {
|
||||
const chunkHash = createHash('sha256').update(currentChunk).digest('hex');
|
||||
|
||||
if (this.isLoopDetectedForChunk(currentChunk, chunkHash)) {
|
||||
this.detectedLoopType = 'chanting_identical_sentences';
|
||||
logLoopDetected(
|
||||
this.config,
|
||||
new LoopDetectedEvent(
|
||||
@@ -575,6 +580,7 @@ export class LoopDetectionService {
|
||||
result: Record<string, unknown>,
|
||||
modelName: string,
|
||||
): void {
|
||||
this.detectedLoopType = 'llm_detected_loop';
|
||||
if (
|
||||
typeof result['unproductive_state_analysis'] === 'string' &&
|
||||
result['unproductive_state_analysis']
|
||||
@@ -599,6 +605,13 @@ export class LoopDetectionService {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the type of the most recently detected loop, or null if none.
|
||||
*/
|
||||
getDetectedLoopType(): string | null {
|
||||
return this.detectedLoopType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets all loop detection state.
|
||||
*/
|
||||
@@ -608,6 +621,7 @@ export class LoopDetectionService {
|
||||
this.resetContentTracking();
|
||||
this.resetLlmCheckTracking();
|
||||
this.loopDetected = false;
|
||||
this.detectedLoopType = null;
|
||||
}
|
||||
|
||||
private resetToolCallCount(): void {
|
||||
|
||||
Executable
+223
@@ -0,0 +1,223 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------
|
||||
# Gemini CLI Headless Mode Monitoring Test Script
|
||||
# -----------------------------------------------------------------------------
|
||||
# Purpose:
|
||||
# Runs the Gemini CLI in headless mode across multiple models and output
|
||||
# formats, then displays the monitoring data (auth method, API stats, retries,
|
||||
# loop detection) in a readable summary.
|
||||
#
|
||||
# Prerequisites:
|
||||
# Authentication must already be configured (API key, OAuth, or Vertex AI).
|
||||
# Build the project first: npm run build
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/test_gemini.sh [--prompt "custom prompt"] [--models "model1 model2"]
|
||||
#
|
||||
# Options:
|
||||
# --prompt <text> Override the default test prompt
|
||||
# --models <list> Space-separated list of models to test (quoted)
|
||||
#
|
||||
# Example:
|
||||
# ./scripts/test_gemini.sh
|
||||
# ./scripts/test_gemini.sh --prompt "list files" --models "gemini-2.5-flash"
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
CLI="$REPO_ROOT/packages/cli/dist/index.js"
|
||||
|
||||
# Defaults
|
||||
PROMPT="count how many files are in the current folder"
|
||||
MODELS=(
|
||||
"gemini-2.5-pro"
|
||||
"gemini-2.5-flash"
|
||||
"gemini-3.1-pro-preview"
|
||||
"gemini-3-flash-preview"
|
||||
)
|
||||
|
||||
# Parse args
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--prompt) PROMPT="$2"; shift ;;
|
||||
--models) IFS=' ' read -ra MODELS <<< "$2"; shift ;;
|
||||
*) echo "Unknown option: $1"; exit 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Colors
|
||||
BOLD='\033[1m'
|
||||
DIM='\033[2m'
|
||||
GREEN='\033[32m'
|
||||
YELLOW='\033[33m'
|
||||
RED='\033[31m'
|
||||
CYAN='\033[36m'
|
||||
RESET='\033[0m'
|
||||
|
||||
# Check prerequisites
|
||||
if [[ ! -f "$CLI" ]]; then
|
||||
echo -e "${RED}CLI not found at $CLI${RESET}"
|
||||
echo "Run 'npm run build' from the repo root first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq &>/dev/null; then
|
||||
echo -e "${RED}jq is required but not installed.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
separator() {
|
||||
echo -e "${DIM}$(printf '%.0s─' {1..72})${RESET}"
|
||||
}
|
||||
|
||||
# Header
|
||||
echo ""
|
||||
echo -e "${BOLD}Gemini CLI Headless Monitoring Test${RESET}"
|
||||
separator
|
||||
echo -e "${DIM}Prompt:${RESET} $PROMPT"
|
||||
echo -e "${DIM}Models:${RESET} ${MODELS[*]}"
|
||||
echo -e "${DIM}CLI:${RESET} $CLI"
|
||||
separator
|
||||
echo ""
|
||||
|
||||
total_models=${#MODELS[@]}
|
||||
pass_count=0
|
||||
fail_count=0
|
||||
|
||||
for model in "${MODELS[@]}"; do
|
||||
echo -e "${BOLD}${CYAN}[$model]${RESET}"
|
||||
echo ""
|
||||
|
||||
# ── stream-json run ──────────────────────────────────────────────────
|
||||
TMPFILE=$(mktemp)
|
||||
STDERRFILE=$(mktemp)
|
||||
exit_code=0
|
||||
|
||||
echo -e " ${DIM}Running with -o stream-json -d ...${RESET}"
|
||||
node "$CLI" -p "$PROMPT" -y -m "$model" -o stream-json -d \
|
||||
>"$TMPFILE" 2>"$STDERRFILE" || exit_code=$?
|
||||
|
||||
if [[ $exit_code -ne 0 ]]; then
|
||||
echo -e " ${RED}FAILED${RESET} (exit code $exit_code)"
|
||||
echo ""
|
||||
if [[ -s "$STDERRFILE" ]]; then
|
||||
echo -e " ${DIM}stderr:${RESET}"
|
||||
sed 's/^/ /' "$STDERRFILE"
|
||||
echo ""
|
||||
fi
|
||||
((fail_count++))
|
||||
rm -f "$TMPFILE" "$STDERRFILE"
|
||||
separator
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
((pass_count++))
|
||||
|
||||
# Parse init event
|
||||
init_line=$(jq -c 'select(.type=="init")' "$TMPFILE" 2>/dev/null | head -1)
|
||||
auth_method=$(echo "$init_line" | jq -r '.auth_method // "not set"' 2>/dev/null)
|
||||
user_tier=$(echo "$init_line" | jq -r '.user_tier // "not set"' 2>/dev/null)
|
||||
session_id=$(echo "$init_line" | jq -r '.session_id // "?"' 2>/dev/null)
|
||||
|
||||
# Parse result event
|
||||
result_line=$(jq -c 'select(.type=="result")' "$TMPFILE" 2>/dev/null | tail -1)
|
||||
status=$(echo "$result_line" | jq -r '.status // "?"' 2>/dev/null)
|
||||
api_requests=$(echo "$result_line" | jq -r '.stats.api_requests // "?"' 2>/dev/null)
|
||||
api_errors=$(echo "$result_line" | jq -r '.stats.api_errors // "?"' 2>/dev/null)
|
||||
retry_count=$(echo "$result_line" | jq -r '.stats.retry_count // 0' 2>/dev/null)
|
||||
total_tokens=$(echo "$result_line" | jq -r '.stats.total_tokens // "?"' 2>/dev/null)
|
||||
input_tokens=$(echo "$result_line" | jq -r '.stats.input_tokens // "?"' 2>/dev/null)
|
||||
output_tokens=$(echo "$result_line" | jq -r '.stats.output_tokens // "?"' 2>/dev/null)
|
||||
cached=$(echo "$result_line" | jq -r '.stats.cached // "?"' 2>/dev/null)
|
||||
tool_calls=$(echo "$result_line" | jq -r '.stats.tool_calls // 0' 2>/dev/null)
|
||||
duration_ms=$(echo "$result_line" | jq -r '.stats.duration_ms // "?"' 2>/dev/null)
|
||||
|
||||
# Count retries and loop events
|
||||
retry_events=$(jq -c 'select(.type=="retry")' "$TMPFILE" 2>/dev/null | wc -l | tr -d ' ')
|
||||
loop_events=$(jq -c 'select(.type=="loop_detected")' "$TMPFILE" 2>/dev/null)
|
||||
if [[ -n "$loop_events" ]]; then
|
||||
loop_count=$(echo "$loop_events" | wc -l | tr -d ' ')
|
||||
loop_type=$(echo "$loop_events" | jq -r '.loop_type // empty' 2>/dev/null | head -1)
|
||||
else
|
||||
loop_count=0
|
||||
loop_type=""
|
||||
fi
|
||||
|
||||
# Extract assistant response (concatenate deltas)
|
||||
response=$(jq -r 'select(.type=="message" and .role=="assistant") | .content' "$TMPFILE" 2>/dev/null | tr -d '\n')
|
||||
# Truncate for display
|
||||
if [[ ${#response} -gt 120 ]]; then
|
||||
response="${response:0:120}..."
|
||||
fi
|
||||
|
||||
# Format duration
|
||||
if [[ "$duration_ms" != "?" ]]; then
|
||||
duration_s=$(echo "scale=1; $duration_ms / 1000" | bc 2>/dev/null || echo "$duration_ms ms")
|
||||
duration_display="${duration_s}s"
|
||||
else
|
||||
duration_display="?"
|
||||
fi
|
||||
|
||||
# Display
|
||||
echo -e " ${BOLD}Auth & Session${RESET}"
|
||||
echo -e " auth_method: ${GREEN}$auth_method${RESET}"
|
||||
echo -e " user_tier: $user_tier"
|
||||
echo -e " session_id: ${DIM}$session_id${RESET}"
|
||||
echo ""
|
||||
|
||||
echo -e " ${BOLD}API Stats${RESET}"
|
||||
echo -e " status: $([ "$status" = "success" ] && echo "${GREEN}$status${RESET}" || echo "${RED}$status${RESET}")"
|
||||
echo -e " api_requests: $api_requests"
|
||||
echo -e " api_errors: $([ "$api_errors" = "0" ] && echo "$api_errors" || echo "${RED}$api_errors${RESET}")"
|
||||
echo -e " retry_count: $([ "$retry_count" = "0" ] && echo "$retry_count" || echo "${YELLOW}$retry_count${RESET}")"
|
||||
echo -e " duration: $duration_display"
|
||||
echo ""
|
||||
|
||||
echo -e " ${BOLD}Tokens${RESET}"
|
||||
echo -e " total: $total_tokens (in: $input_tokens, out: $output_tokens, cached: $cached)"
|
||||
echo -e " tools: $tool_calls calls"
|
||||
echo ""
|
||||
|
||||
if [[ "$retry_events" -gt 0 ]]; then
|
||||
echo -e " ${BOLD}${YELLOW}Retries ($retry_events)${RESET}"
|
||||
jq -r 'select(.type=="retry") | " attempt \(.attempt)/\(.max_attempts) delay=\(.delay_ms)ms \(.error // "")"' "$TMPFILE" 2>/dev/null
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [[ "$loop_count" -gt 0 ]]; then
|
||||
echo -e " ${BOLD}${RED}Loop Detected${RESET}"
|
||||
echo -e " type: ${loop_type:-unknown}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo -e " ${BOLD}Response${RESET}"
|
||||
echo -e " ${DIM}$response${RESET}"
|
||||
echo ""
|
||||
|
||||
# Show stderr if any
|
||||
stderr_content=$(cat "$STDERRFILE")
|
||||
if [[ -n "$stderr_content" ]]; then
|
||||
echo -e " ${BOLD}Stderr${RESET}"
|
||||
echo "$stderr_content" | sed 's/^/ /'
|
||||
echo ""
|
||||
fi
|
||||
|
||||
rm -f "$TMPFILE" "$STDERRFILE"
|
||||
separator
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Summary
|
||||
echo -e "${BOLD}Summary${RESET}"
|
||||
echo -e " Models tested: $total_models"
|
||||
echo -e " Passed: ${GREEN}$pass_count${RESET}"
|
||||
if [[ $fail_count -gt 0 ]]; then
|
||||
echo -e " Failed: ${RED}$fail_count${RESET}"
|
||||
else
|
||||
echo -e " Failed: $fail_count"
|
||||
fi
|
||||
echo ""
|
||||
Reference in New Issue
Block a user