Compare commits

...

9 Commits

Author SHA1 Message Date
Dmitry Lyalin 7bb9d8bd92 Merge branch 'main' into gemini-cli-headless-monitor 2026-02-27 14:01:00 -05:00
Dmitry Lyalin 457793c279 Merge branch 'main' into gemini-cli-headless-monitor 2026-02-26 08:56:48 -08:00
Dmitry Lyalin 058fb6c64e Merge branch 'main' into gemini-cli-headless-monitor 2026-02-25 20:44:18 -08:00
Dmitry Lyalin 43eb0a9df2 test(cli): use AuthType enum in nonInteractive diagnostics tests 2026-02-25 10:34:50 -08:00
Dmitry Lyalin 67dc645b10 Merge branch 'main' into gemini-cli-headless-monitor 2026-02-25 10:24:33 -08:00
Dmitry Lyalin 96aa6004fb fix(headless): complete debug diagnostics parity for json and stream-json 2026-02-25 10:23:16 -08:00
Dmitry Lyalin 8fb2f1e7f8 Merge branch 'main' into gemini-cli-headless-monitor 2026-02-25 09:52:48 -08:00
Dmitry Lyalin 941a479855 feat(headless): gate diagnostic output behind --debug flag
Diagnostic monitoring data (auth_method, user_tier, api_requests,
api_errors, retry_count, RETRY events, LOOP_DETECTED events, and
stderr warnings) is now only emitted when --debug / -d is passed.

Without the flag, headless output is identical to before — no new
fields, no new events, no stderr noise. This keeps default output
clean for piped workflows while making diagnostics available on demand.
2026-02-25 09:52:14 -08:00
Dmitry Lyalin a4b3229513 feat(headless): surface diagnostic monitoring data in non-interactive output
When running Gemini CLI in headless mode (-p), critical diagnostic data
like auth method, API retry attempts, loop detection, and request stats
was invisible despite being tracked internally. This change surfaces
that data across all three output formats (stream-json, json, text).

Changes:
- Add RETRY and LOOP_DETECTED event types to stream-json output
- Include auth_method and user_tier in init events and JSON output
- Add api_requests, api_errors, and retry_count to result stats
- Track and expose detected loop type (tool call, chanting, LLM-detected)
- Emit [RETRY] and [WARNING] messages to stderr in text mode
- Listen to CoreEvent.RetryAttempt in non-interactive CLI
- Add test script (scripts/test_gemini.sh) for manual verification
2026-02-24 23:37:39 -08:00
12 changed files with 771 additions and 12 deletions
+21 -1
View File
@@ -17,8 +17,22 @@ You can specify the output format using the `--output-format` flag.
Returns a single JSON object containing the response and usage statistics.
- **Schema:**
- `session_id`: (string, optional) Session ID.
- `auth_method`: (string, optional) Authentication method. Emitted with
`--debug`.
- `user_tier`: (string, optional) User tier name. Emitted with `--debug`.
- `response`: (string) The model's final answer.
- `stats`: (object) Token usage and API latency metrics.
- `stats.api_requests`: (number, optional) Total API requests. Emitted with
`--debug`.
- `stats.api_errors`: (number, optional) Total API errors. Emitted with
`--debug`.
- `stats.retry_count`: (number, optional) Total retries. Emitted with
`--debug`.
- `stats.loop_detected`: (boolean, optional) Whether a loop was detected.
Emitted with `--debug`.
- `stats.loop_type`: (string, optional) Loop classification. Emitted with
`--debug`.
- `error`: (object, optional) Error details if the request failed.
#### Streaming JSON output
@@ -26,13 +40,19 @@ Returns a single JSON object containing the response and usage statistics.
Returns a stream of newline-delimited JSON (JSONL) events.
- **Event types:**
- `init`: Session metadata (session ID, model).
- `init`: Session metadata (session ID, model). Includes `auth_method` and
`user_tier` with `--debug`.
- `message`: User and assistant message chunks.
- `tool_use`: Tool call requests with arguments.
- `tool_result`: Output from executed tools.
- `error`: Non-fatal warnings and system errors.
- `retry`: Retry attempt diagnostics. Emitted with `--debug`.
- `loop_detected`: Loop detection diagnostics. Emitted with `--debug`.
- `result`: Final outcome with aggregated statistics.
In debug mode (`--debug`), `result.stats` also includes `api_requests`,
`api_errors`, and `retry_count`.
## Exit codes
The CLI returns standard exit codes to indicate the result of the headless
+152
View File
@@ -14,6 +14,7 @@ import type {
UserFeedbackPayload,
} from '@google/gemini-cli-core';
import {
AuthType,
ToolErrorType,
GeminiEventType,
OutputFormat,
@@ -183,6 +184,7 @@ describe('runNonInteractive', () => {
getIdeMode: vi.fn().mockReturnValue(false),
getContentGeneratorConfig: vi.fn().mockReturnValue({}),
getUserTierName: vi.fn().mockReturnValue(undefined),
getDebugMode: vi.fn().mockReturnValue(false),
getOutputFormat: vi.fn().mockReturnValue('text'),
getModel: vi.fn().mockReturnValue('test-model'),
@@ -733,6 +735,79 @@ describe('runNonInteractive', () => {
);
});
it('should include debug diagnostics in JSON output when debug mode is enabled', async () => {
const events: ServerGeminiStreamEvent[] = [
{ type: GeminiEventType.Content, value: 'Hello World' },
{
type: GeminiEventType.Finished,
value: { reason: undefined, usageMetadata: { totalTokenCount: 10 } },
},
];
const debugMetrics: SessionMetrics = {
models: {
'gemini-2.5-pro': {
api: {
totalRequests: 2,
totalErrors: 1,
totalLatencyMs: 1234,
},
tokens: {
input: 10,
prompt: 10,
candidates: 5,
total: 15,
cached: 0,
thoughts: 0,
tool: 0,
},
roles: {},
},
},
tools: {
totalCalls: 0,
totalSuccess: 0,
totalFail: 0,
totalDurationMs: 0,
totalDecisions: {
accept: 0,
reject: 0,
modify: 0,
auto_accept: 0,
},
byName: {},
},
files: {
totalLinesAdded: 0,
totalLinesRemoved: 0,
},
};
mockGeminiClient.sendMessageStream.mockReturnValue(
createStreamFromEvents(events),
);
vi.mocked(mockConfig.getOutputFormat).mockReturnValue(OutputFormat.JSON);
vi.mocked(mockConfig.getDebugMode).mockReturnValue(true);
vi.mocked(mockConfig.getContentGeneratorConfig).mockReturnValue({
authType: AuthType.USE_GEMINI,
});
vi.mocked(mockConfig.getUserTierName).mockReturnValue('free');
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(debugMetrics);
await runNonInteractive({
config: mockConfig,
settings: mockSettings,
input: 'Test input',
prompt_id: 'prompt-id-json-debug',
});
const parsed = JSON.parse(getWrittenOutput());
expect(parsed.auth_method).toBe('gemini-api-key');
expect(parsed.user_tier).toBe('free');
expect(parsed.stats.api_requests).toBe(2);
expect(parsed.stats.api_errors).toBe(1);
expect(parsed.stats.retry_count).toBe(0);
});
it('should write JSON output with stats for tool-only commands (no text response)', async () => {
// Test the scenario where a command completes successfully with only tool calls
// but no text response - this would have caught the original bug
@@ -1415,6 +1490,10 @@ describe('runNonInteractive', () => {
CoreEvent.UserFeedback,
expect.any(Function),
);
expect(mockCoreEvents.on).toHaveBeenCalledWith(
CoreEvent.RetryAttempt,
expect.any(Function),
);
expect(mockCoreEvents.drainBacklogs).toHaveBeenCalledTimes(1);
});
@@ -1440,6 +1519,10 @@ describe('runNonInteractive', () => {
CoreEvent.UserFeedback,
expect.any(Function),
);
expect(mockCoreEvents.off).toHaveBeenCalledWith(
CoreEvent.RetryAttempt,
expect.any(Function),
);
});
it('logs to process.stderr when UserFeedback event is received', async () => {
@@ -1723,6 +1806,75 @@ describe('runNonInteractive', () => {
},
);
it('should emit loop_detected and legacy warning error events in debug stream-json mode', async () => {
vi.mocked(mockConfig.getOutputFormat).mockReturnValue(
OutputFormat.STREAM_JSON,
);
vi.mocked(mockConfig.getDebugMode).mockReturnValue(true);
vi.mocked(mockConfig.getContentGeneratorConfig).mockReturnValue({
authType: AuthType.LOGIN_WITH_GOOGLE,
});
vi.mocked(mockConfig.getUserTierName).mockReturnValue('pro');
vi.mocked(uiTelemetryService.getMetrics).mockReturnValue(
MOCK_SESSION_METRICS,
);
const events: ServerGeminiStreamEvent[] = [
{
type: GeminiEventType.LoopDetected,
value: { loopType: 'llm_detected_loop' },
},
{
type: GeminiEventType.Finished,
value: { reason: undefined, usageMetadata: { totalTokenCount: 0 } },
},
];
mockGeminiClient.sendMessageStream.mockReturnValue(
createStreamFromEvents(events),
);
await runNonInteractive({
config: mockConfig,
settings: mockSettings,
input: 'Loop debug test',
prompt_id: 'prompt-id-loop-debug',
});
const outputLines = getWrittenOutput()
.trim()
.split('\n')
.map((line) => JSON.parse(line));
expect(outputLines[0]).toMatchObject({
type: 'init',
auth_method: 'oauth-personal',
user_tier: 'pro',
});
expect(outputLines).toContainEqual(
expect.objectContaining({
type: 'loop_detected',
loop_type: 'llm_detected_loop',
}),
);
expect(outputLines).toContainEqual(
expect.objectContaining({
type: 'error',
severity: 'warning',
message: 'Loop detected, stopping execution',
}),
);
expect(outputLines).toContainEqual(
expect.objectContaining({
type: 'result',
stats: expect.objectContaining({
api_requests: 0,
api_errors: 0,
retry_count: 0,
}),
}),
);
});
it('should log error when tool recording fails', async () => {
const toolCallEvent: ServerGeminiStreamEvent = {
type: GeminiEventType.ToolCallRequest,
+96 -4
View File
@@ -9,6 +9,7 @@ import type {
ToolCallRequestInfo,
ResumedSessionData,
UserFeedbackPayload,
RetryAttemptPayload,
} from '@google/gemini-cli-core';
import { isSlashCommand } from './ui/utils/commandUtils.js';
import type { LoadedSettings } from './config/settings.js';
@@ -94,6 +95,10 @@ export async function runNonInteractive({
};
const startTime = Date.now();
let retryCount = 0;
let loopDetected = false;
let detectedLoopType: string | undefined;
const debugMode = config.getDebugMode();
const streamFormatter =
config.getOutputFormat() === OutputFormat.STREAM_JSON
? new StreamJsonFormatter()
@@ -181,6 +186,27 @@ export async function runNonInteractive({
}
};
const handleRetryAttempt = (payload: RetryAttemptPayload) => {
retryCount++;
if (!debugMode) return;
if (streamFormatter) {
streamFormatter.emitEvent({
type: JsonStreamEventType.RETRY,
timestamp: new Date().toISOString(),
attempt: payload.attempt,
max_attempts: payload.maxAttempts,
delay_ms: payload.delayMs,
error: payload.error,
model: payload.model,
});
} else if (config.getOutputFormat() === OutputFormat.TEXT) {
const errorSuffix = payload.error ? `: ${payload.error}` : '';
process.stderr.write(
`[RETRY] Attempt ${payload.attempt}/${payload.maxAttempts} for model ${payload.model} (delay: ${payload.delayMs}ms)${errorSuffix}\n`,
);
}
};
let errorToHandle: unknown | undefined;
try {
consolePatcher.patch();
@@ -199,6 +225,8 @@ export async function runNonInteractive({
setupStdinCancellation();
coreEvents.on(CoreEvent.UserFeedback, handleUserFeedback);
coreEvents.on(CoreEvent.RetryAttempt, handleRetryAttempt);
coreEvents.drainBacklogs();
// Handle EPIPE errors when the output is piped to a command that closes early.
@@ -228,12 +256,18 @@ export async function runNonInteractive({
}
// Emit init event for streaming JSON
const authMethod = debugMode
? config.getContentGeneratorConfig()?.authType
: undefined;
const userTier = debugMode ? config.getUserTierName() : undefined;
if (streamFormatter) {
streamFormatter.emitEvent({
type: JsonStreamEventType.INIT,
timestamp: new Date().toISOString(),
session_id: config.getSessionId(),
model: config.getModel(),
...(authMethod && { auth_method: authMethod }),
...(userTier && { user_tier: userTier }),
});
}
@@ -345,7 +379,33 @@ export async function runNonInteractive({
}
toolCallRequests.push(event.value);
} else if (event.type === GeminiEventType.LoopDetected) {
if (streamFormatter) {
const loopType = event.value?.loopType;
loopDetected = true;
if (loopType) {
detectedLoopType = loopType;
}
if (debugMode) {
if (streamFormatter) {
streamFormatter.emitEvent({
type: JsonStreamEventType.LOOP_DETECTED,
timestamp: new Date().toISOString(),
...(loopType && { loop_type: loopType }),
});
// Keep emitting the legacy warning event for existing parsers.
streamFormatter.emitEvent({
type: JsonStreamEventType.ERROR,
timestamp: new Date().toISOString(),
severity: 'warning',
message: 'Loop detected, stopping execution',
});
} else if (config.getOutputFormat() === OutputFormat.TEXT) {
const loopTypeStr = loopType ? ` (${loopType})` : '';
process.stderr.write(
`[WARNING] Loop detected${loopTypeStr}, stopping execution\n`,
);
}
} else if (streamFormatter) {
streamFormatter.emitEvent({
type: JsonStreamEventType.ERROR,
timestamp: new Date().toISOString(),
@@ -380,6 +440,7 @@ export async function runNonInteractive({
stats: streamFormatter.convertToStreamStats(
metrics,
durationMs,
{ retryCount, includeDiagnostics: debugMode },
),
});
}
@@ -479,13 +540,27 @@ export async function runNonInteractive({
stats: streamFormatter.convertToStreamStats(
metrics,
durationMs,
{ retryCount, includeDiagnostics: debugMode },
),
});
} else if (config.getOutputFormat() === OutputFormat.JSON) {
const formatter = new JsonFormatter();
const stats = uiTelemetryService.getMetrics();
textOutput.write(
formatter.format(config.getSessionId(), responseText, stats),
formatter.format(
config.getSessionId(),
responseText,
stats,
undefined,
authMethod,
userTier,
{
includeDiagnostics: debugMode,
retryCount,
loopDetected,
loopType: detectedLoopType,
},
),
);
} else {
textOutput.ensureTrailingNewline(); // Ensure a final newline
@@ -503,13 +578,29 @@ export async function runNonInteractive({
type: JsonStreamEventType.RESULT,
timestamp: new Date().toISOString(),
status: 'success',
stats: streamFormatter.convertToStreamStats(metrics, durationMs),
stats: streamFormatter.convertToStreamStats(metrics, durationMs, {
retryCount,
includeDiagnostics: debugMode,
}),
});
} else if (config.getOutputFormat() === OutputFormat.JSON) {
const formatter = new JsonFormatter();
const stats = uiTelemetryService.getMetrics();
textOutput.write(
formatter.format(config.getSessionId(), responseText, stats),
formatter.format(
config.getSessionId(),
responseText,
stats,
undefined,
authMethod,
userTier,
{
includeDiagnostics: debugMode,
retryCount,
loopDetected,
loopType: detectedLoopType,
},
),
);
} else {
textOutput.ensureTrailingNewline(); // Ensure a final newline
@@ -524,6 +615,7 @@ export async function runNonInteractive({
cleanupStdinCancellation();
consolePatcher.cleanup();
coreEvents.off(CoreEvent.RetryAttempt, handleRetryAttempt);
coreEvents.off(CoreEvent.UserFeedback, handleUserFeedback);
}
+10 -2
View File
@@ -636,7 +636,11 @@ export class GeminiClient {
const loopDetected = await this.loopDetector.turnStarted(signal);
if (loopDetected) {
yield { type: GeminiEventType.LoopDetected };
const loopType = this.loopDetector.getDetectedLoopType();
yield {
type: GeminiEventType.LoopDetected,
value: loopType ? { loopType } : undefined,
};
return turn;
}
@@ -689,7 +693,11 @@ export class GeminiClient {
for await (const event of resultStream) {
if (this.loopDetector.addAndCheck(event)) {
yield { type: GeminiEventType.LoopDetected };
const loopType = this.loopDetector.getDetectedLoopType();
yield {
type: GeminiEventType.LoopDetected,
value: loopType ? { loopType } : undefined,
};
controller.abort();
return turn;
}
+1
View File
@@ -207,6 +207,7 @@ export type ServerGeminiFinishedEvent = {
export type ServerGeminiLoopDetectedEvent = {
type: GeminiEventType.LoopDetected;
value?: { loopType: string };
};
export type ServerGeminiCitationEvent = {
@@ -41,6 +41,25 @@ describe('JsonFormatter', () => {
expect(parsed.response).toBe('Red text and Green text');
});
it('should include auth method and user tier when provided', () => {
const formatter = new JsonFormatter();
const formatted = formatter.format(
'test-session-id',
'hello',
undefined,
undefined,
'gemini-api-key',
'free',
);
expect(JSON.parse(formatted)).toEqual({
session_id: 'test-session-id',
auth_method: 'gemini-api-key',
user_tier: 'free',
response: 'hello',
});
});
it('should strip control characters from response text', () => {
const formatter = new JsonFormatter();
const responseWithControlChars =
@@ -138,6 +157,87 @@ describe('JsonFormatter', () => {
expect(JSON.parse(formatted)).toEqual(expected);
});
it('should include debug diagnostic stats when enabled', () => {
const formatter = new JsonFormatter();
const stats: SessionMetrics = {
models: {
'gemini-2.5-pro': {
api: {
totalRequests: 2,
totalErrors: 1,
totalLatencyMs: 1234,
},
tokens: {
input: 10,
prompt: 10,
candidates: 5,
total: 15,
cached: 0,
thoughts: 0,
tool: 0,
},
roles: {},
},
'gemini-2.5-flash': {
api: {
totalRequests: 3,
totalErrors: 0,
totalLatencyMs: 2345,
},
tokens: {
input: 10,
prompt: 10,
candidates: 5,
total: 15,
cached: 0,
thoughts: 0,
tool: 0,
},
roles: {},
},
},
tools: {
totalCalls: 0,
totalSuccess: 0,
totalFail: 0,
totalDurationMs: 0,
totalDecisions: {
accept: 0,
reject: 0,
modify: 0,
auto_accept: 0,
},
byName: {},
},
files: {
totalLinesAdded: 0,
totalLinesRemoved: 0,
},
};
const formatted = formatter.format(
'test-session-id',
'hello',
stats,
undefined,
'oauth-personal',
'pro',
{
includeDiagnostics: true,
retryCount: 0,
loopDetected: true,
loopType: 'llm_detected_loop',
},
);
const parsed = JSON.parse(formatted);
expect(parsed.stats.api_requests).toBe(5);
expect(parsed.stats.api_errors).toBe(1);
expect(parsed.stats.retry_count).toBe(0);
expect(parsed.stats.loop_detected).toBe(true);
expect(parsed.stats.loop_type).toBe('llm_detected_loop');
});
it('should format error as JSON', () => {
const formatter = new JsonFormatter();
const error: JsonError = {
+41 -2
View File
@@ -6,7 +6,14 @@
import stripAnsi from 'strip-ansi';
import type { SessionMetrics } from '../telemetry/uiTelemetry.js';
import type { JsonError, JsonOutput } from './types.js';
import type { JsonError, JsonOutput, JsonOutputStats } from './types.js';
type JsonFormatDiagnostics = {
includeDiagnostics?: boolean;
retryCount?: number;
loopDetected?: boolean;
loopType?: string;
};
export class JsonFormatter {
format(
@@ -14,6 +21,9 @@ export class JsonFormatter {
response?: string,
stats?: SessionMetrics,
error?: JsonError,
authMethod?: string,
userTier?: string,
diagnostics?: JsonFormatDiagnostics,
): string {
const output: JsonOutput = {};
@@ -21,12 +31,41 @@ export class JsonFormatter {
output.session_id = sessionId;
}
if (authMethod) {
output.auth_method = authMethod;
}
if (userTier) {
output.user_tier = userTier;
}
if (response !== undefined) {
output.response = stripAnsi(response);
}
if (stats) {
output.stats = stats;
const outputStats: JsonOutputStats = { ...stats };
if (diagnostics?.includeDiagnostics) {
let apiRequests = 0;
let apiErrors = 0;
for (const modelMetrics of Object.values(stats.models)) {
apiRequests += modelMetrics.api.totalRequests;
apiErrors += modelMetrics.api.totalErrors;
}
outputStats.api_requests = apiRequests;
outputStats.api_errors = apiErrors;
outputStats.retry_count = diagnostics.retryCount ?? 0;
if (diagnostics.loopDetected) {
outputStats.loop_detected = true;
}
if (diagnostics.loopType) {
outputStats.loop_type = diagnostics.loopType;
}
}
output.stats = outputStats;
}
if (error) {
@@ -473,6 +473,68 @@ describe('StreamJsonFormatter', () => {
expect(result.duration_ms).toBe(5000);
});
it('should include diagnostic stats when enabled', () => {
const metrics = createMockMetrics();
metrics.models['gemini-pro'] = {
api: { totalRequests: 2, totalErrors: 1, totalLatencyMs: 1000 },
tokens: {
input: 10,
prompt: 10,
candidates: 5,
total: 15,
cached: 0,
thoughts: 0,
tool: 0,
},
roles: {},
};
metrics.models['gemini-flash'] = {
api: { totalRequests: 3, totalErrors: 0, totalLatencyMs: 2000 },
tokens: {
input: 20,
prompt: 20,
candidates: 10,
total: 30,
cached: 0,
thoughts: 0,
tool: 0,
},
roles: {},
};
const result = formatter.convertToStreamStats(metrics, 750, {
includeDiagnostics: true,
retryCount: 0,
});
expect(result.api_requests).toBe(5);
expect(result.api_errors).toBe(1);
expect(result.retry_count).toBe(0);
});
it('should not include diagnostic stats when disabled', () => {
const metrics = createMockMetrics();
metrics.models['gemini-pro'] = {
api: { totalRequests: 2, totalErrors: 1, totalLatencyMs: 1000 },
tokens: {
input: 10,
prompt: 10,
candidates: 5,
total: 15,
cached: 0,
thoughts: 0,
tool: 0,
},
roles: {},
};
const result = formatter.convertToStreamStats(metrics, 750);
expect(result.api_requests).toBeUndefined();
expect(result.api_errors).toBeUndefined();
expect(result.retry_count).toBeUndefined();
});
});
describe('JSON validity', () => {
@@ -39,6 +39,7 @@ export class StreamJsonFormatter {
convertToStreamStats(
metrics: SessionMetrics,
durationMs: number,
options?: { retryCount?: number; includeDiagnostics?: boolean },
): StreamStats {
let totalTokens = 0;
let inputTokens = 0;
@@ -55,7 +56,7 @@ export class StreamJsonFormatter {
input += modelMetrics.tokens.input;
}
return {
const stats: StreamStats = {
total_tokens: totalTokens,
input_tokens: inputTokens,
output_tokens: outputTokens,
@@ -64,5 +65,19 @@ export class StreamJsonFormatter {
duration_ms: durationMs,
tool_calls: metrics.tools.totalCalls,
};
if (options?.includeDiagnostics) {
let apiRequests = 0;
let apiErrors = 0;
for (const modelMetrics of Object.values(metrics.models)) {
apiRequests += modelMetrics.api.totalRequests;
apiErrors += modelMetrics.api.totalErrors;
}
stats.api_requests = apiRequests;
stats.api_errors = apiErrors;
stats.retry_count = options.retryCount ?? 0;
}
return stats;
}
}
+35 -2
View File
@@ -18,10 +18,20 @@ export interface JsonError {
code?: string | number;
}
export interface JsonOutputStats extends SessionMetrics {
api_requests?: number;
api_errors?: number;
retry_count?: number;
loop_detected?: boolean;
loop_type?: string;
}
export interface JsonOutput {
session_id?: string;
auth_method?: string;
user_tier?: string;
response?: string;
stats?: SessionMetrics;
stats?: JsonOutputStats;
error?: JsonError;
}
@@ -33,6 +43,8 @@ export enum JsonStreamEventType {
TOOL_RESULT = 'tool_result',
ERROR = 'error',
RESULT = 'result',
RETRY = 'retry',
LOOP_DETECTED = 'loop_detected',
}
export interface BaseJsonStreamEvent {
@@ -44,6 +56,8 @@ export interface InitEvent extends BaseJsonStreamEvent {
type: JsonStreamEventType.INIT;
session_id: string;
model: string;
auth_method?: string;
user_tier?: string;
}
export interface MessageEvent extends BaseJsonStreamEvent {
@@ -86,6 +100,9 @@ export interface StreamStats {
input: number;
duration_ms: number;
tool_calls: number;
api_requests?: number;
api_errors?: number;
retry_count?: number;
}
export interface ResultEvent extends BaseJsonStreamEvent {
@@ -98,10 +115,26 @@ export interface ResultEvent extends BaseJsonStreamEvent {
stats?: StreamStats;
}
export interface RetryEvent extends BaseJsonStreamEvent {
type: JsonStreamEventType.RETRY;
attempt: number;
max_attempts: number;
delay_ms: number;
error?: string;
model: string;
}
export interface LoopDetectedStreamEvent extends BaseJsonStreamEvent {
type: JsonStreamEventType.LOOP_DETECTED;
loop_type?: string;
}
export type JsonStreamEvent =
| InitEvent
| MessageEvent
| ToolUseEvent
| ToolResultEvent
| ErrorEvent
| ResultEvent;
| ResultEvent
| RetryEvent
| LoopDetectedStreamEvent;
@@ -118,6 +118,9 @@ export class LoopDetectionService {
private llmCheckInterval = DEFAULT_LLM_CHECK_INTERVAL;
private lastCheckTurn = 0;
// Detected loop type tracking
private detectedLoopType: string | null = null;
// Session-level disable flag
private disabledForSession = false;
@@ -208,6 +211,7 @@ export class LoopDetectionService {
this.toolCallRepetitionCount = 1;
}
if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) {
this.detectedLoopType = 'consecutive_identical_tool_calls';
logLoopDetected(
this.config,
new LoopDetectedEvent(
@@ -321,6 +325,7 @@ export class LoopDetectionService {
const chunkHash = createHash('sha256').update(currentChunk).digest('hex');
if (this.isLoopDetectedForChunk(currentChunk, chunkHash)) {
this.detectedLoopType = 'chanting_identical_sentences';
logLoopDetected(
this.config,
new LoopDetectedEvent(
@@ -575,6 +580,7 @@ export class LoopDetectionService {
result: Record<string, unknown>,
modelName: string,
): void {
this.detectedLoopType = 'llm_detected_loop';
if (
typeof result['unproductive_state_analysis'] === 'string' &&
result['unproductive_state_analysis']
@@ -599,6 +605,13 @@ export class LoopDetectionService {
);
}
/**
* Returns the type of the most recently detected loop, or null if none.
*/
getDetectedLoopType(): string | null {
return this.detectedLoopType;
}
/**
* Resets all loop detection state.
*/
@@ -608,6 +621,7 @@ export class LoopDetectionService {
this.resetContentTracking();
this.resetLlmCheckTracking();
this.loopDetected = false;
this.detectedLoopType = null;
}
private resetToolCallCount(): void {
+223
View File
@@ -0,0 +1,223 @@
#!/bin/bash
# -----------------------------------------------------------------------------
# Gemini CLI Headless Mode Monitoring Test Script
# -----------------------------------------------------------------------------
# Purpose:
# Runs the Gemini CLI in headless mode across multiple models and output
# formats, then displays the monitoring data (auth method, API stats, retries,
# loop detection) in a readable summary.
#
# Prerequisites:
# Authentication must already be configured (API key, OAuth, or Vertex AI).
# Build the project first: npm run build
#
# Usage:
# ./scripts/test_gemini.sh [--prompt "custom prompt"] [--models "model1 model2"]
#
# Options:
# --prompt <text> Override the default test prompt
# --models <list> Space-separated list of models to test (quoted)
#
# Example:
# ./scripts/test_gemini.sh
# ./scripts/test_gemini.sh --prompt "list files" --models "gemini-2.5-flash"
# -----------------------------------------------------------------------------
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
CLI="$REPO_ROOT/packages/cli/dist/index.js"
# Defaults
PROMPT="count how many files are in the current folder"
MODELS=(
"gemini-2.5-pro"
"gemini-2.5-flash"
"gemini-3.1-pro-preview"
"gemini-3-flash-preview"
)
# Parse args
while [[ "$#" -gt 0 ]]; do
case "$1" in
--prompt) PROMPT="$2"; shift ;;
--models) IFS=' ' read -ra MODELS <<< "$2"; shift ;;
*) echo "Unknown option: $1"; exit 1 ;;
esac
shift
done
# Colors
BOLD='\033[1m'
DIM='\033[2m'
GREEN='\033[32m'
YELLOW='\033[33m'
RED='\033[31m'
CYAN='\033[36m'
RESET='\033[0m'
# Check prerequisites
if [[ ! -f "$CLI" ]]; then
echo -e "${RED}CLI not found at $CLI${RESET}"
echo "Run 'npm run build' from the repo root first."
exit 1
fi
if ! command -v jq &>/dev/null; then
echo -e "${RED}jq is required but not installed.${RESET}"
exit 1
fi
separator() {
echo -e "${DIM}$(printf '%.0s─' {1..72})${RESET}"
}
# Header
echo ""
echo -e "${BOLD}Gemini CLI Headless Monitoring Test${RESET}"
separator
echo -e "${DIM}Prompt:${RESET} $PROMPT"
echo -e "${DIM}Models:${RESET} ${MODELS[*]}"
echo -e "${DIM}CLI:${RESET} $CLI"
separator
echo ""
total_models=${#MODELS[@]}
pass_count=0
fail_count=0
for model in "${MODELS[@]}"; do
echo -e "${BOLD}${CYAN}[$model]${RESET}"
echo ""
# ── stream-json run ──────────────────────────────────────────────────
TMPFILE=$(mktemp)
STDERRFILE=$(mktemp)
exit_code=0
echo -e " ${DIM}Running with -o stream-json -d ...${RESET}"
node "$CLI" -p "$PROMPT" -y -m "$model" -o stream-json -d \
>"$TMPFILE" 2>"$STDERRFILE" || exit_code=$?
if [[ $exit_code -ne 0 ]]; then
echo -e " ${RED}FAILED${RESET} (exit code $exit_code)"
echo ""
if [[ -s "$STDERRFILE" ]]; then
echo -e " ${DIM}stderr:${RESET}"
sed 's/^/ /' "$STDERRFILE"
echo ""
fi
((fail_count++))
rm -f "$TMPFILE" "$STDERRFILE"
separator
echo ""
continue
fi
((pass_count++))
# Parse init event
init_line=$(jq -c 'select(.type=="init")' "$TMPFILE" 2>/dev/null | head -1)
auth_method=$(echo "$init_line" | jq -r '.auth_method // "not set"' 2>/dev/null)
user_tier=$(echo "$init_line" | jq -r '.user_tier // "not set"' 2>/dev/null)
session_id=$(echo "$init_line" | jq -r '.session_id // "?"' 2>/dev/null)
# Parse result event
result_line=$(jq -c 'select(.type=="result")' "$TMPFILE" 2>/dev/null | tail -1)
status=$(echo "$result_line" | jq -r '.status // "?"' 2>/dev/null)
api_requests=$(echo "$result_line" | jq -r '.stats.api_requests // "?"' 2>/dev/null)
api_errors=$(echo "$result_line" | jq -r '.stats.api_errors // "?"' 2>/dev/null)
retry_count=$(echo "$result_line" | jq -r '.stats.retry_count // 0' 2>/dev/null)
total_tokens=$(echo "$result_line" | jq -r '.stats.total_tokens // "?"' 2>/dev/null)
input_tokens=$(echo "$result_line" | jq -r '.stats.input_tokens // "?"' 2>/dev/null)
output_tokens=$(echo "$result_line" | jq -r '.stats.output_tokens // "?"' 2>/dev/null)
cached=$(echo "$result_line" | jq -r '.stats.cached // "?"' 2>/dev/null)
tool_calls=$(echo "$result_line" | jq -r '.stats.tool_calls // 0' 2>/dev/null)
duration_ms=$(echo "$result_line" | jq -r '.stats.duration_ms // "?"' 2>/dev/null)
# Count retries and loop events
retry_events=$(jq -c 'select(.type=="retry")' "$TMPFILE" 2>/dev/null | wc -l | tr -d ' ')
loop_events=$(jq -c 'select(.type=="loop_detected")' "$TMPFILE" 2>/dev/null)
if [[ -n "$loop_events" ]]; then
loop_count=$(echo "$loop_events" | wc -l | tr -d ' ')
loop_type=$(echo "$loop_events" | jq -r '.loop_type // empty' 2>/dev/null | head -1)
else
loop_count=0
loop_type=""
fi
# Extract assistant response (concatenate deltas)
response=$(jq -r 'select(.type=="message" and .role=="assistant") | .content' "$TMPFILE" 2>/dev/null | tr -d '\n')
# Truncate for display
if [[ ${#response} -gt 120 ]]; then
response="${response:0:120}..."
fi
# Format duration
if [[ "$duration_ms" != "?" ]]; then
duration_s=$(echo "scale=1; $duration_ms / 1000" | bc 2>/dev/null || echo "$duration_ms ms")
duration_display="${duration_s}s"
else
duration_display="?"
fi
# Display
echo -e " ${BOLD}Auth & Session${RESET}"
echo -e " auth_method: ${GREEN}$auth_method${RESET}"
echo -e " user_tier: $user_tier"
echo -e " session_id: ${DIM}$session_id${RESET}"
echo ""
echo -e " ${BOLD}API Stats${RESET}"
echo -e " status: $([ "$status" = "success" ] && echo "${GREEN}$status${RESET}" || echo "${RED}$status${RESET}")"
echo -e " api_requests: $api_requests"
echo -e " api_errors: $([ "$api_errors" = "0" ] && echo "$api_errors" || echo "${RED}$api_errors${RESET}")"
echo -e " retry_count: $([ "$retry_count" = "0" ] && echo "$retry_count" || echo "${YELLOW}$retry_count${RESET}")"
echo -e " duration: $duration_display"
echo ""
echo -e " ${BOLD}Tokens${RESET}"
echo -e " total: $total_tokens (in: $input_tokens, out: $output_tokens, cached: $cached)"
echo -e " tools: $tool_calls calls"
echo ""
if [[ "$retry_events" -gt 0 ]]; then
echo -e " ${BOLD}${YELLOW}Retries ($retry_events)${RESET}"
jq -r 'select(.type=="retry") | " attempt \(.attempt)/\(.max_attempts) delay=\(.delay_ms)ms \(.error // "")"' "$TMPFILE" 2>/dev/null
echo ""
fi
if [[ "$loop_count" -gt 0 ]]; then
echo -e " ${BOLD}${RED}Loop Detected${RESET}"
echo -e " type: ${loop_type:-unknown}"
echo ""
fi
echo -e " ${BOLD}Response${RESET}"
echo -e " ${DIM}$response${RESET}"
echo ""
# Show stderr if any
stderr_content=$(cat "$STDERRFILE")
if [[ -n "$stderr_content" ]]; then
echo -e " ${BOLD}Stderr${RESET}"
echo "$stderr_content" | sed 's/^/ /'
echo ""
fi
rm -f "$TMPFILE" "$STDERRFILE"
separator
echo ""
done
# Summary
echo -e "${BOLD}Summary${RESET}"
echo -e " Models tested: $total_models"
echo -e " Passed: ${GREEN}$pass_count${RESET}"
if [[ $fail_count -gt 0 ]]; then
echo -e " Failed: ${RED}$fail_count${RESET}"
else
echo -e " Failed: $fail_count"
fi
echo ""