feat(cli): enable activity logging for non-interactive mode and evals (#17703)

This commit is contained in:
Sandy Tao
2026-01-28 09:02:41 -08:00
committed by GitHub
parent 25ae1a1b54
commit 9e09db1ddb
4 changed files with 92 additions and 16 deletions

View File

@@ -34,6 +34,10 @@ export type EvalPolicy = 'ALWAYS_PASSES' | 'USUALLY_PASSES';
export function evalTest(policy: EvalPolicy, evalCase: EvalCase) { export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
const fn = async () => { const fn = async () => {
const rig = new TestRig(); const rig = new TestRig();
const { logDir, sanitizedName } = await prepareLogDir(evalCase.name);
const activityLogFile = path.join(logDir, `${sanitizedName}.jsonl`);
const logFile = path.join(logDir, `${sanitizedName}.log`);
let isSuccess = false;
try { try {
rig.setup(evalCase.name, evalCase.params); rig.setup(evalCase.name, evalCase.params);
@@ -62,6 +66,9 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
const result = await rig.run({ const result = await rig.run({
args: evalCase.prompt, args: evalCase.prompt,
approvalMode: evalCase.approvalMode ?? 'yolo', approvalMode: evalCase.approvalMode ?? 'yolo',
env: {
GEMINI_CLI_ACTIVITY_LOG_FILE: activityLogFile,
},
}); });
const unauthorizedErrorPrefix = const unauthorizedErrorPrefix =
@@ -73,9 +80,16 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
} }
await evalCase.assert(rig, result); await evalCase.assert(rig, result);
isSuccess = true;
} finally { } finally {
await logToFile( if (isSuccess) {
evalCase.name, await fs.promises.unlink(activityLogFile).catch((err) => {
if (err.code !== 'ENOENT') throw err;
});
}
await fs.promises.writeFile(
logFile,
JSON.stringify(rig.readToolLogs(), null, 2), JSON.stringify(rig.readToolLogs(), null, 2),
); );
await rig.cleanup(); await rig.cleanup();
@@ -89,6 +103,13 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
} }
} }
async function prepareLogDir(name: string) {
const logDir = 'evals/logs';
await fs.promises.mkdir(logDir, { recursive: true });
const sanitizedName = name.replace(/[^a-z0-9]/gi, '_').toLowerCase();
return { logDir, sanitizedName };
}
export interface EvalCase { export interface EvalCase {
name: string; name: string;
params?: Record<string, any>; params?: Record<string, any>;
@@ -97,11 +118,3 @@ export interface EvalCase {
approvalMode?: 'default' | 'auto_edit' | 'yolo' | 'plan'; approvalMode?: 'default' | 'auto_edit' | 'yolo' | 'plan';
assert: (rig: TestRig, result: string) => Promise<void>; assert: (rig: TestRig, result: string) => Promise<void>;
} }
async function logToFile(name: string, content: string) {
const logDir = 'evals/logs';
await fs.promises.mkdir(logDir, { recursive: true });
const sanitizedName = name.replace(/[^a-z0-9]/gi, '_').toLowerCase();
const logFile = `${logDir}/${sanitizedName}.log`;
await fs.promises.writeFile(logFile, content);
}

View File

@@ -38,6 +38,11 @@ import type { LoadedSettings } from './config/settings.js';
// Mock core modules // Mock core modules
vi.mock('./ui/hooks/atCommandProcessor.js'); vi.mock('./ui/hooks/atCommandProcessor.js');
const mockRegisterActivityLogger = vi.hoisted(() => vi.fn());
vi.mock('./utils/activityLogger.js', () => ({
registerActivityLogger: mockRegisterActivityLogger,
}));
const mockCoreEvents = vi.hoisted(() => ({ const mockCoreEvents = vi.hoisted(() => ({
on: vi.fn(), on: vi.fn(),
off: vi.fn(), off: vi.fn(),
@@ -259,6 +264,52 @@ describe('runNonInteractive', () => {
// so we no longer expect shutdownTelemetry to be called directly here // so we no longer expect shutdownTelemetry to be called directly here
}); });
it('should register activity logger when GEMINI_CLI_ACTIVITY_LOG_FILE is set', async () => {
vi.stubEnv('GEMINI_CLI_ACTIVITY_LOG_FILE', '/tmp/test.jsonl');
const events: ServerGeminiStreamEvent[] = [
{
type: GeminiEventType.Finished,
value: { reason: undefined, usageMetadata: { totalTokenCount: 0 } },
},
];
mockGeminiClient.sendMessageStream.mockReturnValue(
createStreamFromEvents(events),
);
await runNonInteractive({
config: mockConfig,
settings: mockSettings,
input: 'test',
prompt_id: 'prompt-id-activity-logger',
});
expect(mockRegisterActivityLogger).toHaveBeenCalledWith(mockConfig);
vi.unstubAllEnvs();
});
it('should not register activity logger when GEMINI_CLI_ACTIVITY_LOG_FILE is not set', async () => {
vi.stubEnv('GEMINI_CLI_ACTIVITY_LOG_FILE', '');
const events: ServerGeminiStreamEvent[] = [
{
type: GeminiEventType.Finished,
value: { reason: undefined, usageMetadata: { totalTokenCount: 0 } },
},
];
mockGeminiClient.sendMessageStream.mockReturnValue(
createStreamFromEvents(events),
);
await runNonInteractive({
config: mockConfig,
settings: mockSettings,
input: 'test',
prompt_id: 'prompt-id-activity-logger-off',
});
expect(mockRegisterActivityLogger).not.toHaveBeenCalled();
vi.unstubAllEnvs();
});
it('should handle a single tool call and respond', async () => { it('should handle a single tool call and respond', async () => {
const toolCallEvent: ServerGeminiStreamEvent = { const toolCallEvent: ServerGeminiStreamEvent = {
type: GeminiEventType.ToolCallRequest, type: GeminiEventType.ToolCallRequest,

View File

@@ -70,6 +70,14 @@ export async function runNonInteractive({
coreEvents.emitConsoleLog(msg.type, msg.content); coreEvents.emitConsoleLog(msg.type, msg.content);
}, },
}); });
if (config.storage && process.env['GEMINI_CLI_ACTIVITY_LOG_FILE']) {
const { registerActivityLogger } = await import(
'./utils/activityLogger.js'
);
registerActivityLogger(config);
}
const { stdout: workingStdout } = createWorkingStdio(); const { stdout: workingStdout } = createWorkingStdio();
const textOutput = new TextOutput(workingStdout); const textOutput = new TextOutput(workingStdout);

View File

@@ -323,13 +323,17 @@ export class ActivityLogger extends EventEmitter {
} }
/** /**
* Registers the activity logger if debug mode and interactive session are enabled. * Registers the activity logger.
* Captures network and console logs to a session-specific JSONL file. * Captures network and console logs to a session-specific JSONL file.
* *
* The log file location can be overridden via the GEMINI_CLI_ACTIVITY_LOG_FILE
* environment variable. If not set, defaults to logs/session-{sessionId}.jsonl
* in the project's temp directory.
*
* @param config The CLI configuration * @param config The CLI configuration
*/ */
export function registerActivityLogger(config: Config) { export function registerActivityLogger(config: Config) {
if (config.isInteractive() && config.storage && config.getDebugMode()) { if (config.storage) {
const capture = ActivityLogger.getInstance(); const capture = ActivityLogger.getInstance();
capture.enable(); capture.enable();
@@ -338,10 +342,10 @@ export function registerActivityLogger(config: Config) {
fs.mkdirSync(logsDir, { recursive: true }); fs.mkdirSync(logsDir, { recursive: true });
} }
const logFile = path.join( const logFile =
logsDir, process.env['GEMINI_CLI_ACTIVITY_LOG_FILE'] ||
`session-${config.getSessionId()}.jsonl`, path.join(logsDir, `session-${config.getSessionId()}.jsonl`);
);
const writeToLog = (type: 'console' | 'network', payload: unknown) => { const writeToLog = (type: 'console' | 'network', payload: unknown) => {
try { try {
const entry = const entry =