mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-03-10 14:10:37 -07:00
feat(cli): enable activity logging for non-interactive mode and evals (#17703)
This commit is contained in:
@@ -34,6 +34,10 @@ export type EvalPolicy = 'ALWAYS_PASSES' | 'USUALLY_PASSES';
|
|||||||
export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
|
export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
|
||||||
const fn = async () => {
|
const fn = async () => {
|
||||||
const rig = new TestRig();
|
const rig = new TestRig();
|
||||||
|
const { logDir, sanitizedName } = await prepareLogDir(evalCase.name);
|
||||||
|
const activityLogFile = path.join(logDir, `${sanitizedName}.jsonl`);
|
||||||
|
const logFile = path.join(logDir, `${sanitizedName}.log`);
|
||||||
|
let isSuccess = false;
|
||||||
try {
|
try {
|
||||||
rig.setup(evalCase.name, evalCase.params);
|
rig.setup(evalCase.name, evalCase.params);
|
||||||
|
|
||||||
@@ -62,6 +66,9 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
|
|||||||
const result = await rig.run({
|
const result = await rig.run({
|
||||||
args: evalCase.prompt,
|
args: evalCase.prompt,
|
||||||
approvalMode: evalCase.approvalMode ?? 'yolo',
|
approvalMode: evalCase.approvalMode ?? 'yolo',
|
||||||
|
env: {
|
||||||
|
GEMINI_CLI_ACTIVITY_LOG_FILE: activityLogFile,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const unauthorizedErrorPrefix =
|
const unauthorizedErrorPrefix =
|
||||||
@@ -73,9 +80,16 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
await evalCase.assert(rig, result);
|
await evalCase.assert(rig, result);
|
||||||
|
isSuccess = true;
|
||||||
} finally {
|
} finally {
|
||||||
await logToFile(
|
if (isSuccess) {
|
||||||
evalCase.name,
|
await fs.promises.unlink(activityLogFile).catch((err) => {
|
||||||
|
if (err.code !== 'ENOENT') throw err;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.promises.writeFile(
|
||||||
|
logFile,
|
||||||
JSON.stringify(rig.readToolLogs(), null, 2),
|
JSON.stringify(rig.readToolLogs(), null, 2),
|
||||||
);
|
);
|
||||||
await rig.cleanup();
|
await rig.cleanup();
|
||||||
@@ -89,6 +103,13 @@ export function evalTest(policy: EvalPolicy, evalCase: EvalCase) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function prepareLogDir(name: string) {
|
||||||
|
const logDir = 'evals/logs';
|
||||||
|
await fs.promises.mkdir(logDir, { recursive: true });
|
||||||
|
const sanitizedName = name.replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
||||||
|
return { logDir, sanitizedName };
|
||||||
|
}
|
||||||
|
|
||||||
export interface EvalCase {
|
export interface EvalCase {
|
||||||
name: string;
|
name: string;
|
||||||
params?: Record<string, any>;
|
params?: Record<string, any>;
|
||||||
@@ -97,11 +118,3 @@ export interface EvalCase {
|
|||||||
approvalMode?: 'default' | 'auto_edit' | 'yolo' | 'plan';
|
approvalMode?: 'default' | 'auto_edit' | 'yolo' | 'plan';
|
||||||
assert: (rig: TestRig, result: string) => Promise<void>;
|
assert: (rig: TestRig, result: string) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function logToFile(name: string, content: string) {
|
|
||||||
const logDir = 'evals/logs';
|
|
||||||
await fs.promises.mkdir(logDir, { recursive: true });
|
|
||||||
const sanitizedName = name.replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
|
||||||
const logFile = `${logDir}/${sanitizedName}.log`;
|
|
||||||
await fs.promises.writeFile(logFile, content);
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -38,6 +38,11 @@ import type { LoadedSettings } from './config/settings.js';
|
|||||||
// Mock core modules
|
// Mock core modules
|
||||||
vi.mock('./ui/hooks/atCommandProcessor.js');
|
vi.mock('./ui/hooks/atCommandProcessor.js');
|
||||||
|
|
||||||
|
const mockRegisterActivityLogger = vi.hoisted(() => vi.fn());
|
||||||
|
vi.mock('./utils/activityLogger.js', () => ({
|
||||||
|
registerActivityLogger: mockRegisterActivityLogger,
|
||||||
|
}));
|
||||||
|
|
||||||
const mockCoreEvents = vi.hoisted(() => ({
|
const mockCoreEvents = vi.hoisted(() => ({
|
||||||
on: vi.fn(),
|
on: vi.fn(),
|
||||||
off: vi.fn(),
|
off: vi.fn(),
|
||||||
@@ -259,6 +264,52 @@ describe('runNonInteractive', () => {
|
|||||||
// so we no longer expect shutdownTelemetry to be called directly here
|
// so we no longer expect shutdownTelemetry to be called directly here
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should register activity logger when GEMINI_CLI_ACTIVITY_LOG_FILE is set', async () => {
|
||||||
|
vi.stubEnv('GEMINI_CLI_ACTIVITY_LOG_FILE', '/tmp/test.jsonl');
|
||||||
|
const events: ServerGeminiStreamEvent[] = [
|
||||||
|
{
|
||||||
|
type: GeminiEventType.Finished,
|
||||||
|
value: { reason: undefined, usageMetadata: { totalTokenCount: 0 } },
|
||||||
|
},
|
||||||
|
];
|
||||||
|
mockGeminiClient.sendMessageStream.mockReturnValue(
|
||||||
|
createStreamFromEvents(events),
|
||||||
|
);
|
||||||
|
|
||||||
|
await runNonInteractive({
|
||||||
|
config: mockConfig,
|
||||||
|
settings: mockSettings,
|
||||||
|
input: 'test',
|
||||||
|
prompt_id: 'prompt-id-activity-logger',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockRegisterActivityLogger).toHaveBeenCalledWith(mockConfig);
|
||||||
|
vi.unstubAllEnvs();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should not register activity logger when GEMINI_CLI_ACTIVITY_LOG_FILE is not set', async () => {
|
||||||
|
vi.stubEnv('GEMINI_CLI_ACTIVITY_LOG_FILE', '');
|
||||||
|
const events: ServerGeminiStreamEvent[] = [
|
||||||
|
{
|
||||||
|
type: GeminiEventType.Finished,
|
||||||
|
value: { reason: undefined, usageMetadata: { totalTokenCount: 0 } },
|
||||||
|
},
|
||||||
|
];
|
||||||
|
mockGeminiClient.sendMessageStream.mockReturnValue(
|
||||||
|
createStreamFromEvents(events),
|
||||||
|
);
|
||||||
|
|
||||||
|
await runNonInteractive({
|
||||||
|
config: mockConfig,
|
||||||
|
settings: mockSettings,
|
||||||
|
input: 'test',
|
||||||
|
prompt_id: 'prompt-id-activity-logger-off',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockRegisterActivityLogger).not.toHaveBeenCalled();
|
||||||
|
vi.unstubAllEnvs();
|
||||||
|
});
|
||||||
|
|
||||||
it('should handle a single tool call and respond', async () => {
|
it('should handle a single tool call and respond', async () => {
|
||||||
const toolCallEvent: ServerGeminiStreamEvent = {
|
const toolCallEvent: ServerGeminiStreamEvent = {
|
||||||
type: GeminiEventType.ToolCallRequest,
|
type: GeminiEventType.ToolCallRequest,
|
||||||
|
|||||||
@@ -70,6 +70,14 @@ export async function runNonInteractive({
|
|||||||
coreEvents.emitConsoleLog(msg.type, msg.content);
|
coreEvents.emitConsoleLog(msg.type, msg.content);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (config.storage && process.env['GEMINI_CLI_ACTIVITY_LOG_FILE']) {
|
||||||
|
const { registerActivityLogger } = await import(
|
||||||
|
'./utils/activityLogger.js'
|
||||||
|
);
|
||||||
|
registerActivityLogger(config);
|
||||||
|
}
|
||||||
|
|
||||||
const { stdout: workingStdout } = createWorkingStdio();
|
const { stdout: workingStdout } = createWorkingStdio();
|
||||||
const textOutput = new TextOutput(workingStdout);
|
const textOutput = new TextOutput(workingStdout);
|
||||||
|
|
||||||
|
|||||||
@@ -323,13 +323,17 @@ export class ActivityLogger extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Registers the activity logger if debug mode and interactive session are enabled.
|
* Registers the activity logger.
|
||||||
* Captures network and console logs to a session-specific JSONL file.
|
* Captures network and console logs to a session-specific JSONL file.
|
||||||
*
|
*
|
||||||
|
* The log file location can be overridden via the GEMINI_CLI_ACTIVITY_LOG_FILE
|
||||||
|
* environment variable. If not set, defaults to logs/session-{sessionId}.jsonl
|
||||||
|
* in the project's temp directory.
|
||||||
|
*
|
||||||
* @param config The CLI configuration
|
* @param config The CLI configuration
|
||||||
*/
|
*/
|
||||||
export function registerActivityLogger(config: Config) {
|
export function registerActivityLogger(config: Config) {
|
||||||
if (config.isInteractive() && config.storage && config.getDebugMode()) {
|
if (config.storage) {
|
||||||
const capture = ActivityLogger.getInstance();
|
const capture = ActivityLogger.getInstance();
|
||||||
capture.enable();
|
capture.enable();
|
||||||
|
|
||||||
@@ -338,10 +342,10 @@ export function registerActivityLogger(config: Config) {
|
|||||||
fs.mkdirSync(logsDir, { recursive: true });
|
fs.mkdirSync(logsDir, { recursive: true });
|
||||||
}
|
}
|
||||||
|
|
||||||
const logFile = path.join(
|
const logFile =
|
||||||
logsDir,
|
process.env['GEMINI_CLI_ACTIVITY_LOG_FILE'] ||
|
||||||
`session-${config.getSessionId()}.jsonl`,
|
path.join(logsDir, `session-${config.getSessionId()}.jsonl`);
|
||||||
);
|
|
||||||
const writeToLog = (type: 'console' | 'network', payload: unknown) => {
|
const writeToLog = (type: 'console' | 'network', payload: unknown) => {
|
||||||
try {
|
try {
|
||||||
const entry =
|
const entry =
|
||||||
|
|||||||
Reference in New Issue
Block a user