MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-04-20 10:10:56 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): add telemetry for subagent execution (#10456)

Browse Source
This commit is contained in:
Abhi
2025-10-08 15:42:33 -04:00
committed by GitHub
parent b45bd5ff7b
commit c0552ceb22
11 changed files with 608 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/src/telemetry/clearcut-logger/clearcut-logger.test.ts
+84
View File
@@ -29,7 +29,10 @@ import {
makeChatCompressionEvent,
ModelRoutingEvent,
ToolCallEvent,
AgentStartEvent,
AgentFinishEvent,
} from '../types.js';
import { AgentTerminateMode } from '../../agents/types.js';
import { GIT_COMMIT_INFO, CLI_VERSION } from '../../generated/git-commit.js';
import { UserAccountManager } from '../../utils/userAccountManager.js';
import { InstallationManager } from '../../utils/installationManager.js';
@@ -721,6 +724,87 @@ describe('ClearcutLogger', () => {
});
});
describe('logAgentStartEvent', () => {
it('logs an event with proper fields', () => {
const { logger } = setup();
const event = new AgentStartEvent('agent-123', 'TestAgent');
logger?.logAgentStartEvent(event);
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.AGENT_START);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_ID,
'agent-123',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_NAME,
'TestAgent',
]);
});
});
describe('logAgentFinishEvent', () => {
it('logs an event with proper fields (success)', () => {
const { logger } = setup();
const event = new AgentFinishEvent(
'agent-123',
'TestAgent',
1000,
5,
AgentTerminateMode.GOAL,
);
logger?.logAgentFinishEvent(event);
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.AGENT_FINISH);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_ID,
'agent-123',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_NAME,
'TestAgent',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_DURATION_MS,
'1000',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_TURN_COUNT,
'5',
]);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_TERMINATE_REASON,
'GOAL',
]);
});
it('logs an event with proper fields (error)', () => {
const { logger } = setup();
const event = new AgentFinishEvent(
'agent-123',
'TestAgent',
500,
2,
AgentTerminateMode.ERROR,
);
logger?.logAgentFinishEvent(event);
const events = getEvents(logger!);
expect(events.length).toBe(1);
expect(events[0]).toHaveEventName(EventNames.AGENT_FINISH);
expect(events[0]).toHaveMetadataValue([
EventMetadataKey.GEMINI_CLI_AGENT_TERMINATE_REASON,
'ERROR',
]);
});
});
describe('logToolCallEvent', () => {
it('logs an event with all diff metadata', () => {
const { logger } = setup();
packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
+48
View File
@@ -33,6 +33,8 @@ import type {
ExtensionDisableEvent,
SmartEditStrategyEvent,
SmartEditCorrectionEvent,
AgentStartEvent,
AgentFinishEvent,
} from '../types.js';
import { EventMetadataKey } from './event-metadata-key.js';
import type { Config } from '../../config/config.js';
@@ -79,6 +81,8 @@ export enum EventNames {
MODEL_SLASH_COMMAND = 'model_slash_command',
SMART_EDIT_STRATEGY = 'smart_edit_strategy',
SMART_EDIT_CORRECTION = 'smart_edit_correction',
AGENT_START = 'agent_start',
AGENT_FINISH = 'agent_finish',
}
export interface LogResponse {
@@ -1063,6 +1067,50 @@ export class ClearcutLogger {
this.flushIfNeeded();
}
logAgentStartEvent(event: AgentStartEvent): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AGENT_ID,
value: event.agent_id,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AGENT_NAME,
value: event.agent_name,
},
];
this.enqueueLogEvent(this.createLogEvent(EventNames.AGENT_START, data));
this.flushIfNeeded();
}
logAgentFinishEvent(event: AgentFinishEvent): void {
const data: EventValue[] = [
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AGENT_ID,
value: event.agent_id,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AGENT_NAME,
value: event.agent_name,
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AGENT_DURATION_MS,
value: event.duration_ms.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AGENT_TURN_COUNT,
value: event.turn_count.toString(),
},
{
gemini_cli_key: EventMetadataKey.GEMINI_CLI_AGENT_TERMINATE_REASON,
value: event.terminate_reason,
},
];
this.enqueueLogEvent(this.createLogEvent(EventNames.AGENT_FINISH, data));
this.flushIfNeeded();
}
/**
* Adds default fields to data, and returns a new data array. This fields
* should exist on all log events.
packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts
+19
View File
@@ -426,4 +426,23 @@ export enum EventMetadataKey {
// Logs an event when the user uses the /model command.
GEMINI_CLI_MODEL_SLASH_COMMAND = 108,
// ==========================================================================
// Agent Event Keys
// ==========================================================================
// Logs the name of the agent.
GEMINI_CLI_AGENT_NAME = 111,
// Logs the unique ID of the agent instance.
GEMINI_CLI_AGENT_ID = 112,
// Logs the duration of the agent execution in milliseconds.
GEMINI_CLI_AGENT_DURATION_MS = 113,
// Logs the number of turns the agent took.
GEMINI_CLI_AGENT_TURN_COUNT = 114,
// Logs the reason for agent termination.
GEMINI_CLI_AGENT_TERMINATE_REASON = 115,
}
packages/core/src/telemetry/constants.ts
+4
View File
@@ -36,6 +36,10 @@ export const EVENT_SMART_EDIT_STRATEGY = 'gemini_cli.smart_edit.strategy';
export const EVENT_MODEL_ROUTING = 'gemini_cli.model_routing';
export const EVENT_SMART_EDIT_CORRECTION = 'gemini_cli.smart_edit.correction';
// Agent Events
export const EVENT_AGENT_START = 'gemini_cli.agent.start';
export const EVENT_AGENT_FINISH = 'gemini_cli.agent.finish';
// Performance Events
export const EVENT_STARTUP_PERFORMANCE = 'gemini_cli.startup.performance';
export const EVENT_MEMORY_USAGE = 'gemini_cli.memory.usage';
packages/core/src/telemetry/loggers.test.ts
+88
View File
@@ -38,6 +38,8 @@ import {
EVENT_EXTENSION_INSTALL,
EVENT_EXTENSION_UNINSTALL,
EVENT_TOOL_OUTPUT_TRUNCATED,
EVENT_AGENT_START,
EVENT_AGENT_FINISH,
} from './constants.js';
import {
logApiRequest,
@@ -56,6 +58,8 @@ import {
logExtensionDisable,
logExtensionInstallEvent,
logExtensionUninstall,
logAgentStart,
logAgentFinish,
} from './loggers.js';
import { ToolCallDecision } from './tool-call-decision.js';
import {
@@ -75,6 +79,8 @@ import {
ExtensionDisableEvent,
ExtensionInstallEvent,
ExtensionUninstallEvent,
AgentStartEvent,
AgentFinishEvent,
} from './types.js';
import * as metrics from './metrics.js';
import {
@@ -93,6 +99,7 @@ import * as uiTelemetry from './uiTelemetry.js';
import { makeFakeConfig } from '../test-utils/config.js';
import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
import { UserAccountManager } from '../utils/userAccountManager.js';
import { AgentTerminateMode } from '../agents/types.js';
describe('loggers', () => {
const mockLogger = {
@@ -1406,4 +1413,85 @@ describe('loggers', () => {
});
});
});
describe('logAgentStart', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getUsageStatisticsEnabled: () => true,
} as unknown as Config;
beforeEach(() => {
vi.spyOn(ClearcutLogger.prototype, 'logAgentStartEvent');
});
it('should log agent start event', () => {
const event = new AgentStartEvent('agent-123', 'TestAgent');
logAgentStart(mockConfig, event);
expect(ClearcutLogger.prototype.logAgentStartEvent).toHaveBeenCalledWith(
event,
);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Agent TestAgent started. ID: agent-123',
attributes: {
'session.id': 'test-session-id',
'user.email': 'test-user@example.com',
'event.name': EVENT_AGENT_START,
'event.timestamp': '2025-01-01T00:00:00.000Z',
agent_id: 'agent-123',
agent_name: 'TestAgent',
},
});
});
});
describe('logAgentFinish', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getUsageStatisticsEnabled: () => true,
} as unknown as Config;
beforeEach(() => {
vi.spyOn(ClearcutLogger.prototype, 'logAgentFinishEvent');
vi.spyOn(metrics, 'recordAgentRunMetrics');
});
it('should log agent finish event and record metrics', () => {
const event = new AgentFinishEvent(
'agent-123',
'TestAgent',
1000,
5,
AgentTerminateMode.GOAL,
);
logAgentFinish(mockConfig, event);
expect(ClearcutLogger.prototype.logAgentFinishEvent).toHaveBeenCalledWith(
event,
);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'Agent TestAgent finished. Reason: GOAL. Duration: 1000ms. Turns: 5.',
attributes: {
'session.id': 'test-session-id',
'user.email': 'test-user@example.com',
'event.name': EVENT_AGENT_FINISH,
'event.timestamp': '2025-01-01T00:00:00.000Z',
agent_id: 'agent-123',
agent_name: 'TestAgent',
duration_ms: 1000,
turn_count: 5,
terminate_reason: 'GOAL',
},
});
expect(metrics.recordAgentRunMetrics).toHaveBeenCalledWith(
mockConfig,
event,
);
});
});
});
packages/core/src/telemetry/loggers.ts
+43
View File
@@ -37,6 +37,8 @@ import {
EVENT_EXTENSION_DISABLE,
EVENT_SMART_EDIT_STRATEGY,
EVENT_SMART_EDIT_CORRECTION,
EVENT_AGENT_START,
EVENT_AGENT_FINISH,
} from './constants.js';
import type {
ApiErrorEvent,
@@ -69,6 +71,8 @@ import type {
ModelSlashCommandEvent,
SmartEditStrategyEvent,
SmartEditCorrectionEvent,
AgentStartEvent,
AgentFinishEvent,
} from './types.js';
import {
recordApiErrorMetrics,
@@ -83,6 +87,7 @@ import {
getConventionAttributes,
recordTokenUsageMetrics,
recordApiResponseMetrics,
recordAgentRunMetrics,
} from './metrics.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import type { UiEvent } from './uiTelemetry.js';
@@ -863,3 +868,41 @@ export function logSmartEditCorrectionEvent(
};
logger.emit(logRecord);
}
export function logAgentStart(config: Config, event: AgentStartEvent): void {
ClearcutLogger.getInstance(config)?.logAgentStartEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_AGENT_START,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Agent ${event.agent_name} started. ID: ${event.agent_id}`,
attributes,
};
logger.emit(logRecord);
}
export function logAgentFinish(config: Config, event: AgentFinishEvent): void {
ClearcutLogger.getInstance(config)?.logAgentFinishEvent(event);
if (!isTelemetrySdkInitialized()) return;
const attributes: LogAttributes = {
...getCommonAttributes(config),
...event,
'event.name': EVENT_AGENT_FINISH,
};
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: `Agent ${event.agent_name} finished. Reason: ${event.terminate_reason}. Duration: ${event.duration_ms}ms. Turns: ${event.turn_count}.`,
attributes,
};
logger.emit(logRecord);
recordAgentRunMetrics(config, event);
}
packages/core/src/telemetry/metrics.test.ts
+58 -1
View File
@@ -20,7 +20,8 @@ import {
ApiRequestPhase,
} from './metrics.js';
import { makeFakeConfig } from '../test-utils/config.js';
import { ModelRoutingEvent } from './types.js';
import { ModelRoutingEvent, AgentFinishEvent } from './types.js';
import { AgentTerminateMode } from '../agents/types.js';
const mockCounterAddFn: Mock<
(value: number, attributes?: Attributes, context?: Context) => void
@@ -89,6 +90,7 @@ describe('Telemetry Metrics', () => {
let recordBaselineComparisonModule: typeof import('./metrics.js').recordBaselineComparison;
let recordGenAiClientTokenUsageModule: typeof import('./metrics.js').recordGenAiClientTokenUsage;
let recordGenAiClientOperationDurationModule: typeof import('./metrics.js').recordGenAiClientOperationDuration;
let recordAgentRunMetricsModule: typeof import('./metrics.js').recordAgentRunMetrics;
beforeEach(async () => {
vi.resetModules();
@@ -121,6 +123,7 @@ describe('Telemetry Metrics', () => {
metricsJsModule.recordGenAiClientTokenUsage;
recordGenAiClientOperationDurationModule =
metricsJsModule.recordGenAiClientOperationDuration;
recordAgentRunMetricsModule = metricsJsModule.recordAgentRunMetrics;
const otelApiModule = await import('@opentelemetry/api');
@@ -439,6 +442,60 @@ describe('Telemetry Metrics', () => {
});
});
describe('recordAgentRunMetrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
} as unknown as Config;
it('should not record metrics if not initialized', () => {
const event = new AgentFinishEvent(
'agent-123',
'TestAgent',
1000,
5,
AgentTerminateMode.GOAL,
);
recordAgentRunMetricsModule(mockConfig, event);
expect(mockCounterAddFn).not.toHaveBeenCalled();
expect(mockHistogramRecordFn).not.toHaveBeenCalled();
});
it('should record agent run metrics', () => {
initializeMetricsModule(mockConfig);
mockCounterAddFn.mockClear();
mockHistogramRecordFn.mockClear();
const event = new AgentFinishEvent(
'agent-123',
'TestAgent',
1000,
5,
AgentTerminateMode.GOAL,
);
recordAgentRunMetricsModule(mockConfig, event);
// Verify agent run counter
expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
'session.id': 'test-session-id',
agent_name: 'TestAgent',
terminate_reason: 'GOAL',
});
// Verify agent duration histogram
expect(mockHistogramRecordFn).toHaveBeenCalledWith(1000, {
'session.id': 'test-session-id',
agent_name: 'TestAgent',
});
// Verify agent turns histogram
expect(mockHistogramRecordFn).toHaveBeenCalledWith(5, {
'session.id': 'test-session-id',
agent_name: 'TestAgent',
});
});
});
describe('OpenTelemetry GenAI Semantic Convention Metrics', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
packages/core/src/telemetry/metrics.ts
+71 -1
View File
@@ -8,7 +8,11 @@ import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api';
import { diag, metrics, ValueType } from '@opentelemetry/api';
import { SERVICE_NAME, EVENT_CHAT_COMPRESSION } from './constants.js';
import type { Config } from '../config/config.js';
import type { ModelRoutingEvent, ModelSlashCommandEvent } from './types.js';
import type {
ModelRoutingEvent,
ModelSlashCommandEvent,
AgentFinishEvent,
} from './types.js';
import { AuthType } from '../core/contentGenerator.js';
const TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
@@ -27,6 +31,11 @@ const MODEL_ROUTING_FAILURE_COUNT = 'gemini_cli.model_routing.failure.count';
const MODEL_SLASH_COMMAND_CALL_COUNT =
'gemini_cli.slash_command.model.call_count';
// Agent Metrics
const AGENT_RUN_COUNT = 'gemini_cli.agent.run.count';
const AGENT_DURATION_MS = 'gemini_cli.agent.duration';
const AGENT_TURNS = 'gemini_cli.agent.turns';
// OpenTelemetry GenAI Semantic Convention Metrics
const GEN_AI_CLIENT_TOKEN_USAGE = 'gen_ai.client.token.usage';
const GEN_AI_CLIENT_OPERATION_DURATION = 'gen_ai.client.operation.duration';
@@ -144,6 +153,15 @@ const COUNTER_DEFINITIONS = {
tokens_after: number;
},
},
[AGENT_RUN_COUNT]: {
description: 'Counts agent runs, tagged by name and termination reason.',
valueType: ValueType.INT,
assign: (c: Counter) => (agentRunCounter = c),
attributes: {} as {
agent_name: string;
terminate_reason: string;
},
},
} as const;
const HISTOGRAM_DEFINITIONS = {
@@ -175,6 +193,24 @@ const HISTOGRAM_DEFINITIONS = {
'routing.decision_source': string;
},
},
[AGENT_DURATION_MS]: {
description: 'Duration of agent runs in milliseconds.',
unit: 'ms',
valueType: ValueType.INT,
assign: (h: Histogram) => (agentDurationHistogram = h),
attributes: {} as {
agent_name: string;
},
},
[AGENT_TURNS]: {
description: 'Number of turns taken by agents.',
unit: 'turns',
valueType: ValueType.INT,
assign: (h: Histogram) => (agentTurnsHistogram = h),
attributes: {} as {
agent_name: string;
},
},
[GEN_AI_CLIENT_TOKEN_USAGE]: {
description: 'Number of input and output tokens used.',
unit: 'token',
@@ -405,6 +441,9 @@ let contentRetryFailureCounter: Counter | undefined;
let modelRoutingLatencyHistogram: Histogram | undefined;
let modelRoutingFailureCounter: Counter | undefined;
let modelSlashCommandCallCounter: Counter | undefined;
let agentRunCounter: Counter | undefined;
let agentDurationHistogram: Histogram | undefined;
let agentTurnsHistogram: Histogram | undefined;
// OpenTelemetry GenAI Semantic Convention Metrics
let genAiClientTokenUsageHistogram: Histogram | undefined;
@@ -626,6 +665,37 @@ export function recordModelRoutingMetrics(
}
}
export function recordAgentRunMetrics(
config: Config,
event: AgentFinishEvent,
): void {
if (
!agentRunCounter ||
!agentDurationHistogram ||
!agentTurnsHistogram ||
!isMetricsInitialized
)
return;
const commonAttributes = baseMetricDefinition.getCommonAttributes(config);
agentRunCounter.add(1, {
...commonAttributes,
agent_name: event.agent_name,
terminate_reason: event.terminate_reason,
});
agentDurationHistogram.record(event.duration_ms, {
...commonAttributes,
agent_name: event.agent_name,
});
agentTurnsHistogram.record(event.turn_count, {
...commonAttributes,
agent_name: event.agent_name,
});
}
// OpenTelemetry GenAI Semantic Convention Recording Functions
export function recordGenAiClientTokenUsage(
packages/core/src/telemetry/types.ts
+44 -1
View File
@@ -19,6 +19,7 @@ import type { FileOperation } from './metrics.js';
export { ToolCallDecision };
import type { ToolRegistry } from '../tools/tool-registry.js';
import type { OutputFormat } from '../output/types.js';
import type { AgentTerminateMode } from '../agents/types.js';
export interface BaseTelemetryEvent {
'event.name': string;
@@ -687,7 +688,9 @@ export type TelemetryEvent =
| ExtensionUninstallEvent
| ModelRoutingEvent
| ToolOutputTruncatedEvent
| ModelSlashCommandEvent;
| ModelSlashCommandEvent
| AgentStartEvent
| AgentFinishEvent;
export class ExtensionDisableEvent implements BaseTelemetryEvent {
'event.name': 'extension_disable';
@@ -726,3 +729,43 @@ export class SmartEditCorrectionEvent implements BaseTelemetryEvent {
this.correction = correction;
}
}
export class AgentStartEvent implements BaseTelemetryEvent {
'event.name': 'agent_start';
'event.timestamp': string;
agent_id: string;
agent_name: string;
constructor(agent_id: string, agent_name: string) {
this['event.name'] = 'agent_start';
this['event.timestamp'] = new Date().toISOString();
this.agent_id = agent_id;
this.agent_name = agent_name;
}
}
export class AgentFinishEvent implements BaseTelemetryEvent {
'event.name': 'agent_finish';
'event.timestamp': string;
agent_id: string;
agent_name: string;
duration_ms: number;
turn_count: number;
terminate_reason: AgentTerminateMode;
constructor(
agent_id: string,
agent_name: string,
duration_ms: number,
turn_count: number,
terminate_reason: AgentTerminateMode,
) {
this['event.name'] = 'agent_finish';
this['event.timestamp'] = new Date().toISOString();
this.agent_id = agent_id;
this.agent_name = agent_name;
this.duration_ms = duration_ms;
this.turn_count = turn_count;
this.terminate_reason = terminate_reason;
}
}