MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-03-10 14:10:37 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add Open Telemetric semantic standard compliant log (#11975)

Browse Source
This commit is contained in:
Christie Warwick (Wilson)
2025-10-28 13:02:46 -07:00
committed by GitHub
parent 44bdd3ad11
commit 70996bfdee
11 changed files with 1371 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
docs/cli/telemetry.md
View File

@@ -298,6 +298,26 @@ Captures tool executions, output truncation, and Smart Edit behavior.
- **Attributes**:
- `correction` ("success" | "failure")
- `gen_ai.client.inference.operation.details`: This event provides detailed
information about the GenAI operation, aligned with [OpenTelemetry GenAI
semantic conventions for events].
- **Attributes**:
- `gen_ai.request.model` (string)
- `gen_ai.provider.name` (string)
- `gen_ai.operation.name` (string)
- `gen_ai.input.messages` (json string)
- `gen_ai.output.messages` (json string)
- `gen_ai.response.finish_reasons` (array of strings)
- `gen_ai.usage.input_tokens` (int)
- `gen_ai.usage.output_tokens` (int)
- `gen_ai.request.temperature` (float)
- `gen_ai.request.top_p` (float)
- `gen_ai.request.top_k` (int)
- `gen_ai.request.max_tokens` (int)
- `gen_ai.system_instructions` (json string)
- `server.address` (string)
- `server.port` (int)
#### Files
Tracks file operations performed by tools.
@@ -735,3 +755,5 @@ standardized observability across GenAI applications:
[OpenTelemetry GenAI semantic conventions]:
https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-metrics.md
[OpenTelemetry GenAI semantic conventions for events]:
https://github.com/open-telemetry/semantic-conventions/blob/8b4f210f43136e57c1f6f47292eb6d38e3bf30bb/docs/gen-ai/gen-ai-events.md

112
packages/core/src/core/loggingContentGenerator.ts
View File

@@ -5,15 +5,18 @@
*/
import type {
Candidate,
Content,
CountTokensParameters,
CountTokensResponse,
EmbedContentParameters,
EmbedContentResponse,
GenerateContentConfig,
GenerateContentParameters,
GenerateContentResponseUsageMetadata,
GenerateContentResponse,
} from '@google/genai';
import type { ServerDetails } from '../telemetry/types.js';
import {
ApiRequestEvent,
ApiResponseEvent,
@@ -26,6 +29,7 @@ import {
logApiResponse,
} from '../telemetry/loggers.js';
import type { ContentGenerator } from './contentGenerator.js';
import { CodeAssistServer } from '../code_assist/server.js';
import { toContents } from '../code_assist/converter.js';
import { isStructuredError } from '../utils/quotaErrorDetection.js';
import { runInDevTraceSpan, type SpanMetadata } from '../telemetry/trace.js';
@@ -59,19 +63,66 @@ export class LoggingContentGenerator implements ContentGenerator {
);
}
private _getEndpointUrl(
req: GenerateContentParameters,
method: 'generateContent' | 'generateContentStream',
): ServerDetails {
// Case 1: Authenticated with a Google account (`gcloud auth login`).
// Requests are routed through the internal CodeAssistServer.
if (this.wrapped instanceof CodeAssistServer) {
const url = new URL(this.wrapped.getMethodUrl(method));
const port = url.port
? parseInt(url.port, 10)
: url.protocol === 'https:'
? 443
: 80;
return { address: url.hostname, port };
}
const genConfig = this.config.getContentGeneratorConfig();
// Case 2: Using an API key for Vertex AI.
if (genConfig?.vertexai) {
const location = process.env['GOOGLE_CLOUD_LOCATION'];
if (location) {
return { address: `${location}-aiplatform.googleapis.com`, port: 443 };
} else {
return { address: 'unknown', port: 0 };
}
}
// Case 3: Default to the public Gemini API endpoint.
// This is used when an API key is provided but not for Vertex AI.
return { address: `generativelanguage.googleapis.com`, port: 443 };
}
private _logApiResponse(
requestContents: Content[],
durationMs: number,
model: string,
prompt_id: string,
responseId: string | undefined,
responseCandidates?: Candidate[],
usageMetadata?: GenerateContentResponseUsageMetadata,
responseText?: string,
generationConfig?: GenerateContentConfig,
serverDetails?: ServerDetails,
): void {
logApiResponse(
this.config,
new ApiResponseEvent(
model,
durationMs,
prompt_id,
{
prompt_id,
contents: requestContents,
generate_content_config: generationConfig,
server: serverDetails,
},
{
candidates: responseCandidates,
response_id: responseId,
},
this.config.getContentGeneratorConfig()?.authType,
usageMetadata,
responseText,
@@ -84,6 +135,9 @@ export class LoggingContentGenerator implements ContentGenerator {
error: unknown,
model: string,
prompt_id: string,
requestContents: Content[],
generationConfig?: GenerateContentConfig,
serverDetails?: ServerDetails,
): void {
const errorMessage = error instanceof Error ? error.message : String(error);
const errorType = error instanceof Error ? error.name : 'unknown';
@@ -94,7 +148,12 @@ export class LoggingContentGenerator implements ContentGenerator {
model,
errorMessage,
durationMs,
prompt_id,
{
prompt_id,
contents: requestContents,
generate_content_config: generationConfig,
server: serverDetails,
},
this.config.getContentGeneratorConfig()?.authType,
errorType,
isStructuredError(error)
@@ -116,7 +175,9 @@ export class LoggingContentGenerator implements ContentGenerator {
spanMetadata.input = { request: req, userPromptId, model: req.model };
const startTime = Date.now();
const contents: Content[] = toContents(req.contents);
this.logApiRequest(toContents(req.contents), req.model, userPromptId);
const serverDetails = this._getEndpointUrl(req, 'generateContent');
try {
const response = await this.wrapped.generateContent(
req,
@@ -128,16 +189,29 @@ export class LoggingContentGenerator implements ContentGenerator {
};
const durationMs = Date.now() - startTime;
this._logApiResponse(
contents,
durationMs,
response.modelVersion || req.model,
userPromptId,
response.responseId,
response.candidates,
response.usageMetadata,
JSON.stringify(response),
req.config,
serverDetails,
);
return response;
} catch (error) {
const durationMs = Date.now() - startTime;
this._logApiError(durationMs, error, req.model, userPromptId);
this._logApiError(
durationMs,
error,
req.model,
userPromptId,
contents,
req.config,
serverDetails,
);
throw error;
}
},
@@ -157,21 +231,33 @@ export class LoggingContentGenerator implements ContentGenerator {
spanMetadata.input = { request: req, userPromptId, model: req.model };
const startTime = Date.now();
this.logApiRequest(toContents(req.contents), req.model, userPromptId);
const serverDetails = this._getEndpointUrl(
req,
'generateContentStream',
);
let stream: AsyncGenerator<GenerateContentResponse>;
try {
stream = await this.wrapped.generateContentStream(req, userPromptId);
} catch (error) {
const durationMs = Date.now() - startTime;
this._logApiError(durationMs, error, req.model, userPromptId);
this._logApiError(
durationMs,
error,
req.model,
userPromptId,
toContents(req.contents),
req.config,
serverDetails,
);
throw error;
}
return this.loggingStreamWrapper(
req,
stream,
startTime,
userPromptId,
req.model,
spanMetadata,
endSpan,
);
@@ -180,16 +266,18 @@ export class LoggingContentGenerator implements ContentGenerator {
}
private async *loggingStreamWrapper(
req: GenerateContentParameters,
stream: AsyncGenerator<GenerateContentResponse>,
startTime: number,
userPromptId: string,
model: string,
spanMetadata: SpanMetadata,
endSpan: () => void,
): AsyncGenerator<GenerateContentResponse> {
const responses: GenerateContentResponse[] = [];
let lastUsageMetadata: GenerateContentResponseUsageMetadata | undefined;
const serverDetails = this._getEndpointUrl(req, 'generateContentStream');
const requestContents: Content[] = toContents(req.contents);
try {
for await (const response of stream) {
responses.push(response);
@@ -201,11 +289,16 @@ export class LoggingContentGenerator implements ContentGenerator {
// Only log successful API response if no error occurred
const durationMs = Date.now() - startTime;
this._logApiResponse(
requestContents,
durationMs,
responses[0]?.modelVersion || model,
responses[0]?.modelVersion || req.model,
userPromptId,
responses[0]?.responseId,
responses.flatMap((response) => response.candidates || []),
lastUsageMetadata,
JSON.stringify(responses),
req.config,
serverDetails,
);
spanMetadata.output = {
streamChunks: responses.map((r) => ({
@@ -220,8 +313,11 @@ export class LoggingContentGenerator implements ContentGenerator {
this._logApiError(
durationMs,
error,
responses[0]?.modelVersion || model,
responses[0]?.modelVersion || req.model,
userPromptId,
requestContents,
req.config,
serverDetails,
);
throw error;
} finally {

10
packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
View File

@@ -634,27 +634,27 @@ export class ClearcutLogger {
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT,
value: JSON.stringify(event.input_token_count),
value: JSON.stringify(event.usage.input_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT,
value: JSON.stringify(event.output_token_count),
value: JSON.stringify(event.usage.output_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT,
value: JSON.stringify(event.cached_content_token_count),
value: JSON.stringify(event.usage.cached_content_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT,
value: JSON.stringify(event.thoughts_token_count),
value: JSON.stringify(event.usage.thoughts_token_count),
},
{
gemini_cli_key:
EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT,
value: JSON.stringify(event.tool_token_count),
value: JSON.stringify(event.usage.tool_token_count),
},
];

249
packages/core/src/telemetry/loggers.test.ts
View File

@@ -20,9 +20,9 @@ import {
} from '../index.js';
import { OutputFormat } from '../output/types.js';
import { logs } from '@opentelemetry/api-logs';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import type { Config } from '../config/config.js';
import {
logApiError,
logApiRequest,
logApiResponse,
logCliConfiguration,
@@ -46,6 +46,7 @@ import {
} from './loggers.js';
import { ToolCallDecision } from './tool-call-decision.js';
import {
EVENT_API_ERROR,
EVENT_API_REQUEST,
EVENT_API_RESPONSE,
EVENT_CLI_CONFIG,
@@ -64,6 +65,7 @@ import {
EVENT_AGENT_START,
EVENT_AGENT_FINISH,
EVENT_WEB_FETCH_FALLBACK_ATTEMPT,
ApiErrorEvent,
ApiRequestEvent,
ApiResponseEvent,
StartSessionEvent,
@@ -87,16 +89,13 @@ import {
EVENT_EXTENSION_UPDATE,
} from './types.js';
import * as metrics from './metrics.js';
import {
FileOperation,
GenAiOperationName,
GenAiProviderName,
} from './metrics.js';
import { FileOperation } from './metrics.js';
import * as sdk from './sdk.js';
import { vi, describe, beforeEach, it, expect, afterEach } from 'vitest';
import type {
CallableTool,
GenerateContentResponseUsageMetadata,
import {
FinishReason,
type CallableTool,
type GenerateContentResponseUsageMetadata,
} from '@google/genai';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import * as uiTelemetry from './uiTelemetry.js';
@@ -316,12 +315,6 @@ describe('loggers', () => {
const mockMetrics = {
recordApiResponseMetrics: vi.fn(),
recordTokenUsageMetrics: vi.fn(),
getConventionAttributes: vi.fn(() => ({
'gen_ai.operation.name': GenAiOperationName.GENERATE_CONTENT,
'gen_ai.provider.name': GenAiProviderName.GCP_VERTEX_AI,
'gen_ai.request.model': 'test-model',
'gen_ai.response.model': 'test-model',
})),
};
beforeEach(() => {
@@ -331,9 +324,6 @@ describe('loggers', () => {
vi.spyOn(metrics, 'recordTokenUsageMetrics').mockImplementation(
mockMetrics.recordTokenUsageMetrics,
);
vi.spyOn(metrics, 'getConventionAttributes').mockImplementation(
mockMetrics.getConventionAttributes,
);
});
it('should log an API response with all fields', () => {
@@ -347,7 +337,47 @@ describe('loggers', () => {
const event = new ApiResponseEvent(
'test-model',
100,
'prompt-id-1',
{
prompt_id: 'prompt-id-1',
contents: [
{
role: 'user',
parts: [{ text: 'Hello' }],
},
],
generate_content_config: {
temperature: 1,
topP: 2,
topK: 3,
responseMimeType: 'text/plain',
candidateCount: 1,
seed: 678,
frequencyPenalty: 10,
maxOutputTokens: 8000,
presencePenalty: 6,
stopSequences: ['stop', 'please stop'],
systemInstruction: {
role: 'model',
parts: [{ text: 'be nice' }],
},
},
server: {
address: 'foo.com',
port: 8080,
},
},
{
response_id: '',
candidates: [
{
content: {
role: 'model',
parts: [{ text: 'candidate 1' }],
},
finishReason: FinishReason.STOP,
},
],
},
AuthType.LOGIN_WITH_GOOGLE,
usageData,
'test-response',
@@ -357,26 +387,40 @@ describe('loggers', () => {
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'API response from test-model. Status: 200. Duration: 100ms.',
attributes: {
'session.id': 'test-session-id',
'user.email': 'test-user@example.com',
'installation.id': 'test-installation-id',
attributes: expect.objectContaining({
'event.name': EVENT_API_RESPONSE,
'event.timestamp': '2025-01-01T00:00:00.000Z',
[SemanticAttributes.HTTP_STATUS_CODE]: 200,
model: 'test-model',
status_code: 200,
duration_ms: 100,
input_token_count: 17,
output_token_count: 50,
cached_content_token_count: 10,
thoughts_token_count: 5,
tool_token_count: 2,
total_token_count: 0,
response_text: 'test-response',
prompt_id: 'prompt-id-1',
auth_type: 'oauth-personal',
},
}),
});
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'GenAI operation details from test-model. Status: 200. Duration: 100ms.',
attributes: expect.objectContaining({
'event.name': 'gen_ai.client.inference.operation.details',
'gen_ai.request.model': 'test-model',
'gen_ai.request.temperature': 1,
'gen_ai.request.top_p': 2,
'gen_ai.request.top_k': 3,
'gen_ai.input.messages':
'[{"role":"user","parts":[{"type":"text","content":"Hello"}]}]',
'gen_ai.output.messages':
'[{"finish_reason":"stop","role":"system","parts":[{"type":"text","content":"candidate 1"}]}]',
'gen_ai.response.finish_reasons': ['stop'],
'gen_ai.response.model': 'test-model',
'gen_ai.usage.input_tokens': 17,
'gen_ai.usage.output_tokens': 50,
'gen_ai.operation.name': 'generate_content',
'gen_ai.output.type': 'text',
'gen_ai.request.choice.count': 1,
'gen_ai.request.seed': 678,
'gen_ai.request.frequency_penalty': 10,
'gen_ai.request.presence_penalty': 6,
'gen_ai.request.max_tokens': 8000,
'server.address': 'foo.com',
'server.port': 8080,
'gen_ai.request.stop_sequences': ['stop', 'please stop'],
'gen_ai.system_instructions': '[{"type":"text","content":"be nice"}]',
}),
});
expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith(
@@ -433,6 +477,137 @@ describe('loggers', () => {
});
});
describe('logApiError', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getUsageStatisticsEnabled: () => true,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
} as Config;
const mockMetrics = {
recordApiResponseMetrics: vi.fn(),
recordApiErrorMetrics: vi.fn(),
recordTokenUsageMetrics: vi.fn(),
};
beforeEach(() => {
vi.spyOn(metrics, 'recordApiResponseMetrics').mockImplementation(
mockMetrics.recordApiResponseMetrics,
);
vi.spyOn(metrics, 'recordApiErrorMetrics').mockImplementation(
mockMetrics.recordApiErrorMetrics,
);
});
it('should log an API error with all fields', () => {
const event = new ApiErrorEvent(
'test-model',
'UNAVAILABLE. {"error":{"code":503,"message":"The model is overloaded. Please try again later.","status":"UNAVAILABLE"}}',
100,
{
prompt_id: 'prompt-id-1',
contents: [
{
role: 'user',
parts: [{ text: 'Hello' }],
},
],
generate_content_config: {
temperature: 1,
topP: 2,
topK: 3,
responseMimeType: 'text/plain',
candidateCount: 1,
seed: 678,
frequencyPenalty: 10,
maxOutputTokens: 8000,
presencePenalty: 6,
stopSequences: ['stop', 'please stop'],
systemInstruction: {
role: 'model',
parts: [{ text: 'be nice' }],
},
},
server: {
address: 'foo.com',
port: 8080,
},
},
AuthType.LOGIN_WITH_GOOGLE,
'ApiError',
503,
);
logApiError(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'API error for test-model. Error: UNAVAILABLE. {"error":{"code":503,"message":"The model is overloaded. Please try again later.","status":"UNAVAILABLE"}}. Duration: 100ms.',
attributes: expect.objectContaining({
'event.name': EVENT_API_ERROR,
prompt_id: 'prompt-id-1',
}),
});
expect(mockLogger.emit).toHaveBeenCalledWith({
body: 'GenAI operation error details from test-model. Error: UNAVAILABLE. {"error":{"code":503,"message":"The model is overloaded. Please try again later.","status":"UNAVAILABLE"}}. Duration: 100ms.',
attributes: expect.objectContaining({
'event.name': 'gen_ai.client.inference.operation.details',
'gen_ai.request.model': 'test-model',
'gen_ai.request.temperature': 1,
'gen_ai.request.top_p': 2,
'gen_ai.request.top_k': 3,
'gen_ai.input.messages':
'[{"role":"user","parts":[{"type":"text","content":"Hello"}]}]',
'gen_ai.operation.name': 'generate_content',
'gen_ai.output.type': 'text',
'gen_ai.request.choice.count': 1,
'gen_ai.request.seed': 678,
'gen_ai.request.frequency_penalty': 10,
'gen_ai.request.presence_penalty': 6,
'gen_ai.request.max_tokens': 8000,
'server.address': 'foo.com',
'server.port': 8080,
'gen_ai.request.stop_sequences': ['stop', 'please stop'],
'gen_ai.system_instructions': '[{"type":"text","content":"be nice"}]',
}),
});
expect(mockMetrics.recordApiErrorMetrics).toHaveBeenCalledWith(
mockConfig,
100,
{
model: 'test-model',
status_code: 503,
error_type: 'ApiError',
},
);
expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith(
mockConfig,
100,
{
model: 'test-model',
status_code: 503,
genAiAttributes: {
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.vertex_ai',
'gen_ai.request.model': 'test-model',
'gen_ai.response.model': 'test-model',
'error.type': 'ApiError',
},
},
);
expect(mockUiEvent.addEvent).toHaveBeenCalledWith({
...event,
'event.name': EVENT_API_ERROR,
'event.timestamp': '2025-01-01T00:00:00.000Z',
});
});
});
describe('logApiRequest', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',

29
packages/core/src/telemetry/loggers.ts
View File

@@ -89,6 +89,7 @@ export function logUserPrompt(config: Config, event: UserPromptEvent): void {
if (!isTelemetrySdkInitialized()) return;
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: event.toLogBody(),
attributes: event.toOpenTelemetryAttributes(config),
@@ -219,11 +220,9 @@ export function logApiError(config: Config, event: ApiErrorEvent): void {
if (!isTelemetrySdkInitialized()) return;
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: event.toLogBody(),
attributes: event.toOpenTelemetryAttributes(config),
};
logger.emit(logRecord);
logger.emit(event.toLogRecord(config));
logger.emit(event.toSemanticLogRecord(config));
recordApiErrorMetrics(config, event.duration_ms, {
model: event.model,
status_code: event.status_code,
@@ -231,12 +230,11 @@ export function logApiError(config: Config, event: ApiErrorEvent): void {
});
// Record GenAI operation duration for errors
const conventionAttributes = getConventionAttributes(event);
recordApiResponseMetrics(config, event.duration_ms, {
model: event.model,
status_code: event.status_code,
genAiAttributes: {
...conventionAttributes,
...getConventionAttributes(event),
'error.type': event.error_type || 'unknown',
},
});
@@ -253,11 +251,8 @@ export function logApiResponse(config: Config, event: ApiResponseEvent): void {
if (!isTelemetrySdkInitialized()) return;
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: event.toLogBody(),
attributes: event.toOpenTelemetryAttributes(config),
};
logger.emit(logRecord);
logger.emit(event.toLogRecord(config));
logger.emit(event.toSemanticLogRecord(config));
const conventionAttributes = getConventionAttributes(event);
@@ -268,11 +263,11 @@ export function logApiResponse(config: Config, event: ApiResponseEvent): void {
});
const tokenUsageData = [
{ count: event.input_token_count, type: 'input' as const },
{ count: event.output_token_count, type: 'output' as const },
{ count: event.cached_content_token_count, type: 'cache' as const },
{ count: event.thoughts_token_count, type: 'thought' as const },
{ count: event.tool_token_count, type: 'tool' as const },
{ count: event.usage.input_token_count, type: 'input' as const },
{ count: event.usage.output_token_count, type: 'output' as const },
{ count: event.usage.cached_content_token_count, type: 'cache' as const },
{ count: event.usage.thoughts_token_count, type: 'thought' as const },
{ count: event.usage.tool_token_count, type: 'tool' as const },
];
for (const { count, type } of tokenUsageData) {

2
packages/core/src/telemetry/metrics.ts
View File

@@ -7,7 +7,6 @@
import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api';
import { diag, metrics, ValueType } from '@opentelemetry/api';
import { SERVICE_NAME } from './constants.js';
import { EVENT_CHAT_COMPRESSION } from './types.js';
import type { Config } from '../config/config.js';
import type {
ModelRoutingEvent,
@@ -17,6 +16,7 @@ import type {
import { AuthType } from '../core/contentGenerator.js';
import { getCommonAttributes } from './telemetryAttributes.js';
const EVENT_CHAT_COMPRESSION = 'gemini_cli.chat_compression';
const TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
const TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
const API_REQUEST_COUNT = 'gemini_cli.api.request.count';

425
packages/core/src/telemetry/semantic.test.ts Normal file
View File

@@ -0,0 +1,425 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import {
toChatMessage,
toInputMessages,
toSystemInstruction,
toOutputMessages,
toFinishReasons,
OTelFinishReason,
toOutputType,
OTelOutputType,
} from './semantic.js';
import {
Language,
type Content,
Outcome,
type Candidate,
FinishReason,
} from '@google/genai';
describe('toChatMessage', () => {
it('should correctly handle text parts', () => {
const content: Content = {
role: 'user',
parts: [{ text: 'Hello' }],
};
expect(toChatMessage(content)).toEqual({
role: 'user',
parts: [
{
type: 'text',
content: 'Hello',
},
],
});
});
it('should correctly handle function call parts', () => {
const content: Content = {
role: 'model',
parts: [
{
functionCall: {
name: 'test-function',
args: {
arg1: 'test-value',
},
id: '12345',
},
// include field not specified in semantic specification that could be present
thoughtSignature: '1234',
},
],
};
expect(toChatMessage(content)).toEqual({
role: 'system',
parts: [
{
type: 'tool_call',
name: 'test-function',
arguments: '{"arg1":"test-value"}',
id: '12345',
},
],
});
});
it('should correctly handle function response parts', () => {
const content: Content = {
role: 'user',
parts: [
{
functionResponse: {
name: 'test-function',
response: {
result: 'success',
},
id: '12345',
},
// include field not specified in semantic specification that could be present
fileData: {
displayName: 'greatfile',
},
},
],
};
expect(toChatMessage(content)).toEqual({
role: 'user',
parts: [
{
type: 'tool_call_response',
response: '{"result":"success"}',
id: '12345',
},
],
});
});
it('should correctly handle reasoning parts with text', () => {
const content: Content = {
role: 'system',
parts: [{ text: 'Hmm', thought: true }],
};
expect(toChatMessage(content)).toEqual({
role: 'system',
parts: [
{
type: 'reasoning',
content: 'Hmm',
},
],
});
});
it('should correctly handle reasoning parts without text', () => {
const content: Content = {
role: 'system',
parts: [
{
thought: true,
// include field not specified in semantic specification that could be present
inlineData: {
displayName: 'wowdata',
},
},
],
};
expect(toChatMessage(content)).toEqual({
role: 'system',
parts: [
{
type: 'reasoning',
content: '',
},
],
});
});
it('should correctly handle text parts that are not reasoning parts', () => {
const content: Content = {
role: 'user',
parts: [{ text: 'what a nice day', thought: false }],
};
expect(toChatMessage(content)).toEqual({
role: 'user',
parts: [
{
type: 'text',
content: 'what a nice day',
},
],
});
});
it('should correctly handle "generic" parts', () => {
const content: Content = {
role: 'model',
parts: [
{
executableCode: {
code: 'print("foo")',
language: Language.PYTHON,
},
},
{
codeExecutionResult: {
outcome: Outcome.OUTCOME_OK,
output: 'foo',
},
// include field not specified in semantic specification that could be present
videoMetadata: {
fps: 5,
},
},
],
};
expect(toChatMessage(content)).toEqual({
role: 'system',
parts: [
{
type: 'executableCode',
code: 'print("foo")',
language: 'PYTHON',
},
{
type: 'codeExecutionResult',
outcome: 'OUTCOME_OK',
output: 'foo',
videoMetadata: {
fps: 5,
},
},
],
});
});
it('should correctly handle unknown parts', () => {
const content: Content = {
role: 'model',
parts: [
{
fileData: {
displayName: 'superfile',
},
},
],
};
expect(toChatMessage(content)).toEqual({
role: 'system',
parts: [
{
type: 'unknown',
fileData: {
displayName: 'superfile',
},
},
],
});
});
});
describe('toSystemInstruction', () => {
it('should correctly handle a string', () => {
const content = 'Hello';
expect(toSystemInstruction(content)).toEqual([
{
type: 'text',
content: 'Hello',
},
]);
});
it('should correctly handle a Content object with a text part', () => {
const content: Content = {
role: 'user',
parts: [{ text: 'Hello' }],
};
expect(toSystemInstruction(content)).toEqual([
{
type: 'text',
content: 'Hello',
},
]);
});
it('should correctly handle a Content object with multiple parts', () => {
const content: Content = {
role: 'user',
parts: [{ text: 'Hello' }, { text: 'Hmm', thought: true }],
};
expect(toSystemInstruction(content)).toEqual([
{
type: 'text',
content: 'Hello',
},
{
type: 'reasoning',
content: 'Hmm',
},
]);
});
});
describe('toInputMessages', () => {
it('should correctly convert an array of Content objects', () => {
const contents: Content[] = [
{
role: 'user',
parts: [{ text: 'Hello' }],
},
{
role: 'model',
parts: [{ text: 'Hi there!' }],
},
];
expect(toInputMessages(contents)).toEqual([
{
role: 'user',
parts: [
{
type: 'text',
content: 'Hello',
},
],
},
{
role: 'system',
parts: [
{
type: 'text',
content: 'Hi there!',
},
],
},
]);
});
});
describe('toOutputMessages', () => {
it('should correctly convert an array of Candidate objects', () => {
const candidates: Candidate[] = [
{
index: 0,
finishReason: FinishReason.STOP,
content: {
role: 'model',
parts: [{ text: 'This is the first candidate.' }],
},
},
{
index: 1,
finishReason: FinishReason.MAX_TOKENS,
content: {
role: 'model',
parts: [{ text: 'This is the second candidate.' }],
},
},
];
expect(toOutputMessages(candidates)).toEqual([
{
role: 'system',
finish_reason: 'stop',
parts: [
{
type: 'text',
content: 'This is the first candidate.',
},
],
},
{
role: 'system',
finish_reason: 'length',
parts: [
{
type: 'text',
content: 'This is the second candidate.',
},
],
},
]);
});
});
describe('toFinishReasons', () => {
it('should return an empty array if candidates is undefined', () => {
expect(toFinishReasons(undefined)).toEqual([]);
});
it('should return an empty array if candidates is an empty array', () => {
expect(toFinishReasons([])).toEqual([]);
});
it('should correctly convert a single candidate', () => {
const candidates: Candidate[] = [
{
index: 0,
finishReason: FinishReason.STOP,
content: {
role: 'model',
parts: [{ text: 'This is the first candidate.' }],
},
},
];
expect(toFinishReasons(candidates)).toEqual([OTelFinishReason.STOP]);
});
it('should correctly convert multiple candidates', () => {
const candidates: Candidate[] = [
{
index: 0,
finishReason: FinishReason.STOP,
content: {
role: 'model',
parts: [{ text: 'This is the first candidate.' }],
},
},
{
index: 1,
finishReason: FinishReason.MAX_TOKENS,
content: {
role: 'model',
parts: [{ text: 'This is the second candidate.' }],
},
},
{
index: 2,
finishReason: FinishReason.SAFETY,
content: {
role: 'model',
parts: [{ text: 'This is the third candidate.' }],
},
},
];
expect(toFinishReasons(candidates)).toEqual([
OTelFinishReason.STOP,
OTelFinishReason.LENGTH,
OTelFinishReason.CONTENT_FILTER,
]);
});
});
describe('toOutputType', () => {
it('should return TEXT for text/plain', () => {
expect(toOutputType('text/plain')).toBe(OTelOutputType.TEXT);
});
it('should return JSON for application/json', () => {
expect(toOutputType('application/json')).toBe(OTelOutputType.JSON);
});
it('should return the custom mime type for other strings', () => {
expect(toOutputType('application/vnd.custom-type')).toBe(
'application/vnd.custom-type',
);
});
it('should return undefined for undefined input', () => {
expect(toOutputType(undefined)).toBeUndefined();
});
});

323
packages/core/src/telemetry/semantic.ts Normal file
View File

@@ -0,0 +1,323 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* This file contains functions and types for converting Gemini API request/response
* formats to the OpenTelemetry semantic conventions for generative AI.
*
* @see https://github.com/open-telemetry/semantic-conventions/blob/8b4f210f43136e57c1f6f47292eb6d38e3bf30bb/docs/gen-ai/gen-ai-events.md
*/
import { FinishReason } from '@google/genai';
import type {
Candidate,
Content,
ContentUnion,
Part,
PartUnion,
} from '@google/genai';
export function toInputMessages(contents: Content[]): InputMessages {
const messages: ChatMessage[] = [];
for (const content of contents) {
messages.push(toChatMessage(content));
}
return messages;
}
function isPart(value: unknown): value is Part {
return (
typeof value === 'object' &&
value !== null &&
!Array.isArray(value) &&
!('parts' in value)
);
}
function toPart(part: PartUnion): Part {
if (typeof part === 'string') {
return { text: part };
}
return part;
}
function toContent(content: ContentUnion): Content | undefined {
if (typeof content === 'string') {
// 1. It's a string
return {
parts: [toPart(content)],
};
} else if (Array.isArray(content)) {
// 2. It's an array of parts (PartUnion[])
return {
parts: content.map(toPart),
};
} else if ('parts' in content) {
// 3. It's a Content object
return content;
} else if (isPart(content)) {
// 4. It's a single Part object (asserted with type guard)
return {
parts: [content],
};
} else {
// 5. Handle any other unexpected case
return undefined;
}
}
export function toSystemInstruction(
systemInstruction?: ContentUnion,
): SystemInstruction | undefined {
const parts: AnyPart[] = [];
if (systemInstruction) {
const content = toContent(systemInstruction);
if (content && content.parts) {
for (const part of content.parts) {
parts.push(toOTelPart(part));
}
}
}
return parts;
}
export function toOutputMessages(candidates?: Candidate[]): OutputMessages {
const messages: OutputMessage[] = [];
if (candidates) {
for (const candidate of candidates) {
messages.push({
finish_reason: toOTelFinishReason(candidate.finishReason),
...toChatMessage(candidate.content),
});
}
}
return messages;
}
export function toFinishReasons(candidates?: Candidate[]): OTelFinishReason[] {
const reasons: OTelFinishReason[] = [];
if (candidates) {
for (const candidate of candidates) {
reasons.push(toOTelFinishReason(candidate.finishReason));
}
}
return reasons;
}
export function toOutputType(requested_mime?: string): string | undefined {
switch (requested_mime) {
// explictly support the known good values of responseMimeType
case 'text/plain':
return OTelOutputType.TEXT;
case 'application/json':
return OTelOutputType.JSON;
default:
// if none of the well-known values applies, a custom value may be used
return requested_mime;
}
}
export function toChatMessage(content?: Content): ChatMessage {
const message: ChatMessage = {
role: undefined,
parts: [],
};
if (content && content.parts) {
message.role = toOTelRole(content.role);
for (const part of content.parts) {
message.parts.push(toOTelPart(part));
}
}
return message;
}
export function toOTelPart(part: Part): AnyPart {
if (part.thought) {
if (part.text) {
return new ReasoningPart(part.text);
} else {
return new ReasoningPart('');
}
} else if (part.text) {
return new TextPart(part.text);
} else if (part.functionCall) {
return new ToolCallRequestPart(
part.functionCall.name,
part.functionCall.id,
JSON.stringify(part.functionCall.args),
);
} else if (part.functionResponse) {
return new ToolCallResponsePart(
JSON.stringify(part.functionResponse.response),
part.functionResponse.id,
);
} else if (part.executableCode) {
const { executableCode, ...unexpectedData } = part;
return new GenericPart('executableCode', {
code: executableCode.code,
language: executableCode.language,
...unexpectedData,
});
} else if (part.codeExecutionResult) {
const { codeExecutionResult, ...unexpectedData } = part;
return new GenericPart('codeExecutionResult', {
outcome: codeExecutionResult.outcome,
output: codeExecutionResult.output,
...unexpectedData,
});
}
// Assuming the above cases capture all the expected parts
// but adding a fallthrough just in case.
return new GenericPart('unknown', { ...part });
}
export enum OTelRole {
SYSTEM = 'system',
USER = 'user',
ASSISTANT = 'assistant',
TOOL = 'tool',
}
export function toOTelRole(role?: string): OTelRole {
switch (role?.toLowerCase()) {
case 'system':
return OTelRole.SYSTEM;
// Our APIs seem to frequently use 'model'
case 'model':
return OTelRole.SYSTEM;
case 'user':
return OTelRole.USER;
case 'assistant':
return OTelRole.ASSISTANT;
case 'tool':
return OTelRole.TOOL;
default:
return OTelRole.SYSTEM;
}
}
export type InputMessages = ChatMessage[];
export enum OTelOutputType {
IMAGE = 'image',
JSON = 'json',
SPEECH = 'speech',
TEXT = 'text',
}
export enum OTelFinishReason {
STOP = 'stop',
LENGTH = 'length',
CONTENT_FILTER = 'content_filter',
TOOL_CALL = 'tool_call',
ERROR = 'error',
}
export function toOTelFinishReason(finishReason?: string): OTelFinishReason {
switch (finishReason) {
// we have significantly more finish reasons than the spec
case FinishReason.FINISH_REASON_UNSPECIFIED:
return OTelFinishReason.STOP;
case FinishReason.STOP:
return OTelFinishReason.STOP;
case FinishReason.MAX_TOKENS:
return OTelFinishReason.LENGTH;
case FinishReason.SAFETY:
return OTelFinishReason.CONTENT_FILTER;
case FinishReason.RECITATION:
return OTelFinishReason.CONTENT_FILTER;
case FinishReason.LANGUAGE:
return OTelFinishReason.CONTENT_FILTER;
case FinishReason.OTHER:
return OTelFinishReason.STOP;
case FinishReason.BLOCKLIST:
return OTelFinishReason.CONTENT_FILTER;
case FinishReason.PROHIBITED_CONTENT:
return OTelFinishReason.CONTENT_FILTER;
case FinishReason.SPII:
return OTelFinishReason.CONTENT_FILTER;
case FinishReason.MALFORMED_FUNCTION_CALL:
return OTelFinishReason.ERROR;
case FinishReason.IMAGE_SAFETY:
return OTelFinishReason.CONTENT_FILTER;
case FinishReason.UNEXPECTED_TOOL_CALL:
return OTelFinishReason.ERROR;
default:
return OTelFinishReason.STOP;
}
}
export interface OutputMessage extends ChatMessage {
finish_reason: FinishReason | string;
}
export type OutputMessages = OutputMessage[];
export type AnyPart =
| TextPart
| ToolCallRequestPart
| ToolCallResponsePart
| ReasoningPart
| GenericPart;
export type SystemInstruction = AnyPart[];
export interface ChatMessage {
role: string | undefined;
parts: AnyPart[];
}
export class TextPart {
readonly type = 'text';
content: string;
constructor(content: string) {
this.content = content;
}
}
export class ToolCallRequestPart {
readonly type = 'tool_call';
name?: string;
id?: string;
arguments?: string;
constructor(name?: string, id?: string, args?: string) {
this.name = name;
this.id = id;
this.arguments = args;
}
}
export class ToolCallResponsePart {
readonly type = 'tool_call_response';
response?: string;
id?: string;
constructor(response?: string, id?: string) {
this.response = response;
this.id = id;
}
}
export class ReasoningPart {
readonly type = 'reasoning';
content: string;
constructor(content: string) {
this.content = content;
}
}
export class GenericPart {
type: string;
[key: string]: unknown;
constructor(type: string, data: { [key: string]: unknown }) {
this.type = type;
Object.assign(this, data);
}
}

212
packages/core/src/telemetry/types.ts
View File

@@ -4,19 +4,24 @@
* SPDX-License-Identifier: Apache-2.0
*/
import type { GenerateContentResponseUsageMetadata } from '@google/genai';
import type {
Candidate,
Content,
GenerateContentConfig,
GenerateContentResponseUsageMetadata,
} from '@google/genai';
import type { Config } from '../config/config.js';
import type { ApprovalMode } from '../config/config.js';
import type { CompletedToolCall } from '../core/coreToolScheduler.js';
import { DiscoveredMCPTool } from '../tools/mcp-tool.js';
import type { FileDiff } from '../tools/tools.js';
import { AuthType } from '../core/contentGenerator.js';
import type { LogAttributes } from '@opentelemetry/api-logs';
import type { LogAttributes, LogRecord } from '@opentelemetry/api-logs';
import {
getDecisionFromOutcome,
ToolCallDecision,
} from './tool-call-decision.js';
import type { FileOperation } from './metrics.js';
import { getConventionAttributes, type FileOperation } from './metrics.js';
export { ToolCallDecision };
import type { ToolRegistry } from '../tools/tool-registry.js';
import type { OutputFormat } from '../output/types.js';
@@ -25,6 +30,13 @@ import type { AgentTerminateMode } from '../agents/types.js';
import { getCommonAttributes } from './telemetryAttributes.js';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
import {
toInputMessages,
toOutputMessages,
toFinishReasons,
toOutputType,
toSystemInstruction,
} from './semantic.js';
export interface BaseTelemetryEvent {
'event.name': string;
@@ -358,18 +370,18 @@ export class ApiErrorEvent implements BaseTelemetryEvent {
'event.name': 'api_error';
'event.timestamp': string;
model: string;
prompt: GenAIPromptDetails;
error: string;
error_type?: string;
status_code?: number | string;
duration_ms: number;
prompt_id: string;
auth_type?: string;
constructor(
model: string,
error: string,
duration_ms: number,
prompt_id: string,
prompt_details: GenAIPromptDetails,
auth_type?: string,
error_type?: string,
status_code?: number | string,
@@ -381,11 +393,11 @@ export class ApiErrorEvent implements BaseTelemetryEvent {
this.error_type = error_type;
this.status_code = status_code;
this.duration_ms = duration_ms;
this.prompt_id = prompt_id;
this.prompt = prompt_details;
this.auth_type = auth_type;
}
toOpenTelemetryAttributes(config: Config): LogAttributes {
toLogRecord(config: Config): LogRecord {
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_API_ERROR,
@@ -397,7 +409,7 @@ export class ApiErrorEvent implements BaseTelemetryEvent {
error: this.error,
status_code: this.status_code,
duration_ms: this.duration_ms,
prompt_id: this.prompt_id,
prompt_id: this.prompt.prompt_id,
auth_type: this.auth_type,
};
@@ -407,69 +419,151 @@ export class ApiErrorEvent implements BaseTelemetryEvent {
if (typeof this.status_code === 'number') {
attributes[SemanticAttributes.HTTP_STATUS_CODE] = this.status_code;
}
return attributes;
const logRecord: LogRecord = {
body: `API error for ${this.model}. Error: ${this.error}. Duration: ${this.duration_ms}ms.`,
attributes,
};
return logRecord;
}
toLogBody(): string {
return `API error for ${this.model}. Error: ${this.error}. Duration: ${this.duration_ms}ms.`;
toSemanticLogRecord(config: Config): LogRecord {
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_GEN_AI_OPERATION_DETAILS,
'event.timestamp': this['event.timestamp'],
...toGenerateContentConfigAttributes(this.prompt.generate_content_config),
...getConventionAttributes(this),
};
if (this.prompt.server) {
attributes['server.address'] = this.prompt.server.address;
attributes['server.port'] = this.prompt.server.port;
}
if (config.getTelemetryLogPromptsEnabled() && this.prompt.contents) {
attributes['gen_ai.input.messages'] = JSON.stringify(
toInputMessages(this.prompt.contents),
);
}
const logRecord: LogRecord = {
body: `GenAI operation error details from ${this.model}. Error: ${this.error}. Duration: ${this.duration_ms}ms.`,
attributes,
};
return logRecord;
}
}
export const EVENT_API_RESPONSE = 'gemini_cli.api_response';
export class ApiResponseEvent implements BaseTelemetryEvent {
'event.name': 'api_response';
'event.timestamp': string;
model: string;
status_code?: number | string;
duration_ms: number;
export interface ServerDetails {
address: string;
port: number;
}
export interface GenAIPromptDetails {
prompt_id: string;
contents: Content[];
generate_content_config?: GenerateContentConfig;
server?: ServerDetails;
}
export interface GenAIResponseDetails {
response_id?: string;
candidates?: Candidate[];
}
export interface GenAIUsageDetails {
input_token_count: number;
output_token_count: number;
cached_content_token_count: number;
thoughts_token_count: number;
tool_token_count: number;
total_token_count: number;
}
export const EVENT_API_RESPONSE = 'gemini_cli.api_response';
export const EVENT_GEN_AI_OPERATION_DETAILS =
'gen_ai.client.inference.operation.details';
function toGenerateContentConfigAttributes(
config?: GenerateContentConfig,
): LogAttributes {
if (!config) {
return {};
}
return {
'gen_ai.request.temperature': config.temperature,
'gen_ai.request.top_p': config.topP,
'gen_ai.request.top_k': config.topK,
'gen_ai.request.choice.count': config.candidateCount,
'gen_ai.request.seed': config.seed,
'gen_ai.request.frequency_penalty': config.frequencyPenalty,
'gen_ai.request.presence_penalty': config.presencePenalty,
'gen_ai.request.max_tokens': config.maxOutputTokens,
'gen_ai.output.type': toOutputType(config.responseMimeType),
'gen_ai.request.stop_sequences': config.stopSequences,
'gen_ai.system_instructions': JSON.stringify(
toSystemInstruction(config.systemInstruction),
),
};
}
export class ApiResponseEvent implements BaseTelemetryEvent {
'event.name': 'api_response';
'event.timestamp': string;
status_code?: number | string;
duration_ms: number;
response_text?: string;
prompt_id: string;
auth_type?: string;
model: string;
prompt: GenAIPromptDetails;
response: GenAIResponseDetails;
usage: GenAIUsageDetails;
constructor(
model: string,
duration_ms: number,
prompt_id: string,
prompt_details: GenAIPromptDetails,
response_details: GenAIResponseDetails,
auth_type?: string,
usage_data?: GenerateContentResponseUsageMetadata,
response_text?: string,
) {
this['event.name'] = 'api_response';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.duration_ms = duration_ms;
this.status_code = 200;
this.input_token_count = usage_data?.promptTokenCount ?? 0;
this.output_token_count = usage_data?.candidatesTokenCount ?? 0;
this.cached_content_token_count = usage_data?.cachedContentTokenCount ?? 0;
this.thoughts_token_count = usage_data?.thoughtsTokenCount ?? 0;
this.tool_token_count = usage_data?.toolUsePromptTokenCount ?? 0;
this.total_token_count = usage_data?.totalTokenCount ?? 0;
this.response_text = response_text;
this.prompt_id = prompt_id;
this.auth_type = auth_type;
this.model = model;
this.prompt = prompt_details;
this.response = response_details;
this.usage = {
input_token_count: usage_data?.promptTokenCount ?? 0,
output_token_count: usage_data?.candidatesTokenCount ?? 0,
cached_content_token_count: usage_data?.cachedContentTokenCount ?? 0,
thoughts_token_count: usage_data?.thoughtsTokenCount ?? 0,
tool_token_count: usage_data?.toolUsePromptTokenCount ?? 0,
total_token_count: usage_data?.totalTokenCount ?? 0,
};
}
toOpenTelemetryAttributes(config: Config): LogAttributes {
toLogRecord(config: Config): LogRecord {
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_API_RESPONSE,
'event.timestamp': this['event.timestamp'],
model: this.model,
duration_ms: this.duration_ms,
input_token_count: this.input_token_count,
output_token_count: this.output_token_count,
cached_content_token_count: this.cached_content_token_count,
thoughts_token_count: this.thoughts_token_count,
tool_token_count: this.tool_token_count,
total_token_count: this.total_token_count,
prompt_id: this.prompt_id,
input_token_count: this.usage.input_token_count,
output_token_count: this.usage.output_token_count,
cached_content_token_count: this.usage.cached_content_token_count,
thoughts_token_count: this.usage.thoughts_token_count,
tool_token_count: this.usage.tool_token_count,
total_token_count: this.usage.total_token_count,
prompt_id: this.prompt.prompt_id,
auth_type: this.auth_type,
status_code: this.status_code,
};
@@ -481,11 +575,51 @@ export class ApiResponseEvent implements BaseTelemetryEvent {
attributes[SemanticAttributes.HTTP_STATUS_CODE] = this.status_code;
}
}
return attributes;
const logRecord: LogRecord = {
body: `API response from ${this.model}. Status: ${this.status_code || 'N/A'}. Duration: ${this.duration_ms}ms.`,
attributes,
};
return logRecord;
}
toLogBody(): string {
return `API response from ${this.model}. Status: ${this.status_code || 'N/A'}. Duration: ${this.duration_ms}ms.`;
toSemanticLogRecord(config: Config): LogRecord {
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_GEN_AI_OPERATION_DETAILS,
'event.timestamp': this['event.timestamp'],
'gen_ai.response.id': this.response.response_id,
'gen_ai.response.finish_reasons': toFinishReasons(
this.response.candidates,
),
'gen_ai.output.messages': JSON.stringify(
toOutputMessages(this.response.candidates),
),
...toGenerateContentConfigAttributes(this.prompt.generate_content_config),
...getConventionAttributes(this),
};
if (this.prompt.server) {
attributes['server.address'] = this.prompt.server.address;
attributes['server.port'] = this.prompt.server.port;
}
if (config.getTelemetryLogPromptsEnabled() && this.prompt.contents) {
attributes['gen_ai.input.messages'] = JSON.stringify(
toInputMessages(this.prompt.contents),
);
}
if (this.usage) {
attributes['gen_ai.usage.input_tokens'] = this.usage.input_token_count;
attributes['gen_ai.usage.output_tokens'] = this.usage.output_token_count;
}
const logRecord: LogRecord = {
body: `GenAI operation details from ${this.model}. Status: ${this.status_code || 'N/A'}. Duration: ${this.duration_ms}ms.`,
attributes,
};
return logRecord;
}
}

154
packages/core/src/telemetry/uiTelemetry.test.ts
View File

@@ -129,12 +129,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
usage: {
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
},
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);
@@ -151,12 +153,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
usage: {
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
},
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);
@@ -185,12 +189,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
usage: {
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
},
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
@@ -198,12 +204,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 600,
input_token_count: 15,
output_token_count: 25,
total_token_count: 40,
cached_content_token_count: 10,
thoughts_token_count: 4,
tool_token_count: 6,
usage: {
input_token_count: 15,
output_token_count: 25,
total_token_count: 40,
cached_content_token_count: 10,
thoughts_token_count: 4,
tool_token_count: 6,
},
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
@@ -235,12 +243,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
usage: {
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
},
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
@@ -248,12 +258,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-flash',
duration_ms: 1000,
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
usage: {
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
},
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
@@ -304,12 +316,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
usage: {
input_token_count: 10,
output_token_count: 20,
total_token_count: 30,
cached_content_token_count: 5,
thoughts_token_count: 2,
tool_token_count: 3,
},
} as ApiResponseEvent & {
'event.name': typeof EVENT_API_RESPONSE;
};
@@ -534,12 +548,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
usage: {
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
},
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);
@@ -559,12 +575,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
usage: {
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
},
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);
@@ -584,12 +602,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
usage: {
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
},
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);
@@ -616,12 +636,14 @@ describe('UiTelemetryService', () => {
'event.name': EVENT_API_RESPONSE,
model: 'gemini-2.5-pro',
duration_ms: 500,
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
usage: {
input_token_count: 100,
output_token_count: 200,
total_token_count: 300,
cached_content_token_count: 50,
thoughts_token_count: 20,
tool_token_count: 30,
},
} as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE };
service.addEvent(event);

12
packages/core/src/telemetry/uiTelemetry.ts
View File

@@ -165,12 +165,12 @@ export class UiTelemetryService extends EventEmitter {
modelMetrics.api.totalRequests++;
modelMetrics.api.totalLatencyMs += event.duration_ms;
modelMetrics.tokens.prompt += event.input_token_count;
modelMetrics.tokens.candidates += event.output_token_count;
modelMetrics.tokens.total += event.total_token_count;
modelMetrics.tokens.cached += event.cached_content_token_count;
modelMetrics.tokens.thoughts += event.thoughts_token_count;
modelMetrics.tokens.tool += event.tool_token_count;
modelMetrics.tokens.prompt += event.usage.input_token_count;
modelMetrics.tokens.candidates += event.usage.output_token_count;
modelMetrics.tokens.total += event.usage.total_token_count;
modelMetrics.tokens.cached += event.usage.cached_content_token_count;
modelMetrics.tokens.thoughts += event.usage.thoughts_token_count;
modelMetrics.tokens.tool += event.usage.tool_token_count;
}
private processApiError(event: ApiErrorEvent) {