From 70996bfdee652fae2d28d02a2c59f77161d0c5ec Mon Sep 17 00:00:00 2001 From: "Christie Warwick (Wilson)" Date: Tue, 28 Oct 2025 13:02:46 -0700 Subject: [PATCH] feat: Add Open Telemetric semantic standard compliant log (#11975) --- docs/cli/telemetry.md | 22 + .../core/src/core/loggingContentGenerator.ts | 112 ++++- .../clearcut-logger/clearcut-logger.ts | 10 +- packages/core/src/telemetry/loggers.test.ts | 249 ++++++++-- packages/core/src/telemetry/loggers.ts | 29 +- packages/core/src/telemetry/metrics.ts | 2 +- packages/core/src/telemetry/semantic.test.ts | 425 ++++++++++++++++++ packages/core/src/telemetry/semantic.ts | 323 +++++++++++++ packages/core/src/telemetry/types.ts | 212 +++++++-- .../core/src/telemetry/uiTelemetry.test.ts | 154 ++++--- packages/core/src/telemetry/uiTelemetry.ts | 12 +- 11 files changed, 1371 insertions(+), 179 deletions(-) create mode 100644 packages/core/src/telemetry/semantic.test.ts create mode 100644 packages/core/src/telemetry/semantic.ts diff --git a/docs/cli/telemetry.md b/docs/cli/telemetry.md index fd59260d2a..4b218cb8bd 100644 --- a/docs/cli/telemetry.md +++ b/docs/cli/telemetry.md @@ -298,6 +298,26 @@ Captures tool executions, output truncation, and Smart Edit behavior. - **Attributes**: - `correction` ("success" | "failure") +- `gen_ai.client.inference.operation.details`: This event provides detailed + information about the GenAI operation, aligned with [OpenTelemetry GenAI + semantic conventions for events]. + - **Attributes**: + - `gen_ai.request.model` (string) + - `gen_ai.provider.name` (string) + - `gen_ai.operation.name` (string) + - `gen_ai.input.messages` (json string) + - `gen_ai.output.messages` (json string) + - `gen_ai.response.finish_reasons` (array of strings) + - `gen_ai.usage.input_tokens` (int) + - `gen_ai.usage.output_tokens` (int) + - `gen_ai.request.temperature` (float) + - `gen_ai.request.top_p` (float) + - `gen_ai.request.top_k` (int) + - `gen_ai.request.max_tokens` (int) + - `gen_ai.system_instructions` (json string) + - `server.address` (string) + - `server.port` (int) + #### Files Tracks file operations performed by tools. @@ -735,3 +755,5 @@ standardized observability across GenAI applications: [OpenTelemetry GenAI semantic conventions]: https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-metrics.md +[OpenTelemetry GenAI semantic conventions for events]: + https://github.com/open-telemetry/semantic-conventions/blob/8b4f210f43136e57c1f6f47292eb6d38e3bf30bb/docs/gen-ai/gen-ai-events.md diff --git a/packages/core/src/core/loggingContentGenerator.ts b/packages/core/src/core/loggingContentGenerator.ts index e73d74513f..3b0768e98d 100644 --- a/packages/core/src/core/loggingContentGenerator.ts +++ b/packages/core/src/core/loggingContentGenerator.ts @@ -5,15 +5,18 @@ */ import type { + Candidate, Content, CountTokensParameters, CountTokensResponse, EmbedContentParameters, EmbedContentResponse, + GenerateContentConfig, GenerateContentParameters, GenerateContentResponseUsageMetadata, GenerateContentResponse, } from '@google/genai'; +import type { ServerDetails } from '../telemetry/types.js'; import { ApiRequestEvent, ApiResponseEvent, @@ -26,6 +29,7 @@ import { logApiResponse, } from '../telemetry/loggers.js'; import type { ContentGenerator } from './contentGenerator.js'; +import { CodeAssistServer } from '../code_assist/server.js'; import { toContents } from '../code_assist/converter.js'; import { isStructuredError } from '../utils/quotaErrorDetection.js'; import { runInDevTraceSpan, type SpanMetadata } from '../telemetry/trace.js'; @@ -59,19 +63,66 @@ export class LoggingContentGenerator implements ContentGenerator { ); } + private _getEndpointUrl( + req: GenerateContentParameters, + method: 'generateContent' | 'generateContentStream', + ): ServerDetails { + // Case 1: Authenticated with a Google account (`gcloud auth login`). + // Requests are routed through the internal CodeAssistServer. + if (this.wrapped instanceof CodeAssistServer) { + const url = new URL(this.wrapped.getMethodUrl(method)); + const port = url.port + ? parseInt(url.port, 10) + : url.protocol === 'https:' + ? 443 + : 80; + return { address: url.hostname, port }; + } + + const genConfig = this.config.getContentGeneratorConfig(); + + // Case 2: Using an API key for Vertex AI. + if (genConfig?.vertexai) { + const location = process.env['GOOGLE_CLOUD_LOCATION']; + if (location) { + return { address: `${location}-aiplatform.googleapis.com`, port: 443 }; + } else { + return { address: 'unknown', port: 0 }; + } + } + + // Case 3: Default to the public Gemini API endpoint. + // This is used when an API key is provided but not for Vertex AI. + return { address: `generativelanguage.googleapis.com`, port: 443 }; + } + private _logApiResponse( + requestContents: Content[], durationMs: number, model: string, prompt_id: string, + responseId: string | undefined, + responseCandidates?: Candidate[], usageMetadata?: GenerateContentResponseUsageMetadata, responseText?: string, + generationConfig?: GenerateContentConfig, + serverDetails?: ServerDetails, ): void { logApiResponse( this.config, new ApiResponseEvent( model, durationMs, - prompt_id, + { + prompt_id, + contents: requestContents, + generate_content_config: generationConfig, + server: serverDetails, + }, + { + candidates: responseCandidates, + response_id: responseId, + }, this.config.getContentGeneratorConfig()?.authType, usageMetadata, responseText, @@ -84,6 +135,9 @@ export class LoggingContentGenerator implements ContentGenerator { error: unknown, model: string, prompt_id: string, + requestContents: Content[], + generationConfig?: GenerateContentConfig, + serverDetails?: ServerDetails, ): void { const errorMessage = error instanceof Error ? error.message : String(error); const errorType = error instanceof Error ? error.name : 'unknown'; @@ -94,7 +148,12 @@ export class LoggingContentGenerator implements ContentGenerator { model, errorMessage, durationMs, - prompt_id, + { + prompt_id, + contents: requestContents, + generate_content_config: generationConfig, + server: serverDetails, + }, this.config.getContentGeneratorConfig()?.authType, errorType, isStructuredError(error) @@ -116,7 +175,9 @@ export class LoggingContentGenerator implements ContentGenerator { spanMetadata.input = { request: req, userPromptId, model: req.model }; const startTime = Date.now(); + const contents: Content[] = toContents(req.contents); this.logApiRequest(toContents(req.contents), req.model, userPromptId); + const serverDetails = this._getEndpointUrl(req, 'generateContent'); try { const response = await this.wrapped.generateContent( req, @@ -128,16 +189,29 @@ export class LoggingContentGenerator implements ContentGenerator { }; const durationMs = Date.now() - startTime; this._logApiResponse( + contents, durationMs, response.modelVersion || req.model, userPromptId, + response.responseId, + response.candidates, response.usageMetadata, JSON.stringify(response), + req.config, + serverDetails, ); return response; } catch (error) { const durationMs = Date.now() - startTime; - this._logApiError(durationMs, error, req.model, userPromptId); + this._logApiError( + durationMs, + error, + req.model, + userPromptId, + contents, + req.config, + serverDetails, + ); throw error; } }, @@ -157,21 +231,33 @@ export class LoggingContentGenerator implements ContentGenerator { spanMetadata.input = { request: req, userPromptId, model: req.model }; const startTime = Date.now(); this.logApiRequest(toContents(req.contents), req.model, userPromptId); + const serverDetails = this._getEndpointUrl( + req, + 'generateContentStream', + ); let stream: AsyncGenerator; try { stream = await this.wrapped.generateContentStream(req, userPromptId); } catch (error) { const durationMs = Date.now() - startTime; - this._logApiError(durationMs, error, req.model, userPromptId); + this._logApiError( + durationMs, + error, + req.model, + userPromptId, + toContents(req.contents), + req.config, + serverDetails, + ); throw error; } return this.loggingStreamWrapper( + req, stream, startTime, userPromptId, - req.model, spanMetadata, endSpan, ); @@ -180,16 +266,18 @@ export class LoggingContentGenerator implements ContentGenerator { } private async *loggingStreamWrapper( + req: GenerateContentParameters, stream: AsyncGenerator, startTime: number, userPromptId: string, - model: string, spanMetadata: SpanMetadata, endSpan: () => void, ): AsyncGenerator { const responses: GenerateContentResponse[] = []; let lastUsageMetadata: GenerateContentResponseUsageMetadata | undefined; + const serverDetails = this._getEndpointUrl(req, 'generateContentStream'); + const requestContents: Content[] = toContents(req.contents); try { for await (const response of stream) { responses.push(response); @@ -201,11 +289,16 @@ export class LoggingContentGenerator implements ContentGenerator { // Only log successful API response if no error occurred const durationMs = Date.now() - startTime; this._logApiResponse( + requestContents, durationMs, - responses[0]?.modelVersion || model, + responses[0]?.modelVersion || req.model, userPromptId, + responses[0]?.responseId, + responses.flatMap((response) => response.candidates || []), lastUsageMetadata, JSON.stringify(responses), + req.config, + serverDetails, ); spanMetadata.output = { streamChunks: responses.map((r) => ({ @@ -220,8 +313,11 @@ export class LoggingContentGenerator implements ContentGenerator { this._logApiError( durationMs, error, - responses[0]?.modelVersion || model, + responses[0]?.modelVersion || req.model, userPromptId, + requestContents, + req.config, + serverDetails, ); throw error; } finally { diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index 93eec836ef..7651c87a15 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -634,27 +634,27 @@ export class ClearcutLogger { { gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_INPUT_TOKEN_COUNT, - value: JSON.stringify(event.input_token_count), + value: JSON.stringify(event.usage.input_token_count), }, { gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_OUTPUT_TOKEN_COUNT, - value: JSON.stringify(event.output_token_count), + value: JSON.stringify(event.usage.output_token_count), }, { gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_CACHED_TOKEN_COUNT, - value: JSON.stringify(event.cached_content_token_count), + value: JSON.stringify(event.usage.cached_content_token_count), }, { gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_THINKING_TOKEN_COUNT, - value: JSON.stringify(event.thoughts_token_count), + value: JSON.stringify(event.usage.thoughts_token_count), }, { gemini_cli_key: EventMetadataKey.GEMINI_CLI_API_RESPONSE_TOOL_TOKEN_COUNT, - value: JSON.stringify(event.tool_token_count), + value: JSON.stringify(event.usage.tool_token_count), }, ]; diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index b6f52af8f1..904e410940 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -20,9 +20,9 @@ import { } from '../index.js'; import { OutputFormat } from '../output/types.js'; import { logs } from '@opentelemetry/api-logs'; -import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; import type { Config } from '../config/config.js'; import { + logApiError, logApiRequest, logApiResponse, logCliConfiguration, @@ -46,6 +46,7 @@ import { } from './loggers.js'; import { ToolCallDecision } from './tool-call-decision.js'; import { + EVENT_API_ERROR, EVENT_API_REQUEST, EVENT_API_RESPONSE, EVENT_CLI_CONFIG, @@ -64,6 +65,7 @@ import { EVENT_AGENT_START, EVENT_AGENT_FINISH, EVENT_WEB_FETCH_FALLBACK_ATTEMPT, + ApiErrorEvent, ApiRequestEvent, ApiResponseEvent, StartSessionEvent, @@ -87,16 +89,13 @@ import { EVENT_EXTENSION_UPDATE, } from './types.js'; import * as metrics from './metrics.js'; -import { - FileOperation, - GenAiOperationName, - GenAiProviderName, -} from './metrics.js'; +import { FileOperation } from './metrics.js'; import * as sdk from './sdk.js'; import { vi, describe, beforeEach, it, expect, afterEach } from 'vitest'; -import type { - CallableTool, - GenerateContentResponseUsageMetadata, +import { + FinishReason, + type CallableTool, + type GenerateContentResponseUsageMetadata, } from '@google/genai'; import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; import * as uiTelemetry from './uiTelemetry.js'; @@ -316,12 +315,6 @@ describe('loggers', () => { const mockMetrics = { recordApiResponseMetrics: vi.fn(), recordTokenUsageMetrics: vi.fn(), - getConventionAttributes: vi.fn(() => ({ - 'gen_ai.operation.name': GenAiOperationName.GENERATE_CONTENT, - 'gen_ai.provider.name': GenAiProviderName.GCP_VERTEX_AI, - 'gen_ai.request.model': 'test-model', - 'gen_ai.response.model': 'test-model', - })), }; beforeEach(() => { @@ -331,9 +324,6 @@ describe('loggers', () => { vi.spyOn(metrics, 'recordTokenUsageMetrics').mockImplementation( mockMetrics.recordTokenUsageMetrics, ); - vi.spyOn(metrics, 'getConventionAttributes').mockImplementation( - mockMetrics.getConventionAttributes, - ); }); it('should log an API response with all fields', () => { @@ -347,7 +337,47 @@ describe('loggers', () => { const event = new ApiResponseEvent( 'test-model', 100, - 'prompt-id-1', + { + prompt_id: 'prompt-id-1', + contents: [ + { + role: 'user', + parts: [{ text: 'Hello' }], + }, + ], + generate_content_config: { + temperature: 1, + topP: 2, + topK: 3, + responseMimeType: 'text/plain', + candidateCount: 1, + seed: 678, + frequencyPenalty: 10, + maxOutputTokens: 8000, + presencePenalty: 6, + stopSequences: ['stop', 'please stop'], + systemInstruction: { + role: 'model', + parts: [{ text: 'be nice' }], + }, + }, + server: { + address: 'foo.com', + port: 8080, + }, + }, + { + response_id: '', + candidates: [ + { + content: { + role: 'model', + parts: [{ text: 'candidate 1' }], + }, + finishReason: FinishReason.STOP, + }, + ], + }, AuthType.LOGIN_WITH_GOOGLE, usageData, 'test-response', @@ -357,26 +387,40 @@ describe('loggers', () => { expect(mockLogger.emit).toHaveBeenCalledWith({ body: 'API response from test-model. Status: 200. Duration: 100ms.', - attributes: { - 'session.id': 'test-session-id', - 'user.email': 'test-user@example.com', - 'installation.id': 'test-installation-id', + attributes: expect.objectContaining({ 'event.name': EVENT_API_RESPONSE, - 'event.timestamp': '2025-01-01T00:00:00.000Z', - [SemanticAttributes.HTTP_STATUS_CODE]: 200, - model: 'test-model', - status_code: 200, - duration_ms: 100, - input_token_count: 17, - output_token_count: 50, - cached_content_token_count: 10, - thoughts_token_count: 5, - tool_token_count: 2, - total_token_count: 0, - response_text: 'test-response', prompt_id: 'prompt-id-1', - auth_type: 'oauth-personal', - }, + }), + }); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'GenAI operation details from test-model. Status: 200. Duration: 100ms.', + attributes: expect.objectContaining({ + 'event.name': 'gen_ai.client.inference.operation.details', + 'gen_ai.request.model': 'test-model', + 'gen_ai.request.temperature': 1, + 'gen_ai.request.top_p': 2, + 'gen_ai.request.top_k': 3, + 'gen_ai.input.messages': + '[{"role":"user","parts":[{"type":"text","content":"Hello"}]}]', + 'gen_ai.output.messages': + '[{"finish_reason":"stop","role":"system","parts":[{"type":"text","content":"candidate 1"}]}]', + 'gen_ai.response.finish_reasons': ['stop'], + 'gen_ai.response.model': 'test-model', + 'gen_ai.usage.input_tokens': 17, + 'gen_ai.usage.output_tokens': 50, + 'gen_ai.operation.name': 'generate_content', + 'gen_ai.output.type': 'text', + 'gen_ai.request.choice.count': 1, + 'gen_ai.request.seed': 678, + 'gen_ai.request.frequency_penalty': 10, + 'gen_ai.request.presence_penalty': 6, + 'gen_ai.request.max_tokens': 8000, + 'server.address': 'foo.com', + 'server.port': 8080, + 'gen_ai.request.stop_sequences': ['stop', 'please stop'], + 'gen_ai.system_instructions': '[{"type":"text","content":"be nice"}]', + }), }); expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith( @@ -433,6 +477,137 @@ describe('loggers', () => { }); }); + describe('logApiError', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getUsageStatisticsEnabled: () => true, + getTelemetryEnabled: () => true, + getTelemetryLogPromptsEnabled: () => true, + } as Config; + + const mockMetrics = { + recordApiResponseMetrics: vi.fn(), + recordApiErrorMetrics: vi.fn(), + recordTokenUsageMetrics: vi.fn(), + }; + + beforeEach(() => { + vi.spyOn(metrics, 'recordApiResponseMetrics').mockImplementation( + mockMetrics.recordApiResponseMetrics, + ); + vi.spyOn(metrics, 'recordApiErrorMetrics').mockImplementation( + mockMetrics.recordApiErrorMetrics, + ); + }); + + it('should log an API error with all fields', () => { + const event = new ApiErrorEvent( + 'test-model', + 'UNAVAILABLE. {"error":{"code":503,"message":"The model is overloaded. Please try again later.","status":"UNAVAILABLE"}}', + 100, + { + prompt_id: 'prompt-id-1', + contents: [ + { + role: 'user', + parts: [{ text: 'Hello' }], + }, + ], + generate_content_config: { + temperature: 1, + topP: 2, + topK: 3, + responseMimeType: 'text/plain', + candidateCount: 1, + seed: 678, + frequencyPenalty: 10, + maxOutputTokens: 8000, + presencePenalty: 6, + stopSequences: ['stop', 'please stop'], + systemInstruction: { + role: 'model', + parts: [{ text: 'be nice' }], + }, + }, + server: { + address: 'foo.com', + port: 8080, + }, + }, + AuthType.LOGIN_WITH_GOOGLE, + 'ApiError', + 503, + ); + + logApiError(mockConfig, event); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'API error for test-model. Error: UNAVAILABLE. {"error":{"code":503,"message":"The model is overloaded. Please try again later.","status":"UNAVAILABLE"}}. Duration: 100ms.', + attributes: expect.objectContaining({ + 'event.name': EVENT_API_ERROR, + prompt_id: 'prompt-id-1', + }), + }); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'GenAI operation error details from test-model. Error: UNAVAILABLE. {"error":{"code":503,"message":"The model is overloaded. Please try again later.","status":"UNAVAILABLE"}}. Duration: 100ms.', + attributes: expect.objectContaining({ + 'event.name': 'gen_ai.client.inference.operation.details', + 'gen_ai.request.model': 'test-model', + 'gen_ai.request.temperature': 1, + 'gen_ai.request.top_p': 2, + 'gen_ai.request.top_k': 3, + 'gen_ai.input.messages': + '[{"role":"user","parts":[{"type":"text","content":"Hello"}]}]', + 'gen_ai.operation.name': 'generate_content', + 'gen_ai.output.type': 'text', + 'gen_ai.request.choice.count': 1, + 'gen_ai.request.seed': 678, + 'gen_ai.request.frequency_penalty': 10, + 'gen_ai.request.presence_penalty': 6, + 'gen_ai.request.max_tokens': 8000, + 'server.address': 'foo.com', + 'server.port': 8080, + 'gen_ai.request.stop_sequences': ['stop', 'please stop'], + 'gen_ai.system_instructions': '[{"type":"text","content":"be nice"}]', + }), + }); + + expect(mockMetrics.recordApiErrorMetrics).toHaveBeenCalledWith( + mockConfig, + 100, + { + model: 'test-model', + status_code: 503, + error_type: 'ApiError', + }, + ); + + expect(mockMetrics.recordApiResponseMetrics).toHaveBeenCalledWith( + mockConfig, + 100, + { + model: 'test-model', + status_code: 503, + genAiAttributes: { + 'gen_ai.operation.name': 'generate_content', + 'gen_ai.provider.name': 'gcp.vertex_ai', + 'gen_ai.request.model': 'test-model', + 'gen_ai.response.model': 'test-model', + 'error.type': 'ApiError', + }, + }, + ); + + expect(mockUiEvent.addEvent).toHaveBeenCalledWith({ + ...event, + 'event.name': EVENT_API_ERROR, + 'event.timestamp': '2025-01-01T00:00:00.000Z', + }); + }); + }); + describe('logApiRequest', () => { const mockConfig = { getSessionId: () => 'test-session-id', diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index 81f8b43029..cf95c69340 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -89,6 +89,7 @@ export function logUserPrompt(config: Config, event: UserPromptEvent): void { if (!isTelemetrySdkInitialized()) return; const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { body: event.toLogBody(), attributes: event.toOpenTelemetryAttributes(config), @@ -219,11 +220,9 @@ export function logApiError(config: Config, event: ApiErrorEvent): void { if (!isTelemetrySdkInitialized()) return; const logger = logs.getLogger(SERVICE_NAME); - const logRecord: LogRecord = { - body: event.toLogBody(), - attributes: event.toOpenTelemetryAttributes(config), - }; - logger.emit(logRecord); + logger.emit(event.toLogRecord(config)); + logger.emit(event.toSemanticLogRecord(config)); + recordApiErrorMetrics(config, event.duration_ms, { model: event.model, status_code: event.status_code, @@ -231,12 +230,11 @@ export function logApiError(config: Config, event: ApiErrorEvent): void { }); // Record GenAI operation duration for errors - const conventionAttributes = getConventionAttributes(event); recordApiResponseMetrics(config, event.duration_ms, { model: event.model, status_code: event.status_code, genAiAttributes: { - ...conventionAttributes, + ...getConventionAttributes(event), 'error.type': event.error_type || 'unknown', }, }); @@ -253,11 +251,8 @@ export function logApiResponse(config: Config, event: ApiResponseEvent): void { if (!isTelemetrySdkInitialized()) return; const logger = logs.getLogger(SERVICE_NAME); - const logRecord: LogRecord = { - body: event.toLogBody(), - attributes: event.toOpenTelemetryAttributes(config), - }; - logger.emit(logRecord); + logger.emit(event.toLogRecord(config)); + logger.emit(event.toSemanticLogRecord(config)); const conventionAttributes = getConventionAttributes(event); @@ -268,11 +263,11 @@ export function logApiResponse(config: Config, event: ApiResponseEvent): void { }); const tokenUsageData = [ - { count: event.input_token_count, type: 'input' as const }, - { count: event.output_token_count, type: 'output' as const }, - { count: event.cached_content_token_count, type: 'cache' as const }, - { count: event.thoughts_token_count, type: 'thought' as const }, - { count: event.tool_token_count, type: 'tool' as const }, + { count: event.usage.input_token_count, type: 'input' as const }, + { count: event.usage.output_token_count, type: 'output' as const }, + { count: event.usage.cached_content_token_count, type: 'cache' as const }, + { count: event.usage.thoughts_token_count, type: 'thought' as const }, + { count: event.usage.tool_token_count, type: 'tool' as const }, ]; for (const { count, type } of tokenUsageData) { diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts index 4123ed5325..6e9f1846ec 100644 --- a/packages/core/src/telemetry/metrics.ts +++ b/packages/core/src/telemetry/metrics.ts @@ -7,7 +7,6 @@ import type { Attributes, Meter, Counter, Histogram } from '@opentelemetry/api'; import { diag, metrics, ValueType } from '@opentelemetry/api'; import { SERVICE_NAME } from './constants.js'; -import { EVENT_CHAT_COMPRESSION } from './types.js'; import type { Config } from '../config/config.js'; import type { ModelRoutingEvent, @@ -17,6 +16,7 @@ import type { import { AuthType } from '../core/contentGenerator.js'; import { getCommonAttributes } from './telemetryAttributes.js'; +const EVENT_CHAT_COMPRESSION = 'gemini_cli.chat_compression'; const TOOL_CALL_COUNT = 'gemini_cli.tool.call.count'; const TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency'; const API_REQUEST_COUNT = 'gemini_cli.api.request.count'; diff --git a/packages/core/src/telemetry/semantic.test.ts b/packages/core/src/telemetry/semantic.test.ts new file mode 100644 index 0000000000..6ea4cf3a3e --- /dev/null +++ b/packages/core/src/telemetry/semantic.test.ts @@ -0,0 +1,425 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect } from 'vitest'; +import { + toChatMessage, + toInputMessages, + toSystemInstruction, + toOutputMessages, + toFinishReasons, + OTelFinishReason, + toOutputType, + OTelOutputType, +} from './semantic.js'; +import { + Language, + type Content, + Outcome, + type Candidate, + FinishReason, +} from '@google/genai'; + +describe('toChatMessage', () => { + it('should correctly handle text parts', () => { + const content: Content = { + role: 'user', + parts: [{ text: 'Hello' }], + }; + expect(toChatMessage(content)).toEqual({ + role: 'user', + parts: [ + { + type: 'text', + content: 'Hello', + }, + ], + }); + }); + + it('should correctly handle function call parts', () => { + const content: Content = { + role: 'model', + parts: [ + { + functionCall: { + name: 'test-function', + args: { + arg1: 'test-value', + }, + id: '12345', + }, + // include field not specified in semantic specification that could be present + thoughtSignature: '1234', + }, + ], + }; + expect(toChatMessage(content)).toEqual({ + role: 'system', + parts: [ + { + type: 'tool_call', + name: 'test-function', + arguments: '{"arg1":"test-value"}', + id: '12345', + }, + ], + }); + }); + + it('should correctly handle function response parts', () => { + const content: Content = { + role: 'user', + parts: [ + { + functionResponse: { + name: 'test-function', + response: { + result: 'success', + }, + id: '12345', + }, + // include field not specified in semantic specification that could be present + fileData: { + displayName: 'greatfile', + }, + }, + ], + }; + expect(toChatMessage(content)).toEqual({ + role: 'user', + parts: [ + { + type: 'tool_call_response', + response: '{"result":"success"}', + id: '12345', + }, + ], + }); + }); + + it('should correctly handle reasoning parts with text', () => { + const content: Content = { + role: 'system', + parts: [{ text: 'Hmm', thought: true }], + }; + expect(toChatMessage(content)).toEqual({ + role: 'system', + parts: [ + { + type: 'reasoning', + content: 'Hmm', + }, + ], + }); + }); + + it('should correctly handle reasoning parts without text', () => { + const content: Content = { + role: 'system', + parts: [ + { + thought: true, + // include field not specified in semantic specification that could be present + inlineData: { + displayName: 'wowdata', + }, + }, + ], + }; + expect(toChatMessage(content)).toEqual({ + role: 'system', + parts: [ + { + type: 'reasoning', + content: '', + }, + ], + }); + }); + + it('should correctly handle text parts that are not reasoning parts', () => { + const content: Content = { + role: 'user', + parts: [{ text: 'what a nice day', thought: false }], + }; + expect(toChatMessage(content)).toEqual({ + role: 'user', + parts: [ + { + type: 'text', + content: 'what a nice day', + }, + ], + }); + }); + + it('should correctly handle "generic" parts', () => { + const content: Content = { + role: 'model', + parts: [ + { + executableCode: { + code: 'print("foo")', + language: Language.PYTHON, + }, + }, + { + codeExecutionResult: { + outcome: Outcome.OUTCOME_OK, + output: 'foo', + }, + // include field not specified in semantic specification that could be present + videoMetadata: { + fps: 5, + }, + }, + ], + }; + expect(toChatMessage(content)).toEqual({ + role: 'system', + parts: [ + { + type: 'executableCode', + code: 'print("foo")', + language: 'PYTHON', + }, + { + type: 'codeExecutionResult', + outcome: 'OUTCOME_OK', + output: 'foo', + videoMetadata: { + fps: 5, + }, + }, + ], + }); + }); + + it('should correctly handle unknown parts', () => { + const content: Content = { + role: 'model', + parts: [ + { + fileData: { + displayName: 'superfile', + }, + }, + ], + }; + expect(toChatMessage(content)).toEqual({ + role: 'system', + parts: [ + { + type: 'unknown', + fileData: { + displayName: 'superfile', + }, + }, + ], + }); + }); +}); + +describe('toSystemInstruction', () => { + it('should correctly handle a string', () => { + const content = 'Hello'; + expect(toSystemInstruction(content)).toEqual([ + { + type: 'text', + content: 'Hello', + }, + ]); + }); + + it('should correctly handle a Content object with a text part', () => { + const content: Content = { + role: 'user', + parts: [{ text: 'Hello' }], + }; + expect(toSystemInstruction(content)).toEqual([ + { + type: 'text', + content: 'Hello', + }, + ]); + }); + + it('should correctly handle a Content object with multiple parts', () => { + const content: Content = { + role: 'user', + parts: [{ text: 'Hello' }, { text: 'Hmm', thought: true }], + }; + expect(toSystemInstruction(content)).toEqual([ + { + type: 'text', + content: 'Hello', + }, + { + type: 'reasoning', + content: 'Hmm', + }, + ]); + }); +}); + +describe('toInputMessages', () => { + it('should correctly convert an array of Content objects', () => { + const contents: Content[] = [ + { + role: 'user', + parts: [{ text: 'Hello' }], + }, + { + role: 'model', + parts: [{ text: 'Hi there!' }], + }, + ]; + expect(toInputMessages(contents)).toEqual([ + { + role: 'user', + parts: [ + { + type: 'text', + content: 'Hello', + }, + ], + }, + { + role: 'system', + parts: [ + { + type: 'text', + content: 'Hi there!', + }, + ], + }, + ]); + }); +}); + +describe('toOutputMessages', () => { + it('should correctly convert an array of Candidate objects', () => { + const candidates: Candidate[] = [ + { + index: 0, + finishReason: FinishReason.STOP, + content: { + role: 'model', + parts: [{ text: 'This is the first candidate.' }], + }, + }, + { + index: 1, + finishReason: FinishReason.MAX_TOKENS, + content: { + role: 'model', + parts: [{ text: 'This is the second candidate.' }], + }, + }, + ]; + expect(toOutputMessages(candidates)).toEqual([ + { + role: 'system', + finish_reason: 'stop', + parts: [ + { + type: 'text', + content: 'This is the first candidate.', + }, + ], + }, + { + role: 'system', + finish_reason: 'length', + parts: [ + { + type: 'text', + content: 'This is the second candidate.', + }, + ], + }, + ]); + }); +}); + +describe('toFinishReasons', () => { + it('should return an empty array if candidates is undefined', () => { + expect(toFinishReasons(undefined)).toEqual([]); + }); + + it('should return an empty array if candidates is an empty array', () => { + expect(toFinishReasons([])).toEqual([]); + }); + + it('should correctly convert a single candidate', () => { + const candidates: Candidate[] = [ + { + index: 0, + finishReason: FinishReason.STOP, + content: { + role: 'model', + parts: [{ text: 'This is the first candidate.' }], + }, + }, + ]; + expect(toFinishReasons(candidates)).toEqual([OTelFinishReason.STOP]); + }); + + it('should correctly convert multiple candidates', () => { + const candidates: Candidate[] = [ + { + index: 0, + finishReason: FinishReason.STOP, + content: { + role: 'model', + parts: [{ text: 'This is the first candidate.' }], + }, + }, + { + index: 1, + finishReason: FinishReason.MAX_TOKENS, + content: { + role: 'model', + parts: [{ text: 'This is the second candidate.' }], + }, + }, + { + index: 2, + finishReason: FinishReason.SAFETY, + content: { + role: 'model', + parts: [{ text: 'This is the third candidate.' }], + }, + }, + ]; + expect(toFinishReasons(candidates)).toEqual([ + OTelFinishReason.STOP, + OTelFinishReason.LENGTH, + OTelFinishReason.CONTENT_FILTER, + ]); + }); +}); + +describe('toOutputType', () => { + it('should return TEXT for text/plain', () => { + expect(toOutputType('text/plain')).toBe(OTelOutputType.TEXT); + }); + + it('should return JSON for application/json', () => { + expect(toOutputType('application/json')).toBe(OTelOutputType.JSON); + }); + + it('should return the custom mime type for other strings', () => { + expect(toOutputType('application/vnd.custom-type')).toBe( + 'application/vnd.custom-type', + ); + }); + + it('should return undefined for undefined input', () => { + expect(toOutputType(undefined)).toBeUndefined(); + }); +}); diff --git a/packages/core/src/telemetry/semantic.ts b/packages/core/src/telemetry/semantic.ts new file mode 100644 index 0000000000..b2ee296793 --- /dev/null +++ b/packages/core/src/telemetry/semantic.ts @@ -0,0 +1,323 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * This file contains functions and types for converting Gemini API request/response + * formats to the OpenTelemetry semantic conventions for generative AI. + * + * @see https://github.com/open-telemetry/semantic-conventions/blob/8b4f210f43136e57c1f6f47292eb6d38e3bf30bb/docs/gen-ai/gen-ai-events.md + */ + +import { FinishReason } from '@google/genai'; +import type { + Candidate, + Content, + ContentUnion, + Part, + PartUnion, +} from '@google/genai'; + +export function toInputMessages(contents: Content[]): InputMessages { + const messages: ChatMessage[] = []; + for (const content of contents) { + messages.push(toChatMessage(content)); + } + return messages; +} + +function isPart(value: unknown): value is Part { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + !('parts' in value) + ); +} + +function toPart(part: PartUnion): Part { + if (typeof part === 'string') { + return { text: part }; + } + return part; +} + +function toContent(content: ContentUnion): Content | undefined { + if (typeof content === 'string') { + // 1. It's a string + return { + parts: [toPart(content)], + }; + } else if (Array.isArray(content)) { + // 2. It's an array of parts (PartUnion[]) + return { + parts: content.map(toPart), + }; + } else if ('parts' in content) { + // 3. It's a Content object + return content; + } else if (isPart(content)) { + // 4. It's a single Part object (asserted with type guard) + return { + parts: [content], + }; + } else { + // 5. Handle any other unexpected case + return undefined; + } +} + +export function toSystemInstruction( + systemInstruction?: ContentUnion, +): SystemInstruction | undefined { + const parts: AnyPart[] = []; + if (systemInstruction) { + const content = toContent(systemInstruction); + if (content && content.parts) { + for (const part of content.parts) { + parts.push(toOTelPart(part)); + } + } + } + return parts; +} + +export function toOutputMessages(candidates?: Candidate[]): OutputMessages { + const messages: OutputMessage[] = []; + if (candidates) { + for (const candidate of candidates) { + messages.push({ + finish_reason: toOTelFinishReason(candidate.finishReason), + ...toChatMessage(candidate.content), + }); + } + } + return messages; +} + +export function toFinishReasons(candidates?: Candidate[]): OTelFinishReason[] { + const reasons: OTelFinishReason[] = []; + if (candidates) { + for (const candidate of candidates) { + reasons.push(toOTelFinishReason(candidate.finishReason)); + } + } + return reasons; +} + +export function toOutputType(requested_mime?: string): string | undefined { + switch (requested_mime) { + // explictly support the known good values of responseMimeType + case 'text/plain': + return OTelOutputType.TEXT; + case 'application/json': + return OTelOutputType.JSON; + default: + // if none of the well-known values applies, a custom value may be used + return requested_mime; + } +} + +export function toChatMessage(content?: Content): ChatMessage { + const message: ChatMessage = { + role: undefined, + parts: [], + }; + if (content && content.parts) { + message.role = toOTelRole(content.role); + for (const part of content.parts) { + message.parts.push(toOTelPart(part)); + } + } + return message; +} + +export function toOTelPart(part: Part): AnyPart { + if (part.thought) { + if (part.text) { + return new ReasoningPart(part.text); + } else { + return new ReasoningPart(''); + } + } else if (part.text) { + return new TextPart(part.text); + } else if (part.functionCall) { + return new ToolCallRequestPart( + part.functionCall.name, + part.functionCall.id, + JSON.stringify(part.functionCall.args), + ); + } else if (part.functionResponse) { + return new ToolCallResponsePart( + JSON.stringify(part.functionResponse.response), + part.functionResponse.id, + ); + } else if (part.executableCode) { + const { executableCode, ...unexpectedData } = part; + return new GenericPart('executableCode', { + code: executableCode.code, + language: executableCode.language, + ...unexpectedData, + }); + } else if (part.codeExecutionResult) { + const { codeExecutionResult, ...unexpectedData } = part; + return new GenericPart('codeExecutionResult', { + outcome: codeExecutionResult.outcome, + output: codeExecutionResult.output, + ...unexpectedData, + }); + } + // Assuming the above cases capture all the expected parts + // but adding a fallthrough just in case. + return new GenericPart('unknown', { ...part }); +} + +export enum OTelRole { + SYSTEM = 'system', + USER = 'user', + ASSISTANT = 'assistant', + TOOL = 'tool', +} + +export function toOTelRole(role?: string): OTelRole { + switch (role?.toLowerCase()) { + case 'system': + return OTelRole.SYSTEM; + // Our APIs seem to frequently use 'model' + case 'model': + return OTelRole.SYSTEM; + case 'user': + return OTelRole.USER; + case 'assistant': + return OTelRole.ASSISTANT; + case 'tool': + return OTelRole.TOOL; + default: + return OTelRole.SYSTEM; + } +} + +export type InputMessages = ChatMessage[]; + +export enum OTelOutputType { + IMAGE = 'image', + JSON = 'json', + SPEECH = 'speech', + TEXT = 'text', +} + +export enum OTelFinishReason { + STOP = 'stop', + LENGTH = 'length', + CONTENT_FILTER = 'content_filter', + TOOL_CALL = 'tool_call', + ERROR = 'error', +} + +export function toOTelFinishReason(finishReason?: string): OTelFinishReason { + switch (finishReason) { + // we have significantly more finish reasons than the spec + case FinishReason.FINISH_REASON_UNSPECIFIED: + return OTelFinishReason.STOP; + case FinishReason.STOP: + return OTelFinishReason.STOP; + case FinishReason.MAX_TOKENS: + return OTelFinishReason.LENGTH; + case FinishReason.SAFETY: + return OTelFinishReason.CONTENT_FILTER; + case FinishReason.RECITATION: + return OTelFinishReason.CONTENT_FILTER; + case FinishReason.LANGUAGE: + return OTelFinishReason.CONTENT_FILTER; + case FinishReason.OTHER: + return OTelFinishReason.STOP; + case FinishReason.BLOCKLIST: + return OTelFinishReason.CONTENT_FILTER; + case FinishReason.PROHIBITED_CONTENT: + return OTelFinishReason.CONTENT_FILTER; + case FinishReason.SPII: + return OTelFinishReason.CONTENT_FILTER; + case FinishReason.MALFORMED_FUNCTION_CALL: + return OTelFinishReason.ERROR; + case FinishReason.IMAGE_SAFETY: + return OTelFinishReason.CONTENT_FILTER; + case FinishReason.UNEXPECTED_TOOL_CALL: + return OTelFinishReason.ERROR; + default: + return OTelFinishReason.STOP; + } +} + +export interface OutputMessage extends ChatMessage { + finish_reason: FinishReason | string; +} + +export type OutputMessages = OutputMessage[]; + +export type AnyPart = + | TextPart + | ToolCallRequestPart + | ToolCallResponsePart + | ReasoningPart + | GenericPart; + +export type SystemInstruction = AnyPart[]; + +export interface ChatMessage { + role: string | undefined; + parts: AnyPart[]; +} + +export class TextPart { + readonly type = 'text'; + content: string; + + constructor(content: string) { + this.content = content; + } +} + +export class ToolCallRequestPart { + readonly type = 'tool_call'; + name?: string; + id?: string; + arguments?: string; + + constructor(name?: string, id?: string, args?: string) { + this.name = name; + this.id = id; + this.arguments = args; + } +} + +export class ToolCallResponsePart { + readonly type = 'tool_call_response'; + response?: string; + id?: string; + + constructor(response?: string, id?: string) { + this.response = response; + this.id = id; + } +} + +export class ReasoningPart { + readonly type = 'reasoning'; + content: string; + + constructor(content: string) { + this.content = content; + } +} + +export class GenericPart { + type: string; + [key: string]: unknown; + + constructor(type: string, data: { [key: string]: unknown }) { + this.type = type; + Object.assign(this, data); + } +} diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index 38e636b128..a6778fc851 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -4,19 +4,24 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type { GenerateContentResponseUsageMetadata } from '@google/genai'; +import type { + Candidate, + Content, + GenerateContentConfig, + GenerateContentResponseUsageMetadata, +} from '@google/genai'; import type { Config } from '../config/config.js'; import type { ApprovalMode } from '../config/config.js'; import type { CompletedToolCall } from '../core/coreToolScheduler.js'; import { DiscoveredMCPTool } from '../tools/mcp-tool.js'; import type { FileDiff } from '../tools/tools.js'; import { AuthType } from '../core/contentGenerator.js'; -import type { LogAttributes } from '@opentelemetry/api-logs'; +import type { LogAttributes, LogRecord } from '@opentelemetry/api-logs'; import { getDecisionFromOutcome, ToolCallDecision, } from './tool-call-decision.js'; -import type { FileOperation } from './metrics.js'; +import { getConventionAttributes, type FileOperation } from './metrics.js'; export { ToolCallDecision }; import type { ToolRegistry } from '../tools/tool-registry.js'; import type { OutputFormat } from '../output/types.js'; @@ -25,6 +30,13 @@ import type { AgentTerminateMode } from '../agents/types.js'; import { getCommonAttributes } from './telemetryAttributes.js'; import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; import { safeJsonStringify } from '../utils/safeJsonStringify.js'; +import { + toInputMessages, + toOutputMessages, + toFinishReasons, + toOutputType, + toSystemInstruction, +} from './semantic.js'; export interface BaseTelemetryEvent { 'event.name': string; @@ -358,18 +370,18 @@ export class ApiErrorEvent implements BaseTelemetryEvent { 'event.name': 'api_error'; 'event.timestamp': string; model: string; + prompt: GenAIPromptDetails; error: string; error_type?: string; status_code?: number | string; duration_ms: number; - prompt_id: string; auth_type?: string; constructor( model: string, error: string, duration_ms: number, - prompt_id: string, + prompt_details: GenAIPromptDetails, auth_type?: string, error_type?: string, status_code?: number | string, @@ -381,11 +393,11 @@ export class ApiErrorEvent implements BaseTelemetryEvent { this.error_type = error_type; this.status_code = status_code; this.duration_ms = duration_ms; - this.prompt_id = prompt_id; + this.prompt = prompt_details; this.auth_type = auth_type; } - toOpenTelemetryAttributes(config: Config): LogAttributes { + toLogRecord(config: Config): LogRecord { const attributes: LogAttributes = { ...getCommonAttributes(config), 'event.name': EVENT_API_ERROR, @@ -397,7 +409,7 @@ export class ApiErrorEvent implements BaseTelemetryEvent { error: this.error, status_code: this.status_code, duration_ms: this.duration_ms, - prompt_id: this.prompt_id, + prompt_id: this.prompt.prompt_id, auth_type: this.auth_type, }; @@ -407,69 +419,151 @@ export class ApiErrorEvent implements BaseTelemetryEvent { if (typeof this.status_code === 'number') { attributes[SemanticAttributes.HTTP_STATUS_CODE] = this.status_code; } - return attributes; + const logRecord: LogRecord = { + body: `API error for ${this.model}. Error: ${this.error}. Duration: ${this.duration_ms}ms.`, + attributes, + }; + return logRecord; } - toLogBody(): string { - return `API error for ${this.model}. Error: ${this.error}. Duration: ${this.duration_ms}ms.`; + toSemanticLogRecord(config: Config): LogRecord { + const attributes: LogAttributes = { + ...getCommonAttributes(config), + 'event.name': EVENT_GEN_AI_OPERATION_DETAILS, + 'event.timestamp': this['event.timestamp'], + ...toGenerateContentConfigAttributes(this.prompt.generate_content_config), + ...getConventionAttributes(this), + }; + + if (this.prompt.server) { + attributes['server.address'] = this.prompt.server.address; + attributes['server.port'] = this.prompt.server.port; + } + + if (config.getTelemetryLogPromptsEnabled() && this.prompt.contents) { + attributes['gen_ai.input.messages'] = JSON.stringify( + toInputMessages(this.prompt.contents), + ); + } + + const logRecord: LogRecord = { + body: `GenAI operation error details from ${this.model}. Error: ${this.error}. Duration: ${this.duration_ms}ms.`, + attributes, + }; + + return logRecord; } } -export const EVENT_API_RESPONSE = 'gemini_cli.api_response'; -export class ApiResponseEvent implements BaseTelemetryEvent { - 'event.name': 'api_response'; - 'event.timestamp': string; - model: string; - status_code?: number | string; - duration_ms: number; +export interface ServerDetails { + address: string; + port: number; +} + +export interface GenAIPromptDetails { + prompt_id: string; + contents: Content[]; + generate_content_config?: GenerateContentConfig; + server?: ServerDetails; +} + +export interface GenAIResponseDetails { + response_id?: string; + candidates?: Candidate[]; +} + +export interface GenAIUsageDetails { input_token_count: number; output_token_count: number; cached_content_token_count: number; thoughts_token_count: number; tool_token_count: number; total_token_count: number; +} + +export const EVENT_API_RESPONSE = 'gemini_cli.api_response'; +export const EVENT_GEN_AI_OPERATION_DETAILS = + 'gen_ai.client.inference.operation.details'; + +function toGenerateContentConfigAttributes( + config?: GenerateContentConfig, +): LogAttributes { + if (!config) { + return {}; + } + return { + 'gen_ai.request.temperature': config.temperature, + 'gen_ai.request.top_p': config.topP, + 'gen_ai.request.top_k': config.topK, + 'gen_ai.request.choice.count': config.candidateCount, + 'gen_ai.request.seed': config.seed, + 'gen_ai.request.frequency_penalty': config.frequencyPenalty, + 'gen_ai.request.presence_penalty': config.presencePenalty, + 'gen_ai.request.max_tokens': config.maxOutputTokens, + 'gen_ai.output.type': toOutputType(config.responseMimeType), + 'gen_ai.request.stop_sequences': config.stopSequences, + 'gen_ai.system_instructions': JSON.stringify( + toSystemInstruction(config.systemInstruction), + ), + }; +} + +export class ApiResponseEvent implements BaseTelemetryEvent { + 'event.name': 'api_response'; + 'event.timestamp': string; + status_code?: number | string; + duration_ms: number; response_text?: string; - prompt_id: string; auth_type?: string; + model: string; + prompt: GenAIPromptDetails; + response: GenAIResponseDetails; + usage: GenAIUsageDetails; + constructor( model: string, duration_ms: number, - prompt_id: string, + prompt_details: GenAIPromptDetails, + response_details: GenAIResponseDetails, auth_type?: string, usage_data?: GenerateContentResponseUsageMetadata, response_text?: string, ) { this['event.name'] = 'api_response'; this['event.timestamp'] = new Date().toISOString(); - this.model = model; this.duration_ms = duration_ms; this.status_code = 200; - this.input_token_count = usage_data?.promptTokenCount ?? 0; - this.output_token_count = usage_data?.candidatesTokenCount ?? 0; - this.cached_content_token_count = usage_data?.cachedContentTokenCount ?? 0; - this.thoughts_token_count = usage_data?.thoughtsTokenCount ?? 0; - this.tool_token_count = usage_data?.toolUsePromptTokenCount ?? 0; - this.total_token_count = usage_data?.totalTokenCount ?? 0; this.response_text = response_text; - this.prompt_id = prompt_id; this.auth_type = auth_type; + + this.model = model; + this.prompt = prompt_details; + this.response = response_details; + this.usage = { + input_token_count: usage_data?.promptTokenCount ?? 0, + output_token_count: usage_data?.candidatesTokenCount ?? 0, + cached_content_token_count: usage_data?.cachedContentTokenCount ?? 0, + thoughts_token_count: usage_data?.thoughtsTokenCount ?? 0, + tool_token_count: usage_data?.toolUsePromptTokenCount ?? 0, + total_token_count: usage_data?.totalTokenCount ?? 0, + }; } - toOpenTelemetryAttributes(config: Config): LogAttributes { + toLogRecord(config: Config): LogRecord { const attributes: LogAttributes = { ...getCommonAttributes(config), 'event.name': EVENT_API_RESPONSE, 'event.timestamp': this['event.timestamp'], model: this.model, duration_ms: this.duration_ms, - input_token_count: this.input_token_count, - output_token_count: this.output_token_count, - cached_content_token_count: this.cached_content_token_count, - thoughts_token_count: this.thoughts_token_count, - tool_token_count: this.tool_token_count, - total_token_count: this.total_token_count, - prompt_id: this.prompt_id, + input_token_count: this.usage.input_token_count, + output_token_count: this.usage.output_token_count, + cached_content_token_count: this.usage.cached_content_token_count, + thoughts_token_count: this.usage.thoughts_token_count, + tool_token_count: this.usage.tool_token_count, + total_token_count: this.usage.total_token_count, + prompt_id: this.prompt.prompt_id, auth_type: this.auth_type, status_code: this.status_code, }; @@ -481,11 +575,51 @@ export class ApiResponseEvent implements BaseTelemetryEvent { attributes[SemanticAttributes.HTTP_STATUS_CODE] = this.status_code; } } - return attributes; + const logRecord: LogRecord = { + body: `API response from ${this.model}. Status: ${this.status_code || 'N/A'}. Duration: ${this.duration_ms}ms.`, + attributes, + }; + return logRecord; } - toLogBody(): string { - return `API response from ${this.model}. Status: ${this.status_code || 'N/A'}. Duration: ${this.duration_ms}ms.`; + toSemanticLogRecord(config: Config): LogRecord { + const attributes: LogAttributes = { + ...getCommonAttributes(config), + 'event.name': EVENT_GEN_AI_OPERATION_DETAILS, + 'event.timestamp': this['event.timestamp'], + 'gen_ai.response.id': this.response.response_id, + 'gen_ai.response.finish_reasons': toFinishReasons( + this.response.candidates, + ), + 'gen_ai.output.messages': JSON.stringify( + toOutputMessages(this.response.candidates), + ), + ...toGenerateContentConfigAttributes(this.prompt.generate_content_config), + ...getConventionAttributes(this), + }; + + if (this.prompt.server) { + attributes['server.address'] = this.prompt.server.address; + attributes['server.port'] = this.prompt.server.port; + } + + if (config.getTelemetryLogPromptsEnabled() && this.prompt.contents) { + attributes['gen_ai.input.messages'] = JSON.stringify( + toInputMessages(this.prompt.contents), + ); + } + + if (this.usage) { + attributes['gen_ai.usage.input_tokens'] = this.usage.input_token_count; + attributes['gen_ai.usage.output_tokens'] = this.usage.output_token_count; + } + + const logRecord: LogRecord = { + body: `GenAI operation details from ${this.model}. Status: ${this.status_code || 'N/A'}. Duration: ${this.duration_ms}ms.`, + attributes, + }; + + return logRecord; } } diff --git a/packages/core/src/telemetry/uiTelemetry.test.ts b/packages/core/src/telemetry/uiTelemetry.test.ts index 9f4aed54e7..aa06e87a9f 100644 --- a/packages/core/src/telemetry/uiTelemetry.test.ts +++ b/packages/core/src/telemetry/uiTelemetry.test.ts @@ -129,12 +129,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 10, - output_token_count: 20, - total_token_count: 30, - cached_content_token_count: 5, - thoughts_token_count: 2, - tool_token_count: 3, + usage: { + input_token_count: 10, + output_token_count: 20, + total_token_count: 30, + cached_content_token_count: 5, + thoughts_token_count: 2, + tool_token_count: 3, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE }; service.addEvent(event); @@ -151,12 +153,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 10, - output_token_count: 20, - total_token_count: 30, - cached_content_token_count: 5, - thoughts_token_count: 2, - tool_token_count: 3, + usage: { + input_token_count: 10, + output_token_count: 20, + total_token_count: 30, + cached_content_token_count: 5, + thoughts_token_count: 2, + tool_token_count: 3, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE }; service.addEvent(event); @@ -185,12 +189,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 10, - output_token_count: 20, - total_token_count: 30, - cached_content_token_count: 5, - thoughts_token_count: 2, - tool_token_count: 3, + usage: { + input_token_count: 10, + output_token_count: 20, + total_token_count: 30, + cached_content_token_count: 5, + thoughts_token_count: 2, + tool_token_count: 3, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE; }; @@ -198,12 +204,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 600, - input_token_count: 15, - output_token_count: 25, - total_token_count: 40, - cached_content_token_count: 10, - thoughts_token_count: 4, - tool_token_count: 6, + usage: { + input_token_count: 15, + output_token_count: 25, + total_token_count: 40, + cached_content_token_count: 10, + thoughts_token_count: 4, + tool_token_count: 6, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE; }; @@ -235,12 +243,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 10, - output_token_count: 20, - total_token_count: 30, - cached_content_token_count: 5, - thoughts_token_count: 2, - tool_token_count: 3, + usage: { + input_token_count: 10, + output_token_count: 20, + total_token_count: 30, + cached_content_token_count: 5, + thoughts_token_count: 2, + tool_token_count: 3, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE; }; @@ -248,12 +258,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-flash', duration_ms: 1000, - input_token_count: 100, - output_token_count: 200, - total_token_count: 300, - cached_content_token_count: 50, - thoughts_token_count: 20, - tool_token_count: 30, + usage: { + input_token_count: 100, + output_token_count: 200, + total_token_count: 300, + cached_content_token_count: 50, + thoughts_token_count: 20, + tool_token_count: 30, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE; }; @@ -304,12 +316,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 10, - output_token_count: 20, - total_token_count: 30, - cached_content_token_count: 5, - thoughts_token_count: 2, - tool_token_count: 3, + usage: { + input_token_count: 10, + output_token_count: 20, + total_token_count: 30, + cached_content_token_count: 5, + thoughts_token_count: 2, + tool_token_count: 3, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE; }; @@ -534,12 +548,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 100, - output_token_count: 200, - total_token_count: 300, - cached_content_token_count: 50, - thoughts_token_count: 20, - tool_token_count: 30, + usage: { + input_token_count: 100, + output_token_count: 200, + total_token_count: 300, + cached_content_token_count: 50, + thoughts_token_count: 20, + tool_token_count: 30, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE }; service.addEvent(event); @@ -559,12 +575,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 100, - output_token_count: 200, - total_token_count: 300, - cached_content_token_count: 50, - thoughts_token_count: 20, - tool_token_count: 30, + usage: { + input_token_count: 100, + output_token_count: 200, + total_token_count: 300, + cached_content_token_count: 50, + thoughts_token_count: 20, + tool_token_count: 30, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE }; service.addEvent(event); @@ -584,12 +602,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 100, - output_token_count: 200, - total_token_count: 300, - cached_content_token_count: 50, - thoughts_token_count: 20, - tool_token_count: 30, + usage: { + input_token_count: 100, + output_token_count: 200, + total_token_count: 300, + cached_content_token_count: 50, + thoughts_token_count: 20, + tool_token_count: 30, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE }; service.addEvent(event); @@ -616,12 +636,14 @@ describe('UiTelemetryService', () => { 'event.name': EVENT_API_RESPONSE, model: 'gemini-2.5-pro', duration_ms: 500, - input_token_count: 100, - output_token_count: 200, - total_token_count: 300, - cached_content_token_count: 50, - thoughts_token_count: 20, - tool_token_count: 30, + usage: { + input_token_count: 100, + output_token_count: 200, + total_token_count: 300, + cached_content_token_count: 50, + thoughts_token_count: 20, + tool_token_count: 30, + }, } as ApiResponseEvent & { 'event.name': typeof EVENT_API_RESPONSE }; service.addEvent(event); diff --git a/packages/core/src/telemetry/uiTelemetry.ts b/packages/core/src/telemetry/uiTelemetry.ts index 2b6a813485..79cb957d7d 100644 --- a/packages/core/src/telemetry/uiTelemetry.ts +++ b/packages/core/src/telemetry/uiTelemetry.ts @@ -165,12 +165,12 @@ export class UiTelemetryService extends EventEmitter { modelMetrics.api.totalRequests++; modelMetrics.api.totalLatencyMs += event.duration_ms; - modelMetrics.tokens.prompt += event.input_token_count; - modelMetrics.tokens.candidates += event.output_token_count; - modelMetrics.tokens.total += event.total_token_count; - modelMetrics.tokens.cached += event.cached_content_token_count; - modelMetrics.tokens.thoughts += event.thoughts_token_count; - modelMetrics.tokens.tool += event.tool_token_count; + modelMetrics.tokens.prompt += event.usage.input_token_count; + modelMetrics.tokens.candidates += event.usage.output_token_count; + modelMetrics.tokens.total += event.usage.total_token_count; + modelMetrics.tokens.cached += event.usage.cached_content_token_count; + modelMetrics.tokens.thoughts += event.usage.thoughts_token_count; + modelMetrics.tokens.tool += event.usage.tool_token_count; } private processApiError(event: ApiErrorEvent) {