diff --git a/packages/core/src/core/loggingContentGenerator.ts b/packages/core/src/core/loggingContentGenerator.ts index 3b0768e98d..de16b8de69 100644 --- a/packages/core/src/core/loggingContentGenerator.ts +++ b/packages/core/src/core/loggingContentGenerator.ts @@ -55,11 +55,22 @@ export class LoggingContentGenerator implements ContentGenerator { contents: Content[], model: string, promptId: string, + generationConfig?: GenerateContentConfig, + serverDetails?: ServerDetails, ): void { const requestText = JSON.stringify(contents); logApiRequest( this.config, - new ApiRequestEvent(model, promptId, requestText), + new ApiRequestEvent( + model, + { + prompt_id: promptId, + contents, + generate_content_config: generationConfig, + server: serverDetails, + }, + requestText, + ), ); } @@ -176,8 +187,14 @@ export class LoggingContentGenerator implements ContentGenerator { const startTime = Date.now(); const contents: Content[] = toContents(req.contents); - this.logApiRequest(toContents(req.contents), req.model, userPromptId); const serverDetails = this._getEndpointUrl(req, 'generateContent'); + this.logApiRequest( + contents, + req.model, + userPromptId, + req.config, + serverDetails, + ); try { const response = await this.wrapped.generateContent( req, @@ -230,11 +247,17 @@ export class LoggingContentGenerator implements ContentGenerator { async ({ metadata: spanMetadata, endSpan }) => { spanMetadata.input = { request: req, userPromptId, model: req.model }; const startTime = Date.now(); - this.logApiRequest(toContents(req.contents), req.model, userPromptId); const serverDetails = this._getEndpointUrl( req, 'generateContentStream', ); + this.logApiRequest( + toContents(req.contents), + req.model, + userPromptId, + req.config, + serverDetails, + ); let stream: AsyncGenerator; try { diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index 9ae976b7b2..bdf5de6865 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -637,50 +637,249 @@ describe('loggers', () => { getTelemetryEnabled: () => true, getTelemetryLogPromptsEnabled: () => true, isInteractive: () => false, + getContentGeneratorConfig: () => ({ + authType: AuthType.LOGIN_WITH_GOOGLE, + }), } as Config; it('should log an API request with request_text', () => { const event = new ApiRequestEvent( 'test-model', - 'prompt-id-7', + { + prompt_id: 'prompt-id-7', + contents: [], + }, 'This is a test request', ); logApiRequest(mockConfig, event); - expect(mockLogger.emit).toHaveBeenCalledWith({ + expect(mockLogger.emit).toHaveBeenNthCalledWith(1, { body: 'API request to test-model.', - attributes: { - 'session.id': 'test-session-id', - 'user.email': 'test-user@example.com', - 'installation.id': 'test-installation-id', + attributes: expect.objectContaining({ 'event.name': EVENT_API_REQUEST, 'event.timestamp': '2025-01-01T00:00:00.000Z', interactive: false, model: 'test-model', request_text: 'This is a test request', prompt_id: 'prompt-id-7', - }, + }), + }); + + expect(mockLogger.emit).toHaveBeenNthCalledWith(2, { + body: 'GenAI operation request details from test-model.', + attributes: expect.objectContaining({ + 'event.name': 'gen_ai.client.inference.operation.details', + 'gen_ai.request.model': 'test-model', + 'gen_ai.provider.name': 'gcp.vertex_ai', + }), }); }); it('should log an API request without request_text', () => { - const event = new ApiRequestEvent('test-model', 'prompt-id-6'); + const event = new ApiRequestEvent('test-model', { + prompt_id: 'prompt-id-6', + contents: [], + }); logApiRequest(mockConfig, event); - expect(mockLogger.emit).toHaveBeenCalledWith({ + expect(mockLogger.emit).toHaveBeenNthCalledWith(1, { body: 'API request to test-model.', - attributes: { - 'session.id': 'test-session-id', - 'user.email': 'test-user@example.com', - 'installation.id': 'test-installation-id', + attributes: expect.objectContaining({ 'event.name': EVENT_API_REQUEST, 'event.timestamp': '2025-01-01T00:00:00.000Z', interactive: false, model: 'test-model', prompt_id: 'prompt-id-6', + }), + }); + + expect(mockLogger.emit).toHaveBeenNthCalledWith(2, { + body: 'GenAI operation request details from test-model.', + attributes: expect.objectContaining({ + 'event.name': 'gen_ai.client.inference.operation.details', + 'gen_ai.request.model': 'test-model', + 'gen_ai.provider.name': 'gcp.vertex_ai', + }), + }); + }); + + it('should log an API request with full semantic details when logPrompts is enabled', () => { + const mockConfigWithPrompts = { + getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getUsageStatisticsEnabled: () => true, + getTelemetryEnabled: () => true, + getTelemetryLogPromptsEnabled: () => true, // Enabled + isInteractive: () => false, + getContentGeneratorConfig: () => ({ + authType: AuthType.USE_GEMINI, + }), + } as Config; + + const promptDetails = { + prompt_id: 'prompt-id-semantic-1', + contents: [ + { + role: 'user', + parts: [{ text: 'Semantic request test' }], + }, + ], + generate_content_config: { + temperature: 0.5, + topP: 0.8, + topK: 10, + responseMimeType: 'application/json', + candidateCount: 1, + stopSequences: ['end'], + systemInstruction: { + role: 'model', + parts: [{ text: 'be helpful' }], + }, }, + server: { + address: 'semantic-api.example.com', + port: 8080, + }, + }; + + const event = new ApiRequestEvent( + 'test-model', + promptDetails, + 'Full semantic request', + ); + + logApiRequest(mockConfigWithPrompts, event); + + // Expect two calls to emit: one for the regular log, one for the semantic log + expect(mockLogger.emit).toHaveBeenCalledTimes(2); + + // Verify the first (original) log record + expect(mockLogger.emit).toHaveBeenNthCalledWith(1, { + body: 'API request to test-model.', + attributes: expect.objectContaining({ + 'event.name': EVENT_API_REQUEST, + prompt_id: 'prompt-id-semantic-1', + }), + }); + + // Verify the second (semantic) log record + expect(mockLogger.emit).toHaveBeenNthCalledWith(2, { + body: 'GenAI operation request details from test-model.', + attributes: expect.objectContaining({ + 'event.name': 'gen_ai.client.inference.operation.details', + 'gen_ai.request.model': 'test-model', + 'gen_ai.request.temperature': 0.5, + 'gen_ai.request.top_p': 0.8, + 'gen_ai.request.top_k': 10, + 'gen_ai.input.messages': JSON.stringify([ + { + role: 'user', + parts: [{ type: 'text', content: 'Semantic request test' }], + }, + ]), + 'server.address': 'semantic-api.example.com', + 'server.port': 8080, + 'gen_ai.operation.name': 'generate_content', + 'gen_ai.provider.name': 'gcp.gen_ai', + 'gen_ai.output.type': 'json', + 'gen_ai.request.stop_sequences': ['end'], + 'gen_ai.system_instructions': JSON.stringify([ + { type: 'text', content: 'be helpful' }, + ]), + }), + }); + }); + + it('should log an API request with semantic details, but without prompts when logPrompts is disabled', () => { + const mockConfigWithoutPrompts = { + getSessionId: () => 'test-session-id', + getTargetDir: () => 'target-dir', + getUsageStatisticsEnabled: () => true, + getTelemetryEnabled: () => true, + getTelemetryLogPromptsEnabled: () => false, // Disabled + isInteractive: () => false, + getContentGeneratorConfig: () => ({ + authType: AuthType.USE_VERTEX_AI, + }), + } as Config; + + const promptDetails = { + prompt_id: 'prompt-id-semantic-2', + contents: [ + { + role: 'user', + parts: [{ text: 'This prompt should be hidden' }], + }, + ], + generate_content_config: {}, + model: 'gemini-1.0-pro', + }; + + const event = new ApiRequestEvent( + 'gemini-1.0-pro', + promptDetails, + 'Request with hidden prompt', + ); + + logApiRequest(mockConfigWithoutPrompts, event); + + // Expect two calls to emit + expect(mockLogger.emit).toHaveBeenCalledTimes(2); + + // Get the arguments of the second (semantic) log call + const semanticLogCall = mockLogger.emit.mock.calls[1][0]; + + // Assert on the body + expect(semanticLogCall.body).toBe( + 'GenAI operation request details from gemini-1.0-pro.', + ); + + // Assert on specific attributes + const attributes = semanticLogCall.attributes; + expect(attributes['event.name']).toBe( + 'gen_ai.client.inference.operation.details', + ); + expect(attributes['gen_ai.request.model']).toBe('gemini-1.0-pro'); + expect(attributes['gen_ai.provider.name']).toBe('gcp.vertex_ai'); + // Ensure prompt messages are NOT included + expect(attributes['gen_ai.input.messages']).toBeUndefined(); + }); + + it('should correctly derive model from prompt details if available in semantic log', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + getTelemetryEnabled: () => true, + getTelemetryLogPromptsEnabled: () => true, + isInteractive: () => false, + getUsageStatisticsEnabled: () => true, + getContentGeneratorConfig: () => ({ + authType: AuthType.USE_GEMINI, + }), + } as Config; + + const promptDetails = { + prompt_id: 'prompt-id-semantic-3', + contents: [], + model: 'my-custom-model', + }; + + const event = new ApiRequestEvent( + 'my-custom-model', + promptDetails, + 'Request with custom model', + ); + + logApiRequest(mockConfig, event); + + // Verify the second (semantic) log record + expect(mockLogger.emit).toHaveBeenNthCalledWith(2, { + body: 'GenAI operation request details from my-custom-model.', + attributes: expect.objectContaining({ + 'event.name': 'gen_ai.client.inference.operation.details', + 'gen_ai.request.model': 'my-custom-model', + }), }); }); }); diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts index 41ff664487..20525e6529 100644 --- a/packages/core/src/telemetry/loggers.ts +++ b/packages/core/src/telemetry/loggers.ts @@ -184,11 +184,8 @@ export function logApiRequest(config: Config, event: ApiRequestEvent): void { if (!isTelemetrySdkInitialized()) return; const logger = logs.getLogger(SERVICE_NAME); - const logRecord: LogRecord = { - body: event.toLogBody(), - attributes: event.toOpenTelemetryAttributes(config), - }; - logger.emit(logRecord); + logger.emit(event.toLogRecord(config)); + logger.emit(event.toSemanticLogRecord(config)); } export function logFlashFallback( diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index fe218ecee1..680b9cd6a1 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -364,30 +364,64 @@ export class ApiRequestEvent implements BaseTelemetryEvent { 'event.name': 'api_request'; 'event.timestamp': string; model: string; - prompt_id: string; + prompt: GenAIPromptDetails; request_text?: string; - constructor(model: string, prompt_id: string, request_text?: string) { + constructor( + model: string, + prompt_details: GenAIPromptDetails, + request_text?: string, + ) { this['event.name'] = 'api_request'; this['event.timestamp'] = new Date().toISOString(); this.model = model; - this.prompt_id = prompt_id; + this.prompt = prompt_details; this.request_text = request_text; } - toOpenTelemetryAttributes(config: Config): LogAttributes { - return { + toLogRecord(config: Config): LogRecord { + const attributes: LogAttributes = { ...getCommonAttributes(config), 'event.name': EVENT_API_REQUEST, 'event.timestamp': this['event.timestamp'], model: this.model, - prompt_id: this.prompt_id, + prompt_id: this.prompt.prompt_id, request_text: this.request_text, }; + return { body: `API request to ${this.model}.`, attributes }; } - toLogBody(): string { - return `API request to ${this.model}.`; + toSemanticLogRecord(config: Config): LogRecord { + const { 'gen_ai.response.model': _, ...requestConventionAttributes } = + getConventionAttributes({ + model: this.model, + auth_type: config.getContentGeneratorConfig()?.authType, + }); + const attributes: LogAttributes = { + ...getCommonAttributes(config), + 'event.name': EVENT_GEN_AI_OPERATION_DETAILS, + 'event.timestamp': this['event.timestamp'], + ...toGenerateContentConfigAttributes(this.prompt.generate_content_config), + ...requestConventionAttributes, + }; + + if (this.prompt.server) { + attributes['server.address'] = this.prompt.server.address; + attributes['server.port'] = this.prompt.server.port; + } + + if (config.getTelemetryLogPromptsEnabled() && this.prompt.contents) { + attributes['gen_ai.input.messages'] = JSON.stringify( + toInputMessages(this.prompt.contents), + ); + } + + const logRecord: LogRecord = { + body: `GenAI operation request details from ${this.model}.`, + attributes, + }; + + return logRecord; } }