MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-03-10 22:21:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): Add Semantic logging for to ApiRequestEvents (#13912)

Browse Source 
Co-authored-by: Shnatu <snatu@google.com>
This commit is contained in:
Shardul Natu
2025-11-30 09:05:24 +05:30
committed by GitHub
parent 576fda18eb
commit bbd61f375f
4 changed files with 282 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
packages/core/src/core/loggingContentGenerator.ts
View File

@@ -55,11 +55,22 @@ export class LoggingContentGenerator implements ContentGenerator {
contents: Content[],
model: string,
promptId: string,
generationConfig?: GenerateContentConfig,
serverDetails?: ServerDetails,
): void {
const requestText = JSON.stringify(contents);
logApiRequest(
this.config,
new ApiRequestEvent(model, promptId, requestText),
new ApiRequestEvent(
model,
{
prompt_id: promptId,
contents,
generate_content_config: generationConfig,
server: serverDetails,
},
requestText,
),
);
}
@@ -176,8 +187,14 @@ export class LoggingContentGenerator implements ContentGenerator {
const startTime = Date.now();
const contents: Content[] = toContents(req.contents);
this.logApiRequest(toContents(req.contents), req.model, userPromptId);
const serverDetails = this._getEndpointUrl(req, 'generateContent');
this.logApiRequest(
contents,
req.model,
userPromptId,
req.config,
serverDetails,
);
try {
const response = await this.wrapped.generateContent(
req,
@@ -230,11 +247,17 @@ export class LoggingContentGenerator implements ContentGenerator {
async ({ metadata: spanMetadata, endSpan }) => {
spanMetadata.input = { request: req, userPromptId, model: req.model };
const startTime = Date.now();
this.logApiRequest(toContents(req.contents), req.model, userPromptId);
const serverDetails = this._getEndpointUrl(
req,
'generateContentStream',
);
this.logApiRequest(
toContents(req.contents),
req.model,
userPromptId,
req.config,
serverDetails,
);
let stream: AsyncGenerator<GenerateContentResponse>;
try {

225
packages/core/src/telemetry/loggers.test.ts
View File

@@ -637,50 +637,249 @@ describe('loggers', () => {
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
isInteractive: () => false,
getContentGeneratorConfig: () => ({
authType: AuthType.LOGIN_WITH_GOOGLE,
}),
} as Config;
it('should log an API request with request_text', () => {
const event = new ApiRequestEvent(
'test-model',
'prompt-id-7',
{
prompt_id: 'prompt-id-7',
contents: [],
},
'This is a test request',
);
logApiRequest(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
expect(mockLogger.emit).toHaveBeenNthCalledWith(1, {
body: 'API request to test-model.',
attributes: {
'session.id': 'test-session-id',
'user.email': 'test-user@example.com',
'installation.id': 'test-installation-id',
attributes: expect.objectContaining({
'event.name': EVENT_API_REQUEST,
'event.timestamp': '2025-01-01T00:00:00.000Z',
interactive: false,
model: 'test-model',
request_text: 'This is a test request',
prompt_id: 'prompt-id-7',
},
}),
});
expect(mockLogger.emit).toHaveBeenNthCalledWith(2, {
body: 'GenAI operation request details from test-model.',
attributes: expect.objectContaining({
'event.name': 'gen_ai.client.inference.operation.details',
'gen_ai.request.model': 'test-model',
'gen_ai.provider.name': 'gcp.vertex_ai',
}),
});
});
it('should log an API request without request_text', () => {
const event = new ApiRequestEvent('test-model', 'prompt-id-6');
const event = new ApiRequestEvent('test-model', {
prompt_id: 'prompt-id-6',
contents: [],
});
logApiRequest(mockConfig, event);
expect(mockLogger.emit).toHaveBeenCalledWith({
expect(mockLogger.emit).toHaveBeenNthCalledWith(1, {
body: 'API request to test-model.',
attributes: {
'session.id': 'test-session-id',
'user.email': 'test-user@example.com',
'installation.id': 'test-installation-id',
attributes: expect.objectContaining({
'event.name': EVENT_API_REQUEST,
'event.timestamp': '2025-01-01T00:00:00.000Z',
interactive: false,
model: 'test-model',
prompt_id: 'prompt-id-6',
}),
});
expect(mockLogger.emit).toHaveBeenNthCalledWith(2, {
body: 'GenAI operation request details from test-model.',
attributes: expect.objectContaining({
'event.name': 'gen_ai.client.inference.operation.details',
'gen_ai.request.model': 'test-model',
'gen_ai.provider.name': 'gcp.vertex_ai',
}),
});
});
it('should log an API request with full semantic details when logPrompts is enabled', () => {
const mockConfigWithPrompts = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getUsageStatisticsEnabled: () => true,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true, // Enabled
isInteractive: () => false,
getContentGeneratorConfig: () => ({
authType: AuthType.USE_GEMINI,
}),
} as Config;
const promptDetails = {
prompt_id: 'prompt-id-semantic-1',
contents: [
{
role: 'user',
parts: [{ text: 'Semantic request test' }],
},
],
generate_content_config: {
temperature: 0.5,
topP: 0.8,
topK: 10,
responseMimeType: 'application/json',
candidateCount: 1,
stopSequences: ['end'],
systemInstruction: {
role: 'model',
parts: [{ text: 'be helpful' }],
},
},
server: {
address: 'semantic-api.example.com',
port: 8080,
},
};
const event = new ApiRequestEvent(
'test-model',
promptDetails,
'Full semantic request',
);
logApiRequest(mockConfigWithPrompts, event);
// Expect two calls to emit: one for the regular log, one for the semantic log
expect(mockLogger.emit).toHaveBeenCalledTimes(2);
// Verify the first (original) log record
expect(mockLogger.emit).toHaveBeenNthCalledWith(1, {
body: 'API request to test-model.',
attributes: expect.objectContaining({
'event.name': EVENT_API_REQUEST,
prompt_id: 'prompt-id-semantic-1',
}),
});
// Verify the second (semantic) log record
expect(mockLogger.emit).toHaveBeenNthCalledWith(2, {
body: 'GenAI operation request details from test-model.',
attributes: expect.objectContaining({
'event.name': 'gen_ai.client.inference.operation.details',
'gen_ai.request.model': 'test-model',
'gen_ai.request.temperature': 0.5,
'gen_ai.request.top_p': 0.8,
'gen_ai.request.top_k': 10,
'gen_ai.input.messages': JSON.stringify([
{
role: 'user',
parts: [{ type: 'text', content: 'Semantic request test' }],
},
]),
'server.address': 'semantic-api.example.com',
'server.port': 8080,
'gen_ai.operation.name': 'generate_content',
'gen_ai.provider.name': 'gcp.gen_ai',
'gen_ai.output.type': 'json',
'gen_ai.request.stop_sequences': ['end'],
'gen_ai.system_instructions': JSON.stringify([
{ type: 'text', content: 'be helpful' },
]),
}),
});
});
it('should log an API request with semantic details, but without prompts when logPrompts is disabled', () => {
const mockConfigWithoutPrompts = {
getSessionId: () => 'test-session-id',
getTargetDir: () => 'target-dir',
getUsageStatisticsEnabled: () => true,
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => false, // Disabled
isInteractive: () => false,
getContentGeneratorConfig: () => ({
authType: AuthType.USE_VERTEX_AI,
}),
} as Config;
const promptDetails = {
prompt_id: 'prompt-id-semantic-2',
contents: [
{
role: 'user',
parts: [{ text: 'This prompt should be hidden' }],
},
],
generate_content_config: {},
model: 'gemini-1.0-pro',
};
const event = new ApiRequestEvent(
'gemini-1.0-pro',
promptDetails,
'Request with hidden prompt',
);
logApiRequest(mockConfigWithoutPrompts, event);
// Expect two calls to emit
expect(mockLogger.emit).toHaveBeenCalledTimes(2);
// Get the arguments of the second (semantic) log call
const semanticLogCall = mockLogger.emit.mock.calls[1][0];
// Assert on the body
expect(semanticLogCall.body).toBe(
'GenAI operation request details from gemini-1.0-pro.',
);
// Assert on specific attributes
const attributes = semanticLogCall.attributes;
expect(attributes['event.name']).toBe(
'gen_ai.client.inference.operation.details',
);
expect(attributes['gen_ai.request.model']).toBe('gemini-1.0-pro');
expect(attributes['gen_ai.provider.name']).toBe('gcp.vertex_ai');
// Ensure prompt messages are NOT included
expect(attributes['gen_ai.input.messages']).toBeUndefined();
});
it('should correctly derive model from prompt details if available in semantic log', () => {
const mockConfig = {
getSessionId: () => 'test-session-id',
getTelemetryEnabled: () => true,
getTelemetryLogPromptsEnabled: () => true,
isInteractive: () => false,
getUsageStatisticsEnabled: () => true,
getContentGeneratorConfig: () => ({
authType: AuthType.USE_GEMINI,
}),
} as Config;
const promptDetails = {
prompt_id: 'prompt-id-semantic-3',
contents: [],
model: 'my-custom-model',
};
const event = new ApiRequestEvent(
'my-custom-model',
promptDetails,
'Request with custom model',
);
logApiRequest(mockConfig, event);
// Verify the second (semantic) log record
expect(mockLogger.emit).toHaveBeenNthCalledWith(2, {
body: 'GenAI operation request details from my-custom-model.',
attributes: expect.objectContaining({
'event.name': 'gen_ai.client.inference.operation.details',
'gen_ai.request.model': 'my-custom-model',
}),
});
});
});

7
packages/core/src/telemetry/loggers.ts
View File

@@ -184,11 +184,8 @@ export function logApiRequest(config: Config, event: ApiRequestEvent): void {
if (!isTelemetrySdkInitialized()) return;
const logger = logs.getLogger(SERVICE_NAME);
const logRecord: LogRecord = {
body: event.toLogBody(),
attributes: event.toOpenTelemetryAttributes(config),
};
logger.emit(logRecord);
logger.emit(event.toLogRecord(config));
logger.emit(event.toSemanticLogRecord(config));
}
export function logFlashFallback(

50
packages/core/src/telemetry/types.ts
View File

@@ -364,30 +364,64 @@ export class ApiRequestEvent implements BaseTelemetryEvent {
'event.name': 'api_request';
'event.timestamp': string;
model: string;
prompt_id: string;
prompt: GenAIPromptDetails;
request_text?: string;
constructor(model: string, prompt_id: string, request_text?: string) {
constructor(
model: string,
prompt_details: GenAIPromptDetails,
request_text?: string,
) {
this['event.name'] = 'api_request';
this['event.timestamp'] = new Date().toISOString();
this.model = model;
this.prompt_id = prompt_id;
this.prompt = prompt_details;
this.request_text = request_text;
}
toOpenTelemetryAttributes(config: Config): LogAttributes {
return {
toLogRecord(config: Config): LogRecord {
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_API_REQUEST,
'event.timestamp': this['event.timestamp'],
model: this.model,
prompt_id: this.prompt_id,
prompt_id: this.prompt.prompt_id,
request_text: this.request_text,
};
return { body: `API request to ${this.model}.`, attributes };
}
toLogBody(): string {
return `API request to ${this.model}.`;
toSemanticLogRecord(config: Config): LogRecord {
const { 'gen_ai.response.model': _, ...requestConventionAttributes } =
getConventionAttributes({
model: this.model,
auth_type: config.getContentGeneratorConfig()?.authType,
});
const attributes: LogAttributes = {
...getCommonAttributes(config),
'event.name': EVENT_GEN_AI_OPERATION_DETAILS,
'event.timestamp': this['event.timestamp'],
...toGenerateContentConfigAttributes(this.prompt.generate_content_config),
...requestConventionAttributes,
};
if (this.prompt.server) {
attributes['server.address'] = this.prompt.server.address;
attributes['server.port'] = this.prompt.server.port;
}
if (config.getTelemetryLogPromptsEnabled() && this.prompt.contents) {
attributes['gen_ai.input.messages'] = JSON.stringify(
toInputMessages(this.prompt.contents),
);
}
const logRecord: LogRecord = {
body: `GenAI operation request details from ${this.model}.`,
attributes,
};
return logRecord;
}
}