diff --git a/integration-tests/telemetry.test.ts b/integration-tests/telemetry.test.ts new file mode 100644 index 0000000000..111f24c866 --- /dev/null +++ b/integration-tests/telemetry.test.ts @@ -0,0 +1,26 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect } from 'vitest'; +import { TestRig } from './test-helper.js'; + +describe('telemetry', () => { + it('should emit a metric and a log event', async () => { + const rig = new TestRig(); + rig.setup('should emit a metric and a log event'); + + // Run a simple command that should trigger telemetry + await rig.run('just saying hi'); + + // Verify that a user_prompt event was logged + const hasUserPromptEvent = await rig.waitForTelemetryEvent('user_prompt'); + expect(hasUserPromptEvent).toBe(true); + + // Verify that a cli_command_count metric was emitted + const cliCommandCountMetric = rig.readMetric('session.count'); + expect(cliCommandCountMetric).not.toBeNull(); + }); +}); diff --git a/integration-tests/test-helper.ts b/integration-tests/test-helper.ts index a894be28b1..817323f648 100644 --- a/integration-tests/test-helper.ts +++ b/integration-tests/test-helper.ts @@ -112,6 +112,23 @@ export function validateModelOutput( return true; } +interface ParsedLog { + attributes?: { + 'event.name'?: string; + function_name?: string; + function_args?: string; + success?: boolean; + duration_ms?: number; + }; + scopeMetrics?: { + metrics: { + descriptor: { + name: string; + }; + }[]; + }[]; +} + export class TestRig { bundlePath: string; testDir: string | null; @@ -418,37 +435,12 @@ export class TestRig { return this.poll( () => { - const logFilePath = join(this.testDir!, 'telemetry.log'); - - if (!logFilePath || !fs.existsSync(logFilePath)) { - return false; - } - - const content = readFileSync(logFilePath, 'utf-8'); - const jsonObjects = content - .split(/}\n{/) - .map((obj, index, array) => { - // Add back the braces we removed during split - if (index > 0) obj = '{' + obj; - if (index < array.length - 1) obj = obj + '}'; - return obj.trim(); - }) - .filter((obj) => obj); - - for (const jsonStr of jsonObjects) { - try { - const logData = JSON.parse(jsonStr); - if ( - logData.attributes && - logData.attributes['event.name'] === `gemini_cli.${eventName}` - ) { - return true; - } - } catch { - // ignore - } - } - return false; + const logs = this._readAndParseTelemetryLog(); + return logs.some( + (logData) => + logData.attributes && + logData.attributes['event.name'] === `gemini_cli.${eventName}`, + ); }, timeout, 100, @@ -645,6 +637,45 @@ export class TestRig { return logs; } + private _readAndParseTelemetryLog(): ParsedLog[] { + // Telemetry is always written to the test directory + const logFilePath = join(this.testDir!, 'telemetry.log'); + + if (!logFilePath || !fs.existsSync(logFilePath)) { + return []; + } + + const content = readFileSync(logFilePath, 'utf-8'); + + // Split the content into individual JSON objects + // They are separated by "}\n{" + const jsonObjects = content + .split(/}\n{/) + .map((obj, index, array) => { + // Add back the braces we removed during split + if (index > 0) obj = '{' + obj; + if (index < array.length - 1) obj = obj + '}'; + return obj.trim(); + }) + .filter((obj) => obj); + + const logs: ParsedLog[] = []; + + for (const jsonStr of jsonObjects) { + try { + const logData = JSON.parse(jsonStr); + logs.push(logData); + } catch (e) { + // Skip objects that aren't valid JSON + if (env.VERBOSE === 'true') { + console.error('Failed to parse telemetry object:', e); + } + } + } + + return logs; + } + readToolLogs() { // For Podman, first check if telemetry file exists and has content // If not, fall back to parsing from stdout @@ -674,33 +705,7 @@ export class TestRig { } } - // Telemetry is always written to the test directory - const logFilePath = join(this.testDir!, 'telemetry.log'); - - if (!logFilePath) { - console.warn(`TELEMETRY_LOG_FILE environment variable not set`); - return []; - } - - // Check if file exists, if not return empty array (file might not be created yet) - if (!fs.existsSync(logFilePath)) { - return []; - } - - const content = readFileSync(logFilePath, 'utf-8'); - - // Split the content into individual JSON objects - // They are separated by "}\n{" - const jsonObjects = content - .split(/}\n{/) - .map((obj, index, array) => { - // Add back the braces we removed during split - if (index > 0) obj = '{' + obj; - if (index < array.length - 1) obj = obj + '}'; - return obj.trim(); - }) - .filter((obj) => obj); - + const parsedLogs = this._readAndParseTelemetryLog(); const logs: { toolRequest: { name: string; @@ -710,29 +715,21 @@ export class TestRig { }; }[] = []; - for (const jsonStr of jsonObjects) { - try { - const logData = JSON.parse(jsonStr); - // Look for tool call logs - if ( - logData.attributes && - logData.attributes['event.name'] === 'gemini_cli.tool_call' - ) { - const toolName = logData.attributes.function_name; - logs.push({ - toolRequest: { - name: toolName, - args: logData.attributes.function_args, - success: logData.attributes.success, - duration_ms: logData.attributes.duration_ms, - }, - }); - } - } catch (e) { - // Skip objects that aren't valid JSON - if (env.VERBOSE === 'true') { - console.error('Failed to parse telemetry object:', e); - } + for (const logData of parsedLogs) { + // Look for tool call logs + if ( + logData.attributes && + logData.attributes['event.name'] === 'gemini_cli.tool_call' + ) { + const toolName = logData.attributes.function_name; + logs.push({ + toolRequest: { + name: toolName, + args: logData.attributes.function_args, + success: logData.attributes.success, + duration_ms: logData.attributes.duration_ms, + }, + }); } } @@ -740,39 +737,29 @@ export class TestRig { } readLastApiRequest(): Record | null { - // Telemetry is always written to the test directory - const logFilePath = join(this.testDir!, 'telemetry.log'); + const logs = this._readAndParseTelemetryLog(); + const apiRequests = logs.filter( + (logData) => + logData.attributes && + logData.attributes['event.name'] === 'gemini_cli.api_request', + ); + return apiRequests.pop() || null; + } - if (!logFilePath || !fs.existsSync(logFilePath)) { - return null; - } - - const content = readFileSync(logFilePath, 'utf-8'); - const jsonObjects = content - .split(/}\n{/) - .map((obj, index, array) => { - if (index > 0) obj = '{' + obj; - if (index < array.length - 1) obj = obj + '}'; - return obj.trim(); - }) - .filter((obj) => obj); - - let lastApiRequest = null; - - for (const jsonStr of jsonObjects) { - try { - const logData = JSON.parse(jsonStr); - if ( - logData.attributes && - logData.attributes['event.name'] === 'gemini_cli.api_request' - ) { - lastApiRequest = logData; + readMetric(metricName: string): Record | null { + const logs = this._readAndParseTelemetryLog(); + for (const logData of logs) { + if (logData.scopeMetrics) { + for (const scopeMetric of logData.scopeMetrics) { + for (const metric of scopeMetric.metrics) { + if (metric.descriptor.name === `gemini_cli.${metricName}`) { + return metric; + } + } } - } catch { - // ignore } } - return lastApiRequest; + return null; } runInteractive(...args: string[]): {