Compare commits

...

10 Commits

Author SHA1 Message Date
Spencer 8a5c470faf Merge branch 'main' into fix-telemetry-truncation 2026-04-23 15:45:50 -04:00
Spencer ae509f1ba5 test(integration): fix stdin buffering race condition in file-system interactive test 2026-04-21 20:44:02 +00:00
Spencer 65c3d7cabb test(integration): fix command loader race condition and test timeouts in interactive compression test 2026-04-21 20:44:02 +00:00
Spencer c72ac00a53 fix(telemetry): resolve OOM risk with Array.from, add surrogate pair protection, and fix try/catch key scope 2026-04-21 20:44:02 +00:00
Spencer 2bc481a3b5 fix(telemetry): strip high-cardinality attributes from metrics
Decouples generic OpenTelemetry attributes from metric-specific
attributes to resolve a Cloud Monitoring 'Internal error encountered'
caused by a cardinality explosion. High-cardinality values like
session.id, installation.id, user.email, and experiments.ids
are now excluded from time series metric labels, but they
are still preserved on traces and logs.

Also refactors getCommonAttributes to use getCommonMetricAttributes
to reduce repetition, and adds robust unit tests covering the
truncation of the experiments.ids array and fallback cases.
2026-04-21 20:44:01 +00:00
Spencer 169f9e457d fix: apply bounded structural truncation to Event Logs 2026-04-21 20:44:01 +00:00
Spencer 8481d977bb fix: add back global string limit gracefully returning a valid json string 2026-04-21 20:44:01 +00:00
Spencer dd4d7301ba fix: remove global string slice to guarantee JSON parseability 2026-04-21 20:44:01 +00:00
Spencer 5357770a3d fix: address structural truncation review comments 2026-04-21 20:44:01 +00:00
Spencer dc3ab31af0 fix(telemetry): implement bounded structural truncation 2026-04-21 20:44:01 +00:00
10 changed files with 337 additions and 46 deletions
@@ -58,7 +58,7 @@ describe.skipIf(skipOnDarwin)('Interactive Mode', () => {
);
await run.expectText('Chat history compressed', 5000);
});
}, 60000);
// TODO: Context compression is broken and doesn't include the system
// instructions or tool counts, so it thinks compression is beneficial when
@@ -102,6 +102,9 @@ describe.skipIf(skipOnDarwin)('Interactive Mode', () => {
});
const run = await rig.runInteractive();
await run.expectText('tips for getting started:', 5000);
// Wait for the async command loaders to finish so /compress is recognized
await new Promise((r) => setTimeout(r, 2000));
await run.type('/compress');
await run.type('\r');
@@ -116,5 +119,5 @@ describe.skipIf(skipOnDarwin)('Interactive Mode', () => {
foundEvent,
'chat_compression telemetry event should not be found for NOOP',
).toBe(false);
});
}, 60000);
});
@@ -33,6 +33,8 @@ describe('Interactive file system', () => {
rig.createFile(fileName, '1.0.0');
const run = await rig.runInteractive();
await run.expectText('tips for getting started:', 5000);
await new Promise((r) => setTimeout(r, 1000));
// Step 1: Read the file
const readPrompt = `Read the version from ${fileName}`;
@@ -56,5 +58,5 @@ describe('Interactive file system', () => {
// Wait for telemetry to flush and file system to sync, especially in sandboxed environments
await rig.waitForTelemetryReady();
});
}, 60000);
});
+4 -2
View File
@@ -121,8 +121,10 @@ describe('Telemetry Metrics', () => {
return actualApi;
});
const { getCommonAttributes } = await import('./telemetryAttributes.js');
(getCommonAttributes as Mock).mockReturnValue({
const { getCommonMetricAttributes } = await import(
'./telemetryAttributes.js'
);
(getCommonMetricAttributes as Mock).mockReturnValue({
'session.id': 'test-session-id',
'installation.id': 'test-installation-id',
'user.email': 'test@example.com',
+2 -2
View File
@@ -24,7 +24,7 @@ import type {
TokenStorageInitializationEvent,
} from './types.js';
import { AuthType } from '../core/contentGenerator.js';
import { getCommonAttributes } from './telemetryAttributes.js';
import { getCommonMetricAttributes } from './telemetryAttributes.js';
import { sanitizeHookName } from './sanitize.js';
const EVENT_CHAT_COMPRESSION = 'gemini_cli.chat_compression';
@@ -104,7 +104,7 @@ const EXIT_FAIL_COUNT = 'gemini_cli.exit.fail.count';
const PLAN_EXECUTION_COUNT = 'gemini_cli.plan.execution.count';
const baseMetricDefinition = {
getCommonAttributes,
getCommonAttributes: getCommonMetricAttributes,
};
const COUNTER_DEFINITIONS = {
@@ -0,0 +1,115 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest';
import {
getCommonAttributes,
getCommonMetricAttributes,
} from './telemetryAttributes.js';
import type { Config } from '../config/config.js';
import { UserAccountManager } from '../utils/userAccountManager.js';
import { InstallationManager } from '../utils/installationManager.js';
vi.mock('../utils/userAccountManager.js');
vi.mock('../utils/installationManager.js');
describe('telemetryAttributes', () => {
let mockConfig: Partial<Config>;
beforeEach(() => {
vi.resetAllMocks();
mockConfig = {
getSessionId: vi.fn().mockReturnValue('mock-session-id'),
isInteractive: vi.fn().mockReturnValue(true),
getExperiments: vi.fn().mockReturnValue(undefined),
getContentGeneratorConfig: vi.fn().mockReturnValue(undefined),
};
(
UserAccountManager.prototype.getCachedGoogleAccount as Mock
).mockReturnValue(undefined);
(InstallationManager.prototype.getInstallationId as Mock).mockReturnValue(
'mock-install-id',
);
});
describe('getCommonMetricAttributes', () => {
it('should return interactive and auth_type when defined', () => {
mockConfig.getContentGeneratorConfig = vi
.fn()
.mockReturnValue({ authType: 'oauth-personal' });
const attributes = getCommonMetricAttributes(mockConfig as Config);
expect(attributes).toEqual({
interactive: true,
auth_type: 'oauth-personal',
});
});
it('should return only interactive when auth_type is not defined', () => {
const attributes = getCommonMetricAttributes(mockConfig as Config);
expect(attributes).toEqual({
interactive: true,
});
});
});
describe('getCommonAttributes', () => {
it('should include all common attributes', () => {
(
UserAccountManager.prototype.getCachedGoogleAccount as Mock
).mockReturnValue('test@google.com');
mockConfig.getExperiments = vi
.fn()
.mockReturnValue({ experimentIds: [123, 456] });
mockConfig.getContentGeneratorConfig = vi
.fn()
.mockReturnValue({ authType: 'adc' });
const attributes = getCommonAttributes(mockConfig as Config);
expect(attributes).toEqual({
'session.id': 'mock-session-id',
'installation.id': 'mock-install-id',
interactive: true,
'user.email': 'test@google.com',
auth_type: 'adc',
'experiments.ids': '123,456',
});
});
it('should safely truncate experiments string to not exceed 1000 characters and not cut mid-ID', () => {
// Generate a list of experiment IDs that will produce a string > 1000 chars
const expIds = [];
for (let i = 0; i < 200; i++) {
// e.g., 100000000 -> 9 chars + 1 comma = 10 chars per ID
expIds.push(100000000 + i);
}
mockConfig.getExperiments = vi
.fn()
.mockReturnValue({ experimentIds: expIds });
const attributes = getCommonAttributes(mockConfig as Config);
const expString = attributes['experiments.ids'] as string;
expect(expString.length).toBeLessThanOrEqual(1000);
// Verify the last ID is complete (not cut off) by checking if it's one of our expected IDs
const ids = expString.split(',');
const lastIdStr = ids[ids.length - 1];
const lastIdNumber = parseInt(lastIdStr, 10);
expect(lastIdNumber).toBeGreaterThanOrEqual(100000000);
expect(lastIdNumber).toBeLessThan(100000200);
// Also ensure no trailing comma
expect(expString.endsWith(',')).toBe(false);
});
});
});
@@ -12,19 +12,36 @@ import { UserAccountManager } from '../utils/userAccountManager.js';
const userAccountManager = new UserAccountManager();
const installationManager = new InstallationManager();
export function getCommonMetricAttributes(config: Config): Attributes {
const authType = config.getContentGeneratorConfig()?.authType;
return {
interactive: config.isInteractive(),
...(authType && { auth_type: authType }),
};
}
export function getCommonAttributes(config: Config): Attributes {
const email = userAccountManager.getCachedGoogleAccount();
const experiments = config.getExperiments();
const authType = config.getContentGeneratorConfig()?.authType;
let experimentsIdsStr = '';
if (experiments && experiments.experimentIds.length > 0) {
experimentsIdsStr = experiments.experimentIds.join(',');
if (experimentsIdsStr.length > 1000) {
experimentsIdsStr = experimentsIdsStr.substring(0, 1000);
const lastCommaIndex = experimentsIdsStr.lastIndexOf(',');
if (lastCommaIndex > 0) {
experimentsIdsStr = experimentsIdsStr.substring(0, lastCommaIndex);
}
}
}
return {
...getCommonMetricAttributes(config),
'session.id': config.getSessionId(),
'installation.id': installationManager.getInstallationId(),
interactive: config.isInteractive(),
...(email && { 'user.email': email }),
...(authType && { auth_type: authType }),
...(experiments &&
experiments.experimentIds.length > 0 && {
'experiments.ids': experiments.experimentIds,
}),
...(experimentsIdsStr && { 'experiments.ids': experimentsIdsStr }),
};
}
+78 -9
View File
@@ -52,26 +52,95 @@ describe('truncateForTelemetry', () => {
});
it('should correctly truncate strings with multi-byte unicode characters (emojis)', () => {
// 5 emojis, each is multiple bytes in UTF-16
// 5 emojis, each is a surrogate pair (2 code units)
const emojis = '👋🌍🚀🔥🎉';
// Truncating to length 5 (which is 2.5 emojis in UTF-16 length terms)
// truncateString will stop after the full grapheme clusters that fit within 5
const result = truncateForTelemetry(emojis, 5);
// Truncating to 2 code units
const result = truncateForTelemetry(emojis, 2);
expect(result).toBe('👋🌍...[TRUNCATED: original length 10]');
expect(result).toBe('👋...[TRUNCATED: original length 10]');
});
it('should stringify and truncate objects if exceeding maxLength', () => {
it('should stringify and structurally truncate objects if exceeding limits', () => {
const obj = { message: 'hello world', nested: { a: 1 } };
const stringified = JSON.stringify(obj);
const result = truncateForTelemetry(obj, 10);
expect(result).toBe(
stringified.substring(0, 10) +
`...[TRUNCATED: original length ${stringified.length}]`,
JSON.stringify({
message: 'hello worl...[TRUNCATED: original length 11]',
nested: { a: 1 },
}),
);
});
it('should structurally truncate arrays and depth', () => {
const obj = {
arr: [1, 2, 3],
deep: { level1: { level2: { level3: { a: 1 } } } },
};
const result = truncateForTelemetry(obj, 100, 2, 3);
expect(result).toBe(
JSON.stringify({
arr: [1, 2, '[TRUNCATED: Array of length 3]'],
deep: { level1: { level2: '[TRUNCATED: Max Depth Reached]' } },
}),
);
});
it('should handle objects with a toJSON method', () => {
const date = new Date('2026-04-13T00:00:00.000Z');
const result = truncateForTelemetry(date);
expect(result).toBe('2026-04-13T00:00:00.000Z');
});
it('should handle getters via direct property access', () => {
const obj = {
get myGetter() {
return 'getter value';
},
get errorGetter() {
throw new Error('getter error');
},
};
const result = truncateForTelemetry(obj);
expect(result).toBe(
JSON.stringify({
myGetter: 'getter value',
errorGetter: '[ERROR: Failed to read property]',
}),
);
});
it('should truncate extremely long keys', () => {
const longKey = 'a'.repeat(150);
const obj = {
[longKey]: 'value',
};
const result = truncateForTelemetry(obj);
const expectedKey = 'a'.repeat(100) + '...[TRUNCATED_KEY]';
expect(result).toBe(
JSON.stringify({
[expectedKey]: 'value',
}),
);
});
it('should enforce a global payload string limit without breaking JSON', () => {
const obj = {
a: 'x'.repeat(100),
b: 'y'.repeat(100),
};
// Capping global string length to 50
const result = truncateForTelemetry(obj, 100, 100, 4, 100, 50) as string;
// It should replace the entire object with a valid JSON string indicating truncation
expect(result).toBe(
'"[TRUNCATED: Payload exceeded global limit of 50 characters. Original length: 215]"',
);
// Prove it remains perfectly parseable JSON
expect(() => JSON.parse(result)).not.toThrow();
});
it('should stringify objects unchanged if within maxLength', () => {
const obj = { a: 1 };
expect(truncateForTelemetry(obj, 100)).toBe(JSON.stringify(obj));
+99 -18
View File
@@ -14,7 +14,6 @@ import {
import { debugLogger } from '../utils/debugLogger.js';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
import { truncateString } from '../utils/textUtils.js';
import {
GEN_AI_AGENT_DESCRIPTION,
GEN_AI_AGENT_NAME,
@@ -45,6 +44,17 @@ export const spanRegistry = new FinalizationRegistry((endSpan: () => void) => {
}
});
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null;
}
function hasToJSON(value: unknown): value is { toJSON: () => unknown } {
if (!isRecord(value)) return false;
if (!('toJSON' in value)) return false;
const toJSONFn = value['toJSON'];
return typeof toJSONFn === 'function';
}
/**
* Truncates a value for inclusion in telemetry attributes.
*
@@ -54,27 +64,98 @@ export const spanRegistry = new FinalizationRegistry((endSpan: () => void) => {
*/
export function truncateForTelemetry(
value: unknown,
maxLength = 10000,
maxStringLength = 10000,
maxArrayLength = 100,
maxDepth = 4,
maxObjectKeys = 100,
maxGlobalStringLength = 50000,
): AttributeValue | undefined {
if (typeof value === 'string') {
return truncateString(
value,
maxLength,
`...[TRUNCATED: original length ${value.length}]`,
) as AttributeValue;
const truncateObj = (v: unknown, depth: number): unknown => {
if (typeof v === 'string') {
if (v.length > maxStringLength) {
let truncatedStr = v.slice(0, maxStringLength);
if (truncatedStr.length > 0 && /[\uD800-\uDBFF]$/.test(truncatedStr)) {
truncatedStr = truncatedStr.slice(0, -1);
}
return truncatedStr + `...[TRUNCATED: original length ${v.length}]`;
}
return v;
}
if (
typeof v === 'number' ||
typeof v === 'boolean' ||
v === null ||
v === undefined
) {
return v;
}
if (hasToJSON(v)) {
try {
return truncateObj(v.toJSON(), depth);
} catch {
// Ignore and fall back to manual structural iteration
}
}
if (typeof v === 'object') {
if (depth >= maxDepth) {
return `[TRUNCATED: Max Depth Reached]`;
}
if (Array.isArray(v)) {
if (v.length > maxArrayLength) {
const truncatedArray = v
.slice(0, maxArrayLength)
.map((item) => truncateObj(item, depth + 1));
truncatedArray.push(`[TRUNCATED: Array of length ${v.length}]`);
return truncatedArray;
}
return v.map((item) => truncateObj(item, depth + 1));
}
const newObj: Record<string, unknown> = {};
let numKeys = 0;
const recordV = isRecord(v) ? v : {};
for (const key in recordV) {
if (!Object.prototype.hasOwnProperty.call(recordV, key)) continue;
if (numKeys >= maxObjectKeys) {
newObj['__truncated'] =
`[TRUNCATED: Object with >${maxObjectKeys} keys]`;
break;
}
const truncatedKey =
key.length > 100 ? key.slice(0, 100) + '...[TRUNCATED_KEY]' : key;
try {
const val = recordV[key];
newObj[truncatedKey] = truncateObj(val, depth + 1);
} catch {
newObj[truncatedKey] = '[ERROR: Failed to read property]';
}
numKeys++;
}
return newObj;
}
return undefined;
};
const truncated = truncateObj(value, 0);
if (
typeof truncated === 'string' ||
typeof truncated === 'number' ||
typeof truncated === 'boolean'
) {
return truncated as AttributeValue;
}
if (typeof value === 'object' && value !== null) {
const stringified = safeJsonStringify(value);
return truncateString(
stringified,
maxLength,
`...[TRUNCATED: original length ${stringified.length}]`,
) as AttributeValue;
if (truncated === null || truncated === undefined) {
return undefined;
}
if (typeof value === 'number' || typeof value === 'boolean') {
return value as AttributeValue;
const stringified = safeJsonStringify(truncated);
if (stringified.length > maxGlobalStringLength) {
return `"[TRUNCATED: Payload exceeded global limit of ${maxGlobalStringLength} characters. Original length: ${stringified.length}]"`;
}
return undefined;
return stringified as AttributeValue;
}
function isAsyncIterable<T>(value: T): value is T & AsyncIterable<unknown> {
+6 -5
View File
@@ -31,6 +31,7 @@ import type { AgentTerminateMode } from '../agents/types.js';
import { getCommonAttributes } from './telemetryAttributes.js';
import { SemanticAttributes } from '@opentelemetry/semantic-conventions';
import { safeJsonStringify } from '../utils/safeJsonStringify.js';
import { truncateForTelemetry } from './trace.js';
import {
toInputMessages,
toOutputMessages,
@@ -451,7 +452,7 @@ export class ApiRequestEvent implements BaseTelemetryEvent {
}
if (shouldIncludePayloads(config) && this.prompt.contents) {
attributes['gen_ai.input.messages'] = JSON.stringify(
attributes['gen_ai.input.messages'] = truncateForTelemetry(
toInputMessages(this.prompt.contents),
);
}
@@ -548,7 +549,7 @@ export class ApiErrorEvent implements BaseTelemetryEvent {
}
if (shouldIncludePayloads(config) && this.prompt.contents) {
attributes['gen_ai.input.messages'] = JSON.stringify(
attributes['gen_ai.input.messages'] = truncateForTelemetry(
toInputMessages(this.prompt.contents),
);
}
@@ -618,7 +619,7 @@ function toGenerateContentConfigAttributes(
'gen_ai.request.max_tokens': config.maxOutputTokens,
'gen_ai.output.type': toOutputType(config.responseMimeType),
'gen_ai.request.stop_sequences': config.stopSequences,
'gen_ai.system_instructions': JSON.stringify(
'gen_ai.system_instructions': truncateForTelemetry(
toSystemInstruction(config.systemInstruction),
),
};
@@ -716,7 +717,7 @@ export class ApiResponseEvent implements BaseTelemetryEvent {
'gen_ai.response.finish_reasons': this.finish_reasons,
...(shouldIncludePayloads(config)
? {
'gen_ai.output.messages': JSON.stringify(
'gen_ai.output.messages': truncateForTelemetry(
toOutputMessages(this.response.candidates),
),
}
@@ -731,7 +732,7 @@ export class ApiResponseEvent implements BaseTelemetryEvent {
}
if (shouldIncludePayloads(config) && this.prompt.contents) {
attributes['gen_ai.input.messages'] = JSON.stringify(
attributes['gen_ai.input.messages'] = truncateForTelemetry(
toInputMessages(this.prompt.contents),
);
}
+1
View File
@@ -457,6 +457,7 @@ export class TestRig {
// Nightly releases sometimes becomes out of sync with local code and
// triggers auto-update, which causes tests to fail.
enableAutoUpdate: false,
enableAutoUpdateNotification: false,
},
telemetry: {
enabled: true,