Compare commits

...

2 Commits

Author SHA1 Message Date
Sri Pasumarthi 3ff7422790 attempts to run the test rig with eventloopdelay measure and other feedback 2026-04-10 12:53:36 -07:00
Sri Pasumarthi e74b093a42 test(perf): add terminal scrolling performance tests and event loop monitoring
Add a new performance test suite to simulate terminal scrolling and diagnose reported UI lag (indicated by eventloopDelay spikes) and high CPU usage. Make event loop monitoring opt-in to avoid overhead in production.

- Make EventLoopMonitor opt-in via GEMINI_EVENT_LOOP_MONITOR_ENABLED.
- Support passing custom environment variables to the child process in TestRig.
- Add CPU utilization percentage to the PerfTestHarness report.
- Inject child process event loop metrics into performance snapshots.
- Add a matrix of scrolling tests across different buffer configurations.
2026-04-10 09:13:28 -07:00
4 changed files with 330 additions and 2 deletions
+6 -2
View File
@@ -367,13 +367,17 @@ export class PerfTestHarness {
// Show CPU breakdown
const cpuMs = `${(result.median.cpuTotalUs / 1000).toFixed(1)} ms`;
const utilization = (
(result.median.cpuTotalUs / 1000 / result.median.wallClockMs) *
100
).toFixed(1);
lines.push(
` CPU: ${cpuMs} (user: ${formatUs(result.median.cpuUserUs)}, system: ${formatUs(result.median.cpuSystemUs)})`,
` CPU: ${cpuMs} (${utilization}%) (user: ${formatUs(result.median.cpuUserUs)}, system: ${formatUs(result.median.cpuSystemUs)})`,
);
if (result.median.eventLoopDelayMaxMs > 0) {
lines.push(
` Event loop (runner): p50=${result.median.eventLoopDelayP50Ms.toFixed(1)}ms p95=${result.median.eventLoopDelayP95Ms.toFixed(1)}ms max=${result.median.eventLoopDelayMaxMs.toFixed(1)}ms`,
` Event loop: p50=${result.median.eventLoopDelayP50Ms.toFixed(1)}ms p95=${result.median.eventLoopDelayP95Ms.toFixed(1)}ms max=${result.median.eventLoopDelayMaxMs.toFixed(1)}ms`,
);
}
+4
View File
@@ -1336,6 +1336,10 @@ export class TestRig {
return logs;
}
readTelemetryLogs(): ParsedLog[] {
return this._readAndParseTelemetryLog();
}
readToolLogs() {
// For Podman, first check if telemetry file exists and has content
// If not, fall back to parsing from stdout
+63
View File
@@ -0,0 +1,63 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { Storage } from '../packages/core/src/config/storage';
import { SessionSelector } from '../packages/cli/src/utils/sessionUtils';
import { join } from 'node:path';
import { mkdirSync, writeFileSync } from 'node:fs';
async function main() {
const testDir = '/private/tmp/perf-scroll-up-true-true';
mkdirSync(testDir, { recursive: true });
const storage = new Storage(testDir);
await storage.initialize();
console.log('Project Temp Dir:', storage.getProjectTempDir());
const chatsDir = join(storage.getProjectTempDir(), 'chats');
mkdirSync(chatsDir, { recursive: true });
const sessionFile = join(chatsDir, 'session-2026-04-10T00-00-test.jsonl');
const metadata = {
sessionId: 'test-session-id',
projectHash: 'test-project-hash',
startTime: new Date().toISOString(),
lastUpdated: new Date().toISOString(),
kind: 'main',
hasUserOrAssistantMessage: true,
messageCount: 1,
summary: 'Test Session',
};
const msg = {
id: 'msg-0',
timestamp: new Date().toISOString(),
type: 'user',
content: 'Message content 0',
};
writeFileSync(
sessionFile,
JSON.stringify(metadata) + '\n' + JSON.stringify(msg) + '\n',
);
console.log('Created file:', sessionFile);
const selector = new SessionSelector(storage);
const sessions = await selector.listSessions();
console.log(
'Sessions found:',
sessions.map((s) => s.id),
);
try {
const resolved = await selector.resolveSession('test-session-id');
console.log('Resolved session:', resolved.sessionData.sessionId);
} catch (e) {
console.error('Failed to resolve:', e);
}
}
main();
+257
View File
@@ -0,0 +1,257 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, beforeAll, afterAll, expect } from 'vitest';
import { TestRig, PerfTestHarness } from '@google/gemini-cli-test-utils';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = dirname(fileURLToPath(import.meta.url));
const BASELINES_PATH = join(__dirname, 'baselines.json');
const TOLERANCE_PERCENT = 15;
const SAMPLE_COUNT = 1;
const WARMUP_COUNT = 0;
describe.each([
{ alt: true, term: true },
{ alt: false, term: false },
{ alt: true, term: false },
{ alt: false, term: true },
])('Scrolling Performance Tests (alt=$alt, term=$term)', ({ alt, term }) => {
let harness: PerfTestHarness;
beforeAll(() => {
harness = new PerfTestHarness({
baselinesPath: BASELINES_PATH,
defaultTolerancePercent: TOLERANCE_PERCENT,
sampleCount: SAMPLE_COUNT,
warmupCount: WARMUP_COUNT,
});
});
afterAll(async () => {
await harness.generateReport();
}, 30000);
async function createRealHistory(rig: TestRig, count: number) {
const run = await rig.runInteractive({
args: [],
env: {
HOME: rig.homeDir!,
GEMINI_API_KEY: 'test-api-key',
},
});
await run.expectText('type your message');
for (let i = 0; i < count; i++) {
// Send the /compress command directly
await run.sendText('/compress');
await new Promise((r) => setTimeout(r, 100));
// First enter to select from autocomplete
await run.sendText('\r');
await new Promise((r) => setTimeout(r, 100));
// Second enter to execute
await run.sendText('\r');
// Wait for the output to confirm execution
await run.expectText('nothing to compress.');
}
// Exit gracefully by sending SIGTERM to ensure telemetry is flushed
await run.kill();
await run.expectExit();
}
function readMetrics(
rig: TestRig,
): { p50: number; p95: number; max: number }[] {
const logs = rig.readTelemetryLogs();
const metrics: { p50: number; p95: number; max: number }[] = [];
for (const log of logs) {
if (log.scopeMetrics) {
for (const sm of log.scopeMetrics) {
for (const m of sm.metrics) {
if (m.descriptor.name === 'gemini_cli.event_loop.delay') {
const p50 = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'p50',
)?.value;
const p95 = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'p95',
)?.value;
const max = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'max',
)?.value;
if (p50 !== undefined || p95 !== undefined || max !== undefined) {
metrics.push({ p50: p50 ?? 0, p95: p95 ?? 0, max: max ?? 0 });
}
}
}
}
}
}
return metrics;
}
it('straight-scroll-up: scrolling all the way up', async () => {
await harness.runScenario(
`straight-scroll-up-alt-${alt}-term-${term}`,
async () => {
const rig = new TestRig();
try {
rig.setup(`perf-scroll-up-${alt}-${term}`, {
settings: {
ui: {
useAlternateBuffer: alt,
terminalBuffer: term,
},
},
});
await createRealHistory(rig, 10);
const run = await rig.runInteractive({
args: ['--resume', '1'],
env: {
GEMINI_EVENT_LOOP_MONITOR_ENABLED: 'true',
GEMINI_MEMORY_MONITOR_INTERVAL: '500',
HOME: rig.homeDir!,
GEMINI_API_KEY: 'test-api-key',
},
});
await run.expectText('type your message');
// We expect to see the history text when resuming
await run.expectText('nothing to compress.', 30000);
const snapshot = await harness.measure(
'straight-scroll',
async () => {
// Send PageUp 10 times to go to top
for (let i = 0; i < 10; i++) {
run.ptyProcess.write('\x1b[5~');
await new Promise((r) => setImmediate(r));
}
await run.expectText('type your message');
},
);
await run.type('/exit');
await run.sendText('\r');
await run.sendText('\r');
await run.expectExit();
run.ptyProcess.kill();
const metrics = readMetrics(rig);
expect(metrics.length).toBeGreaterThan(0);
if (metrics.length > 0) {
snapshot.eventLoopDelayP50Ms = Math.max(
...metrics.map((m) => m.p50),
);
snapshot.eventLoopDelayP95Ms = Math.max(
...metrics.map((m) => m.p95),
);
snapshot.eventLoopDelayMaxMs = Math.max(
...metrics.map((m) => m.max),
);
}
expect(snapshot.eventLoopDelayMaxMs).toBeLessThan(250);
return snapshot;
} finally {
rig.cleanup();
delete process.env['GEMINI_EVENT_LOOP_MONITOR_ENABLED'];
delete process.env['GEMINI_MEMORY_MONITOR_INTERVAL'];
}
},
);
}, 180000);
it('jitter-scroll: random scroll ups and downs', async () => {
await harness.runScenario(
`jitter-scroll-alt-${alt}-term-${term}`,
async () => {
const rig = new TestRig();
try {
rig.setup(`perf-jitter-scroll-${alt}-${term}`, {
settings: {
ui: {
useAlternateBuffer: alt,
terminalBuffer: term,
},
},
});
await createRealHistory(rig, 10);
const run = await rig.runInteractive({
args: ['--resume', '1'],
env: {
GEMINI_EVENT_LOOP_MONITOR_ENABLED: 'true',
GEMINI_MEMORY_MONITOR_INTERVAL: '500',
HOME: rig.homeDir!,
GEMINI_API_KEY: 'test-api-key',
},
});
await run.expectText('type your message');
await run.expectText('nothing to compress.', 30000);
const snapshot = await harness.measure('jitter-scroll', async () => {
// Simulate jitter: 3 up, 1 down. Do it 10 times to ensure we go all the way up.
for (let i = 0; i < 10; i++) {
for (let j = 0; j < 3; j++) {
run.ptyProcess.write('\x1b[5~'); // PageUp
await new Promise((r) => setImmediate(r));
}
run.ptyProcess.write('\x1b[6~'); // PageDown
await new Promise((r) => setImmediate(r));
}
await run.expectText('type your message');
});
await run.type('/exit');
await run.sendText('\r');
await run.sendText('\r');
await run.expectExit();
run.ptyProcess.kill();
const metrics = readMetrics(rig);
expect(metrics.length).toBeGreaterThan(0);
if (metrics.length > 0) {
snapshot.eventLoopDelayP50Ms = Math.max(
...metrics.map((m) => m.p50),
);
snapshot.eventLoopDelayP95Ms = Math.max(
...metrics.map((m) => m.p95),
);
snapshot.eventLoopDelayMaxMs = Math.max(
...metrics.map((m) => m.max),
);
}
expect(snapshot.eventLoopDelayMaxMs).toBeLessThan(250);
return snapshot;
} finally {
rig.cleanup();
delete process.env['GEMINI_EVENT_LOOP_MONITOR_ENABLED'];
delete process.env['GEMINI_MEMORY_MONITOR_INTERVAL'];
}
},
);
}, 180000);
});