attempts to run the test rig with eventloopdelay measure and other feedback

This commit is contained in:
Sri Pasumarthi
2026-04-10 12:05:35 -07:00
committed by Srinivasa Pasumarthi
parent e74b093a42
commit 3ff7422790
3 changed files with 145 additions and 96 deletions
+4
View File
@@ -1336,6 +1336,10 @@ export class TestRig {
return logs;
}
readTelemetryLogs(): ParsedLog[] {
return this._readAndParseTelemetryLog();
}
readToolLogs() {
// For Podman, first check if telemetry file exists and has content
// If not, fall back to parsing from stdout
+63
View File
@@ -0,0 +1,63 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { Storage } from '../packages/core/src/config/storage';
import { SessionSelector } from '../packages/cli/src/utils/sessionUtils';
import { join } from 'node:path';
import { mkdirSync, writeFileSync } from 'node:fs';
async function main() {
const testDir = '/private/tmp/perf-scroll-up-true-true';
mkdirSync(testDir, { recursive: true });
const storage = new Storage(testDir);
await storage.initialize();
console.log('Project Temp Dir:', storage.getProjectTempDir());
const chatsDir = join(storage.getProjectTempDir(), 'chats');
mkdirSync(chatsDir, { recursive: true });
const sessionFile = join(chatsDir, 'session-2026-04-10T00-00-test.jsonl');
const metadata = {
sessionId: 'test-session-id',
projectHash: 'test-project-hash',
startTime: new Date().toISOString(),
lastUpdated: new Date().toISOString(),
kind: 'main',
hasUserOrAssistantMessage: true,
messageCount: 1,
summary: 'Test Session',
};
const msg = {
id: 'msg-0',
timestamp: new Date().toISOString(),
type: 'user',
content: 'Message content 0',
};
writeFileSync(
sessionFile,
JSON.stringify(metadata) + '\n' + JSON.stringify(msg) + '\n',
);
console.log('Created file:', sessionFile);
const selector = new SessionSelector(storage);
const sessions = await selector.listSessions();
console.log(
'Sessions found:',
sessions.map((s) => s.id),
);
try {
const resolved = await selector.resolveSession('test-session-id');
console.log('Resolved session:', resolved.sessionData.sessionId);
} catch (e) {
console.error('Failed to resolve:', e);
}
}
main();
+78 -96
View File
@@ -8,7 +8,6 @@ import { describe, it, beforeAll, afterAll, expect } from 'vitest';
import { TestRig, PerfTestHarness } from '@google/gemini-cli-test-utils';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { existsSync, writeFileSync, mkdirSync, readFileSync } from 'node:fs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const BASELINES_PATH = join(__dirname, 'baselines.json');
@@ -38,91 +37,65 @@ describe.each([
await harness.generateReport();
}, 30000);
function createFakeHistory(rig: TestRig, count: number) {
const testDir = rig.testDir!;
const chatsDir = join(testDir, '.gemini', 'tmp', 'chats');
mkdirSync(chatsDir, { recursive: true });
async function createRealHistory(rig: TestRig, count: number) {
const run = await rig.runInteractive({
args: [],
env: {
HOME: rig.homeDir!,
GEMINI_API_KEY: 'test-api-key',
},
});
const sessionFile = join(chatsDir, 'session-2026-04-10T00-00-test.jsonl');
const metadata = {
sessionId: 'test-session-id',
projectHash: 'test-project-hash',
startTime: new Date().toISOString(),
lastUpdated: new Date().toISOString(),
kind: 'main',
};
let content = JSON.stringify(metadata) + '\n';
await run.expectText('type your message');
for (let i = 0; i < count; i++) {
const msg = {
id: `msg-${i}`,
timestamp: new Date().toISOString(),
type: i % 2 === 0 ? 'user' : 'gemini',
content: `Message content ${i} `.repeat(10),
};
content += JSON.stringify(msg) + '\n';
// Send the /compress command directly
await run.sendText('/compress');
await new Promise((r) => setTimeout(r, 100));
// First enter to select from autocomplete
await run.sendText('\r');
await new Promise((r) => setTimeout(r, 100));
// Second enter to execute
await run.sendText('\r');
// Wait for the output to confirm execution
await run.expectText('nothing to compress.');
}
writeFileSync(sessionFile, content);
return sessionFile;
// Exit gracefully by sending SIGTERM to ensure telemetry is flushed
await run.kill();
await run.expectExit();
}
function readMetrics(
rig: TestRig,
): { p50: number; p95: number; max: number }[] {
const logFilePath = join(rig.homeDir!, 'telemetry.log');
if (!existsSync(logFilePath)) return [];
const content = readFileSync(logFilePath, 'utf-8');
const jsonObjects = content
.split(/}\n{/)
.map((obj, index, array) => {
if (index > 0) obj = '{' + obj;
if (index < array.length - 1) obj = obj + '}';
return obj.trim();
})
.filter((obj) => obj);
const logs = rig.readTelemetryLogs();
const metrics: { p50: number; p95: number; max: number }[] = [];
for (const jsonStr of jsonObjects) {
try {
const log = JSON.parse(jsonStr);
if (log.scopeMetrics) {
for (const sm of log.scopeMetrics) {
for (const m of sm.metrics) {
if (m.descriptor.name === 'gemini_cli.event_loop.delay') {
// Extract values based on attributes
// event-loop-monitor.ts emits p50, p95, max
// They might be in different data points
const p50 = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'p50',
)?.value;
const p95 = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'p95',
)?.value;
const max = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'max',
)?.value;
for (const log of logs) {
if (log.scopeMetrics) {
for (const sm of log.scopeMetrics) {
for (const m of sm.metrics) {
if (m.descriptor.name === 'gemini_cli.event_loop.delay') {
const p50 = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'p50',
)?.value;
const p95 = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'p95',
)?.value;
const max = m.dataPoints.find(
(dp: { attributes: { percentile: string }; value: number }) =>
dp.attributes.percentile === 'max',
)?.value;
if (
p50 !== undefined ||
p95 !== undefined ||
max !== undefined
) {
metrics.push({ p50: p50 ?? 0, p95: p95 ?? 0, max: max ?? 0 });
}
if (p50 !== undefined || p95 !== undefined || max !== undefined) {
metrics.push({ p50: p50 ?? 0, p95: p95 ?? 0, max: max ?? 0 });
}
}
}
}
} catch {
// ignore parse errors
}
}
return metrics;
@@ -143,49 +116,53 @@ describe.each([
},
});
createFakeHistory(rig, 1000);
await createRealHistory(rig, 10);
const run = await rig.runInteractive({
args: ['--resume', '1'],
env: {
GEMINI_EVENT_LOOP_MONITOR_ENABLED: 'true',
GEMINI_MEMORY_MONITOR_INTERVAL: '500',
HOME: rig.homeDir!,
GEMINI_API_KEY: 'test-api-key',
},
});
// Wait for prompt or some text indicating readiness
await new Promise((r) => setTimeout(r, 3000)); // Wait for history to load
await run.expectText('type your message');
// We expect to see the history text when resuming
await run.expectText('nothing to compress.', 30000);
const snapshot = await harness.measure(
'straight-scroll',
async () => {
// Send PageUp 50 times
for (let i = 0; i < 50; i++) {
// Send PageUp 10 times to go to top
for (let i = 0; i < 10; i++) {
run.ptyProcess.write('\x1b[5~');
await new Promise((r) => setTimeout(r, 50));
await new Promise((r) => setImmediate(r));
}
// Wait for rendering to settle
await new Promise((r) => setTimeout(r, 1000));
await run.expectText('type your message');
},
);
run.ptyProcess.write('exit\n');
await new Promise((r) => setTimeout(r, 3000));
await run.type('/exit');
await run.sendText('\r');
await run.sendText('\r');
await run.expectExit();
run.ptyProcess.kill();
const metrics = readMetrics(rig);
console.log('Event Loop Metrics (Straight Scroll):', metrics);
expect(metrics.length).toBeGreaterThan(0);
if (metrics.length > 0) {
snapshot.eventLoopDelayP50Ms = Math.max(
...metrics.map((m) => m.p50.max),
...metrics.map((m) => m.p50),
);
snapshot.eventLoopDelayP95Ms = Math.max(
...metrics.map((m) => m.p95.max),
...metrics.map((m) => m.p95),
);
snapshot.eventLoopDelayMaxMs = Math.max(
...metrics.map((m) => m.max.max),
...metrics.map((m) => m.max),
);
}
@@ -216,48 +193,53 @@ describe.each([
},
});
createFakeHistory(rig, 1000);
await createRealHistory(rig, 10);
const run = await rig.runInteractive({
args: ['--resume', '1'],
env: {
GEMINI_EVENT_LOOP_MONITOR_ENABLED: 'true',
GEMINI_MEMORY_MONITOR_INTERVAL: '500',
HOME: rig.homeDir!,
GEMINI_API_KEY: 'test-api-key',
},
});
await new Promise((r) => setTimeout(r, 3000));
await run.expectText('type your message');
await run.expectText('nothing to compress.', 30000);
const snapshot = await harness.measure('jitter-scroll', async () => {
// Simulate jitter: 3 up, 1 down
for (let i = 0; i < 20; i++) {
// Simulate jitter: 3 up, 1 down. Do it 10 times to ensure we go all the way up.
for (let i = 0; i < 10; i++) {
for (let j = 0; j < 3; j++) {
run.ptyProcess.write('\x1b[5~'); // PageUp
await new Promise((r) => setTimeout(r, 50));
await new Promise((r) => setImmediate(r));
}
run.ptyProcess.write('\x1b[6~'); // PageDown
await new Promise((r) => setTimeout(r, 50));
await new Promise((r) => setImmediate(r));
}
await new Promise((r) => setTimeout(r, 1000));
await run.expectText('type your message');
});
run.ptyProcess.write('exit\n');
await new Promise((r) => setTimeout(r, 3000));
await run.type('/exit');
await run.sendText('\r');
await run.sendText('\r');
await run.expectExit();
run.ptyProcess.kill();
const metrics = readMetrics(rig);
console.log('Event Loop Metrics (Jitter Scroll):', metrics);
expect(metrics.length).toBeGreaterThan(0);
if (metrics.length > 0) {
snapshot.eventLoopDelayP50Ms = Math.max(
...metrics.map((m) => m.p50.max),
...metrics.map((m) => m.p50),
);
snapshot.eventLoopDelayP95Ms = Math.max(
...metrics.map((m) => m.p95.max),
...metrics.map((m) => m.p95),
);
snapshot.eventLoopDelayMaxMs = Math.max(
...metrics.map((m) => m.max.max),
...metrics.map((m) => m.max),
);
}