feat(test): add high-volume shell test and refine perf harness (#24983)

packages/core/src/telemetry/activity-monitor.ts

@@ -50,6 +50,7 @@ export const DEFAULT_ACTIVITY_CONFIG: ActivityMonitorConfig = {
     ActivityType.USER_INPUT_START,
     ActivityType.MESSAGE_ADDED,
     ActivityType.TOOL_CALL_SCHEDULED,
+    ActivityType.TOOL_CALL_COMPLETED,
     ActivityType.STREAM_START,
   ],
 };

packages/core/src/telemetry/event-loop-monitor.ts

@@ -0,0 +1,99 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import process from 'node:process';
+import { monitorEventLoopDelay, type IntervalHistogram } from 'node:perf_hooks';
+import type { Config } from '../config/config.js';
+import {
+  recordEventLoopDelay,
+  isPerformanceMonitoringActive,
+} from './metrics.js';
+
+export class EventLoopMonitor {
+  private eventLoopHistogram: IntervalHistogram | null = null;
+  private intervalId: NodeJS.Timeout | null = null;
+  private isRunning = false;
+
+  start(config: Config, intervalMs: number = 10000): void {
+    const isEnabled =
+      process.env['GEMINI_EVENT_LOOP_MONITOR_ENABLED'] === 'true';
+    if (!isEnabled || !isPerformanceMonitoringActive() || this.isRunning) {
+      return;
+    }
+
+    this.isRunning = true;
+    this.eventLoopHistogram = monitorEventLoopDelay({ resolution: 10 });
+    this.eventLoopHistogram.enable();
+
+    this.intervalId = setInterval(() => {
+      this.takeSnapshot(config);
+    }, intervalMs).unref();
+  }
+
+  stop(): void {
+    if (!this.isRunning) {
+      return;
+    }
+
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+
+    if (this.eventLoopHistogram) {
+      this.eventLoopHistogram.disable();
+      this.eventLoopHistogram = null;
+    }
+
+    this.isRunning = false;
+  }
+
+  private takeSnapshot(config: Config): void {
+    if (!this.eventLoopHistogram) {
+      return;
+    }
+
+    const p50 = this.eventLoopHistogram.percentile(50) / 1e6;
+    const p95 = this.eventLoopHistogram.percentile(95) / 1e6;
+    const max = this.eventLoopHistogram.max / 1e6;
+
+    recordEventLoopDelay(config, p50, {
+      percentile: 'p50',
+      component: 'event_loop_monitor',
+    });
+    recordEventLoopDelay(config, p95, {
+      percentile: 'p95',
+      component: 'event_loop_monitor',
+    });
+    recordEventLoopDelay(config, max, {
+      percentile: 'max',
+      component: 'event_loop_monitor',
+    });
+  }
+}
+
+let globalEventLoopMonitor: EventLoopMonitor | null = null;
+
+export function startGlobalEventLoopMonitoring(
+  config: Config,
+  intervalMs?: number,
+): void {
+  if (!globalEventLoopMonitor) {
+    globalEventLoopMonitor = new EventLoopMonitor();
+  }
+  globalEventLoopMonitor.start(config, intervalMs);
+}
+
+export function stopGlobalEventLoopMonitoring(): void {
+  if (globalEventLoopMonitor) {
+    globalEventLoopMonitor.stop();
+    globalEventLoopMonitor = null;
+  }
+}
+
+export function getEventLoopMonitor(): EventLoopMonitor | null {
+  return globalEventLoopMonitor;
+}

packages/core/src/telemetry/index.ts

@@ -93,6 +93,12 @@ export {
   stopGlobalMemoryMonitoring,
 } from './memory-monitor.js';
 export type { MemorySnapshot, ProcessMetrics } from './memory-monitor.js';
+export {
+  EventLoopMonitor,
+  startGlobalEventLoopMonitoring,
+  stopGlobalEventLoopMonitoring,
+  getEventLoopMonitor,
+} from './event-loop-monitor.js';
 export { HighWaterMarkTracker } from './high-water-mark-tracker.js';
 export { RateLimiter } from './rate-limiter.js';
 export { ActivityType } from './activity-types.js';
@@ -133,6 +139,7 @@ export {
   recordStartupPerformance,
   recordMemoryUsage,
   recordCpuUsage,
+  recordEventLoopDelay,
   recordToolQueueDepth,
   recordToolExecutionBreakdown,
   recordTokenEfficiency,

packages/core/src/telemetry/metrics.ts

@@ -88,6 +88,7 @@ const GEN_AI_CLIENT_OPERATION_DURATION = 'gen_ai.client.operation.duration';
 const STARTUP_TIME = 'gemini_cli.startup.duration';
 const MEMORY_USAGE = 'gemini_cli.memory.usage';
 const CPU_USAGE = 'gemini_cli.cpu.usage';
+const EVENT_LOOP_DELAY = 'gemini_cli.event_loop.delay';
 const TOOL_QUEUE_DEPTH = 'gemini_cli.tool.queue.depth';
 const TOOL_EXECUTION_BREAKDOWN = 'gemini_cli.tool.execution.breakdown';
 const TOKEN_EFFICIENCY = 'gemini_cli.token.efficiency';
@@ -608,6 +609,17 @@ const PERFORMANCE_HISTOGRAM_DEFINITIONS = {
       component?: string;
     },
   },
+  [EVENT_LOOP_DELAY]: {
+    description: 'Event loop delay in milliseconds.',
+    unit: 'ms',
+    valueType: ValueType.DOUBLE,
+    assign: (h: Histogram) => (eventLoopDelayHistogram = h),
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    attributes: {} as {
+      percentile: string;
+      component?: string;
+    },
+  },
   [TOOL_QUEUE_DEPTH]: {
     description: 'Number of tools in execution queue.',
     unit: 'count',
@@ -806,6 +818,7 @@ let genAiClientOperationDurationHistogram: Histogram | undefined;
 let startupTimeHistogram: Histogram | undefined;
 let memoryUsageGauge: Histogram | undefined; // Using Histogram until ObservableGauge is available
 let cpuUsageGauge: Histogram | undefined;
+let eventLoopDelayHistogram: Histogram | undefined;
 let toolQueueDepthGauge: Histogram | undefined;
 let toolExecutionBreakdownHistogram: Histogram | undefined;
 let tokenEfficiencyHistogram: Histogram | undefined;
@@ -1339,6 +1352,21 @@ export function recordCpuUsage(
   cpuUsageGauge.record(percentage, metricAttributes);
 }
 
+export function recordEventLoopDelay(
+  config: Config,
+  delayMs: number,
+  attributes: MetricDefinitions[typeof EVENT_LOOP_DELAY]['attributes'],
+): void {
+  if (!eventLoopDelayHistogram || !isPerformanceMonitoringEnabled) return;
+
+  const metricAttributes: Attributes = {
+    ...baseMetricDefinition.getCommonAttributes(config),
+    ...attributes,
+  };
+
+  eventLoopDelayHistogram.record(delayMs, metricAttributes);
+}
+
 export function recordToolQueueDepth(config: Config, queueDepth: number): void {
   if (!toolQueueDepthGauge || !isPerformanceMonitoringEnabled) return;
 

packages/core/src/telemetry/sdk.ts

@@ -52,6 +52,11 @@ import {
 } from './gcp-exporters.js';
 import { TelemetryTarget } from './index.js';
 import { debugLogger } from '../utils/debugLogger.js';
+import {
+  startGlobalMemoryMonitoring,
+  getMemoryMonitor,
+} from './memory-monitor.js';
+import { startGlobalEventLoopMonitoring } from './event-loop-monitor.js';
 import { authEvents } from '../code_assist/oauth2.js';
 import { coreEvents, CoreEvent } from '../utils/events.js';
 import {
@@ -91,6 +96,7 @@ diag.setLogger(new DiagLoggerAdapter(), DiagLogLevel.INFO);
 let sdk: NodeSDK | undefined;
 let spanProcessor: BatchSpanProcessor | undefined;
 let logRecordProcessor: BatchLogRecordProcessor | undefined;
+let metricReader: PeriodicExportingMetricReader | undefined;
 let telemetryInitialized = false;
 let callbackRegistered = false;
 let authListener: ((newCredentials: JWTInput) => Promise<void>) | undefined =
@@ -258,7 +264,6 @@ export async function initializeTelemetry(
     | GcpLogExporter
     | FileLogExporter
     | ConsoleLogRecordExporter;
-  let metricReader: PeriodicExportingMetricReader;
 
   if (useDirectGcpExport) {
     debugLogger.log(
@@ -346,6 +351,26 @@ export async function initializeTelemetry(
     }
     activeTelemetryEmail = credentials?.client_email;
     initializeMetrics(config);
+
+    // Start memory monitoring if interval is specified via environment variable
+    const monitorInterval = process.env['GEMINI_MEMORY_MONITOR_INTERVAL'];
+    debugLogger.log(
+      `[TELEMETRY] GEMINI_MEMORY_MONITOR_INTERVAL: ${monitorInterval}`,
+    );
+    if (monitorInterval) {
+      const intervalMs = parseInt(monitorInterval, 10);
+      if (!isNaN(intervalMs) && intervalMs > 0) {
+        startGlobalMemoryMonitoring(config, intervalMs);
+        startGlobalEventLoopMonitoring(config, intervalMs);
+        // Disable enhanced monitoring (rate limiting/high water mark) in tests
+        // to ensure we get regular snapshots regardless of growth.
+        const monitor = getMemoryMonitor();
+        if (monitor) {
+          monitor.setEnhancedMonitoring(false);
+        }
+      }
+    }
+
     telemetryInitialized = true;
     void flushTelemetryBuffer();
   } catch (error) {
@@ -378,6 +403,7 @@ export async function flushTelemetry(config: Config): Promise<void> {
     await Promise.all([
       spanProcessor.forceFlush(),
       logRecordProcessor.forceFlush(),
+      metricReader ? metricReader.forceFlush() : Promise.resolve(),
     ]);
     if (config.getDebugMode()) {
       debugLogger.log('OpenTelemetry SDK flushed successfully.');