Merge branch 'main' into test-utils-model-fallback

packages/test-utils/GEMINI.md

@@ -10,6 +10,58 @@ published to npm.
 - `src/file-system-test-helpers.ts`: Helpers for creating temporary file system
   fixtures.
 - `src/mock-utils.ts`: Common mock utilities.
+- `src/test-mcp-server.ts`: Helper for building test MCP servers for tests.
+- `src/test-mcp-server-template.mjs`: Generic template script for running
+  isolated MCP processes.
+
+## Test MCP Servers
+
+The `TestRig` provides a fully isolated, compliant way to test tool triggers and
+workflows using local test MCP servers. This isolates your tests from live API
+endpoints and rate-limiting.
+
+### Usage
+
+1. **Programmatic Builder:**
+
+   ```typescript
+   import { TestMcpServerBuilder } from '@google/gemini-cli-test-utils';
+
+   const builder = new TestMcpServerBuilder('weather-server').addTool(
+     'get_weather',
+     'Get weather',
+     'It is rainy',
+   );
+
+   rig.addTestMcpServer('weather-server', builder.build());
+   ```
+
+2. **Predefined configurations via JSON:** Place a configuration file in
+   `packages/test-utils/assets/test-servers/google-workspace.json` and load it
+   by title:
+
+   ```typescript
+   rig.addTestMcpServer('workspace-server', 'google-workspace');
+   ```
+
+   **JSON Format Structure (`TestMcpConfig`):**
+
+   ```json
+   {
+     "name": "string (Fallback server name)",
+     "tools": [
+       {
+         "name": "string (Tool execution name)",
+         "description": "string (Helpful summary for router)",
+         "inputSchema": {
+           "type": "object",
+           "properties": { ... }
+         },
+         "response": "string | object (The forced reply payload)"
+       }
+     ]
+   }
+   ```
 
 ## Usage
 

packages/test-utils/src/fixtures/agents.ts

@@ -0,0 +1,152 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Represents a test agent used in evaluations and tests.
+ */
+export interface TestAgent {
+  /** The unique name of the agent. */
+  readonly name: string;
+  /** The full YAML/Markdown definition of the agent. */
+  readonly definition: string;
+  /** The standard path where this agent should be saved in a test project. */
+  readonly path: string;
+  /** A helper to spread this agent directly into a 'files' object for evalTest. */
+  readonly asFile: () => Record<string, string>;
+}
+
+/**
+ * Helper to create a TestAgent with consistent formatting and pathing.
+ */
+function createAgent(options: {
+  name: string;
+  description: string;
+  tools: string[];
+  body: string;
+}): TestAgent {
+  const definition = `---
+name: ${options.name}
+description: ${options.description}
+tools:
+${options.tools.map((t) => `  - ${t}`).join('\n')}
+---
+${options.body}
+`;
+
+  const path = `.gemini/agents/${options.name}.md`;
+
+  return {
+    name: options.name,
+    definition,
+    path,
+    asFile: () => ({ [path]: definition }),
+  };
+}
+
+/**
+ * A collection of predefined test agents for use in evaluations and tests.
+ */
+export const TEST_AGENTS = {
+  /**
+   * An agent with expertise in updating documentation.
+   */
+  DOCS_AGENT: createAgent({
+    name: 'docs-agent',
+    description: 'An agent with expertise in updating documentation.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the docs agent. Update documentation clearly and accurately.',
+  }),
+
+  /**
+   * An agent with expertise in writing and updating tests.
+   */
+  TESTING_AGENT: createAgent({
+    name: 'testing-agent',
+    description: 'An agent with expertise in writing and updating tests.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the test agent. Add or update tests.',
+  }),
+  /**
+   * An agent with expertise in database schemas, SQL, and creating database migrations.
+   */
+  DATABASE_AGENT: createAgent({
+    name: 'database-agent',
+    description:
+      'An expert in database schemas, SQL, and creating database migrations.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the database agent. Create and update SQL migrations.',
+  }),
+
+  /**
+   * An agent with expertise in CSS, styling, and UI design.
+   */
+  CSS_AGENT: createAgent({
+    name: 'css-agent',
+    description: 'An expert in CSS, styling, and UI design.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the CSS agent.',
+  }),
+
+  /**
+   * An agent with expertise in internationalization and translations.
+   */
+  I18N_AGENT: createAgent({
+    name: 'i18n-agent',
+    description: 'An expert in internationalization and translations.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the i18n agent.',
+  }),
+
+  /**
+   * An agent with expertise in security audits and vulnerability patches.
+   */
+  SECURITY_AGENT: createAgent({
+    name: 'security-agent',
+    description: 'An expert in security audits and vulnerability patches.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the security agent.',
+  }),
+
+  /**
+   * An agent with expertise in CI/CD, Docker, and deployment scripts.
+   */
+  DEVOPS_AGENT: createAgent({
+    name: 'devops-agent',
+    description: 'An expert in CI/CD, Docker, and deployment scripts.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the devops agent.',
+  }),
+
+  /**
+   * An agent with expertise in tracking, analytics, and metrics.
+   */
+  ANALYTICS_AGENT: createAgent({
+    name: 'analytics-agent',
+    description: 'An expert in tracking, analytics, and metrics.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the analytics agent.',
+  }),
+
+  /**
+   * An agent with expertise in web accessibility and ARIA roles.
+   */
+  ACCESSIBILITY_AGENT: createAgent({
+    name: 'accessibility-agent',
+    description: 'An expert in web accessibility and ARIA roles.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the accessibility agent.',
+  }),
+
+  /**
+   * An agent with expertise in React Native and mobile app development.
+   */
+  MOBILE_AGENT: createAgent({
+    name: 'mobile-agent',
+    description: 'An expert in React Native and mobile app development.',
+    tools: ['read_file', 'write_file'],
+    body: 'You are the mobile agent.',
+  }),
+} as const;

packages/test-utils/src/index.ts

@@ -5,5 +5,7 @@
  */
 
 export * from './file-system-test-helpers.js';
-export * from './test-rig.js';
+export * from './fixtures/agents.js';
 export * from './mock-utils.js';
+export * from './test-mcp-server.js';
+export * from './test-rig.js';

packages/test-utils/src/test-mcp-server-template.mjs

@@ -0,0 +1,69 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import {
+  ListToolsRequestSchema,
+  CallToolRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+import fs from 'fs';
+
+const configPath = process.argv[2];
+if (!configPath) {
+  console.error('Usage: node template.mjs <config-path>');
+  process.exit(1);
+}
+
+const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+
+const server = new Server(
+  {
+    name: config.name,
+    version: config.version || '1.0.0',
+  },
+  {
+    capabilities: {
+      tools: {},
+    },
+  },
+);
+
+// Add tools handler
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  return {
+    tools: (config.tools || []).map((tool) => ({
+      name: tool.name,
+      description: tool.description,
+      inputSchema: tool.inputSchema || { type: 'object', properties: {} },
+    })),
+  };
+});
+
+// Add call handler
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const toolName = request.params.name;
+  const tool = (config.tools || []).find((t) => t.name === toolName);
+
+  if (!tool) {
+    return {
+      content: [
+        {
+          type: 'text',
+          text: `Error: Tool ${toolName} not found`,
+        },
+      ],
+      isError: true,
+    };
+  }
+
+  return tool.response;
+});
+
+const transport = new StdioServerTransport();
+await server.connect(transport);
+// server.connect resolves when transport connects, but listening continues
+console.error(`Test MCP Server '${config.name}' connected and listening.`);

packages/test-utils/src/test-mcp-server.ts

@@ -0,0 +1,75 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Response structure for a test tool call.
+ */
+export interface TestToolResponse {
+  content: { type: 'text'; text: string }[];
+  isError?: boolean;
+}
+
+/**
+ * Definition of a test tool.
+ */
+export interface TestTool {
+  name: string;
+  description: string;
+  /** JSON Schema for input arguments */
+  inputSchema?: Record<string, unknown>;
+  response: TestToolResponse;
+}
+
+/**
+ * Configuration structure for the generic test MCP server template.
+ */
+export interface TestMcpConfig {
+  name: string;
+  version?: string;
+  tools: TestTool[];
+}
+
+/**
+ * Builder to easily configure a Test MCP Server in tests.
+ */
+export class TestMcpServerBuilder {
+  private config: TestMcpConfig;
+
+  constructor(name: string) {
+    this.config = { name, tools: [] };
+  }
+
+  /**
+   * Adds a tool to the test server configuration.
+   * @param name Tool name
+   * @param description Tool description
+   * @param response The response to return. Can be a string for simple text responses.
+   * @param inputSchema Optional JSON Schema for validation/documentation
+   */
+  addTool(
+    name: string,
+    description: string,
+    response: TestToolResponse | string,
+    inputSchema?: Record<string, unknown>,
+  ): this {
+    const responseObj =
+      typeof response === 'string'
+        ? { content: [{ type: 'text' as const, text: response }] }
+        : response;
+
+    this.config.tools.push({
+      name,
+      description,
+      inputSchema,
+      response: responseObj,
+    });
+    return this;
+  }
+
+  build(): TestMcpConfig {
+    return this.config;
+  }
+}

packages/test-utils/src/test-rig.ts

@@ -16,6 +16,7 @@ export { GEMINI_DIR };
 import * as pty from '@lydell/node-pty';
 import stripAnsi from 'strip-ansi';
 import * as os from 'node:os';
+import type { TestMcpConfig } from './test-mcp-server.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const BUNDLE_PATH = join(__dirname, '..', '..', '..', 'bundle/gemini.js');
@@ -484,7 +485,7 @@ export class TestRig {
         general: {
           // Nightly releases sometimes becomes out of sync with local code and
           // triggers auto-update, which causes tests to fail.
-          disableAutoUpdate: true,
+          enableAutoUpdate: false,
         },
         telemetry: {
           enabled: true,
@@ -595,7 +596,95 @@ export class TestRig {
     }
     const scriptPath = join(this.testDir, fileName);
     writeFileSync(scriptPath, content);
-    return normalizePath(scriptPath);
+    return normalizePath(scriptPath)!;
+  }
+
+  /**
+   * Adds a test MCP server to the test workspace.
+   * @param name The name of the server
+   * @param config Configuration object or name of predefined config (e.g. 'github')
+   */
+  addTestMcpServer(name: string, config: TestMcpConfig | string) {
+    if (!this.testDir) {
+      throw new Error(
+        'TestRig.setup must be called before adding test servers',
+      );
+    }
+
+    let testConfig: TestMcpConfig;
+    if (typeof config === 'string') {
+      const assetsDir = join(__dirname, '..', 'assets', 'test-servers');
+      const configPath = join(assetsDir, `${config}.json`);
+      if (!fs.existsSync(configPath)) {
+        throw new Error(
+          `Predefined test server config not found: ${configPath}`,
+        );
+      }
+      testConfig = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+      testConfig.name = name; // Override name
+    } else {
+      testConfig = config;
+    }
+
+    const configFileName = `test-mcp-${name}.json`;
+    const scriptFileName = `test-mcp-${name}.mjs`;
+
+    const configFilePath = join(this.testDir, configFileName);
+    const scriptFilePath = join(this.testDir, scriptFileName);
+
+    // Write config
+    fs.writeFileSync(configFilePath, JSON.stringify(testConfig, null, 2));
+
+    // Copy template script
+    const templatePath = join(__dirname, 'test-mcp-server-template.mjs');
+    if (!fs.existsSync(templatePath)) {
+      throw new Error(`Test template not found at ${templatePath}`);
+    }
+
+    fs.copyFileSync(templatePath, scriptFilePath);
+
+    // Calculate path to monorepo node_modules
+    const monorepoNodeModules = join(
+      __dirname,
+      '..',
+      '..',
+      '..',
+      'node_modules',
+    );
+
+    // Create symlink to node_modules in testDir for ESM resolution
+    const testNodeModules = join(this.testDir, 'node_modules');
+    if (!fs.existsSync(testNodeModules)) {
+      fs.symlinkSync(monorepoNodeModules, testNodeModules, 'dir');
+    }
+
+    // Update settings in workspace and home
+    const updateSettings = (dir: string) => {
+      const settingsPath = join(dir, GEMINI_DIR, 'settings.json');
+      let settings: any = {};
+      if (fs.existsSync(settingsPath)) {
+        settings = JSON.parse(fs.readFileSync(settingsPath, 'utf-8'));
+      } else {
+        fs.mkdirSync(join(dir, GEMINI_DIR), { recursive: true });
+      }
+
+      if (!settings.mcpServers) {
+        settings.mcpServers = {};
+      }
+
+      settings.mcpServers[name] = {
+        command: 'node',
+        args: [scriptFilePath, configFilePath],
+        // Removed env.NODE_PATH as it is ignored in ESM
+      };
+
+      fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2));
+    };
+
+    updateSettings(this.testDir);
+    if (this.homeDir) {
+      updateSettings(this.homeDir);
+    }
   }
 
   private _getCleanEnv(