Verify npm release by running integration tests (#10174)

.github/actions/publish-release/action.yml

@@ -45,6 +45,9 @@ inputs:
     type: 'boolean'
     required: false
     default: false
+  gemini_api_key:
+    description: 'The API key for running integration tests.'
+    required: true
 
 runs:
   using: 'composite'
@@ -141,6 +144,7 @@ runs:
         npm-package: '@google/gemini-cli@${{ inputs.release-version }}'
         expected-version: '${{ inputs.release-version }}'
         ref: '${{ steps.release_branch.outputs.BRANCH_NAME }}'
+        gemini_api_key: '${{ inputs.gemini_api_key }}'
 
     - name: '🏷️ Tag release'
       uses: './.github/actions/tag-npm-release'

.github/actions/verify-release/action.yml

@@ -9,6 +9,9 @@ inputs:
   expected-version:
     description: 'Expected version'
     required: true
+  gemini_api_key:
+    description: 'The API key for running integration tests.'
+    required: true
   ref:
     description: 'The branch, tag, or SHA to release from.'
     required: false
@@ -18,10 +21,6 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - name: '📝 Print Inputs'
-      shell: 'bash'
-      run: |
-        echo "${{ toJSON(inputs) }}"
     - name: 'Checkout'
       uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v4
       with:
@@ -37,21 +36,25 @@ runs:
         max_attempts: 10
         command: 'cd ./verify && pkg="${{ inputs.npm-package }}" && npm install --prefer-online --no-cache -g "$pkg"'
 
-    # This provides a very basic smoke test for Gemini CLI
-    - name: 'Run Gemini CLI'
-      id: 'gemini_cli'
+    - name: 'Basic smoke test'
       shell: 'bash'
       working-directory: './verify'
       run: |-
-        echo "gemini_version=$(gemini --version)" >> $GITHUB_OUTPUT
+        gemini_version=$(gemini --version)
+        if [ "$gemini_version" != "${{ inputs.expected-version }}" ]; then
+          echo "❌ Version mismatch: Got $gemini_version from ${{ inputs.npm-package }}, expected ${{ inputs.expected-version }}"
+          exit 1
+        fi
 
-    # Force a failure if it doesn't match
-    - name: 'Fail workflow if version does not match'
-      if: '${{ steps.gemini_cli.outputs.gemini_version != inputs.expected-version }}'
+    - name: 'Install dependencies for integration tests'
       shell: 'bash'
       working-directory: './verify'
-      run: |-
-        echo '❌ Got ${{ steps.gemini_cli.outputs.gemini_version }} from ${{ inputs.npm-package }}'
-        echo '❌ Expected Version ${{ inputs.expected-version }}'
+      run: 'npm install'
 
-        exit 1
+    - name: '🔬 Run integration tests against NPM release'
+      working-directory: './verify'
+      env:
+        GEMINI_API_KEY: '${{ inputs.gemini_api_key }}'
+        INTEGRATION_TEST_USE_INSTALLED_GEMINI: 'true'
+      shell: 'bash'
+      run: 'npm run test:integration:sandbox:none'

.github/workflows/release-manual.yml

@@ -99,6 +99,7 @@ jobs:
           previous-tag: '${{ steps.release_info.outputs.PREVIOUS_TAG }}'
           skip-github-release: '${{ github.event.inputs.skip_github_release }}'
           working-directory: './release'
+          gemini_api_key: '${{ secrets.GEMINI_API_KEY }}'
 
       - name: 'Create Issue on Failure'
         if: '${{ failure() && github.event.inputs.dry_run == false }}'

.github/workflows/release-nightly.yml

@@ -103,6 +103,7 @@ jobs:
           previous-tag: '${{ steps.nightly_version.outputs.PREVIOUS_TAG }}'
           working-directory: './release'
           skip-branch-cleanup: true
+          gemini_api_key: '${{ secrets.GEMINI_API_KEY }}'
 
       - name: 'Create and Merge Pull Request'
         uses: './.github/actions/create-pull-request'
@@ -114,7 +115,6 @@ jobs:
           dry-run: '${{ github.event.inputs.dry_run }}'
           working-directory: './release'
 
-
       - name: 'Create Issue on Failure'
         if: "${{ failure() && github.event.inputs.dry_run != 'true' }}"
         env:

.github/workflows/release-patch-3-release.yml

@@ -162,6 +162,7 @@ jobs:
           dry-run: '${{ github.event.inputs.dry_run }}'
           previous-tag: '${{ steps.patch_version.outputs.PREVIOUS_TAG }}'
           working-directory: './release'
+          gemini_api_key: '${{ secrets.GEMINI_API_KEY }}'
 
       - name: 'Create Issue on Failure'
         if: '${{ failure() && github.event.inputs.dry_run == false }}'

.github/workflows/release-promote.yml

@@ -190,6 +190,7 @@ jobs:
           dry-run: '${{ github.event.inputs.dry_run }}'
           previous-tag: '${{ needs.calculate-versions.outputs.PREVIOUS_PREVIEW_TAG }}'
           working-directory: './release'
+          gemini_api_key: '${{ secrets.GEMINI_API_KEY }}'
 
       - name: 'Create Issue on Failure'
         if: '${{ failure() && github.event.inputs.dry_run == false }}'
@@ -246,6 +247,7 @@ jobs:
           dry-run: '${{ github.event.inputs.dry_run }}'
           previous-tag: '${{ needs.calculate-versions.outputs.PREVIOUS_STABLE_TAG }}'
           working-directory: './release'
+          gemini_api_key: '${{ secrets.GEMINI_API_KEY }}'
 
       - name: 'Create Issue on Failure'
         if: '${{ failure() && github.event.inputs.dry_run == false }}'

.github/workflows/verify-release.yml

@@ -29,3 +29,4 @@ jobs:
           npm-package: '${{github.event.inputs.npm-package}}'
           expected-version: '${{github.event.inputs.version}}'
           ref: '${{github.event.inputs.ref}}'
+          gemini_api_key: '${{ secrets.GEMINI_API_KEY }}'

integration-tests/test-helper.ts

@@ -195,13 +195,32 @@ export class TestRig {
     execSync('sync', { cwd: this.testDir! });
   }
 
+  /**
+   * The command and args to use to invoke Gemini CLI. Allows us to switch
+   * between using the bundled gemini.js (the default) and using the installed
+   * 'gemini' (used to verify npm bundles).
+   */
+  private _getCommandAndArgs(extraInitialArgs: string[] = []): {
+    command: string;
+    initialArgs: string[];
+  } {
+    const isNpmReleaseTest =
+      process.env.INTEGRATION_TEST_USE_INSTALLED_GEMINI === 'true';
+    const command = isNpmReleaseTest ? 'gemini' : 'node';
+    const initialArgs = isNpmReleaseTest
+      ? extraInitialArgs
+      : [this.bundlePath, ...extraInitialArgs];
+    return { command, initialArgs };
+  }
+
   run(
     promptOrOptions:
       | string
       | { prompt?: string; stdin?: string; stdinDoesNotEnd?: boolean },
     ...args: string[]
   ): Promise<string> {
-    const commandArgs = [this.bundlePath, '--yolo'];
+    const { command, initialArgs } = this._getCommandAndArgs(['--yolo']);
+    const commandArgs = [...initialArgs];
     const execOptions: {
       cwd: string;
       encoding: 'utf-8';
@@ -227,7 +246,7 @@ export class TestRig {
 
     commandArgs.push(...args);
 
-    const child = spawn('node', commandArgs, {
+    const child = spawn(command, commandArgs, {
       cwd: this.testDir!,
       stdio: 'pipe',
       env: process.env,
@@ -331,9 +350,10 @@ export class TestRig {
     args: string[],
     options: { stdin?: string } = {},
   ): Promise<string> {
-    const commandArgs = [this.bundlePath, ...args];
+    const { command, initialArgs } = this._getCommandAndArgs();
+    const commandArgs = [...initialArgs, ...args];
 
-    const child = spawn('node', commandArgs, {
+    const child = spawn(command, commandArgs, {
       cwd: this.testDir!,
       stdio: 'pipe',
     });
@@ -766,9 +786,10 @@ export class TestRig {
     ptyProcess: pty.IPty;
     promise: Promise<{ exitCode: number; signal?: number; output: string }>;
   } {
-    const commandArgs = [this.bundlePath, '--yolo', ...args];
+    const { command, initialArgs } = this._getCommandAndArgs(['--yolo']);
+    const commandArgs = [...initialArgs, ...args];
 
-    const ptyProcess = pty.spawn('node', commandArgs, {
+    const ptyProcess = pty.spawn(command, commandArgs, {
       name: 'xterm-color',
       cols: 80,
       rows: 30,