From 3d1b0df0fa9b64d2dac5092ec309e60b0393c07d Mon Sep 17 00:00:00 2001 From: Tommaso Sciortino Date: Tue, 30 Sep 2025 11:24:12 -0700 Subject: [PATCH] Verify npm release by running integration tests (#10174) --- .github/actions/publish-release/action.yml | 4 +++ .github/actions/verify-release/action.yml | 33 ++++++++++--------- .github/workflows/release-manual.yml | 1 + .github/workflows/release-nightly.yml | 2 +- .github/workflows/release-patch-3-release.yml | 1 + .github/workflows/release-promote.yml | 2 ++ .github/workflows/verify-release.yml | 1 + integration-tests/test-helper.ts | 33 +++++++++++++++---- 8 files changed, 55 insertions(+), 22 deletions(-) diff --git a/.github/actions/publish-release/action.yml b/.github/actions/publish-release/action.yml index 08520fac8c..5217ca763a 100644 --- a/.github/actions/publish-release/action.yml +++ b/.github/actions/publish-release/action.yml @@ -45,6 +45,9 @@ inputs: type: 'boolean' required: false default: false + gemini_api_key: + description: 'The API key for running integration tests.' + required: true runs: using: 'composite' @@ -141,6 +144,7 @@ runs: npm-package: '@google/gemini-cli@${{ inputs.release-version }}' expected-version: '${{ inputs.release-version }}' ref: '${{ steps.release_branch.outputs.BRANCH_NAME }}' + gemini_api_key: '${{ inputs.gemini_api_key }}' - name: '🏷️ Tag release' uses: './.github/actions/tag-npm-release' diff --git a/.github/actions/verify-release/action.yml b/.github/actions/verify-release/action.yml index b867864694..9ce089789a 100644 --- a/.github/actions/verify-release/action.yml +++ b/.github/actions/verify-release/action.yml @@ -9,6 +9,9 @@ inputs: expected-version: description: 'Expected version' required: true + gemini_api_key: + description: 'The API key for running integration tests.' + required: true ref: description: 'The branch, tag, or SHA to release from.' required: false @@ -18,10 +21,6 @@ inputs: runs: using: 'composite' steps: - - name: '📝 Print Inputs' - shell: 'bash' - run: | - echo "${{ toJSON(inputs) }}" - name: 'Checkout' uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v4 with: @@ -37,21 +36,25 @@ runs: max_attempts: 10 command: 'cd ./verify && pkg="${{ inputs.npm-package }}" && npm install --prefer-online --no-cache -g "$pkg"' - # This provides a very basic smoke test for Gemini CLI - - name: 'Run Gemini CLI' - id: 'gemini_cli' + - name: 'Basic smoke test' shell: 'bash' working-directory: './verify' run: |- - echo "gemini_version=$(gemini --version)" >> $GITHUB_OUTPUT + gemini_version=$(gemini --version) + if [ "$gemini_version" != "${{ inputs.expected-version }}" ]; then + echo "❌ Version mismatch: Got $gemini_version from ${{ inputs.npm-package }}, expected ${{ inputs.expected-version }}" + exit 1 + fi - # Force a failure if it doesn't match - - name: 'Fail workflow if version does not match' - if: '${{ steps.gemini_cli.outputs.gemini_version != inputs.expected-version }}' + - name: 'Install dependencies for integration tests' shell: 'bash' working-directory: './verify' - run: |- - echo '❌ Got ${{ steps.gemini_cli.outputs.gemini_version }} from ${{ inputs.npm-package }}' - echo '❌ Expected Version ${{ inputs.expected-version }}' + run: 'npm install' - exit 1 + - name: '🔬 Run integration tests against NPM release' + working-directory: './verify' + env: + GEMINI_API_KEY: '${{ inputs.gemini_api_key }}' + INTEGRATION_TEST_USE_INSTALLED_GEMINI: 'true' + shell: 'bash' + run: 'npm run test:integration:sandbox:none' diff --git a/.github/workflows/release-manual.yml b/.github/workflows/release-manual.yml index e56406a73c..3d08610bb8 100644 --- a/.github/workflows/release-manual.yml +++ b/.github/workflows/release-manual.yml @@ -99,6 +99,7 @@ jobs: previous-tag: '${{ steps.release_info.outputs.PREVIOUS_TAG }}' skip-github-release: '${{ github.event.inputs.skip_github_release }}' working-directory: './release' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - name: 'Create Issue on Failure' if: '${{ failure() && github.event.inputs.dry_run == false }}' diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index 4803d94d2e..a9518d5734 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -103,6 +103,7 @@ jobs: previous-tag: '${{ steps.nightly_version.outputs.PREVIOUS_TAG }}' working-directory: './release' skip-branch-cleanup: true + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - name: 'Create and Merge Pull Request' uses: './.github/actions/create-pull-request' @@ -114,7 +115,6 @@ jobs: dry-run: '${{ github.event.inputs.dry_run }}' working-directory: './release' - - name: 'Create Issue on Failure' if: "${{ failure() && github.event.inputs.dry_run != 'true' }}" env: diff --git a/.github/workflows/release-patch-3-release.yml b/.github/workflows/release-patch-3-release.yml index ac4f70e01d..8238afa206 100644 --- a/.github/workflows/release-patch-3-release.yml +++ b/.github/workflows/release-patch-3-release.yml @@ -162,6 +162,7 @@ jobs: dry-run: '${{ github.event.inputs.dry_run }}' previous-tag: '${{ steps.patch_version.outputs.PREVIOUS_TAG }}' working-directory: './release' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - name: 'Create Issue on Failure' if: '${{ failure() && github.event.inputs.dry_run == false }}' diff --git a/.github/workflows/release-promote.yml b/.github/workflows/release-promote.yml index 07acbc81af..ac134629b6 100644 --- a/.github/workflows/release-promote.yml +++ b/.github/workflows/release-promote.yml @@ -190,6 +190,7 @@ jobs: dry-run: '${{ github.event.inputs.dry_run }}' previous-tag: '${{ needs.calculate-versions.outputs.PREVIOUS_PREVIEW_TAG }}' working-directory: './release' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - name: 'Create Issue on Failure' if: '${{ failure() && github.event.inputs.dry_run == false }}' @@ -246,6 +247,7 @@ jobs: dry-run: '${{ github.event.inputs.dry_run }}' previous-tag: '${{ needs.calculate-versions.outputs.PREVIOUS_STABLE_TAG }}' working-directory: './release' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' - name: 'Create Issue on Failure' if: '${{ failure() && github.event.inputs.dry_run == false }}' diff --git a/.github/workflows/verify-release.yml b/.github/workflows/verify-release.yml index 7313cf5d40..ef3b70742c 100644 --- a/.github/workflows/verify-release.yml +++ b/.github/workflows/verify-release.yml @@ -29,3 +29,4 @@ jobs: npm-package: '${{github.event.inputs.npm-package}}' expected-version: '${{github.event.inputs.version}}' ref: '${{github.event.inputs.ref}}' + gemini_api_key: '${{ secrets.GEMINI_API_KEY }}' diff --git a/integration-tests/test-helper.ts b/integration-tests/test-helper.ts index 817323f648..ccebae614d 100644 --- a/integration-tests/test-helper.ts +++ b/integration-tests/test-helper.ts @@ -195,13 +195,32 @@ export class TestRig { execSync('sync', { cwd: this.testDir! }); } + /** + * The command and args to use to invoke Gemini CLI. Allows us to switch + * between using the bundled gemini.js (the default) and using the installed + * 'gemini' (used to verify npm bundles). + */ + private _getCommandAndArgs(extraInitialArgs: string[] = []): { + command: string; + initialArgs: string[]; + } { + const isNpmReleaseTest = + process.env.INTEGRATION_TEST_USE_INSTALLED_GEMINI === 'true'; + const command = isNpmReleaseTest ? 'gemini' : 'node'; + const initialArgs = isNpmReleaseTest + ? extraInitialArgs + : [this.bundlePath, ...extraInitialArgs]; + return { command, initialArgs }; + } + run( promptOrOptions: | string | { prompt?: string; stdin?: string; stdinDoesNotEnd?: boolean }, ...args: string[] ): Promise { - const commandArgs = [this.bundlePath, '--yolo']; + const { command, initialArgs } = this._getCommandAndArgs(['--yolo']); + const commandArgs = [...initialArgs]; const execOptions: { cwd: string; encoding: 'utf-8'; @@ -227,7 +246,7 @@ export class TestRig { commandArgs.push(...args); - const child = spawn('node', commandArgs, { + const child = spawn(command, commandArgs, { cwd: this.testDir!, stdio: 'pipe', env: process.env, @@ -331,9 +350,10 @@ export class TestRig { args: string[], options: { stdin?: string } = {}, ): Promise { - const commandArgs = [this.bundlePath, ...args]; + const { command, initialArgs } = this._getCommandAndArgs(); + const commandArgs = [...initialArgs, ...args]; - const child = spawn('node', commandArgs, { + const child = spawn(command, commandArgs, { cwd: this.testDir!, stdio: 'pipe', }); @@ -766,9 +786,10 @@ export class TestRig { ptyProcess: pty.IPty; promise: Promise<{ exitCode: number; signal?: number; output: string }>; } { - const commandArgs = [this.bundlePath, '--yolo', ...args]; + const { command, initialArgs } = this._getCommandAndArgs(['--yolo']); + const commandArgs = [...initialArgs, ...args]; - const ptyProcess = pty.spawn('node', commandArgs, { + const ptyProcess = pty.spawn(command, commandArgs, { name: 'xterm-color', cols: 80, rows: 30,