From 8df12069991e635274690bc0f5ea4d3c78e7ec79 Mon Sep 17 00:00:00 2001 From: "A.K.M. Adib" Date: Mon, 23 Mar 2026 11:10:50 -0400 Subject: [PATCH] test --- .github/workflows/deflake.yml | 56 +++++++++++++++++++---------- integration-tests/plan-mode.test.ts | 39 +++++--------------- 2 files changed, 45 insertions(+), 50 deletions(-) diff --git a/.github/workflows/deflake.yml b/.github/workflows/deflake.yml index 98635dbda7..8ad6c9d2ca 100644 --- a/.github/workflows/deflake.yml +++ b/.github/workflows/deflake.yml @@ -4,14 +4,14 @@ on: workflow_dispatch: inputs: branch_ref: - description: 'Branch to run on' - required: true - default: 'main' - type: 'string' - test_name_pattern: - description: 'The test name pattern to use' + description: 'Branch to run on (defaults to selected branch)' required: false type: 'string' + test_filter: + description: 'Test name pattern to run (passed to vitest -t)' + required: false + default: 'should allow write_file only in the plans directory in plan mode' + type: 'string' runs: description: 'The number of runs' required: false @@ -27,7 +27,6 @@ jobs: deflake_e2e_linux: name: 'E2E Test (Linux) - ${{ matrix.sandbox }}' runs-on: 'gemini-cli-ubuntu-16-core' - if: "github.repository == 'google-gemini/gemini-cli'" strategy: fail-fast: false matrix: @@ -41,7 +40,7 @@ jobs: - name: 'Checkout' uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 with: - ref: '${{ github.event.pull_request.head.sha }}' + ref: '${{ github.event.inputs.branch_ref || github.ref }}' repository: '${{ github.repository }}' - name: 'Set up Node.js ${{ matrix.node-version }}' @@ -62,28 +61,37 @@ jobs: - name: 'Run E2E tests' env: GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' + DEBUG_SCHEDULER: 'true' IS_DOCKER: "${{ matrix.sandbox == 'sandbox:docker' }}" KEEP_OUTPUT: 'true' RUNS: '${{ github.event.inputs.runs }}' - TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + TEST_FILTER: '${{ github.event.inputs.test_filter }}' VERBOSE: 'true' shell: 'bash' run: | + # Construct the inner command if [[ "${IS_DOCKER}" == "true" ]]; then - npm run deflake:test:integration:sandbox:docker -- --runs="${RUNS}" -- --testNamePattern "'${TEST_NAME_PATTERN}'" + INNER_CMD="npm run test:integration:sandbox:docker -- --retry=0" else - npm run deflake:test:integration:sandbox:none -- --runs="${RUNS}" -- --testNamePattern "'${TEST_NAME_PATTERN}'" + INNER_CMD="npm run test:integration:sandbox:none -- --retry=0" fi + # Append test filter if provided + if [[ -n "${TEST_FILTER}" ]]; then + INNER_CMD="${INNER_CMD} -t \"${TEST_FILTER}\"" + fi + + # Run deflake script directly + node scripts/deflake.js --command="${INNER_CMD}" --runs="${RUNS}" + deflake_e2e_mac: name: 'E2E Test (macOS)' runs-on: 'macos-latest' - if: "github.repository == 'google-gemini/gemini-cli'" steps: - name: 'Checkout' uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 with: - ref: '${{ github.event.pull_request.head.sha }}' + ref: '${{ github.event.inputs.branch_ref || github.ref }}' repository: '${{ github.repository }}' - name: 'Set up Node.js 20.x' @@ -105,23 +113,28 @@ jobs: if: "runner.os != 'Windows'" env: GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' + DEBUG_SCHEDULER: 'true' KEEP_OUTPUT: 'true' RUNS: '${{ github.event.inputs.runs }}' SANDBOX: 'sandbox:none' - TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + TEST_FILTER: '${{ github.event.inputs.test_filter }}' VERBOSE: 'true' run: | - npm run deflake:test:integration:sandbox:none -- --runs="${RUNS}" -- --testNamePattern "'${TEST_NAME_PATTERN}'" + INNER_CMD="npm run test:integration:sandbox:none -- --retry=0" + if [[ -n "${TEST_FILTER}" ]]; then + INNER_CMD="${INNER_CMD} -t \"${TEST_FILTER}\"" + fi + node scripts/deflake.js --command="${INNER_CMD}" --runs="${RUNS}" deflake_e2e_windows: name: 'Slow E2E - Win' runs-on: 'gemini-cli-windows-16-core' - if: "github.repository == 'google-gemini/gemini-cli'" + steps: - name: 'Checkout' uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 with: - ref: '${{ github.event.pull_request.head.sha }}' + ref: '${{ github.event.inputs.branch_ref || github.ref }}' repository: '${{ github.repository }}' - name: 'Set up Node.js 20.x' @@ -159,6 +172,7 @@ jobs: - name: 'Run E2E tests' env: GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' + DEBUG_SCHEDULER: 'true' KEEP_OUTPUT: 'true' SANDBOX: 'sandbox:none' VERBOSE: 'true' @@ -166,7 +180,11 @@ jobs: UV_THREADPOOL_SIZE: '32' NODE_ENV: 'test' RUNS: '${{ github.event.inputs.runs }}' - TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + TEST_FILTER: '${{ github.event.inputs.test_filter }}' shell: 'pwsh' run: | - npm run deflake:test:integration:sandbox:none -- --runs="$env:RUNS" -- --testNamePattern "'$env:TEST_NAME_PATTERN'" + $InnerCmd = "npm run test:integration:sandbox:none -- --retry=0" + if ($env:TEST_FILTER) { + $InnerCmd = "$InnerCmd -t `"$env:TEST_FILTER`"" + } + node scripts/deflake.js --command "$InnerCmd" --runs $env:RUNS \ No newline at end of file diff --git a/integration-tests/plan-mode.test.ts b/integration-tests/plan-mode.test.ts index 8709aac189..fffc8b782e 100644 --- a/integration-tests/plan-mode.test.ts +++ b/integration-tests/plan-mode.test.ts @@ -4,10 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { writeFileSync } from 'node:fs'; -import { join } from 'node:path'; import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { TestRig, checkModelOutputContent, GEMINI_DIR } from './test-helper.js'; +import { TestRig, checkModelOutputContent } from './test-helper.js'; describe('Plan Mode', () => { let rig: TestRig; @@ -84,21 +82,14 @@ describe('Plan Mode', () => { }, }); - // Disable the interactive terminal setup prompt in tests - writeFileSync( - join(rig.homeDir!, GEMINI_DIR, 'state.json'), - JSON.stringify({ terminalSetupPromptShown: true }, null, 2), - ); - - const run = await rig.runInteractive({ + await rig.run({ approvalMode: 'plan', + stdin: 'Create a file called plan.md in the plans directory.', }); - await run.type('Create a file called plan.md in the plans directory.'); - await run.type('\r'); - - await rig.expectToolCallSuccess(['write_file'], 30000, (args) => - args.includes('plan.md'), + const writeFileCallFound = await rig.waitForToolCall('write_file_command'); + expect(writeFileCallFound, 'Expected write_file_command to fail').toBe( + false, ); const toolLogs = rig.readToolLogs(); @@ -131,19 +122,11 @@ describe('Plan Mode', () => { }, }); - // Disable the interactive terminal setup prompt in tests - writeFileSync( - join(rig.homeDir!, GEMINI_DIR, 'state.json'), - JSON.stringify({ terminalSetupPromptShown: true }, null, 2), - ); - - const run = await rig.runInteractive({ + await rig.run({ approvalMode: 'plan', + stdin: 'Create a file called hello.txt in the current directory.', }); - await run.type('Create a file called hello.txt in the current directory.'); - await run.type('\r'); - const toolLogs = rig.readToolLogs(); const writeLog = toolLogs.find( (l) => @@ -169,12 +152,6 @@ describe('Plan Mode', () => { }, }); - // Disable the interactive terminal setup prompt in tests - writeFileSync( - join(rig.homeDir!, GEMINI_DIR, 'state.json'), - JSON.stringify({ terminalSetupPromptShown: true }, null, 2), - ); - // Start in default mode and ask to enter plan mode. await rig.run({ approvalMode: 'default',