diff --git a/.github/workflows/deflake.yml b/.github/workflows/deflake.yml index 3d9e9a3fe8..c9f4c3d59f 100644 --- a/.github/workflows/deflake.yml +++ b/.github/workflows/deflake.yml @@ -24,13 +24,147 @@ concurrency: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') }} jobs: - deflake: - name: 'Deflake' + deflake_e2e_linux: + name: 'E2E Test (Linux) - ${{ matrix.sandbox }}' runs-on: 'gemini-cli-ubuntu-16-core' strategy: fail-fast: false + matrix: + sandbox: + - 'sandbox:none' + - 'sandbox:docker' + node-version: + - '20.x' + steps: - - name: 'Deflake' + - name: 'Checkout' + uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 + with: + ref: '${{ github.event.pull_request.head.sha }}' + repository: '${{ github.repository }}' + + - name: 'Set up Node.js ${{ matrix.node-version }}' + uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 + with: + node-version: '${{ matrix.node-version }}' + + - name: 'Install dependencies' + run: 'npm ci' + + - name: 'Build project' + run: 'npm run build' + + - name: 'Set up Docker' + if: "matrix.sandbox == 'sandbox:docker'" + uses: 'docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435' # ratchet:docker/setup-buildx-action@v3 + + - name: 'Run E2E tests' + env: + GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' + IS_DOCKER: "${{ matrix.sandbox == 'sandbox:docker' }}" + KEEP_OUTPUT: 'true' + RUNS: '${{ github.event.inputs.runs }}' + TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + VERBOSE: 'true' shell: 'bash' run: | - ECHO 'DEFLAKE WORKFLOW' + if [[ "${{ env.IS_DOCKER }}" == "true" ]]; then + npm run deflake:test:integration:sandbox:docker -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'" + else + npm run deflake:test:integration:sandbox:none -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'" + fi + + deflake_e2e_mac: + name: 'E2E Test (macOS)' + runs-on: 'macos-latest' + steps: + - name: 'Checkout' + uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 + with: + ref: '${{ github.event.pull_request.head.sha }}' + repository: '${{ github.repository }}' + + - name: 'Set up Node.js 20.x' + uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 + with: + node-version: '20.x' + + - name: 'Install dependencies' + run: 'npm ci' + + - name: 'Build project' + run: 'npm run build' + + - name: 'Fix rollup optional dependencies on macOS' + if: "runner.os == 'macOS'" + run: | + npm cache clean --force + - name: 'Run E2E tests (non-Windows)' + if: "runner.os != 'Windows'" + env: + GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' + KEEP_OUTPUT: 'true' + RUNS: '${{ github.event.inputs.runs }}' + SANDBOX: 'sandbox:none' + TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + VERBOSE: 'true' + run: | + npm run deflake:test:integration:sandbox:none -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'" + + deflake_e2e_windows: + name: 'Slow E2E - Win' + runs-on: 'gemini-cli-windows-16-core' + + steps: + - name: 'Checkout' + uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5 + with: + ref: '${{ github.event.pull_request.head.sha }}' + repository: '${{ github.repository }}' + + - name: 'Set up Node.js 20.x' + uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4 + with: + node-version: '20.x' + cache: 'npm' + + - name: 'Configure Windows Defender exclusions' + run: | + Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE -Force + Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\node_modules" -Force + Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\packages" -Force + Add-MpPreference -ExclusionPath "$env:TEMP" -Force + shell: 'pwsh' + + - name: 'Configure npm for Windows performance' + run: | + npm config set progress false + npm config set audit false + npm config set fund false + npm config set loglevel error + npm config set maxsockets 32 + npm config set registry https://registry.npmjs.org/ + shell: 'pwsh' + + - name: 'Install dependencies' + run: 'npm ci' + shell: 'pwsh' + + - name: 'Build project' + run: 'npm run build' + shell: 'pwsh' + + - name: 'Run E2E tests' + env: + GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' + KEEP_OUTPUT: 'true' + SANDBOX: 'sandbox:none' + VERBOSE: 'true' + NODE_OPTIONS: '--max-old-space-size=32768 --max-semi-space-size=256' + UV_THREADPOOL_SIZE: '32' + NODE_ENV: 'test' + RUNS: '${{ github.event.inputs.runs }}' + TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + shell: 'pwsh' + run: | + npm run deflake:test:integration:sandbox:none -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'" diff --git a/docs/integration-tests.md b/docs/integration-tests.md index b6b8c1ec51..24377c1934 100644 --- a/docs/integration-tests.md +++ b/docs/integration-tests.md @@ -59,12 +59,20 @@ npm run test:e2e -- --test-name-pattern "reads a file" ### Deflaking a test Before adding a **new** integration test, you should test it at least 5 times -with the deflake script to make sure that it is not flaky. +with the deflake script or workflow to make sure that it is not flaky. + +### Deflake script ```bash npm run deflake -- --runs=5 --command="npm run test:e2e -- -- --test-name-pattern ''" ``` +#### Deflake Workflow + +```bash +gh workflow run deflake.yml --ref -f test_name_pattern="" +``` + ### Running all tests To run the entire suite of integration tests, use the following command: diff --git a/package.json b/package.json index d972533488..416d83d0e0 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ "start:a2a-server": "CODER_AGENT_PORT=41242 npm run start --workspace @google/gemini-cli-a2a-server", "debug": "cross-env DEBUG=1 node --inspect-brk scripts/start.js", "deflake": "node scripts/deflake.js", - "deflake:test:integration:sandbox:none": "npm run deflake -- --command=\"npm run test:integration:sandbox:none -- --retry=0", - "deflake:test:integration:sandbox:docker": "npm run deflake -- --command=\"npm run test:integration:sandbox:docker -- --retry=0", + "deflake:test:integration:sandbox:none": "npm run deflake -- --command=\"npm run test:integration:sandbox:none -- --retry=0\"", + "deflake:test:integration:sandbox:docker": "npm run deflake -- --command=\"npm run test:integration:sandbox:docker -- --retry=0\"", "auth:npm": "npx google-artifactregistry-auth", "auth:docker": "gcloud auth configure-docker us-west1-docker.pkg.dev", "auth": "npm run auth:npm && npm run auth:docker",