feat(infra) - Create a workflow for deflake (#11535)

Co-authored-by: gemini-cli-robot <gemini-cli-robot@google.com>

.github/workflows/deflake.yml

@@ -24,13 +24,147 @@ concurrency:
     ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') }}
 
 jobs:
-  deflake:
+  deflake_e2e_linux:
-    name: 'Deflake'
+    name: 'E2E Test (Linux) - ${{ matrix.sandbox }}'
     runs-on: 'gemini-cli-ubuntu-16-core'
     strategy:
       fail-fast: false
+      matrix:
+        sandbox:
+          - 'sandbox:none'
+          - 'sandbox:docker'
+        node-version:
+          - '20.x'
+
     steps:
-      - name: 'Deflake'
+      - name: 'Checkout'
+        uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
+        with:
+          ref: '${{ github.event.pull_request.head.sha }}'
+          repository: '${{ github.repository }}'
+
+      - name: 'Set up Node.js ${{ matrix.node-version }}'
+        uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
+        with:
+          node-version: '${{ matrix.node-version }}'
+
+      - name: 'Install dependencies'
+        run: 'npm ci'
+
+      - name: 'Build project'
+        run: 'npm run build'
+
+      - name: 'Set up Docker'
+        if: "matrix.sandbox == 'sandbox:docker'"
+        uses: 'docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435' # ratchet:docker/setup-buildx-action@v3
+
+      - name: 'Run E2E tests'
+        env:
+          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
+          IS_DOCKER: "${{ matrix.sandbox == 'sandbox:docker' }}"
+          KEEP_OUTPUT: 'true'
+          RUNS: '${{ github.event.inputs.runs }}'
+          TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
+          VERBOSE: 'true'
         shell: 'bash'
         run: |
-          ECHO 'DEFLAKE WORKFLOW'
+          if [[ "${{ env.IS_DOCKER }}" == "true" ]]; then
+            npm run deflake:test:integration:sandbox:docker -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'"
+          else
+            npm run deflake:test:integration:sandbox:none -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'"
+          fi
+
+  deflake_e2e_mac:
+    name: 'E2E Test (macOS)'
+    runs-on: 'macos-latest'
+    steps:
+      - name: 'Checkout'
+        uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
+        with:
+          ref: '${{ github.event.pull_request.head.sha }}'
+          repository: '${{ github.repository }}'
+
+      - name: 'Set up Node.js 20.x'
+        uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
+        with:
+          node-version: '20.x'
+
+      - name: 'Install dependencies'
+        run: 'npm ci'
+
+      - name: 'Build project'
+        run: 'npm run build'
+
+      - name: 'Fix rollup optional dependencies on macOS'
+        if: "runner.os == 'macOS'"
+        run: |
+          npm cache clean --force
+      - name: 'Run E2E tests (non-Windows)'
+        if: "runner.os != 'Windows'"
+        env:
+          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
+          KEEP_OUTPUT: 'true'
+          RUNS: '${{ github.event.inputs.runs }}'
+          SANDBOX: 'sandbox:none'
+          TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
+          VERBOSE: 'true'
+        run: |
+          npm run deflake:test:integration:sandbox:none -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'"
+
+  deflake_e2e_windows:
+    name: 'Slow E2E - Win'
+    runs-on: 'gemini-cli-windows-16-core'
+
+    steps:
+      - name: 'Checkout'
+        uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
+        with:
+          ref: '${{ github.event.pull_request.head.sha }}'
+          repository: '${{ github.repository }}'
+
+      - name: 'Set up Node.js 20.x'
+        uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
+        with:
+          node-version: '20.x'
+          cache: 'npm'
+
+      - name: 'Configure Windows Defender exclusions'
+        run: |
+          Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE -Force
+          Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\node_modules" -Force
+          Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\packages" -Force
+          Add-MpPreference -ExclusionPath "$env:TEMP" -Force
+        shell: 'pwsh'
+
+      - name: 'Configure npm for Windows performance'
+        run: |
+          npm config set progress false
+          npm config set audit false
+          npm config set fund false
+          npm config set loglevel error
+          npm config set maxsockets 32
+          npm config set registry https://registry.npmjs.org/
+        shell: 'pwsh'
+
+      - name: 'Install dependencies'
+        run: 'npm ci'
+        shell: 'pwsh'
+
+      - name: 'Build project'
+        run: 'npm run build'
+        shell: 'pwsh'
+
+      - name: 'Run E2E tests'
+        env:
+          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
+          KEEP_OUTPUT: 'true'
+          SANDBOX: 'sandbox:none'
+          VERBOSE: 'true'
+          NODE_OPTIONS: '--max-old-space-size=32768 --max-semi-space-size=256'
+          UV_THREADPOOL_SIZE: '32'
+          NODE_ENV: 'test'
+          RUNS: '${{ github.event.inputs.runs }}'
+          TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
+        shell: 'pwsh'
+        run: |
+          npm run deflake:test:integration:sandbox:none -- --runs="${{ env.RUNS }}" -- --testNamePattern "'${{ env.TEST_NAME_PATTERN }}'"

docs/integration-tests.md

@@ -59,12 +59,20 @@ npm run test:e2e -- --test-name-pattern "reads a file"
 ### Deflaking a test
 
 Before adding a **new** integration test, you should test it at least 5 times
-with the deflake script to make sure that it is not flaky.
+with the deflake script or workflow to make sure that it is not flaky.
+
+### Deflake script
 
 ```bash
 npm run deflake -- --runs=5 --command="npm run test:e2e -- -- --test-name-pattern '<your-new-test-name>'"
 ```
 
+#### Deflake Workflow
+
+```bash
+gh workflow run deflake.yml --ref <your-branch> -f test_name_pattern="<your-test-name-pattern>"
+```
+
 ### Running all tests
 
 To run the entire suite of integration tests, use the following command:

package.json

@@ -21,8 +21,8 @@
     "start:a2a-server": "CODER_AGENT_PORT=41242 npm run start --workspace @google/gemini-cli-a2a-server",
     "debug": "cross-env DEBUG=1 node --inspect-brk scripts/start.js",
     "deflake": "node scripts/deflake.js",
-    "deflake:test:integration:sandbox:none": "npm run deflake -- --command=\"npm run test:integration:sandbox:none -- --retry=0",
+    "deflake:test:integration:sandbox:none": "npm run deflake -- --command=\"npm run test:integration:sandbox:none -- --retry=0\"",
-    "deflake:test:integration:sandbox:docker": "npm run deflake -- --command=\"npm run test:integration:sandbox:docker -- --retry=0",
+    "deflake:test:integration:sandbox:docker": "npm run deflake -- --command=\"npm run test:integration:sandbox:docker -- --retry=0\"",
     "auth:npm": "npx google-artifactregistry-auth",
     "auth:docker": "gcloud auth configure-docker us-west1-docker.pkg.dev",
     "auth": "npm run auth:npm && npm run auth:docker",