chore(evals): add jitter and max-parallel to mitigate 500s

chore(evals): increase nightly runs to 10 for precision baseline
2026-07-31 20:21:01 -07:00 · 2026-03-24 16:05:23 -07:00 · 2026-03-24 14:43:05 -07:00
1 changed files with 7 additions and 1 deletions
@@ -26,6 +26,7 @@ jobs:
    if: "github.repository == 'google-gemini/gemini-cli'"
    strategy:
      fail-fast: false
+      max-parallel: 10
      matrix:
        model:
          - 'gemini-3.1-pro-preview-customtools'
@@ -34,7 +35,7 @@ jobs:
          - 'gemini-2.5-pro'
          - 'gemini-2.5-flash'
          - 'gemini-2.5-flash-lite'
-        run_attempt: [1, 2, 3]
+        run_attempt: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8' # ratchet:actions/checkout@v5
@@ -63,6 +64,11 @@ jobs:
          TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
          VITEST_RETRY: 0
        run: |
+          # Add jitter to prevent all jobs from hitting API simultaneously
+          SLEEP_TIME=$((RANDOM % 60))
+          echo "Jitter: Sleeping for $SLEEP_TIME seconds..."
+          sleep $SLEEP_TIME
+
          CMD="npm run test:all_evals"
          PATTERN="${TEST_NAME_PATTERN}"
Author	SHA1	Message	Date
Alisa Novikova	8ebc0864db	chore(evals): add jitter and max-parallel to mitigate 500s	2026-03-24 16:05:23 -07:00
Alisa Novikova	9bc176bd09	chore(evals): increase nightly runs to 10 for precision baseline	2026-03-24 14:43:05 -07:00