From 28935d1e6bb76ad3931af26092c46e2fbe014bb1 Mon Sep 17 00:00:00 2001
From: Christian Gunderman <gundermanc@gmail.com>
Date: Sat, 21 Mar 2026 02:52:19 +0000
Subject: [PATCH] Retry evals on API error. (#23322)

---
 .github/workflows/evals-nightly.yml | 1 +
 evals/vitest.config.ts              | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/.github/workflows/evals-nightly.yml b/.github/workflows/evals-nightly.yml
index c5b3709c75..ee17a95121 100644
--- a/.github/workflows/evals-nightly.yml
+++ b/.github/workflows/evals-nightly.yml
@@ -61,6 +61,7 @@ jobs:
           GEMINI_MODEL: '${{ matrix.model }}'
           RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}"
           TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
+          VITEST_RETRY: 0
         run: |
           CMD="npm run test:all_evals"
           PATTERN="${TEST_NAME_PATTERN}"
diff --git a/evals/vitest.config.ts b/evals/vitest.config.ts
index 50733a999c..3231f31a10 100644
--- a/evals/vitest.config.ts
+++ b/evals/vitest.config.ts
@@ -16,6 +16,10 @@ export default defineConfig({
   },
   test: {
     testTimeout: 300000, // 5 minutes
+    // Retry in CI but not nightly to avoid blocking on API error.
+    retry: process.env['VITEST_RETRY']
+      ? parseInt(process.env['VITEST_RETRY'], 10)
+      : 3,
     reporters: ['default', 'json'],
     outputFile: {
       json: 'evals/logs/report.json',