Retry evals on API error. (#23322)

.github/workflows/evals-nightly.yml

@@ -61,6 +61,7 @@ jobs:
           GEMINI_MODEL: '${{ matrix.model }}'
           RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}"
           TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
+          VITEST_RETRY: 0
         run: |
           CMD="npm run test:all_evals"
           PATTERN="${TEST_NAME_PATTERN}"

evals/vitest.config.ts

@@ -16,6 +16,10 @@ export default defineConfig({
   },
   test: {
     testTimeout: 300000, // 5 minutes
+    // Retry in CI but not nightly to avoid blocking on API error.
+    retry: process.env['VITEST_RETRY']
+      ? parseInt(process.env['VITEST_RETRY'], 10)
+      : 3,
     reporters: ['default', 'json'],
     outputFile: {
       json: 'evals/logs/report.json',