diff --git a/.github/workflows/evals-nightly.yml b/.github/workflows/evals-nightly.yml index c5b3709c75..ee17a95121 100644 --- a/.github/workflows/evals-nightly.yml +++ b/.github/workflows/evals-nightly.yml @@ -61,6 +61,7 @@ jobs: GEMINI_MODEL: '${{ matrix.model }}' RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}" TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + VITEST_RETRY: 0 run: | CMD="npm run test:all_evals" PATTERN="${TEST_NAME_PATTERN}" diff --git a/evals/vitest.config.ts b/evals/vitest.config.ts index 50733a999c..3231f31a10 100644 --- a/evals/vitest.config.ts +++ b/evals/vitest.config.ts @@ -16,6 +16,10 @@ export default defineConfig({ }, test: { testTimeout: 300000, // 5 minutes + // Retry in CI but not nightly to avoid blocking on API error. + retry: process.env['VITEST_RETRY'] + ? parseInt(process.env['VITEST_RETRY'], 10) + : 3, reporters: ['default', 'json'], outputFile: { json: 'evals/logs/report.json',