feat(evals): add reliability harvester and 500/503 retry support (#23626)

2026-04-21 02:24:09 -07:00 · 2026-03-25 18:48:45 -07:00
parent c1e4dbd157
commit 2e03e3aed5
6 changed files with 509 additions and 75 deletions
@@ -61,6 +61,8 @@ jobs:
          GEMINI_MODEL: '${{ matrix.model }}'
          RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}"
          TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
+          # Disable Vitest internal retries to avoid double-retrying;
+          # custom retry logic is handled in evals/test-helper.ts
          VITEST_RETRY: 0
        run: |
          CMD="npm run test:all_evals"