feat(evals): add reliability harvester and 500/503 retry support

2026-07-21 23:40:44 -07:00 · 2026-03-23 20:37:08 -07:00
parent fc03891a11
commit 20004fb526
5 changed files with 506 additions and 71 deletions
@@ -61,6 +61,9 @@ jobs:
          GEMINI_MODEL: '${{ matrix.model }}'
          RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}"
          TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}'
+          # Disable Vitest internal retries to avoid double-retrying;
+          # custom retry logic is handled in evals/test-helper.ts
+          VITEST_RETRY: 0
        run: |
          CMD="npm run test:all_evals"
          PATTERN="${TEST_NAME_PATTERN}"