diff --git a/.github/workflows/evals-nightly.yml b/.github/workflows/evals-nightly.yml index 76e4da22a3..08cb90f072 100644 --- a/.github/workflows/evals-nightly.yml +++ b/.github/workflows/evals-nightly.yml @@ -9,6 +9,10 @@ on: description: 'Run all evaluations (including usually passing)' type: 'boolean' default: true + test_name_pattern: + description: 'Test name pattern or file name' + required: false + type: 'string' permissions: contents: 'read' @@ -53,7 +57,20 @@ jobs: GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' GEMINI_MODEL: '${{ matrix.model }}' RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}" - run: 'npm run test:all_evals' + TEST_NAME_PATTERN: '${{ github.event.inputs.test_name_pattern }}' + run: | + CMD="npm run test:all_evals" + PATTERN="${{ env.TEST_NAME_PATTERN }}" + + if [[ -n "$PATTERN" ]]; then + if [[ "$PATTERN" == *.ts || "$PATTERN" == *.js || "$PATTERN" == */* ]]; then + $CMD -- "$PATTERN" + else + $CMD -- -t "$PATTERN" + fi + else + $CMD + fi - name: 'Upload Logs' if: 'always()'