mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-05-16 14:53:19 -07:00
Alisa Novikova
114 lines
3.6 KiB
YAML
114 lines
3.6 KiB
YAML
name: 'Evals: PR Impact'
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- 'packages/core/src/prompts/**'
|
|
- 'packages/core/src/tools/**'
|
|
- 'packages/core/src/agents/**'
|
|
- 'evals/**'
|
|
|
|
permissions:
|
|
contents: 'read'
|
|
checks: 'write'
|
|
pull-requests: 'write'
|
|
|
|
jobs:
|
|
eval-impact:
|
|
name: 'Eval Impact Analysis'
|
|
runs-on: 'gemini-cli-ubuntu-16-core'
|
|
if: "github.repository == 'google-gemini/gemini-cli'"
|
|
steps:
|
|
- name: 'Checkout'
|
|
uses: 'actions/checkout@v4'
|
|
|
|
- name: 'Set up Node.js'
|
|
uses: 'actions/setup-node@v4'
|
|
with:
|
|
node-version-file: '.nvmrc'
|
|
cache: 'npm'
|
|
|
|
- name: 'Install dependencies'
|
|
run: 'npm ci'
|
|
|
|
- name: 'Build project'
|
|
run: 'npm run build'
|
|
|
|
- name: 'Run Evals (3 Attempts with Clean State)'
|
|
env:
|
|
GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
|
|
RUN_EVALS: 'true'
|
|
run: |
|
|
MODELS=("gemini-3.1-pro-preview-customtools" "gemini-3-flash-preview")
|
|
# Create a persistent logs dir outside the workspace that won't be git-cleaned
|
|
FINAL_LOGS_DIR="/tmp/eval-impact-logs"
|
|
mkdir -p "$FINAL_LOGS_DIR"
|
|
|
|
for model in "${MODELS[@]}"; do
|
|
for attempt in {1..3}; do
|
|
echo "::group::Running $model (Attempt $attempt)"
|
|
DIR_NAME="eval-logs-$model-$attempt"
|
|
|
|
# Run the tests
|
|
GEMINI_MODEL=$model npm run test:all_evals -- --outputFile.json="report.json" || true
|
|
|
|
# Move the report to the persistent location
|
|
mkdir -p "$FINAL_LOGS_DIR/$DIR_NAME"
|
|
if [ -f "report.json" ]; then
|
|
mv report.json "$FINAL_LOGS_DIR/$DIR_NAME/report.json"
|
|
fi
|
|
|
|
# FORCE CLEAN: Return to a perfectly pristine state
|
|
git clean -xfd
|
|
npm run build
|
|
|
|
echo "::endgroup::"
|
|
done
|
|
done
|
|
|
|
# Move all logs back into the workspace for the aggregation script
|
|
mkdir -p evals/logs
|
|
cp -r "$FINAL_LOGS_DIR"/* evals/logs/
|
|
|
|
- name: 'Generate Impact Report'
|
|
id: 'generate-report'
|
|
if: 'always()'
|
|
env:
|
|
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
|
run: |
|
|
if [ ! -d "evals/logs" ]; then
|
|
echo "No logs found, skipping report generation."
|
|
exit 0
|
|
fi
|
|
echo "<!-- eval-impact-report -->" > report.md
|
|
node scripts/aggregate_evals.js evals/logs --compare-main --pr-comment >> report.md
|
|
cat report.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# Check for blockers in the report
|
|
if grep -q "🔴" report.md; then
|
|
echo "BLOCKER_DETECTED=true" >> "$GITHUB_ENV"
|
|
fi
|
|
|
|
- name: 'Comment on PR'
|
|
if: 'always()'
|
|
env:
|
|
GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
|
|
run: |
|
|
if [ ! -f "report.md" ]; then
|
|
exit 0
|
|
fi
|
|
PR_NUMBER=${{ github.event.pull_request.number }}
|
|
EXISTING_COMMENT=$(gh pr view $PR_NUMBER --json comments --jq '.comments[] | select(.body | contains("eval-impact-report")) | .id' | head -n 1)
|
|
|
|
if [ -n "$EXISTING_COMMENT" ]; then
|
|
gh pr comment $PR_NUMBER --body-file report.md
|
|
else
|
|
gh pr comment $PR_NUMBER --body-file report.md
|
|
fi
|
|
|
|
- name: 'Block PR on Stable Regression'
|
|
if: "env.BLOCKER_DETECTED == 'true'"
|
|
run: |
|
|
echo "Fatal regressions detected in ALWAYS_PASSES behavioral evaluations."
|
|
exit 1
|