diff --git a/.github/workflows/eval-pr.yml b/.github/workflows/eval-pr.yml new file mode 100644 index 0000000000..f2c7402d4f --- /dev/null +++ b/.github/workflows/eval-pr.yml @@ -0,0 +1,88 @@ +name: 'Evals: PR Impact' + +on: + pull_request: + paths: + - 'packages/core/src/prompts/**' + - 'packages/core/src/tools/**' + - 'packages/core/src/agents/**' + - 'evals/**' + +permissions: + contents: 'read' + checks: 'write' + pull-requests: 'write' + +jobs: + eval-impact: + name: 'Eval Impact Analysis' + runs-on: 'gemini-cli-ubuntu-16-core' + if: "github.repository == 'google-gemini/gemini-cli'" + steps: + - name: 'Checkout' + uses: 'actions/checkout@v4' + + - name: 'Set up Node.js' + uses: 'actions/setup-node@v4' + with: + node-version-file: '.nvmrc' + cache: 'npm' + + - name: 'Install dependencies' + run: 'npm ci' + + - name: 'Build project' + run: 'npm run build' + + - name: 'Create logs directory' + run: 'mkdir -p evals/logs' + + - name: 'Run Evals (Lite)' + env: + GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}' + RUN_EVALS: 'true' + run: | + # Run 1 attempt on critical models for speed in PR + MODELS=("gemini-3.1-pro-preview-customtools" "gemini-3-flash-preview") + for model in "${MODELS[@]}"; do + echo "Running evals for $model..." + mkdir -p "evals/logs/eval-logs-$model-1" + # We redirect to report.json in a subfolder so aggregate_evals.js can find it via subfolder name + GEMINI_MODEL=$model npm run test:all_evals -- --outputFile.json="evals/logs/eval-logs-$model-1/report.json" + done + + - name: 'Generate Impact Report' + id: 'generate-report' + env: + GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + run: | + echo "" > report.md + node scripts/aggregate_evals.js evals/logs --compare-main --pr-comment >> report.md + cat report.md >> "$GITHUB_STEP_SUMMARY" + + # Check if there are any regressions (🔴) + if grep -q "🔴" report.md; then + echo "REGRESSION_DETECTED=true" >> "$GITHUB_ENV" + fi + + - name: 'Comment on PR' + env: + GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + run: | + PR_NUMBER=${{ github.event.pull_request.number }} + # Try to find existing comment to avoid spamming + EXISTING_COMMENT=$(gh pr view $PR_NUMBER --json comments --jq '.comments[] | select(.body | contains("eval-impact-report")) | .id' | head -n 1) + + if [ -n "$EXISTING_COMMENT" ]; then + # We can't easily edit by ID with 'gh pr comment', so we just post a new one for now + # but in the future we could use 'gh api' to update. + gh pr comment $PR_NUMBER --body-file report.md + else + gh pr comment $PR_NUMBER --body-file report.md + fi + + - name: 'Fail if regression detected' + if: "env.REGRESSION_DETECTED == 'true'" + run: | + echo "Behavioral evaluation regressions detected. Please check the impact report." + exit 1 diff --git a/evals/README.md b/evals/README.md index 6cfecbad07..b2d50b4939 100644 --- a/evals/README.md +++ b/evals/README.md @@ -200,10 +200,30 @@ Results for evaluations are available on GitHub Actions: - **CI Evals**: Included in the [E2E (Chained)](https://github.com/google-gemini/gemini-cli/actions/workflows/chained_e2e.yml) workflow. These must pass 100% for every PR. +- **PR Impact Analysis**: Run automatically on PRs that modify prompts, tools, + or agent logic via the + [Evals: PR Impact](https://github.com/google-gemini/gemini-cli/actions/workflows/eval-pr.yml) + workflow. This provides a "before and after" comparison of behavioral eval + stability and will fail the PR if regressions are detected. - **Nightly Evals**: Run daily via the [Evals: Nightly](https://github.com/google-gemini/gemini-cli/actions/workflows/evals-nightly.yml) workflow. These track the long-term health and stability of model steering. +### PR Impact Report + +When a PR triggers the impact analysis, a bot will post a table to the PR +commenting on the change in pass rates for critical models (e.g. Gemini 3.1 Pro +and Gemini 3 Flash). + +- **Baseline**: The pass rate from the latest successful nightly run on `main`. +- **Current**: The pass rate on the PR branch (based on a "lite" run of 1 + attempt). +- **Impact**: A delta showing if stability improved (🟢), regressed (🔴), or + remained stable (⚪). + +A PR that introduces a regression (🔴) in any behavioral evaluation will fail +this check and require investigation before merging. + ### Nightly Report Format The nightly workflow executes the full evaluation suite multiple times diff --git a/scripts/aggregate_evals.js b/scripts/aggregate_evals.js index 263660a25a..6594b05630 100644 --- a/scripts/aggregate_evals.js +++ b/scripts/aggregate_evals.js @@ -11,8 +11,10 @@ import path from 'node:path'; import { execSync } from 'node:child_process'; import os from 'node:os'; -const artifactsDir = process.argv[2] || '.'; -const MAX_HISTORY = 10; +const args = process.argv.slice(2); +const artifactsDir = args.find((arg) => !arg.startsWith('--')) || '.'; +const isPrComment = args.includes('--pr-comment'); +const MAX_HISTORY = isPrComment ? 1 : 10; // Find all report.json files recursively function findReports(dir) { @@ -87,8 +89,20 @@ function fetchHistoricalData() { const history = []; try { - // Determine branch + // Check if gh is available + try { + execSync('gh --version', { stdio: 'ignore' }); + } catch { + if (!isPrComment) { + console.warn( + 'Warning: GitHub CLI (gh) not found. Historical data will be unavailable.', + ); + } + return history; + } + const branch = 'main'; + // ... rest of the function ... // Get recent runs const cmd = `gh run list --workflow evals-nightly.yml --branch "${branch}" --limit ${ @@ -145,18 +159,23 @@ function fetchHistoricalData() { } function generateMarkdown(currentStatsByModel, history) { - console.log('### Evals Nightly Summary\n'); - console.log( - 'See [evals/README.md](https://github.com/google-gemini/gemini-cli/tree/main/evals) for more details.\n', - ); + if (isPrComment) { + console.log('### 🤖 Model Steering Impact Report\n'); + console.log( + "This PR modifies files that affect the model's behavior. Below is the impact on behavioral evaluations.\n", + ); + } else { + console.log('### Evals Nightly Summary\n'); + console.log( + 'See [evals/README.md](https://github.com/google-gemini/gemini-cli/tree/main/evals) for more details.\n', + ); + } - // Reverse history to show oldest first const reversedHistory = [...history].reverse(); - const models = Object.keys(currentStatsByModel).sort(); const getPassRate = (statsForModel) => { - if (!statsForModel) return '-'; + if (!statsForModel) return null; const totalStats = Object.values(statsForModel).reduce( (acc, stats) => { acc.passed += stats.passed; @@ -166,67 +185,89 @@ function generateMarkdown(currentStatsByModel, history) { { passed: 0, total: 0 }, ); return totalStats.total > 0 - ? ((totalStats.passed / totalStats.total) * 100).toFixed(1) + '%' - : '-'; + ? (totalStats.passed / totalStats.total) * 100 + : null; }; + const formatPassRate = (rate) => + rate === null ? '-' : rate.toFixed(1) + '%'; + for (const model of models) { const currentStats = currentStatsByModel[model]; - const totalPassRate = getPassRate(currentStats); + const currentPassRate = getPassRate(currentStats); + + // Baseline is the most recent history entry + const baselineStats = + reversedHistory.length > 0 + ? reversedHistory[reversedHistory.length - 1].stats[model] + : null; + const baselinePassRate = getPassRate(baselineStats); console.log(`#### Model: ${model}`); - console.log(`**Total Pass Rate: ${totalPassRate}**\n`); + if (isPrComment && baselinePassRate !== null) { + const delta = currentPassRate - baselinePassRate; + const deltaStr = + delta === 0 + ? ' (No Change)' + : ` (${delta > 0 ? '↑' : '↓'} ${Math.abs(delta).toFixed(1)}%)`; + console.log( + `**Pass Rate: ${formatPassRate(currentPassRate)}** from ${formatPassRate(baselinePassRate)}${deltaStr}\n`, + ); + } else { + console.log(`**Total Pass Rate: ${formatPassRate(currentPassRate)}**\n`); + } // Header let header = '| Test Name |'; let separator = '| :--- |'; - let passRateRow = '| **Overall Pass Rate** |'; - for (const item of reversedHistory) { - header += ` [${item.run.databaseId}](${item.run.url}) |`; + if (!isPrComment) { + for (const item of reversedHistory) { + header += ` [${item.run.databaseId}](${item.run.url}) |`; + separator += ' :---: |'; + } + } else if (baselinePassRate !== null) { + header += ' Baseline |'; separator += ' :---: |'; - passRateRow += ` **${getPassRate(item.stats[model])}** |`; } - // Add Current column last - header += ' Current |'; - separator += ' :---: |'; - passRateRow += ` **${totalPassRate}** |`; + header += ' Current | Impact |'; + separator += ' :---: | :---: |'; console.log(header); console.log(separator); - console.log(passRateRow); - // Collect all test names for this model const allTestNames = new Set(Object.keys(currentStats)); - for (const item of reversedHistory) { - if (item.stats[model]) { - Object.keys(item.stats[model]).forEach((name) => - allTestNames.add(name), - ); - } + if (baselineStats) { + Object.keys(baselineStats).forEach((name) => allTestNames.add(name)); } for (const name of Array.from(allTestNames).sort()) { const searchUrl = `https://github.com/search?q=repo%3Agoogle-gemini%2Fgemini-cli%20%22${encodeURIComponent(name)}%22&type=code`; let row = `| [${name}](${searchUrl}) |`; - // History - for (const item of reversedHistory) { - const stat = item.stats[model] ? item.stats[model][name] : null; - if (stat) { - const passRate = ((stat.passed / stat.total) * 100).toFixed(0) + '%'; - row += ` ${passRate} |`; - } else { - row += ' - |'; + const curr = currentStats[name]; + const base = baselineStats ? baselineStats[name] : null; + + if (!isPrComment) { + for (const item of reversedHistory) { + const stat = item.stats[model] ? item.stats[model][name] : null; + row += ` ${stat ? ((stat.passed / stat.total) * 100).toFixed(0) + '%' : '-'} |`; } + } else if (baselinePassRate !== null) { + row += ` ${base ? ((base.passed / base.total) * 100).toFixed(0) + '%' : '-'} |`; } - // Current - const curr = currentStats[name]; - if (curr) { - const passRate = ((curr.passed / curr.total) * 100).toFixed(0) + '%'; - row += ` ${passRate} |`; + const currRate = curr ? (curr.passed / curr.total) * 100 : null; + const baseRate = base ? (base.passed / base.total) * 100 : null; + + row += ` ${formatPassRate(currRate)} |`; + + if (currRate !== null && baseRate !== null) { + const delta = currRate - baseRate; + if (delta > 0) row += ` 🟢 +${delta.toFixed(0)}% |`; + else if (delta < 0) row += ` 🔴 ${delta.toFixed(0)}% |`; + else row += ' ⚪ 0% |'; } else { row += ' - |'; } @@ -242,9 +283,6 @@ function generateMarkdown(currentStatsByModel, history) { const currentReports = findReports(artifactsDir); if (currentReports.length === 0) { console.log('No reports found.'); - // We don't exit here because we might still want to see history if available, - // but practically if current has no reports, something is wrong. - // Sticking to original behavior roughly, but maybe we can continue. process.exit(0); }