Merge branch 'main' into feature/eval-pr-impact

.github/workflows/eval-pr.yml

@@ -0,0 +1,113 @@
+name: 'Evals: PR Impact'
+
+on:
+  pull_request:
+    paths:
+      - 'packages/core/src/prompts/**'
+      - 'packages/core/src/tools/**'
+      - 'packages/core/src/agents/**'
+      - 'evals/**'
+
+permissions:
+  contents: 'read'
+  checks: 'write'
+  pull-requests: 'write'
+
+jobs:
+  eval-impact:
+    name: 'Eval Impact Analysis'
+    runs-on: 'gemini-cli-ubuntu-16-core'
+    if: "github.repository == 'google-gemini/gemini-cli'"
+    steps:
+      - name: 'Checkout'
+        uses: 'actions/checkout@v4'
+
+      - name: 'Set up Node.js'
+        uses: 'actions/setup-node@v4'
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: 'Install dependencies'
+        run: 'npm ci'
+
+      - name: 'Build project'
+        run: 'npm run build'
+
+      - name: 'Run Evals (3 Attempts with Clean State)'
+        env:
+          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
+          RUN_EVALS: 'true'
+        run: |
+          MODELS=("gemini-3.1-pro-preview-customtools" "gemini-3-flash-preview")
+          # Create a persistent logs dir outside the workspace that won't be git-cleaned
+          FINAL_LOGS_DIR="/tmp/eval-impact-logs"
+          mkdir -p "$FINAL_LOGS_DIR"
+          
+          for model in "${MODELS[@]}"; do
+            for attempt in {1..3}; do
+              echo "::group::Running $model (Attempt $attempt)"
+              DIR_NAME="eval-logs-$model-$attempt"
+              
+              # Run the tests
+              GEMINI_MODEL=$model npm run test:all_evals -- --outputFile.json="report.json" || true
+              
+              # Move the report to the persistent location
+              mkdir -p "$FINAL_LOGS_DIR/$DIR_NAME"
+              if [ -f "report.json" ]; then
+                mv report.json "$FINAL_LOGS_DIR/$DIR_NAME/report.json"
+              fi
+              
+              # FORCE CLEAN: Return to a perfectly pristine state
+              git clean -xfd
+              npm run build
+              
+              echo "::endgroup::"
+            done
+          done
+          
+          # Move all logs back into the workspace for the aggregation script
+          mkdir -p evals/logs
+          cp -r "$FINAL_LOGS_DIR"/* evals/logs/
+
+      - name: 'Generate Impact Report'
+        id: 'generate-report'
+        if: 'always()'
+        env:
+          GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        run: |
+          if [ ! -d "evals/logs" ]; then
+            echo "No logs found, skipping report generation."
+            exit 0
+          fi
+          echo "<!-- eval-impact-report -->" > report.md
+          node scripts/aggregate_evals.js evals/logs --compare-main --pr-comment >> report.md
+          cat report.md >> "$GITHUB_STEP_SUMMARY"
+          
+          # Check for blockers in the report
+          if grep -q "🔴" report.md; then
+            echo "BLOCKER_DETECTED=true" >> "$GITHUB_ENV"
+          fi
+
+      - name: 'Comment on PR'
+        if: 'always()'
+        env:
+          GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        run: |
+          if [ ! -f "report.md" ]; then
+            exit 0
+          fi
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          EXISTING_COMMENT=$(gh pr view $PR_NUMBER --json comments --jq '.comments[] | select(.body | contains("eval-impact-report")) | .id' | head -n 1)
+          
+          if [ -n "$EXISTING_COMMENT" ]; then
+            gh pr comment $PR_NUMBER --body-file report.md
+          else
+            gh pr comment $PR_NUMBER --body-file report.md
+          fi
+
+      - name: 'Block PR on Stable Regression'
+        if: "env.BLOCKER_DETECTED == 'true'"
+        run: |
+          echo "Fatal regressions detected in ALWAYS_PASSES behavioral evaluations."
+          exit 1

evals/README.md

@@ -202,10 +202,30 @@ Results for evaluations are available on GitHub Actions:
 - **CI Evals**: Included in the
   [E2E (Chained)](https://github.com/google-gemini/gemini-cli/actions/workflows/chained_e2e.yml)
   workflow. These must pass 100% for every PR.
+- **PR Impact Analysis**: Run automatically on PRs that modify prompts, tools,
+  or agent logic via the
+  [Evals: PR Impact](https://github.com/google-gemini/gemini-cli/actions/workflows/eval-pr.yml)
+  workflow. This provides a "before and after" comparison of behavioral eval
+  stability and will fail the PR if regressions are detected.
 - **Nightly Evals**: Run daily via the
   [Evals: Nightly](https://github.com/google-gemini/gemini-cli/actions/workflows/evals-nightly.yml)
   workflow. These track the long-term health and stability of model steering.
 
+### PR Impact Report
+
+When a PR triggers the impact analysis, a bot will post a table to the PR
+commenting on the change in pass rates for critical models (e.g. Gemini 3.1 Pro
+and Gemini 3 Flash).
+
+- **Baseline**: The pass rate from the latest successful nightly run on `main`.
+- **Current**: The pass rate on the PR branch (based on a "lite" run of 1
+  attempt).
+- **Impact**: A delta showing if stability improved (🟢), regressed (🔴), or
+  remained stable (⚪).
+
+A PR that introduces a regression (🔴) in any behavioral evaluation will fail
+this check and require investigation before merging.
+
 ### Nightly Report Format
 
 The nightly workflow executes the full evaluation suite multiple times

scripts/aggregate_evals.js

@@ -11,8 +11,32 @@ import path from 'node:path';
 import { execSync } from 'node:child_process';
 import os from 'node:os';
 
-const artifactsDir = process.argv[2] || '.';
-const MAX_HISTORY = 10;
+const args = process.argv.slice(2);
+const artifactsDir = args.find((arg) => !arg.startsWith('--')) || '.';
+const isPrComment = args.includes('--pr-comment');
+const MAX_HISTORY = 7;
+
+// Extract policies from the source code
+function getTestPolicies() {
+  const policies = {};
+  try {
+    const evalFiles = fs
+      .readdirSync('evals')
+      .filter((f) => f.endsWith('.eval.ts'));
+    for (const file of evalFiles) {
+      const content = fs.readFileSync(path.join('evals', file), 'utf-8');
+      const matches = content.matchAll(
+        /evalTest\s*\(\s*['"](ALWAYS_PASSES|USUALLY_PASSES)['"]\s*,\s*\{\s*name:\s*['"](.+?)['"]/g,
+      );
+      for (const match of matches) {
+        policies[match[2]] = match[1];
+      }
+    }
+  } catch {
+    // Ignore errors in policy extraction
+  }
+  return policies;
+}
 
 // Find all report.json files recursively
 function findReports(dir) {
@@ -34,7 +58,6 @@ function findReports(dir) {
 
 function getModelFromPath(reportPath) {
   const parts = reportPath.split(path.sep);
-  // Find the part that starts with 'eval-logs-'
   const artifactDir = parts.find((p) => p.startsWith('eval-logs-'));
   if (!artifactDir) return 'unknown';
 
@@ -42,13 +65,12 @@ function getModelFromPath(reportPath) {
   if (matchNew) return matchNew[1];
 
   const matchOld = artifactDir.match(/^eval-logs-(\d+)$/);
-  if (matchOld) return 'gemini-2.5-pro'; // Legacy default
+  if (matchOld) return 'gemini-2.5-pro';
 
   return 'unknown';
 }
 
 function getStats(reports) {
-  // Structure: { [model]: { [testName]: { passed, failed, total } } }
   const statsByModel = {};
 
   for (const reportPath of reports) {
@@ -87,35 +109,41 @@ function fetchHistoricalData() {
   const history = [];
 
   try {
-    // Determine branch
+    try {
+      execSync('gh --version', { stdio: 'ignore' });
+    } catch {
+      if (!isPrComment) {
+        console.warn(
+          'Warning: GitHub CLI (gh) not found. Historical data will be unavailable.',
+        );
+      }
+      return history;
+    }
+
     const branch = 'main';
 
-    // Get recent runs
     const cmd = `gh run list --workflow evals-nightly.yml --branch "${branch}" --limit ${
-      MAX_HISTORY + 5
+      MAX_HISTORY + 10
     } --json databaseId,createdAt,url,displayTitle,status,conclusion`;
     const runsJson = execSync(cmd, { encoding: 'utf-8' });
     let runs = JSON.parse(runsJson);
 
-    // Filter out current run
     const currentRunId = process.env.GITHUB_RUN_ID;
     if (currentRunId) {
       runs = runs.filter((r) => r.databaseId.toString() !== currentRunId);
     }
 
-    // Filter for runs that likely have artifacts (completed) and take top N
-    // We accept 'failure' too because we want to see stats.
-    runs = runs.filter((r) => r.status === 'completed').slice(0, MAX_HISTORY);
+    runs = runs
+      .filter((r) => r.status === 'completed')
+      .slice(0, MAX_HISTORY + 5);
 
-    // Fetch artifacts for each run
     for (const run of runs) {
+      if (history.length >= MAX_HISTORY) break;
+
       const tmpDir = fs.mkdtempSync(
         path.join(os.tmpdir(), `gemini-evals-${run.databaseId}-`),
       );
       try {
-        // Download report.json files.
-        // The artifacts are named 'eval-logs-X' or 'eval-logs-MODEL-X'.
-        // We use -p to match pattern.
         execSync(
           `gh run download ${run.databaseId} -p "eval-logs-*" -D "${tmpDir}"`,
           { stdio: 'ignore' },
@@ -123,16 +151,25 @@ function fetchHistoricalData() {
 
         const runReports = findReports(tmpDir);
         if (runReports.length > 0) {
-          history.push({
-            run,
-            stats: getStats(runReports), // Now returns stats grouped by model
+          const stats = getStats(runReports);
+
+          let totalPassed = 0;
+          let totalTests = 0;
+          Object.values(stats).forEach((modelStats) => {
+            Object.values(modelStats).forEach((s) => {
+              totalPassed += s.passed;
+              totalTests += s.total;
+            });
           });
+
+          if (totalTests > 0 && totalPassed === 0) {
+            continue;
+          }
+
+          history.push({ run, stats });
         }
-      } catch (error) {
-        console.error(
-          `Failed to download or process artifacts for run ${run.databaseId}:`,
-          error,
-        );
+      } catch {
+        // Ignore download errors
       } finally {
         fs.rmSync(tmpDir, { recursive: true, force: true });
       }
@@ -145,18 +182,28 @@ function fetchHistoricalData() {
 }
 
 function generateMarkdown(currentStatsByModel, history) {
-  console.log('### Evals Nightly Summary\n');
-  console.log(
-    'See [evals/README.md](https://github.com/google-gemini/gemini-cli/tree/main/evals) for more details.\n',
-  );
-
-  // Reverse history to show oldest first
   const reversedHistory = [...history].reverse();
-
   const models = Object.keys(currentStatsByModel).sort();
+  const policies = getTestPolicies();
+
+  const getConsolidatedBaseline = (model) => {
+    const consolidated = {};
+    for (const item of history) {
+      const stats = item.stats[model];
+      if (!stats) continue;
+      for (const [name, stat] of Object.entries(stats)) {
+        if (!consolidated[name]) {
+          consolidated[name] = { passed: 0, total: 0 };
+        }
+        consolidated[name].passed += stat.passed;
+        consolidated[name].total += stat.total;
+      }
+    }
+    return Object.keys(consolidated).length > 0 ? consolidated : null;
+  };
 
   const getPassRate = (statsForModel) => {
-    if (!statsForModel) return '-';
+    if (!statsForModel) return null;
     const totalStats = Object.values(statsForModel).reduce(
       (acc, stats) => {
         acc.passed += stats.passed;
@@ -166,85 +213,188 @@ function generateMarkdown(currentStatsByModel, history) {
       { passed: 0, total: 0 },
     );
     return totalStats.total > 0
-      ? ((totalStats.passed / totalStats.total) * 100).toFixed(1) + '%'
-      : '-';
+      ? (totalStats.passed / totalStats.total) * 100
+      : null;
   };
 
+  const formatPassRate = (rate) =>
+    rate === null ? '-' : rate.toFixed(1) + '%';
+
+  if (isPrComment) {
+    console.log('### 🤖 Model Steering Impact Report\n');
+
+    let blockerRegression = false;
+    for (const model of models) {
+      const currentStats = currentStatsByModel[model];
+      const baselineStats = getConsolidatedBaseline(model);
+      for (const [name, curr] of Object.entries(currentStats)) {
+        const policy = policies[name] || 'USUALLY_PASSES';
+        const currRate = (curr.passed / curr.total) * 100;
+        const base = baselineStats ? baselineStats[name] : null;
+        const baseRate = base ? (base.passed / base.total) * 100 : null;
+
+        if (policy === 'ALWAYS_PASSES' && currRate < 100) {
+          blockerRegression = true;
+        } else if (
+          policy === 'USUALLY_PASSES' &&
+          baseRate !== null &&
+          baseRate > 90 &&
+          currRate < 60
+        ) {
+          blockerRegression = true; // Significant drop in a highly stable test
+        }
+      }
+    }
+
+    if (blockerRegression) {
+      console.log('**Status: 🔴 Regression Detected (Blocking)**\n');
+      console.log(
+        'This PR has introduced regressions in stable behavioral evaluations. These must be resolved before merging.\n',
+      );
+    } else {
+      console.log('**Status: ✅ Stable**\n');
+      console.log(
+        'Behavioral evaluations remain stable compared to the `main` baseline.\n',
+      );
+    }
+
+    console.log(
+      `> **Note:** Baseline is averaged from the last ${history.length} healthy nightly runs on \`main\`.\n`,
+    );
+  } else {
+    console.log('### Evals Nightly Summary\n');
+  }
+
   for (const model of models) {
     const currentStats = currentStatsByModel[model];
-    const totalPassRate = getPassRate(currentStats);
+    const currentPassRate = getPassRate(currentStats);
+    const baselineStats = getConsolidatedBaseline(model);
+    const baselinePassRate = getPassRate(baselineStats);
 
     console.log(`#### Model: ${model}`);
-    console.log(`**Total Pass Rate: ${totalPassRate}**\n`);
 
-    // Header
-    let header = '| Test Name |';
-    let separator = '| :--- |';
-    let passRateRow = '| **Overall Pass Rate** |';
-
-    for (const item of reversedHistory) {
-      header += ` [${item.run.databaseId}](${item.run.url}) |`;
-      separator += ' :---: |';
-      passRateRow += ` **${getPassRate(item.stats[model])}** |`;
-    }
-
-    // Add Current column last
-    header += ' Current |';
-    separator += ' :---: |';
-    passRateRow += ` **${totalPassRate}** |`;
-
-    console.log(header);
-    console.log(separator);
-    console.log(passRateRow);
-
-    // Collect all test names for this model
     const allTestNames = new Set(Object.keys(currentStats));
-    for (const item of reversedHistory) {
-      if (item.stats[model]) {
-        Object.keys(item.stats[model]).forEach((name) =>
-          allTestNames.add(name),
-        );
-      }
+    if (baselineStats) {
+      Object.keys(baselineStats).forEach((name) => allTestNames.add(name));
     }
 
+    const rows = [];
+    let stableCount = 0;
+
     for (const name of Array.from(allTestNames).sort()) {
+      const policy = policies[name] || 'USUALLY_PASSES';
       const searchUrl = `https://github.com/search?q=repo%3Agoogle-gemini%2Fgemini-cli%20%22${encodeURIComponent(name)}%22&type=code`;
-      let row = `| [${name}](${searchUrl}) |`;
+      const curr = currentStats[name];
+      const base = baselineStats ? baselineStats[name] : null;
 
-      // History
-      for (const item of reversedHistory) {
-        const stat = item.stats[model] ? item.stats[model][name] : null;
-        if (stat) {
-          const passRate = ((stat.passed / stat.total) * 100).toFixed(0) + '%';
-          row += ` ${passRate} |`;
-        } else {
-          row += ' - |';
+      const currRate = curr ? (curr.passed / curr.total) * 100 : null;
+      const baseRate = base ? (base.passed / base.total) * 100 : null;
+      const delta =
+        currRate !== null && baseRate !== null ? currRate - baseRate : null;
+
+      // Smart Noise Filtering
+      let status = '⚪ Stable';
+      let isInteresting = false;
+
+      if (policy === 'ALWAYS_PASSES') {
+        if (currRate !== null && currRate < 100) {
+          status = '🔴 Regression';
+          isInteresting = true;
+        }
+      } else {
+        // USUALLY_PASSES: Only interesting if drop is > 30% OR it's a new failure
+        if (delta !== null && delta < -30) {
+          status = '🔴 Regression';
+          isInteresting = true;
+        } else if (delta !== null && delta > 30) {
+          status = '🟢 Improved';
+          isInteresting = true;
+        } else if (baseRate !== null && baseRate > 80 && currRate === 0) {
+          status = '🔴 Regression';
+          isInteresting = true;
         }
       }
 
-      // Current
-      const curr = currentStats[name];
-      if (curr) {
-        const passRate = ((curr.passed / curr.total) * 100).toFixed(0) + '%';
-        row += ` ${passRate} |`;
-      } else {
-        row += ' - |';
+      // Always show new or missing tests
+      if (currRate === null || baseRate === null) isInteresting = true;
+
+      if (isPrComment && !isInteresting) {
+        stableCount++;
+        continue;
       }
 
-      console.log(row);
+      let row = `| [${name}](${searchUrl}) | ${policy === 'ALWAYS_PASSES' ? '🔒' : '🎲'} |`;
+
+      if (!isPrComment) {
+        for (const item of reversedHistory) {
+          const stat = item.stats[model] ? item.stats[model][name] : null;
+          row += ` ${stat ? ((stat.passed / stat.total) * 100).toFixed(0) + '%' : '-'} |`;
+        }
+      } else if (baselinePassRate !== null) {
+        row += ` ${formatPassRate(baseRate)} |`;
+      }
+
+      row += ` ${formatPassRate(currRate)} | ${status} |`;
+      rows.push(row);
+    }
+
+    if (isPrComment && baselinePassRate !== null) {
+      const delta = currentPassRate - baselinePassRate;
+      const deltaStr =
+        Math.abs(delta) < 5
+          ? ' (Stable)'
+          : ` (${delta > 0 ? '↑' : '↓'} ${Math.abs(delta).toFixed(1)}%)`;
+      console.log(
+        `**Pass Rate: ${formatPassRate(currentPassRate)}** vs. ${formatPassRate(baselinePassRate)} Baseline${deltaStr}\n`,
+      );
+    }
+
+    if (isPrComment && rows.length === 0) {
+      console.log('✅ All behavioral evaluations are stable.\n');
+      continue;
+    }
+
+    let header = `| Test Name | Policy |`;
+    let separator = `| :--- | :---: |`;
+
+    if (!isPrComment) {
+      for (const item of reversedHistory) {
+        header += ` [${item.run.databaseId}](${item.run.url}) |`;
+        separator += ' :---: |';
+      }
+    } else if (baselinePassRate !== null) {
+      header += ' Baseline |';
+      separator += ' :---: |';
+    }
+
+    header += ' Current | Impact |';
+    separator += ' :---: | :---: |';
+
+    console.log(header);
+    console.log(separator);
+    rows.forEach((row) => console.log(row));
+
+    if (isPrComment && stableCount > 0) {
+      console.log(
+        `\n> **Note:** ${stableCount} stable tests were hidden from this report.\n`,
+      );
     }
     console.log('\n');
   }
-}
 
-// --- Main ---
+  if (isPrComment) {
+    console.log(
+      '---\n💡 **Policy Key:** 🔒 `ALWAYS_PASSES` (PR Blocker) | 🎲 `USUALLY_PASSES` (Informational)\n',
+    );
+    console.log(
+      '💡 To investigate regressions locally, run: `gemini /fix-behavioral-eval`',
+    );
+  }
+}
 
 const currentReports = findReports(artifactsDir);
 if (currentReports.length === 0) {
   console.log('No reports found.');
-  // We don't exit here because we might still want to see history if available,
-  // but practically if current has no reports, something is wrong.
-  // Sticking to original behavior roughly, but maybe we can continue.
   process.exit(0);
 }