feat(evals): implement production-grade PR impact analysis with isolation and policy-aware filtering

.github/workflows/eval-pr.yml

@@ -34,24 +34,41 @@ jobs:
       - name: 'Build project'
         run: 'npm run build'
 
-      - name: 'Run Evals (3 Attempts)'
+      - name: 'Run Evals (3 Attempts with Clean State)'
         env:
           GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
           RUN_EVALS: 'true'
         run: |
           MODELS=("gemini-3.1-pro-preview-customtools" "gemini-3-flash-preview")
-          mkdir -p evals/logs
+          # Create a persistent logs dir outside the workspace that won't be git-cleaned
+          FINAL_LOGS_DIR="/tmp/eval-impact-logs"
+          mkdir -p "$FINAL_LOGS_DIR"
           
           for model in "${MODELS[@]}"; do
             for attempt in {1..3}; do
               echo "::group::Running $model (Attempt $attempt)"
               DIR_NAME="eval-logs-$model-$attempt"
-              mkdir -p "evals/logs/$DIR_NAME"
-              # Run sequentially to keep one clean job in the UI
-              GEMINI_MODEL=$model npm run test:all_evals -- --outputFile.json="evals/logs/$DIR_NAME/report.json" || true
+              
+              # Run the tests
+              GEMINI_MODEL=$model npm run test:all_evals -- --outputFile.json="report.json" || true
+              
+              # Move the report to the persistent location
+              mkdir -p "$FINAL_LOGS_DIR/$DIR_NAME"
+              if [ -f "report.json" ]; then
+                mv report.json "$FINAL_LOGS_DIR/$DIR_NAME/report.json"
+              fi
+              
+              # FORCE CLEAN: Return to a perfectly pristine state
+              git clean -xfd
+              npm run build
+              
               echo "::endgroup::"
             done
           done
+          
+          # Move all logs back into the workspace for the aggregation script
+          mkdir -p evals/logs
+          cp -r "$FINAL_LOGS_DIR"/* evals/logs/
 
       - name: 'Generate Impact Report'
         id: 'generate-report'
@@ -66,6 +83,11 @@ jobs:
           echo "<!-- eval-impact-report -->" > report.md
           node scripts/aggregate_evals.js evals/logs --compare-main --pr-comment >> report.md
           cat report.md >> "$GITHUB_STEP_SUMMARY"
+          
+          # Check for blockers in the report
+          if grep -q "🔴" report.md; then
+            echo "BLOCKER_DETECTED=true" >> "$GITHUB_ENV"
+          fi
 
       - name: 'Comment on PR'
         if: 'always()'
@@ -83,3 +105,9 @@ jobs:
           else
             gh pr comment $PR_NUMBER --body-file report.md
           fi
+
+      - name: 'Block PR on Stable Regression'
+        if: "env.BLOCKER_DETECTED == 'true'"
+        run: |
+          echo "Fatal regressions detected in ALWAYS_PASSES behavioral evaluations."
+          exit 1