mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-05-17 07:13:07 -07:00
Alisa Novikova
404 lines
12 KiB
JavaScript
404 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* @license
|
|
* Copyright 2026 Google LLC
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { execSync } from 'node:child_process';
|
|
import os from 'node:os';
|
|
|
|
const args = process.argv.slice(2);
|
|
const artifactsDir = args.find((arg) => !arg.startsWith('--')) || '.';
|
|
const isPrComment = args.includes('--pr-comment');
|
|
const MAX_HISTORY = 7;
|
|
|
|
// Extract policies from the source code
|
|
function getTestPolicies() {
|
|
const policies = {};
|
|
try {
|
|
const evalFiles = fs
|
|
.readdirSync('evals')
|
|
.filter((f) => f.endsWith('.eval.ts'));
|
|
for (const file of evalFiles) {
|
|
const content = fs.readFileSync(path.join('evals', file), 'utf-8');
|
|
const matches = content.matchAll(
|
|
/evalTest\s*\(\s*['"](ALWAYS_PASSES|USUALLY_PASSES)['"]\s*,\s*\{\s*name:\s*['"](.+?)['"]/g,
|
|
);
|
|
for (const match of matches) {
|
|
policies[match[2]] = match[1];
|
|
}
|
|
}
|
|
} catch {
|
|
// Ignore errors in policy extraction
|
|
}
|
|
return policies;
|
|
}
|
|
|
|
// Find all report.json files recursively
|
|
function findReports(dir) {
|
|
const reports = [];
|
|
if (!fs.existsSync(dir)) return reports;
|
|
|
|
const files = fs.readdirSync(dir);
|
|
for (const file of files) {
|
|
const fullPath = path.join(dir, file);
|
|
const stat = fs.statSync(fullPath);
|
|
if (stat.isDirectory()) {
|
|
reports.push(...findReports(fullPath));
|
|
} else if (file === 'report.json') {
|
|
reports.push(fullPath);
|
|
}
|
|
}
|
|
return reports;
|
|
}
|
|
|
|
function getModelFromPath(reportPath) {
|
|
const parts = reportPath.split(path.sep);
|
|
const artifactDir = parts.find((p) => p.startsWith('eval-logs-'));
|
|
if (!artifactDir) return 'unknown';
|
|
|
|
const matchNew = artifactDir.match(/^eval-logs-(.+)-(\d+)$/);
|
|
if (matchNew) return matchNew[1];
|
|
|
|
const matchOld = artifactDir.match(/^eval-logs-(\d+)$/);
|
|
if (matchOld) return 'gemini-2.5-pro';
|
|
|
|
return 'unknown';
|
|
}
|
|
|
|
function getStats(reports) {
|
|
const statsByModel = {};
|
|
|
|
for (const reportPath of reports) {
|
|
try {
|
|
const model = getModelFromPath(reportPath);
|
|
if (!statsByModel[model]) {
|
|
statsByModel[model] = {};
|
|
}
|
|
const testStats = statsByModel[model];
|
|
|
|
const content = fs.readFileSync(reportPath, 'utf-8');
|
|
const json = JSON.parse(content);
|
|
|
|
for (const testResult of json.testResults) {
|
|
for (const assertion of testResult.assertionResults) {
|
|
const name = assertion.title;
|
|
if (!testStats[name]) {
|
|
testStats[name] = { passed: 0, failed: 0, total: 0 };
|
|
}
|
|
testStats[name].total++;
|
|
if (assertion.status === 'passed') {
|
|
testStats[name].passed++;
|
|
} else {
|
|
testStats[name].failed++;
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error(`Error processing report at ${reportPath}:`, error);
|
|
}
|
|
}
|
|
return statsByModel;
|
|
}
|
|
|
|
function fetchHistoricalData() {
|
|
const history = [];
|
|
|
|
try {
|
|
try {
|
|
execSync('gh --version', { stdio: 'ignore' });
|
|
} catch {
|
|
if (!isPrComment) {
|
|
console.warn(
|
|
'Warning: GitHub CLI (gh) not found. Historical data will be unavailable.',
|
|
);
|
|
}
|
|
return history;
|
|
}
|
|
|
|
const branch = 'main';
|
|
|
|
const cmd = `gh run list --workflow evals-nightly.yml --branch "${branch}" --limit ${
|
|
MAX_HISTORY + 10
|
|
} --json databaseId,createdAt,url,displayTitle,status,conclusion`;
|
|
const runsJson = execSync(cmd, { encoding: 'utf-8' });
|
|
let runs = JSON.parse(runsJson);
|
|
|
|
const currentRunId = process.env.GITHUB_RUN_ID;
|
|
if (currentRunId) {
|
|
runs = runs.filter((r) => r.databaseId.toString() !== currentRunId);
|
|
}
|
|
|
|
runs = runs
|
|
.filter((r) => r.status === 'completed')
|
|
.slice(0, MAX_HISTORY + 5);
|
|
|
|
for (const run of runs) {
|
|
if (history.length >= MAX_HISTORY) break;
|
|
|
|
const tmpDir = fs.mkdtempSync(
|
|
path.join(os.tmpdir(), `gemini-evals-${run.databaseId}-`),
|
|
);
|
|
try {
|
|
execSync(
|
|
`gh run download ${run.databaseId} -p "eval-logs-*" -D "${tmpDir}"`,
|
|
{ stdio: 'ignore' },
|
|
);
|
|
|
|
const runReports = findReports(tmpDir);
|
|
if (runReports.length > 0) {
|
|
const stats = getStats(runReports);
|
|
|
|
let totalPassed = 0;
|
|
let totalTests = 0;
|
|
Object.values(stats).forEach((modelStats) => {
|
|
Object.values(modelStats).forEach((s) => {
|
|
totalPassed += s.passed;
|
|
totalTests += s.total;
|
|
});
|
|
});
|
|
|
|
if (totalTests > 0 && totalPassed === 0) {
|
|
continue;
|
|
}
|
|
|
|
history.push({ run, stats });
|
|
}
|
|
} catch {
|
|
// Ignore download errors
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('Failed to fetch historical data:', error);
|
|
}
|
|
|
|
return history;
|
|
}
|
|
|
|
function generateMarkdown(currentStatsByModel, history) {
|
|
const reversedHistory = [...history].reverse();
|
|
const models = Object.keys(currentStatsByModel).sort();
|
|
const policies = getTestPolicies();
|
|
|
|
const getConsolidatedBaseline = (model) => {
|
|
const consolidated = {};
|
|
for (const item of history) {
|
|
const stats = item.stats[model];
|
|
if (!stats) continue;
|
|
for (const [name, stat] of Object.entries(stats)) {
|
|
if (!consolidated[name]) {
|
|
consolidated[name] = { passed: 0, total: 0 };
|
|
}
|
|
consolidated[name].passed += stat.passed;
|
|
consolidated[name].total += stat.total;
|
|
}
|
|
}
|
|
return Object.keys(consolidated).length > 0 ? consolidated : null;
|
|
};
|
|
|
|
const getPassRate = (statsForModel) => {
|
|
if (!statsForModel) return null;
|
|
const totalStats = Object.values(statsForModel).reduce(
|
|
(acc, stats) => {
|
|
acc.passed += stats.passed;
|
|
acc.total += stats.total;
|
|
return acc;
|
|
},
|
|
{ passed: 0, total: 0 },
|
|
);
|
|
return totalStats.total > 0
|
|
? (totalStats.passed / totalStats.total) * 100
|
|
: null;
|
|
};
|
|
|
|
const formatPassRate = (rate) =>
|
|
rate === null ? '-' : rate.toFixed(1) + '%';
|
|
|
|
if (isPrComment) {
|
|
console.log('### 🤖 Model Steering Impact Report\n');
|
|
|
|
let blockerRegression = false;
|
|
for (const model of models) {
|
|
const currentStats = currentStatsByModel[model];
|
|
const baselineStats = getConsolidatedBaseline(model);
|
|
for (const [name, curr] of Object.entries(currentStats)) {
|
|
const policy = policies[name] || 'USUALLY_PASSES';
|
|
const currRate = (curr.passed / curr.total) * 100;
|
|
const base = baselineStats ? baselineStats[name] : null;
|
|
const baseRate = base ? (base.passed / base.total) * 100 : null;
|
|
|
|
if (policy === 'ALWAYS_PASSES' && currRate < 100) {
|
|
blockerRegression = true;
|
|
} else if (
|
|
policy === 'USUALLY_PASSES' &&
|
|
baseRate !== null &&
|
|
baseRate > 90 &&
|
|
currRate < 60
|
|
) {
|
|
blockerRegression = true; // Significant drop in a highly stable test
|
|
}
|
|
}
|
|
}
|
|
|
|
if (blockerRegression) {
|
|
console.log('**Status: 🔴 Regression Detected (Blocking)**\n');
|
|
console.log(
|
|
'This PR has introduced regressions in stable behavioral evaluations. These must be resolved before merging.\n',
|
|
);
|
|
} else {
|
|
console.log('**Status: ✅ Stable**\n');
|
|
console.log(
|
|
'Behavioral evaluations remain stable compared to the `main` baseline.\n',
|
|
);
|
|
}
|
|
|
|
console.log(
|
|
`> **Note:** Baseline is averaged from the last ${history.length} healthy nightly runs on \`main\`.\n`,
|
|
);
|
|
} else {
|
|
console.log('### Evals Nightly Summary\n');
|
|
}
|
|
|
|
for (const model of models) {
|
|
const currentStats = currentStatsByModel[model];
|
|
const currentPassRate = getPassRate(currentStats);
|
|
const baselineStats = getConsolidatedBaseline(model);
|
|
const baselinePassRate = getPassRate(baselineStats);
|
|
|
|
console.log(`#### Model: ${model}`);
|
|
|
|
const allTestNames = new Set(Object.keys(currentStats));
|
|
if (baselineStats) {
|
|
Object.keys(baselineStats).forEach((name) => allTestNames.add(name));
|
|
}
|
|
|
|
const rows = [];
|
|
let stableCount = 0;
|
|
|
|
for (const name of Array.from(allTestNames).sort()) {
|
|
const policy = policies[name] || 'USUALLY_PASSES';
|
|
const searchUrl = `https://github.com/search?q=repo%3Agoogle-gemini%2Fgemini-cli%20%22${encodeURIComponent(name)}%22&type=code`;
|
|
const curr = currentStats[name];
|
|
const base = baselineStats ? baselineStats[name] : null;
|
|
|
|
const currRate = curr ? (curr.passed / curr.total) * 100 : null;
|
|
const baseRate = base ? (base.passed / base.total) * 100 : null;
|
|
const delta =
|
|
currRate !== null && baseRate !== null ? currRate - baseRate : null;
|
|
|
|
// Smart Noise Filtering
|
|
let status = '⚪ Stable';
|
|
let isInteresting = false;
|
|
|
|
if (policy === 'ALWAYS_PASSES') {
|
|
if (currRate !== null && currRate < 100) {
|
|
status = '🔴 Regression';
|
|
isInteresting = true;
|
|
}
|
|
} else {
|
|
// USUALLY_PASSES: Only interesting if drop is > 30% OR it's a new failure
|
|
if (delta !== null && delta < -30) {
|
|
status = '🔴 Regression';
|
|
isInteresting = true;
|
|
} else if (delta !== null && delta > 30) {
|
|
status = '🟢 Improved';
|
|
isInteresting = true;
|
|
} else if (baseRate !== null && baseRate > 80 && currRate === 0) {
|
|
status = '🔴 Regression';
|
|
isInteresting = true;
|
|
}
|
|
}
|
|
|
|
// Always show new or missing tests
|
|
if (currRate === null || baseRate === null) isInteresting = true;
|
|
|
|
if (isPrComment && !isInteresting) {
|
|
stableCount++;
|
|
continue;
|
|
}
|
|
|
|
let row = `| [${name}](${searchUrl}) | ${policy === 'ALWAYS_PASSES' ? '🔒' : '🎲'} |`;
|
|
|
|
if (!isPrComment) {
|
|
for (const item of reversedHistory) {
|
|
const stat = item.stats[model] ? item.stats[model][name] : null;
|
|
row += ` ${stat ? ((stat.passed / stat.total) * 100).toFixed(0) + '%' : '-'} |`;
|
|
}
|
|
} else if (baselinePassRate !== null) {
|
|
row += ` ${formatPassRate(baseRate)} |`;
|
|
}
|
|
|
|
row += ` ${formatPassRate(currRate)} | ${status} |`;
|
|
rows.push(row);
|
|
}
|
|
|
|
if (isPrComment && baselinePassRate !== null) {
|
|
const delta = currentPassRate - baselinePassRate;
|
|
const deltaStr =
|
|
Math.abs(delta) < 5
|
|
? ' (Stable)'
|
|
: ` (${delta > 0 ? '↑' : '↓'} ${Math.abs(delta).toFixed(1)}%)`;
|
|
console.log(
|
|
`**Pass Rate: ${formatPassRate(currentPassRate)}** vs. ${formatPassRate(baselinePassRate)} Baseline${deltaStr}\n`,
|
|
);
|
|
}
|
|
|
|
if (isPrComment && rows.length === 0) {
|
|
console.log('✅ All behavioral evaluations are stable.\n');
|
|
continue;
|
|
}
|
|
|
|
let header = `| Test Name | Policy |`;
|
|
let separator = `| :--- | :---: |`;
|
|
|
|
if (!isPrComment) {
|
|
for (const item of reversedHistory) {
|
|
header += ` [${item.run.databaseId}](${item.run.url}) |`;
|
|
separator += ' :---: |';
|
|
}
|
|
} else if (baselinePassRate !== null) {
|
|
header += ' Baseline |';
|
|
separator += ' :---: |';
|
|
}
|
|
|
|
header += ' Current | Impact |';
|
|
separator += ' :---: | :---: |';
|
|
|
|
console.log(header);
|
|
console.log(separator);
|
|
rows.forEach((row) => console.log(row));
|
|
|
|
if (isPrComment && stableCount > 0) {
|
|
console.log(
|
|
`\n> **Note:** ${stableCount} stable tests were hidden from this report.\n`,
|
|
);
|
|
}
|
|
console.log('\n');
|
|
}
|
|
|
|
if (isPrComment) {
|
|
console.log(
|
|
'---\n💡 **Policy Key:** 🔒 `ALWAYS_PASSES` (PR Blocker) | 🎲 `USUALLY_PASSES` (Informational)\n',
|
|
);
|
|
console.log(
|
|
'💡 To investigate regressions locally, run: `gemini /fix-behavioral-eval`',
|
|
);
|
|
}
|
|
}
|
|
|
|
const currentReports = findReports(artifactsDir);
|
|
if (currentReports.length === 0) {
|
|
console.log('No reports found.');
|
|
process.exit(0);
|
|
}
|
|
|
|
const currentStats = getStats(currentReports);
|
|
const history = fetchHistoricalData();
|
|
generateMarkdown(currentStats, history);
|