MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-03-10 14:10:37 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(evals): add overall pass rate row to eval nightly summary table (#20905)

Browse Source
This commit is contained in:
Christian Gunderman
2026-03-04 18:58:18 +00:00
committed by GitHub
parent 717660997d
commit 54885214a1
1 changed files with 14 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
scripts/aggregate_evals.js
View File

@@ -155,9 +155,9 @@ function generateMarkdown(currentStatsByModel, history) {
const models = Object.keys(currentStatsByModel).sort();
for (const model of models) {
const currentStats = currentStatsByModel[model];
const totalStats = Object.values(currentStats).reduce(
const getPassRate = (statsForModel) => {
if (!statsForModel) return '-';
const totalStats = Object.values(statsForModel).reduce(
(acc, stats) => {
acc.passed += stats.passed;
acc.total += stats.total;
@@ -165,11 +165,14 @@ function generateMarkdown(currentStatsByModel, history) {
},
{ passed: 0, total: 0 },
);
return totalStats.total > 0
? ((totalStats.passed / totalStats.total) * 100).toFixed(1) + '%'
: '-';
};
const totalPassRate =
totalStats.total > 0
? ((totalStats.passed / totalStats.total) * 100).toFixed(1) + '%'
: 'N/A';
for (const model of models) {
const currentStats = currentStatsByModel[model];
const totalPassRate = getPassRate(currentStats);
console.log(`#### Model: ${model}`);
console.log(`**Total Pass Rate: ${totalPassRate}**\n`);
@@ -177,18 +180,22 @@ function generateMarkdown(currentStatsByModel, history) {
// Header
let header = '| Test Name |';
let separator = '| :--- |';
let passRateRow = '| **Overall Pass Rate** |';
for (const item of reversedHistory) {
header += ` [${item.run.databaseId}](${item.run.url}) |`;
separator += ' :---: |';
passRateRow += ` **${getPassRate(item.stats[model])}** |`;
}
// Add Current column last
header += ' Current |';
separator += ' :---: |';
passRateRow += ` **${totalPassRate}** |`;
console.log(header);
console.log(separator);
console.log(passRateRow);
// Collect all test names for this model
const allTestNames = new Set(Object.keys(currentStats));