Run evals for all models. (#17123)

.github/workflows/evals-nightly.yml

@@ -22,6 +22,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        model:
+          - 'gemini-3-pro-preview'
+          - 'gemini-3-flash-preview'
+          - 'gemini-2.5-pro'
+          - 'gemini-2.5-flash'
+          - 'gemini-2.5-flash-lite'
         run_attempt: [1, 2, 3]
     steps:
       - name: 'Checkout'
@@ -45,6 +51,7 @@ jobs:
       - name: 'Run Evals'
         env:
           GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
+          GEMINI_MODEL: '${{ matrix.model }}'
           RUN_EVALS: "${{ github.event.inputs.run_all != 'false' }}"
         run: 'npm run test:all_evals'
 
@@ -52,7 +59,7 @@ jobs:
         if: 'always()'
         uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
         with:
-          name: 'eval-logs-${{ matrix.run_attempt }}'
+          name: 'eval-logs-${{ matrix.model }}-${{ matrix.run_attempt }}'
           path: 'evals/logs'
           retention-days: 7
 

scripts/aggregate_evals.js

@@ -32,11 +32,33 @@ function findReports(dir) {
   return reports;
 }
 
+function getModelFromPath(reportPath) {
+  const parts = reportPath.split(path.sep);
+  // Find the part that starts with 'eval-logs-'
+  const artifactDir = parts.find((p) => p.startsWith('eval-logs-'));
+  if (!artifactDir) return 'unknown';
+
+  const matchNew = artifactDir.match(/^eval-logs-(.+)-(\d+)$/);
+  if (matchNew) return matchNew[1];
+
+  const matchOld = artifactDir.match(/^eval-logs-(\d+)$/);
+  if (matchOld) return 'gemini-2.5-pro'; // Legacy default
+
+  return 'unknown';
+}
+
 function getStats(reports) {
-  const testStats = {};
+  // Structure: { [model]: { [testName]: { passed, failed, total } } }
+  const statsByModel = {};
 
   for (const reportPath of reports) {
     try {
+      const model = getModelFromPath(reportPath);
+      if (!statsByModel[model]) {
+        statsByModel[model] = {};
+      }
+      const testStats = statsByModel[model];
+
       const content = fs.readFileSync(reportPath, 'utf-8');
       const json = JSON.parse(content);
 
@@ -58,7 +80,7 @@ function getStats(reports) {
       console.error(`Error processing report at ${reportPath}:`, error);
     }
   }
-  return testStats;
+  return statsByModel;
 }
 
 function fetchHistoricalData() {
@@ -92,7 +114,7 @@ function fetchHistoricalData() {
       );
       try {
         // Download report.json files.
-        // The artifacts are named 'eval-logs-X'.
+        // The artifacts are named 'eval-logs-X' or 'eval-logs-MODEL-X'.
         // We use -p to match pattern.
         execSync(
           `gh run download ${run.databaseId} -p "eval-logs-*" -D "${tmpDir}"`,
@@ -103,7 +125,7 @@ function fetchHistoricalData() {
         if (runReports.length > 0) {
           history.push({
             run,
-            stats: getStats(runReports),
+            stats: getStats(runReports), // Now returns stats grouped by model
           });
         }
       } catch (error) {
@@ -122,7 +144,19 @@ function fetchHistoricalData() {
   return history;
 }
 
-function generateMarkdown(currentStats, history) {
+function generateMarkdown(currentStatsByModel, history) {
+  console.log('### Evals Nightly Summary\n');
+  console.log(
+    'See [evals/README.md](https://github.com/google-gemini/gemini-cli/tree/main/evals) for more details.\n',
+  );
+
+  // Reverse history to show oldest first
+  const reversedHistory = [...history].reverse();
+
+  const models = Object.keys(currentStatsByModel).sort();
+
+  for (const model of models) {
+    const currentStats = currentStatsByModel[model];
     const totalStats = Object.values(currentStats).reduce(
       (acc, stats) => {
         acc.passed += stats.passed;
@@ -137,14 +171,8 @@ function generateMarkdown(currentStats, history) {
         ? ((totalStats.passed / totalStats.total) * 100).toFixed(1) + '%'
         : 'N/A';
 
-  console.log('### Evals Nightly Summary');
+    console.log(`#### Model: ${model}`);
     console.log(`**Total Pass Rate: ${totalPassRate}**\n`);
-  console.log(
-    'See [evals/README.md](https://github.com/google-gemini/gemini-cli/tree/main/evals) for more details.\n',
-  );
-
-  // Reverse history to show oldest first
-  const reversedHistory = [...history].reverse();
 
     // Header
     let header = '| Test Name |';
@@ -162,10 +190,14 @@ function generateMarkdown(currentStats, history) {
     console.log(header);
     console.log(separator);
 
-  // Collect all test names
+    // Collect all test names for this model
     const allTestNames = new Set(Object.keys(currentStats));
     for (const item of reversedHistory) {
-    Object.keys(item.stats).forEach((name) => allTestNames.add(name));
+      if (item.stats[model]) {
+        Object.keys(item.stats[model]).forEach((name) =>
+          allTestNames.add(name),
+        );
+      }
     }
 
     for (const name of Array.from(allTestNames).sort()) {
@@ -174,7 +206,7 @@ function generateMarkdown(currentStats, history) {
 
       // History
       for (const item of reversedHistory) {
-      const stat = item.stats[name];
+        const stat = item.stats[model] ? item.stats[model][name] : null;
         if (stat) {
           const passRate = ((stat.passed / stat.total) * 100).toFixed(0) + '%';
           row += ` ${passRate} |`;
@@ -194,6 +226,8 @@ function generateMarkdown(currentStats, history) {
 
       console.log(row);
     }
+    console.log('\n');
+  }
 }
 
 // --- Main ---