mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-05-14 05:42:54 -07:00
feat(ci): isolate workflow evals into independent nightly job
- Splits 'Evals: Nightly' into 'evals' (general capabilities) and 'workflow-evals' (specific workflow simulations). - 'workflow-evals' runs only on 'gemini-2.5-pro' (the target model). - 'evals' excludes workflow tests to prevent noise/skewed metrics on other models. - Removes code-level 'targetModels' restrictions in favor of CI configuration. - Updates aggregation script to handle skipped tests correctly (though exclusion avoids them).
This commit is contained in:
@@ -9,7 +9,6 @@ import { evalTest } from '../test-helper.js';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
import { WORKFLOW_TARGET_MODELS } from './constants.js';
|
||||
|
||||
// Read the workflow file to extract the prompt and settings
|
||||
const workflowPath = path.join(
|
||||
@@ -62,7 +61,6 @@ describe('dedup_refresh_agent', () => {
|
||||
params: {
|
||||
settings: REFRESH_SETTINGS,
|
||||
},
|
||||
targetModels: WORKFLOW_TARGET_MODELS,
|
||||
assert: async (rig: any, result) => {
|
||||
// result is the JSON output
|
||||
const output = JSON.parse(result);
|
||||
|
||||
Reference in New Issue
Block a user