feat(ci): isolate workflow evals into independent nightly job

- Splits 'Evals: Nightly' into 'evals' (general capabilities) and 'workflow-evals' (specific workflow simulations).
- 'workflow-evals' runs only on 'gemini-2.5-pro' (the target model).
- 'evals' excludes workflow tests to prevent noise/skewed metrics on other models.
- Removes code-level 'targetModels' restrictions in favor of CI configuration.
- Updates aggregation script to handle skipped tests correctly (though exclusion avoids them).
This commit is contained in:
cocosheng-g
2026-02-03 22:37:15 -05:00
parent d23499db90
commit 9da1542071
6 changed files with 42 additions and 47 deletions
-7
View File
@@ -9,7 +9,6 @@ import { evalTest } from '../test-helper.js';
import fs from 'node:fs/promises';
import path from 'node:path';
import yaml from 'js-yaml';
import { WORKFLOW_TARGET_MODELS } from './constants.js';
// Read the workflow file to extract the prompt
const workflowPath = path.join(
@@ -159,7 +158,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(101, 'area/core'),
});
@@ -177,7 +175,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(102, 'area/platform'),
});
@@ -200,7 +197,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: async (rig: any, result) => {
// Assert issue 103 has area/core
await assertHasIssueLabel(103, 'area/core')(rig, result);
@@ -223,7 +219,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(105, 'status/need-retesting'),
});
@@ -241,7 +236,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(106, 'status/need-information'),
});
@@ -259,7 +253,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: async (rig: any, result) => {
await assertHasIssueLabel(107, 'area/core')(rig, result);
await assertHasIssueLabel(108, 'area/platform')(rig, result);