MediaMetz/gemini-cli
1
0
Fork 0
mirror of https://github.com/google-gemini/gemini-cli.git synced 2026-04-25 12:34:38 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(ci): isolate workflow evals into independent nightly job

Browse Source 
- Splits 'Evals: Nightly' into 'evals' (general capabilities) and 'workflow-evals' (specific workflow simulations).
- 'workflow-evals' runs only on 'gemini-2.5-pro' (the target model).
- 'evals' excludes workflow tests to prevent noise/skewed metrics on other models.
- Removes code-level 'targetModels' restrictions in favor of CI configuration.
- Updates aggregation script to handle skipped tests correctly (though exclusion avoids them).
This commit is contained in:
cocosheng-g
2026-02-03 22:37:15 -05:00
parent d23499db90
commit 9da1542071
6 changed files with 42 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evals/workflows/batch_triage.eval.ts
-7
View File
@@ -9,7 +9,6 @@ import { evalTest } from '../test-helper.js';
import fs from 'node:fs/promises';
import path from 'node:path';
import yaml from 'js-yaml';
import { WORKFLOW_TARGET_MODELS } from './constants.js';
// Read the workflow file to extract the prompt
const workflowPath = path.join(
@@ -159,7 +158,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(101, 'area/core'),
});
@@ -177,7 +175,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(102, 'area/platform'),
});
@@ -200,7 +197,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: async (rig: any, result) => {
// Assert issue 103 has area/core
await assertHasIssueLabel(103, 'area/core')(rig, result);
@@ -223,7 +219,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(105, 'status/need-retesting'),
});
@@ -241,7 +236,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasIssueLabel(106, 'status/need-information'),
});
@@ -259,7 +253,6 @@ describe('batch_triage_agent', () => {
]),
},
params: { settings: BATCH_TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: async (rig: any, result) => {
await assertHasIssueLabel(107, 'area/core')(rig, result);
await assertHasIssueLabel(108, 'area/platform')(rig, result);
evals/workflows/constants.ts
-10
View File
@@ -1,10 +0,0 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
// The models that workflow evals should target.
// These workflows (triage, dedup) run in GitHub Actions using the default CLI model.
// We restrict evals to this model to avoid noise from other models in the nightly matrix.
export const WORKFLOW_TARGET_MODELS = ['gemini-2.5-pro'];
evals/workflows/dedup.eval.ts
-6
View File
@@ -9,7 +9,6 @@ import { evalTest } from '../test-helper.js';
import fs from 'node:fs/promises';
import path from 'node:path';
import yaml from 'js-yaml';
import { WORKFLOW_TARGET_MODELS } from './constants.js';
// Read the workflow file to extract the prompt and settings
const workflowPath = path.join(
@@ -66,7 +65,6 @@ describe('dedup_agent', () => {
params: {
settings: DEDUP_SETTINGS,
},
targetModels: WORKFLOW_TARGET_MODELS,
files: {
github_env: '',
// Mock gh binary
@@ -147,7 +145,6 @@ if (args.includes('issue view')) {
params: {
settings: DEDUP_SETTINGS,
},
targetModels: WORKFLOW_TARGET_MODELS,
files: {
github_env: '',
'bin/gh': `#!/usr/bin/env node
@@ -219,7 +216,6 @@ if (args.includes('issue view')) {
params: {
settings: DEDUP_SETTINGS,
},
targetModels: WORKFLOW_TARGET_MODELS,
files: {
github_env: '',
'bin/gh': `#!/usr/bin/env node
@@ -290,7 +286,6 @@ if (args.includes('issue view')) {
params: {
settings: DEDUP_SETTINGS,
},
targetModels: WORKFLOW_TARGET_MODELS,
files: {
github_env: '',
'bin/gh': `#!/usr/bin/env node
@@ -361,7 +356,6 @@ if (args.includes('issue view')) {
params: {
settings: DEDUP_SETTINGS,
},
targetModels: WORKFLOW_TARGET_MODELS,
files: {
github_env: '',
'bin/gh': `#!/usr/bin/env node
evals/workflows/dedup_refresh.eval.ts
-2
View File
@@ -9,7 +9,6 @@ import { evalTest } from '../test-helper.js';
import fs from 'node:fs/promises';
import path from 'node:path';
import yaml from 'js-yaml';
import { WORKFLOW_TARGET_MODELS } from './constants.js';
// Read the workflow file to extract the prompt and settings
const workflowPath = path.join(
@@ -62,7 +61,6 @@ describe('dedup_refresh_agent', () => {
params: {
settings: REFRESH_SETTINGS,
},
targetModels: WORKFLOW_TARGET_MODELS,
assert: async (rig: any, result) => {
// result is the JSON output
const output = JSON.parse(result);
evals/workflows/triage.eval.ts
-20
View File
@@ -9,7 +9,6 @@ import { evalTest } from '../test-helper.js';
import fs from 'node:fs/promises';
import path from 'node:path';
import yaml from 'js-yaml';
import { WORKFLOW_TARGET_MODELS } from './constants.js';
// Read the workflow file to extract the prompt
const workflowPath = path.join(
@@ -148,7 +147,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/core'),
});
@@ -164,7 +162,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/platform'),
});
@@ -180,7 +177,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/platform'),
});
@@ -196,7 +192,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/core'),
});
@@ -212,7 +207,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/platform'),
});
@@ -228,7 +222,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/core'),
});
@@ -244,7 +237,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/agent'),
});
@@ -260,7 +252,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/extensions'),
});
@@ -273,7 +264,6 @@ describe('triage_agent', () => {
createPrompt('Buy cheap rolex', 'Click here for discount.'),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/unknown'),
});
@@ -289,7 +279,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/core'),
});
@@ -305,7 +294,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/agent'),
});
@@ -321,7 +309,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/enterprise'),
});
@@ -334,7 +321,6 @@ describe('triage_agent', () => {
createPrompt('It does not work', 'I tried to use it and it failed.'),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/unknown'),
});
@@ -350,7 +336,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/security'),
});
@@ -366,7 +351,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/non-interactive'),
});
@@ -382,7 +366,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/agent'),
});
@@ -398,7 +381,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/core'),
});
@@ -414,7 +396,6 @@ describe('triage_agent', () => {
),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/security'),
});
@@ -427,7 +408,6 @@ describe('triage_agent', () => {
createPrompt('asdfasdf', 'qwerqwer zxcvbnm'),
],
params: { settings: TRIAGE_SETTINGS },
targetModels: WORKFLOW_TARGET_MODELS,
assert: assertHasLabel('area/unknown'),
});
});