chore(evals): add steering demonstration with deliberate failure

This commit is contained in:
Alisa Novikova
2026-04-08 21:25:12 -07:00
parent 6ca4147462
commit a591cd9f8f
5 changed files with 41 additions and 2 deletions
+22
View File
@@ -0,0 +1,22 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, expect } from 'vitest';
import { evalTest, TestRig } from './test-helper.js';
describe('Related Eval Demonstration', () => {
evalTest('ALWAYS_PASSES', {
name: 'should fail deliberately to demonstrate regression reporting',
files: {
'test.txt': 'Hello world',
},
prompt: 'What is in test.txt?',
assert: async (rig: TestRig, result: string) => {
// This assertion is designed to fail.
expect(result).toContain('THE MODEL WILL NEVER SAY THIS EXACT SENTENCE');
},
});
});
+4 -2
View File
@@ -104,13 +104,15 @@
"packages/core/src/prompts/**",
"evals/answer-vs-act.eval.ts",
"evals/model_steering.eval.ts",
"evals/redundant_casts.eval.ts"
"evals/redundant_casts.eval.ts",
"evals/failing_steering.eval.ts"
],
"evals": [
"ALL_ALWAYS_PASSING",
"evals/answer-vs-act.eval.ts",
"evals/model_steering.eval.ts",
"evals/redundant_casts.eval.ts"
"evals/redundant_casts.eval.ts",
"evals/failing_steering.eval.ts"
]
},
"edit_fidelity": {