mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-07-12 02:50:41 -07:00
Abhijit Balaji
9bdf5d5995
- Flatten directory structure by moving masking and evals to scripts root. - Merge evaluation metrics into scripts/optimization/evals. - Restore and verify extraction tests for the new structure.
63 lines
2.0 KiB
TypeScript
63 lines
2.0 KiB
TypeScript
/**
|
|
* @license
|
|
* Copyright 2026 Google LLC
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
import { debugLogger } from '../../../../../packages/core/src/utils/debugLogger.js';
|
|
import { DEFAULT_EVAL_CONFIG } from '../config.js';
|
|
import { MetricObjective } from '../types.js';
|
|
import type { MetricResult } from '../types.js';
|
|
|
|
/**
|
|
* Evaluates the brevity of a model's response using a tiered 4-step word-count function.
|
|
* Focuses on rewarding succinctness and providing a non-zero gradient for verbose models.
|
|
*/
|
|
export function evaluateBrevity(
|
|
prediction: { output_text?: string },
|
|
config = DEFAULT_EVAL_CONFIG.objectives.brevity,
|
|
): MetricResult {
|
|
const chatter = (prediction.output_text ?? '').trim();
|
|
|
|
// Simple word count: split by whitespace and filter out empty strings
|
|
const wordCount = chatter === '' ? 0 : chatter.split(/\s+/).length;
|
|
|
|
debugLogger.debug(
|
|
`[Eval:Brevity] Measuring output text word count: ${wordCount} words.`,
|
|
);
|
|
|
|
let score: number;
|
|
let reason: string;
|
|
|
|
if (wordCount <= config.succinctThresholdWords) {
|
|
score = config.succinctScore;
|
|
reason = `Succinct: Response is within ${config.succinctThresholdWords} words.`;
|
|
} else if (wordCount <= config.acceptableThresholdWords) {
|
|
score = config.acceptableScore;
|
|
reason = `Acceptable: Response is slightly verbose (${wordCount} words), exceeding ${config.succinctThresholdWords} words.`;
|
|
} else if (wordCount <= config.verboseThresholdWords) {
|
|
score = config.verboseScore;
|
|
reason = `Verbose: Response contains ${wordCount} words, exceeding acceptable limit of ${config.acceptableThresholdWords} words.`;
|
|
} else {
|
|
score = config.heavyScore;
|
|
reason = `Heavy: Response is excessively verbose (${wordCount} words).`;
|
|
}
|
|
|
|
return {
|
|
score,
|
|
objective: MetricObjective.BREVITY,
|
|
reason,
|
|
metadata: {
|
|
wordCount,
|
|
tier:
|
|
score === 1.0
|
|
? 'succinct'
|
|
: score === 0.7
|
|
? 'acceptable'
|
|
: score === 0.4
|
|
? 'verbose'
|
|
: 'heavy',
|
|
},
|
|
};
|
|
}
|