mirror of
https://github.com/google-gemini/gemini-cli.git
synced 2026-03-13 15:40:57 -07:00
61b35ff745
Major upgrade to the agent's self-validation, safety, and project integrity
capabilities through five iterations of system prompt enhancements:
Workflow & Quality Mandates:
1. Incremental Validation: Mandates building, linting, and testing after
every significant file change to maintain a "green" state.
2. Mandatory Reproduction: Requires creating a failing test case to confirm
a bug before fixing, and explicitly verifying the failure (Negative Verification).
3. Test Persistence & Locality: Requires integrating repro cases into the
permanent test suite, preferably by amending existing related test files.
4. Script Discovery: Mandates identifying project-specific validation
commands from configuration files (package.json, Makefile, etc.).
5. Self-Review: Mandates running `git diff` after every edit, using
`--name-only` for large changes to preserve context window tokens.
6. Fast-Path Validation: Prioritizes lightweight checks (e.g., `tsc --noEmit`)
for frequent feedback, reserving heavy builds for final verification.
7. Output Verification: Requires checking command output (not just exit codes)
to prevent false-positives from empty test runs or hidden warnings.
Semantic Integrity & Dependency Safety:
8. Global Usage Discovery: Mandates searching the entire workspace for all
usages (via `grep_search`) before modifying exported symbols or APIs.
9. Dependency Integrity: Requires verifying that new imports are explicitly
declared in the project's dependency manifest (e.g., package.json).
10. Configuration Sync: Mandates updating build/environment configs
(tsconfig, Dockerfile, etc.) to support new file types or entry points.
11. Documentation Sync: Requires searching for and updating documentation
references when public APIs or CLI interfaces change.
12. Anti-Silencing Mandate: Prohibits using `any`, `@ts-ignore`, or lint
suppressions to resolve validation errors.
Diagnostics, Safety & Runtime Verification:
13. Error Grounding: Mandates reading full error logs and stack traces upon
failure. Includes Smart Log Navigation to prioritize the tail of large files.
14. Scope Isolation: Instructs the agent to focus only on errors introduced
by its changes and ignore unrelated legacy technical debt.
15. Destructive Safety: Mandates a `git status` check before deleting files
or modifying critical project configurations.
16. Non-Blocking Smoke Tests: Requires briefly running applications to
verify boot stability, using background/timeout strategies for servers.
Includes 15 new behavioral evaluations verifying these mandates and updated
snapshots in packages/core/src/core/prompts.test.ts.
103 lines
3.0 KiB
TypeScript
103 lines
3.0 KiB
TypeScript
/**
|
|
* @license
|
|
* Copyright 2026 Google LLC
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
import { describe, expect } from 'vitest';
|
|
import { evalTest } from './test-helper.js';
|
|
|
|
describe('Error Grounding and Scope Isolation', () => {
|
|
/**
|
|
* Verifies that the agent reads the error log when validation fails.
|
|
*/
|
|
evalTest('USUALLY_PASSES', {
|
|
name: 'should read the full error message when validation fails',
|
|
files: {
|
|
'src/app.ts': 'export const x: number = "string"; // Error',
|
|
'package.json': JSON.stringify({
|
|
name: 'test-project',
|
|
type: 'module',
|
|
scripts: {
|
|
typecheck: 'tsc --noEmit > error.log 2>&1',
|
|
},
|
|
}),
|
|
'tsconfig.json': JSON.stringify({
|
|
compilerOptions: { strict: true, module: 'ESNext', target: 'ESNext' },
|
|
}),
|
|
},
|
|
prompt:
|
|
'Run typecheck and fix the error in src/app.ts. Use redirection to a file if needed.',
|
|
assert: async (rig) => {
|
|
const toolLogs = rig.readToolLogs();
|
|
|
|
// Check if it read the error log after running the command
|
|
const ranTypecheck = toolLogs.some(
|
|
(log) =>
|
|
log.toolRequest.name === 'run_shell_command' &&
|
|
log.toolRequest.args.includes('typecheck'),
|
|
);
|
|
|
|
const readErrorLog = toolLogs.some(
|
|
(log) =>
|
|
log.toolRequest.name === 'read_file' &&
|
|
(log.toolRequest.args.includes('error.log') ||
|
|
log.toolRequest.args.includes('app.ts')),
|
|
);
|
|
|
|
expect(ranTypecheck, 'Agent should have run the typecheck command').toBe(
|
|
true,
|
|
);
|
|
expect(
|
|
readErrorLog,
|
|
'Agent should have read the error log or the file to understand the error grounding',
|
|
).toBe(true);
|
|
},
|
|
});
|
|
|
|
/**
|
|
* Verifies that the agent ignores pre-existing technical debt.
|
|
*/
|
|
evalTest('USUALLY_PASSES', {
|
|
name: 'should ignore unrelated pre-existing technical debt during validation',
|
|
files: {
|
|
'src/legacy.ts':
|
|
'export const legacy: any = 1; // Unrelated technical debt',
|
|
'src/new.ts': 'export const current = 42;',
|
|
'package.json': JSON.stringify({
|
|
name: 'test-project',
|
|
type: 'module',
|
|
scripts: {
|
|
lint: 'eslint .',
|
|
},
|
|
}),
|
|
'eslint.config.js':
|
|
'export default [{ rules: { "no-explicit-any": "error" } }];',
|
|
},
|
|
prompt:
|
|
'Rename "current" to "updated" in src/new.ts. Ignore pre-existing lint errors in other files.',
|
|
assert: async (rig) => {
|
|
const toolLogs = rig.readToolLogs();
|
|
|
|
const editedLegacy = toolLogs.some((log) =>
|
|
log.toolRequest.args.includes('src/legacy.ts'),
|
|
);
|
|
|
|
expect(
|
|
editedLegacy,
|
|
'Agent should NOT have edited src/legacy.ts to fix unrelated pre-existing debt',
|
|
).toBe(false);
|
|
|
|
const editedNew = toolLogs.some(
|
|
(log) =>
|
|
log.toolRequest.args.includes('src/new.ts') &&
|
|
log.toolRequest.args.includes('updated'),
|
|
);
|
|
expect(
|
|
editedNew,
|
|
'Agent should have successfully refactored src/new.ts',
|
|
).toBe(true);
|
|
},
|
|
});
|
|
});
|