fix(core): enforce non-interactive flags for scaffolding commands

- Updated `newApplicationSteps` system prompt instructions to strongly require non-interactive flags (e.g. `--yes`, `-y`, or `--template`) when executing application scaffolding CLI tools.
- Warned the model that omitting these flags for interactive tools will cause the environment to hang.
- Added `ALWAYS_PASSES` evaluation case to `evals/interactive-hang.eval.ts` to assert that non-interactive flags are successfully provided for `npm create` and similar commands.
This commit is contained in:
Taylor Mullen
2026-02-11 01:25:46 -08:00
parent 884acda2dc
commit 8d27a29053
2 changed files with 33 additions and 5 deletions
+27
View File
@@ -44,4 +44,31 @@ describe('interactive_commands', () => {
).toMatch(/\b(run|--run)\b/);
},
});
/**
* Validates that the agent uses non-interactive flags when scaffolding a new project.
*/
evalTest('ALWAYS_PASSES', {
name: 'should use non-interactive flags when scaffolding a new app',
prompt: 'Create a new react application named my-app using vite.',
assert: async (rig, result) => {
const logs = rig.readToolLogs();
const scaffoldCall = logs.find(
(l) =>
l.toolRequest.name === 'run_shell_command' &&
/npm (init|create)|npx create-|yarn create|pnpm create/.test(
l.toolRequest.args,
),
);
expect(
scaffoldCall,
'Agent should have called a scaffolding command (e.g., npm create)',
).toBeDefined();
expect(
scaffoldCall?.toolRequest.args,
'Agent should have passed a non-interactive flag (-y, --yes, or a specific --template)',
).toMatch(/(?:^|\s)(--yes|-y|--template\s+\S+)(?:\s|$|\\|")/);
},
});
});