feat: introduce UX Extension and Base Folder Strategy

evals/plan_mode.eval.ts

@@ -18,18 +18,6 @@ describe('plan_mode', () => {
     experimental: { plan: true },
   };
 
-  const getWriteTargets = (logs: any[]) =>
-    logs
-      .filter((log) => ['write_file', 'replace'].includes(log.toolRequest.name))
-      .map((log) => {
-        try {
-          return JSON.parse(log.toolRequest.args).file_path as string;
-        } catch {
-          return '';
-        }
-      })
-      .filter(Boolean);
-
   evalTest('ALWAYS_PASSES', {
     name: 'should refuse file modification when in plan mode',
     approvalMode: ApprovalMode.PLAN,
@@ -44,23 +32,27 @@ describe('plan_mode', () => {
       await rig.waitForTelemetryReady();
       const toolLogs = rig.readToolLogs();
 
-      const exitPlanIndex = toolLogs.findIndex(
-        (log) => log.toolRequest.name === 'exit_plan_mode',
-      );
-
-      const writeTargetsBeforeExitPlan = getWriteTargets(
-        toolLogs.slice(0, exitPlanIndex !== -1 ? exitPlanIndex : undefined),
-      );
+      const writeTargets = toolLogs
+        .filter((log) =>
+          ['write_file', 'replace'].includes(log.toolRequest.name),
+        )
+        .map((log) => {
+          try {
+            return JSON.parse(log.toolRequest.args).file_path;
+          } catch {
+            return null;
+          }
+        });
 
       expect(
-        writeTargetsBeforeExitPlan,
+        writeTargets,
         'Should not attempt to modify README.md in plan mode',
       ).not.toContain('README.md');
 
       assertModelHasOutput(result);
       checkModelOutputContent(result, {
         expectedContent: [/plan mode|read-only|cannot modify|refuse|exiting/i],
-        testName: `${TEST_PREFIX}should refuse file modification in plan mode`,
+        testName: `${TEST_PREFIX}should refuse file modification`,
       });
     },
   });
@@ -77,20 +69,24 @@ describe('plan_mode', () => {
       await rig.waitForTelemetryReady();
       const toolLogs = rig.readToolLogs();
 
-      const exitPlanIndex = toolLogs.findIndex(
-        (log) => log.toolRequest.name === 'exit_plan_mode',
-      );
-
-      const writeTargetsBeforeExit = getWriteTargets(
-        toolLogs.slice(0, exitPlanIndex !== -1 ? exitPlanIndex : undefined),
-      );
+      const writeTargets = toolLogs
+        .filter((log) =>
+          ['write_file', 'replace'].includes(log.toolRequest.name),
+        )
+        .map((log) => {
+          try {
+            return JSON.parse(log.toolRequest.args).file_path;
+          } catch {
+            return null;
+          }
+        });
 
       // It should NOT write to the docs folder or any other repo path
-      const hasRepoWriteBeforeExit = writeTargetsBeforeExit.some(
+      const hasRepoWrite = writeTargets.some(
         (path) => path && !path.includes('/plans/'),
       );
       expect(
-        hasRepoWriteBeforeExit,
+        hasRepoWrite,
         'Should not attempt to create files in the repository while in plan mode',
       ).toBe(false);
 
@@ -170,65 +166,4 @@ describe('plan_mode', () => {
       assertModelHasOutput(result);
     },
   });
-
-  evalTest('USUALLY_PASSES', {
-    name: 'should create a plan in plan mode and implement it for a refactoring task',
-    params: {
-      settings,
-    },
-    files: {
-      'src/mathUtils.ts':
-        'export const sum = (a: number, b: number) => a + b;\nexport const multiply = (a: number, b: number) => a * b;',
-      'src/main.ts':
-        'import { sum } from "./mathUtils";\nconsole.log(sum(1, 2));',
-    },
-    prompt:
-      'I want to refactor our math utilities. Move the `sum` function from `src/mathUtils.ts` to a new file `src/basicMath.ts` and update `src/main.ts` to use the new file. Please create a detailed implementation plan first, then execute it.',
-    assert: async (rig, result) => {
-      const enterPlanCalled = await rig.waitForToolCall('enter_plan_mode');
-      expect(
-        enterPlanCalled,
-        'Expected enter_plan_mode tool to be called',
-      ).toBe(true);
-
-      const exitPlanCalled = await rig.waitForToolCall('exit_plan_mode');
-      expect(exitPlanCalled, 'Expected exit_plan_mode tool to be called').toBe(
-        true,
-      );
-
-      await rig.waitForTelemetryReady();
-      const toolLogs = rig.readToolLogs();
-
-      // Check if plan was written
-      const planWrite = toolLogs.find(
-        (log) =>
-          log.toolRequest.name === 'write_file' &&
-          log.toolRequest.args.includes('/plans/'),
-      );
-      expect(
-        planWrite,
-        'Expected a plan file to be written in the plans directory',
-      ).toBeDefined();
-
-      // Check for implementation files
-      const newFileWrite = toolLogs.find(
-        (log) =>
-          log.toolRequest.name === 'write_file' &&
-          log.toolRequest.args.includes('src/basicMath.ts'),
-      );
-      expect(
-        newFileWrite,
-        'Expected src/basicMath.ts to be created',
-      ).toBeDefined();
-
-      const mainUpdate = toolLogs.find(
-        (log) =>
-          ['write_file', 'replace'].includes(log.toolRequest.name) &&
-          log.toolRequest.args.includes('src/main.ts'),
-      );
-      expect(mainUpdate, 'Expected src/main.ts to be updated').toBeDefined();
-
-      assertModelHasOutput(result);
-    },
-  });
 });