refactor(context): update projector, tokenCalculator and mapper to use flat ship arrays

packages/core/src/context/ir/fromIr.ts

@@ -5,35 +5,68 @@
  */
 
 import type { Content, Part } from '@google/genai';
-import type { Episode, EpisodeStep, UserPrompt, AgentYield } from './types.js';
-import { isAgentThought, isToolExecution, isUserPrompt } from './graphUtils.js';
+import type {
+  ConcreteNode,
+  UserPrompt,
+  AgentThought,
+  ToolExecution,
+  AgentYield,
+  MaskedTool,
+  Snapshot,
+  RollingSummary,
+} from './types.js';
 
-export function fromIr(episodes: Episode[]): Content[] {
+export function fromIr(ship: ReadonlyArray<ConcreteNode>): Content[] {
   const history: Content[] = [];
+  const agentParts: Part[] = [];
 
-  for (const ep of episodes) {
-    if (isUserPrompt(ep.trigger)) {
-      const triggerContent = serializeTrigger(ep.trigger);
-      if (triggerContent) history.push(triggerContent);
+  const flushAgentParts = () => {
+    if (agentParts.length > 0) {
+      history.push({ role: 'model', parts: [...agentParts] });
+      agentParts.length = 0;
     }
+  };
 
-    const stepContents = serializeSteps(ep.steps);
-    history.push(...stepContents);
-
-    if (ep.yield) {
-      history.push(serializeYield(ep.yield));
+  for (const node of ship) {
+    if (node.type === 'USER_PROMPT') {
+      flushAgentParts();
+      const content = serializeUserPrompt(node as UserPrompt);
+      if (content) history.push(content);
+    } else if (node.type === 'SYSTEM_EVENT') {
+      flushAgentParts();
+      // System events do not map strictly to Gemini Content parts unless synthesized.
+    } else if (node.type === 'AGENT_THOUGHT') {
+      agentParts.push(serializeAgentThought(node as AgentThought));
+    } else if (node.type === 'TOOL_EXECUTION') {
+      const parts = serializeToolExecution(node as ToolExecution);
+      agentParts.push(parts.call);
+      flushAgentParts();
+      history.push({ role: 'user', parts: [parts.response] });
+    } else if (node.type === 'MASKED_TOOL') {
+      const parts = serializeMaskedTool(node as MaskedTool);
+      agentParts.push(parts.call);
+      flushAgentParts();
+      history.push({ role: 'user', parts: [parts.response] });
+    } else if (node.type === 'AGENT_YIELD') {
+      agentParts.push(serializeAgentYield(node as AgentYield));
+      flushAgentParts();
+    } else if (node.type === 'SNAPSHOT') {
+      flushAgentParts();
+      history.push({ role: 'user', parts: [{ text: (node as Snapshot).text }] });
+    } else if (node.type === 'ROLLING_SUMMARY') {
+      flushAgentParts();
+      history.push({ role: 'user', parts: [{ text: (node as RollingSummary).text }] });
     }
   }
 
+  flushAgentParts();
   return history;
 }
 
-function serializeTrigger(trigger: UserPrompt): Content | null {
+function serializeUserPrompt(prompt: UserPrompt): Content | null {
   const parts: Part[] = [];
-  for (const sp of trigger.semanticParts) {
-    if (sp.presentation) {
-      parts.push({ text: sp.presentation.text });
-    } else if (sp.type === 'text') {
+  for (const sp of prompt.semanticParts) {
+    if (sp.type === 'text') {
       parts.push({ text: sp.text });
     } else if (sp.type === 'inline_data') {
       parts.push({
@@ -50,59 +83,52 @@ function serializeTrigger(trigger: UserPrompt): Content | null {
   return parts.length > 0 ? { role: 'user', parts } : null;
 }
 
-function serializeSteps(steps: EpisodeStep[]): Content[] {
-  const history: Content[] = [];
-  let pendingModelParts: Part[] = [];
-  let pendingUserParts: Part[] = [];
-
-  const flushPending = () => {
-    if (pendingModelParts.length > 0) {
-      history.push({ role: 'model', parts: [...pendingModelParts] });
-      pendingModelParts = [];
-    }
-    if (pendingUserParts.length > 0) {
-      history.push({ role: 'user', parts: [...pendingUserParts] });
-      pendingUserParts = [];
-    }
-  };
-
-  for (const step of steps) {
-    if (isAgentThought(step)) {
-      if (pendingUserParts.length > 0) flushPending();
-      pendingModelParts.push({
-        text: step.presentation?.text ?? step.text,
-      });
-    } else if (isToolExecution(step)) {
-      pendingModelParts.push({
-        functionCall: {
-          name: step.toolName,
-          args: step.intent,
-          id: step.id,
-        },
-      });
-      const observation = step.presentation
-        ? step.presentation.observation
-        : step.observation;
-      pendingUserParts.push({
-        functionResponse: {
-          name: step.toolName,
-          response:
-            typeof observation === 'string'
-              ? { message: observation }
-              : observation,
-          id: step.id,
-        },
-      });
-    }
-  }
-  flushPending();
-
-  return history;
+function serializeAgentThought(thought: AgentThought): Part {
+  return { text: thought.text };
 }
 
-function serializeYield(yieldNode: AgentYield): Content {
+function serializeToolExecution(
+  tool: ToolExecution,
+): { call: Part; response: Part } {
   return {
-    role: 'model',
-    parts: [{ text: yieldNode.presentation?.text ?? yieldNode.text }],
+    call: {
+      functionCall: {
+        id: tool.id,
+        name: tool.toolName,
+        args: tool.intent,
+      },
+    },
+    response: {
+      functionResponse: {
+        id: tool.id,
+        name: tool.toolName,
+        response: typeof tool.observation === 'string' ? { message: tool.observation } : tool.observation as object,
+      },
+    },
   };
 }
+
+function serializeMaskedTool(
+  tool: MaskedTool,
+): { call: Part; response: Part } {
+  return {
+    call: {
+      functionCall: {
+        id: tool.id,
+        name: tool.toolName,
+        args: tool.intent ?? {},
+      },
+    },
+    response: {
+      functionResponse: {
+        id: tool.id,
+        name: tool.toolName,
+        response: typeof tool.observation === 'string' ? { message: tool.observation } : (tool.observation ?? {}),
+      },
+    },
+  };
+}
+
+function serializeAgentYield(yieldNode: AgentYield): Part {
+  return { text: yieldNode.text };
+}

packages/core/src/context/ir/mapper.ts

@@ -5,7 +5,7 @@
  */
 
 import type { Content } from '@google/genai';
-import type { Episode } from './types.js';
+import type { Episode, ConcreteNode } from './types.js';
 import { toIr } from './toIr.js';
 import { fromIr } from './fromIr.js';
 import type { ContextTokenCalculator } from '../utils/contextTokenCalculator.js';
@@ -23,9 +23,9 @@ export class IrMapper {
   }
 
   /**
-   * Re-serializes the Episodic IR back into a flat Gemini Content[] array.
+   * Re-serializes a flat array of ConcreteNodes back into a flat Gemini Content[] array.
    */
-  static fromIr(episodes: Episode[]): Content[] {
-    return fromIr(episodes);
+  static fromIr(ship: ReadonlyArray<ConcreteNode>): Content[] {
+    return fromIr(ship);
   }
 }

packages/core/src/context/ir/projector.ts

@@ -6,7 +6,7 @@
 
 import type { Content } from '@google/genai';
 import { IrMapper } from './mapper.js';
-import type { Episode } from './types.js';
+import type { ConcreteNode } from './types.js';
 import { debugLogger } from '../../utils/debugLogger.js';
 import type {
   ContextEnvironment,
@@ -17,11 +17,11 @@ import type { SidecarConfig } from '../sidecar/types.js';
 
 export class IrProjector {
   /**
-   * Orchestrates the final projection: takes a working buffer view,
+   * Orchestrates the final projection: takes a working buffer view (The Ship),
    * applies the Immediate Sanitization pipeline, and enforces token boundaries.
    */
   static async project(
-    workingBuffer: Episode[],
+    ship: ReadonlyArray<ConcreteNode>,
     orchestrator: PipelineOrchestrator,
     sidecar: SidecarConfig,
     tracer: ContextTracer,
@@ -29,7 +29,7 @@ export class IrProjector {
     protectedIds: Set<string>,
   ): Promise<Content[]> {
     if (!sidecar.budget) {
-      const contents = IrMapper.fromIr(workingBuffer);
+      const contents = IrMapper.fromIr(ship);
       tracer.logEvent('IrProjector', 'Projected Context to LLM (No Budget)', {
         projectedContext: contents,
       });
@@ -38,14 +38,14 @@ export class IrProjector {
 
     const maxTokens = sidecar.budget.maxTokens;
     const currentTokens =
-      env.tokenCalculator.calculateEpisodeListTokens(workingBuffer);
+      env.tokenCalculator.calculateConcreteListTokens(ship);
 
     if (currentTokens <= maxTokens) {
       tracer.logEvent(
         'IrProjector',
         `View is within maxTokens (${currentTokens} <= ${maxTokens}). Returning view.`,
       );
-      const contents = IrMapper.fromIr(workingBuffer);
+      const contents = IrMapper.fromIr(ship);
       tracer.logEvent('IrProjector', 'Projected Context to LLM', {
         projectedContext: contents,
       });
@@ -64,34 +64,31 @@ export class IrProjector {
     const agedOutNodes = new Set<string>();
     let rollingTokens = 0;
     // Start from newest and count backwards
-    for (let i = workingBuffer.length - 1; i >= 0; i--) {
-      const ep = workingBuffer[i];
-      const epTokens = env.tokenCalculator.calculateEpisodeListTokens([ep]);
-      rollingTokens += epTokens;
+    for (let i = ship.length - 1; i >= 0; i--) {
+      const node = ship[i];
+      const nodeTokens = node.metadata.currentTokens;
+      rollingTokens += nodeTokens;
       if (rollingTokens > sidecar.budget.retainedTokens) {
-        agedOutNodes.add(ep.id);
-        agedOutNodes.add(ep.trigger.id);
-        for (const step of ep.steps) agedOutNodes.add(step.id);
-        if (ep.yield) agedOutNodes.add(ep.yield.id);
+        agedOutNodes.add(node.id);
       }
     }
 
-    const processedEpisodes = await orchestrator.executeTriggerSync(
+    const processedShip = await orchestrator.executeTriggerSync(
       'gc_backstop',
-      workingBuffer,
+      ship,
+      agedOutNodes,
       {
         currentTokens,
         maxTokens: sidecar.budget.maxTokens,
         retainedTokens: sidecar.budget.retainedTokens,
         deficitTokens: Math.max(0, currentTokens - sidecar.budget.maxTokens),
-        protectedEpisodeIds: protectedIds,
+        protectedLogicalIds: protectedIds,
         isBudgetSatisfied: currentTokens <= sidecar.budget.maxTokens,
-        targetNodeIds: agedOutNodes,
       },
     );
 
     const finalTokens =
-      env.tokenCalculator.calculateEpisodeListTokens(processedEpisodes);
+      env.tokenCalculator.calculateConcreteListTokens(processedShip);
     tracer.logEvent(
       'IrProjector',
       `Finished projection. Final token count: ${finalTokens}.`,
@@ -100,7 +97,17 @@ export class IrProjector {
       `Context Manager finished. Final actual token count: ${finalTokens}.`,
     );
 
-    const contents = IrMapper.fromIr(processedEpisodes);
+    // Apply skipList logic to abstract over summarized nodes
+    const skipList = new Set<string>();
+    for (const node of processedShip) {
+      if (node.abstractsIds) {
+         for (const id of node.abstractsIds) skipList.add(id);
+      }
+    }
+
+    const visibleShip = processedShip.filter(n => !skipList.has(n.id));
+
+    const contents = IrMapper.fromIr(visibleShip);
     tracer.logEvent('IrProjector', 'Projected Sanitized Context to LLM', {
       projectedContextSanitized: contents,
     });

packages/core/src/context/utils/contextTokenCalculator.ts

@@ -6,13 +6,13 @@
 
 import type { Part } from '@google/genai';
 import { estimateTokenCountSync as baseEstimate } from '../../utils/tokenCalculation.js';
-import type { Episode } from '../ir/types.js';
+import { BASE_MULTIMODAL_TOKEN_COST } from '../ir/types.js';
 
 /**
  * The flat token cost assigned to a single multi-modal asset (like an image tile)
  * by the Gemini API. We use this as a baseline heuristic for inlineData/fileData.
  */
-const BASE_MULTIMODAL_TOKEN_COST = 258;
+import type { ConcreteNode } from '../ir/types.js';
 
 export class ContextTokenCalculator {
   constructor(private readonly charsPerToken: number) {}
@@ -33,17 +33,13 @@ export class ContextTokenCalculator {
   }
 
   /**
-   * Calculates the total token count for a complete Episodic IR graph.
+   * Calculates the total token count for a flat array of ConcreteNodes (The Ship).
    * This is fast because it relies on pre-computed metadata where available.
    */
-  calculateEpisodeListTokens(episodes: Episode[]): number {
+  calculateConcreteListTokens(ship: ReadonlyArray<ConcreteNode>): number {
     let tokens = 0;
-    for (const ep of episodes) {
-      if (ep.trigger) tokens += ep.trigger.metadata.currentTokens;
-      for (const step of ep.steps) {
-        tokens += step.metadata.currentTokens;
-      }
-      if (ep.yield) tokens += ep.yield.metadata.currentTokens;
+    for (const node of ship) {
+       tokens += node.metadata.currentTokens;
     }
     return tokens;
   }