feat(context): Introduce adaptive token calculator to more accurately calculate content sizes. (#26888)

This commit is contained in:
joshualitt
2026-05-12 08:51:20 -07:00
committed by GitHub
parent 7a9ed4c20a
commit 07792f98cd
26 changed files with 856 additions and 164 deletions
@@ -66,12 +66,12 @@ describe('BlobDegradationProcessor', () => {
const node1 = createDummyNode('ep1', NodeType.USER_PROMPT, 100, {
payload: {
fileData: { mimeType: 'video/mp4', fileUri: 'gs://test1' },
fileData: { mimeType: 'image/png', fileUri: 'gs://test1' },
},
});
const node2 = createDummyNode('ep1', NodeType.USER_PROMPT, 100, {
payload: {
fileData: { mimeType: 'video/mp4', fileUri: 'gs://test2' },
fileData: { mimeType: 'image/png', fileUri: 'gs://test2' },
},
});
@@ -52,7 +52,7 @@ describe('StateSnapshotAsyncProcessor', () => {
'PROPOSED_SNAPSHOT',
expect.objectContaining({
newText:
'{"active_tasks":[],"discovered_facts":[],"constraints_and_preferences":[],"recent_arc":[]}',
'{"active_tasks":[],"discovered_facts":[],"constraints_and_preferences":[],"recent_arc":["Mock LLM summary response"]}',
consumedIds: ['node-A', 'node-B'],
type: 'point-in-time',
}),
@@ -107,7 +107,7 @@ describe('StateSnapshotAsyncProcessor', () => {
'PROPOSED_SNAPSHOT',
expect.objectContaining({
newText:
'{"active_tasks":[],"discovered_facts":[],"constraints_and_preferences":[],"recent_arc":[]}',
'{"active_tasks":[],"discovered_facts":[],"constraints_and_preferences":[],"recent_arc":["Mock LLM summary response"]}',
consumedIds: ['node-A', 'node-B', 'node-C'], // Aggregated!
type: 'accumulate',
}),