refactoring continued

This commit is contained in:
Your Name
2026-04-06 22:27:32 +00:00
parent cf6866c38d
commit fc4439ce03
13 changed files with 130 additions and 97 deletions
@@ -9,17 +9,19 @@ import { BlobDegradationProcessor } from './blobDegradationProcessor.js';
import type { Episode, UserPrompt } from '../ir/types.js';
import type { ContextAccountingState } from '../pipeline.js';
import { randomUUID } from 'node:crypto';
import * as fsPromises from 'node:fs/promises';
vi.mock('node:fs/promises');
import type { ContextEnvironment } from '../sidecar/environment.js';
import { InMemoryFileSystem } from '../system/InMemoryFileSystem.js';
describe('BlobDegradationProcessor', () => {
let processor: BlobDegradationProcessor;
let env: ContextEnvironment;
let fileSystem: InMemoryFileSystem;
beforeEach(() => {
vi.resetAllMocks();
processor = new BlobDegradationProcessor(createMockEnvironment());
env = createMockEnvironment();
fileSystem = env.fileSystem as InMemoryFileSystem;
processor = new BlobDegradationProcessor(env);
});
const getDummyState = (
@@ -61,7 +63,6 @@ describe('BlobDegradationProcessor', () => {
steps: [],
};
// Fake token calculator says inlineData costs 258 tokens, text costs 10
const state = getDummyState(false, 500, new Set());
const result = await processor.process([ep], state);
@@ -79,7 +80,11 @@ describe('BlobDegradationProcessor', () => {
'degraded to text to preserve context window',
);
expect(fsPromises.writeFile).toHaveBeenCalledTimes(1);
// Verify it was written to fake FS
expect(fileSystem.getFiles().size).toBeGreaterThan(0);
const files = Array.from(fileSystem.getFiles().keys());
expect(files[0]).toContain('.gemini/tool-outputs/degraded-blobs/session-mock-session/blob_');
expect(result[0].trigger.metadata.transformations.length).toBe(1);
});
@@ -118,6 +123,6 @@ describe('BlobDegradationProcessor', () => {
'Original URI: gs://fake-bucket/doc.pdf',
);
expect(fsPromises.writeFile).not.toHaveBeenCalled();
expect(fileSystem.getFiles().size).toBe(0);
});
});
@@ -8,8 +8,6 @@ import type { ContextAccountingState, ContextProcessor } from '../pipeline.js';
import type { ContextEnvironment } from '../sidecar/environment.js';
import { sanitizeFilenamePart } from '../../utils/fileUtils.js';
import * as fsPromises from 'node:fs/promises';
import path from 'node:path';
import type { Part } from '@google/genai';
export class BlobDegradationProcessor implements ContextProcessor {
@@ -32,13 +30,13 @@ export class BlobDegradationProcessor implements ContextProcessor {
const newEpisodes = [...episodes];
let directoryCreated = false;
let blobOutputsDir = path.join(
let blobOutputsDir = this.env.fileSystem.join(
this.env.projectTempDir,
'degraded-blobs',
);
const sessionId = this.env.sessionId;
if (sessionId) {
blobOutputsDir = path.join(
blobOutputsDir = this.env.fileSystem.join(
blobOutputsDir,
`session-${sanitizeFilenamePart(sessionId)}`,
);
@@ -46,7 +44,7 @@ export class BlobDegradationProcessor implements ContextProcessor {
const ensureDir = async () => {
if (!directoryCreated) {
await fsPromises.mkdir(blobOutputsDir, { recursive: true });
await this.env.fileSystem.mkdir(blobOutputsDir, { recursive: true });
directoryCreated = true;
}
};
@@ -69,12 +67,12 @@ export class BlobDegradationProcessor implements ContextProcessor {
if (part.type === 'inline_data') {
await ensureDir();
const ext = part.mimeType.split('/')[1] || 'bin';
const fileName = `blob_${Date.now()}_${Math.random().toString(36).substring(7)}.${ext}`;
const filePath = path.join(blobOutputsDir, fileName);
const fileName = `blob_${Date.now()}_${this.env.idGenerator.generateId()}.${ext}`;
const filePath = this.env.fileSystem.join(blobOutputsDir, fileName);
// Base64 to buffer
const buffer = Buffer.from(part.data, 'base64');
await fsPromises.writeFile(filePath, buffer);
await this.env.fileSystem.writeFile(filePath, buffer);
const mb = (buffer.byteLength / 1024 / 1024).toFixed(2);
newText = `[Multi-Modal Blob (${part.mimeType}, ${mb}MB) degraded to text to preserve context window. Saved to: ${filePath}]`;
@@ -37,7 +37,7 @@ export class SemanticCompressionProcessor implements ContextProcessor {
const semanticConfig = this.options;
const limitTokens = semanticConfig.nodeThresholdTokens;
const thresholdChars = limitTokens * this.env.charsPerToken;
const thresholdChars = this.env.tokenCalculator.tokensToChars(limitTokens);
this.modelToUse = 'gemini-2.5-flash';
let currentDeficit = state.deficitTokens;
@@ -10,17 +10,20 @@ import { ToolMaskingProcessor } from './toolMaskingProcessor.js';
import type { Episode, ToolExecution } from '../ir/types.js';
import type { ContextAccountingState } from '../pipeline.js';
import { randomUUID } from 'node:crypto';
import * as fsPromises from 'node:fs/promises';
vi.mock('node:fs/promises');
import type { ContextEnvironment } from '../sidecar/environment.js';
import { InMemoryFileSystem } from '../system/InMemoryFileSystem.js';
describe('ToolMaskingProcessor', () => {
let processor: ToolMaskingProcessor;
let env: ContextEnvironment;
let fileSystem: InMemoryFileSystem;
beforeEach(() => {
vi.resetAllMocks();
env = createMockEnvironment();
fileSystem = env.fileSystem as InMemoryFileSystem;
processor = new ToolMaskingProcessor(createMockEnvironment(), {
processor = new ToolMaskingProcessor(env, {
stringLengthThresholdTokens: 100,
});
});
@@ -76,10 +79,6 @@ describe('ToolMaskingProcessor', () => {
const state = getDummyState(true);
const result = await processor.process(episodes, state);
require('fs').appendFileSync(
'/tmp/debug.json',
'\n\n' + JSON.stringify({ res: result[0].steps[0] }, null, 2),
);
expect(result).toStrictEqual(episodes);
expect((result[0].steps[0] as ToolExecution).presentation).toBeUndefined();
@@ -124,7 +123,7 @@ describe('ToolMaskingProcessor', () => {
);
expect((maskedObs as { error: string }).error).toBeNull();
// Check disk writes occurred
expect(fsPromises.writeFile).toHaveBeenCalledTimes(2);
// Check disk writes occurred to fake FS
expect(fileSystem.getFiles().size).toBe(2);
});
});
@@ -8,8 +8,6 @@ import type { ContextAccountingState, ContextProcessor } from '../pipeline.js';
import type { ContextEnvironment } from '../sidecar/environment.js';
import { sanitizeFilenamePart } from '../../utils/fileUtils.js';
import * as fsPromises from 'node:fs/promises';
import path from 'node:path';
import {
ACTIVATE_SKILL_TOOL_NAME,
MEMORY_TOOL_NAME,
@@ -50,15 +48,15 @@ export class ToolMaskingProcessor implements ContextProcessor {
const newEpisodes = [...episodes];
let currentDeficit = state.deficitTokens;
const limitChars = maskingConfig.stringLengthThresholdTokens * this.env.charsPerToken;
const limitChars = this.env.tokenCalculator.tokensToChars(maskingConfig.stringLengthThresholdTokens);
let toolOutputsDir = path.join(
let toolOutputsDir = this.env.fileSystem.join(
this.env.projectTempDir,
'tool-outputs',
);
const sessionId = this.env.sessionId;
if (sessionId) {
toolOutputsDir = path.join(
toolOutputsDir = this.env.fileSystem.join(
toolOutputsDir,
`session-${sanitizeFilenamePart(sessionId)}`,
);
@@ -75,14 +73,14 @@ export class ToolMaskingProcessor implements ContextProcessor {
nodeType: string,
): Promise<string> => {
if (!directoryCreated) {
await fsPromises.mkdir(toolOutputsDir, { recursive: true });
await this.env.fileSystem.mkdir(toolOutputsDir, { recursive: true });
directoryCreated = true;
}
const fileName = `${sanitizeFilenamePart(toolName).toLowerCase()}_${sanitizeFilenamePart(callId).toLowerCase()}_${nodeType}_${Math.random().toString(36).substring(7)}.txt`;
const filePath = path.join(toolOutputsDir, fileName);
const fileName = `${sanitizeFilenamePart(toolName).toLowerCase()}_${sanitizeFilenamePart(callId).toLowerCase()}_${nodeType}_${this.env.idGenerator.generateId()}.txt`;
const filePath = this.env.fileSystem.join(toolOutputsDir, fileName);
await fsPromises.writeFile(filePath, content, 'utf-8');
await this.env.fileSystem.writeFile(filePath, content);
const fileSizeMB = (
Buffer.byteLength(content, 'utf8') /
@@ -1,13 +1,13 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import * as fs from 'node:fs';
import { describe, it, expect, beforeEach } from 'vitest';
import { SidecarLoader } from './SidecarLoader.js';
import { defaultSidecarProfile } from './profiles.js';
import { InMemoryFileSystem } from '../system/InMemoryFileSystem.js';
vi.mock('node:fs');
describe('SidecarLoader (Fake FS)', () => {
let fileSystem: InMemoryFileSystem;
describe('SidecarLoader', () => {
beforeEach(() => {
vi.resetAllMocks();
fileSystem = new InMemoryFileSystem();
});
const mockConfig = {
@@ -15,48 +15,38 @@ describe('SidecarLoader', () => {
} as any;
it('returns default profile if file does not exist', () => {
vi.mocked(fs.existsSync).mockReturnValue(false);
const result = SidecarLoader.fromConfig(mockConfig);
const result = SidecarLoader.fromConfig(mockConfig, fileSystem);
expect(result).toBe(defaultSidecarProfile);
});
it('returns default profile if file exists but is 0 bytes', () => {
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.statSync).mockReturnValue({ size: 0 } as any);
const result = SidecarLoader.fromConfig(mockConfig);
fileSystem.setFile('/path/to/sidecar.json', '');
const result = SidecarLoader.fromConfig(mockConfig, fileSystem);
expect(result).toBe(defaultSidecarProfile);
});
it('throws an error if file is empty whitespace', () => {
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.statSync).mockReturnValue({ size: 5 } as any);
vi.mocked(fs.readFileSync).mockReturnValue(' \n ');
expect(() => SidecarLoader.fromConfig(mockConfig)).toThrow('is empty');
fileSystem.setFile('/path/to/sidecar.json', ' \n ');
expect(() => SidecarLoader.fromConfig(mockConfig, fileSystem)).toThrow('is empty');
});
it('returns parsed config if file is valid', () => {
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.statSync).mockReturnValue({ size: 100 } as any);
const validConfig = {
budget: { retainedTokens: 1000, maxTokens: 2000 },
gcBackstop: { strategy: 'truncate', target: 'max' },
pipelines: []
};
vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(validConfig));
const result = SidecarLoader.fromConfig(mockConfig);
fileSystem.setFile('/path/to/sidecar.json', JSON.stringify(validConfig));
const result = SidecarLoader.fromConfig(mockConfig, fileSystem);
expect(result).toEqual(validConfig);
});
it('throws an error if schema validation fails', () => {
vi.mocked(fs.existsSync).mockReturnValue(true);
vi.mocked(fs.statSync).mockReturnValue({ size: 100 } as any);
const invalidConfig = {
budget: { retainedTokens: "invalid string" }, // Invalid type
pipelines: []
};
vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(invalidConfig));
expect(() => SidecarLoader.fromConfig(mockConfig)).toThrow('Validation error:');
fileSystem.setFile('/path/to/sidecar.json', JSON.stringify(invalidConfig));
expect(() => SidecarLoader.fromConfig(mockConfig, fileSystem)).toThrow('Validation error:');
});
});
@@ -4,19 +4,21 @@
* SPDX-License-Identifier: Apache-2.0
*/
import * as fs from 'node:fs';
import type { Config } from '../../config/config.js';
import type { SidecarConfig } from './types.js';
import { defaultSidecarProfile } from './profiles.js';
import { SchemaValidator } from '../../utils/schemaValidator.js';
import { sidecarConfigSchema } from './schema.js';
import type { IFileSystem } from '../system/IFileSystem.js';
import { NodeFileSystem } from '../system/NodeFileSystem.js';
export class SidecarLoader {
/**
* Loads and validates a sidecar config from a specific file path.
* Throws an error if the file cannot be read, parsed, or fails schema validation.
*/
static loadFromFile(sidecarPath: string): SidecarConfig {
const fileContent = fs.readFileSync(sidecarPath, 'utf8');
static loadFromFile(sidecarPath: string, fileSystem: IFileSystem = new NodeFileSystem()): SidecarConfig {
const fileContent = fileSystem.readFileSync(sidecarPath, 'utf8');
if (!fileContent.trim()) {
throw new Error(`Sidecar configuration file at ${sidecarPath} is empty.`);
@@ -49,18 +51,18 @@ export class SidecarLoader {
* Generates a Sidecar JSON graph from the experimental config file path or defaults.
* If a config file is present but invalid, this will THROW to prevent silent misconfiguration.
*/
static fromConfig(config: Config): SidecarConfig {
static fromConfig(config: Config, fileSystem: IFileSystem = new NodeFileSystem()): SidecarConfig {
const sidecarPath = config.getExperimentalContextSidecarConfig();
if (sidecarPath && fs.existsSync(sidecarPath)) {
const stat = fs.statSync(sidecarPath);
if (sidecarPath && fileSystem.existsSync(sidecarPath)) {
const size = fileSystem.statSyncSize(sidecarPath);
// If the file exists but is completely empty (0 bytes), it's safe to fallback.
if (stat.size === 0) {
if (size === 0) {
return defaultSidecarProfile;
}
// If the file has content, enforce strict validation and throw on failure.
return this.loadFromFile(sidecarPath);
return this.loadFromFile(sidecarPath, fileSystem);
}
return defaultSidecarProfile;
@@ -4,20 +4,25 @@
* SPDX-License-Identifier: Apache-2.0
*/
import type { BaseLlmClient } from '../../core/baseLlmClient.js';
import type { ContextTracer } from '../tracer.js';
import type { ContextEventBus } from '../eventBus.js';
import type { ContextEventBus } from '../eventBus.js';
import type { ContextTokenCalculator } from '../utils/contextTokenCalculator.js';
export type { ContextTracer, ContextEventBus };
import type { ContextTracer } from '../tracer.js';
import type { IFileSystem } from '../system/IFileSystem.js';
import type { IIdGenerator } from '../system/IIdGenerator.js';
export interface ContextEnvironment {
readonly llmClient: BaseLlmClient;
readonly promptId: string;
readonly sessionId: string;
readonly traceDir: string;
readonly projectTempDir: string;
readonly tracer: ContextTracer;
readonly charsPerToken: number;
readonly tokenCalculator: ContextTokenCalculator;
eventBus: ContextEventBus;
export type { ContextTracer, ContextEventBus };
export interface ContextEnvironment {
readonly llmClient: BaseLlmClient;
readonly promptId: string;
readonly sessionId: string;
readonly traceDir: string;
readonly projectTempDir: string;
readonly tracer: ContextTracer;
readonly charsPerToken: number;
readonly tokenCalculator: ContextTokenCalculator;
readonly fileSystem: IFileSystem;
readonly idGenerator: IIdGenerator;
eventBus: ContextEventBus;
}
@@ -11,9 +11,15 @@ import type { ContextEnvironment } from './environment.js';
import type { ContextEventBus } from '../eventBus.js';
import { ContextTokenCalculator } from '../utils/contextTokenCalculator.js';
import type { IFileSystem } from '../system/IFileSystem.js';
import { NodeFileSystem } from '../system/NodeFileSystem.js';
import type { IIdGenerator } from '../system/IIdGenerator.js';
import { NodeIdGenerator } from '../system/NodeIdGenerator.js';
export class ContextEnvironmentImpl implements ContextEnvironment {
public readonly tokenCalculator: ContextTokenCalculator;
public readonly fileSystem: IFileSystem;
public readonly idGenerator: IIdGenerator;
constructor(
public readonly llmClient: BaseLlmClient,
@@ -24,7 +30,11 @@ export class ContextEnvironmentImpl implements ContextEnvironment {
public readonly tracer: ContextTracer,
public readonly charsPerToken: number,
public readonly eventBus: ContextEventBus,
fileSystem?: IFileSystem,
idGenerator?: IIdGenerator,
) {
this.tokenCalculator = new ContextTokenCalculator(this.charsPerToken);
this.fileSystem = fileSystem || new NodeFileSystem();
this.idGenerator = idGenerator || new NodeIdGenerator();
}
}
@@ -8,9 +8,13 @@ export interface IFileSystem {
existsSync(path: string): boolean;
statSyncSize(path: string): number;
readFileSync(path: string, encoding: 'utf8'): string;
writeFileSync(path: string, data: string, encoding: 'utf-8'): void;
writeFileSync(path: string, data: string | Buffer, encoding?: 'utf-8'): void;
appendFileSync(path: string, data: string, encoding: 'utf-8'): void;
mkdirSync(path: string, options?: { recursive?: boolean }): void;
writeFile(path: string, data: string | Buffer): Promise<void>;
mkdir(path: string, options?: { recursive?: boolean }): Promise<void>;
join(...paths: string[]): string;
dirname(path: string): string;
}
@@ -7,21 +7,17 @@
import type { IFileSystem } from './IFileSystem.js';
export class InMemoryFileSystem implements IFileSystem {
private files = new Map<string, string>();
private files = new Map<string, string | Buffer>();
// Helper for tests
getFiles(): ReadonlyMap<string, string> {
getFiles(): ReadonlyMap<string, string | Buffer> {
return this.files;
}
// Helper for tests
setFile(path: string, content: string) {
setFile(path: string, content: string | Buffer) {
this.files.set(this.normalize(path), content);
}
private normalize(p: string): string {
// A very naive normalization for testing purposes.
// Ensures '/foo/bar' and '/foo//bar' map to the same key.
return p.replace(/\/+/g, '/');
}
@@ -34,7 +30,7 @@ export class InMemoryFileSystem implements IFileSystem {
if (content === undefined) {
throw new Error(`ENOENT: no such file or directory, stat '${p}'`);
}
return content.length; // Naive char length = byte size for testing
return Buffer.isBuffer(content) ? content.byteLength : Buffer.byteLength(content, 'utf8');
}
readFileSync(p: string, encoding: 'utf8'): string {
@@ -42,23 +38,31 @@ export class InMemoryFileSystem implements IFileSystem {
if (content === undefined) {
throw new Error(`ENOENT: no such file or directory, open '${p}'`);
}
if (Buffer.isBuffer(content)) {
return content.toString(encoding);
}
return content;
}
writeFileSync(p: string, data: string, encoding: 'utf-8'): void {
writeFileSync(p: string, data: string | Buffer, encoding?: 'utf-8'): void {
this.files.set(this.normalize(p), data);
}
appendFileSync(p: string, data: string, encoding: 'utf-8'): void {
const norm = this.normalize(p);
const existing = this.files.get(norm) || '';
this.files.set(norm, existing + data);
const existingStr = Buffer.isBuffer(existing) ? existing.toString('utf8') : existing;
this.files.set(norm, existingStr + data);
}
mkdirSync(p: string, options?: { recursive?: boolean }): void {
// In-memory fake doesn't track directories separately from files for our simple use cases
mkdirSync(p: string, options?: { recursive?: boolean }): void {}
async writeFile(p: string, data: string | Buffer): Promise<void> {
this.writeFileSync(p, data);
}
async mkdir(p: string, options?: { recursive?: boolean }): Promise<void> {}
join(...paths: string[]): string {
return this.normalize(paths.join('/'));
}
@@ -5,6 +5,7 @@
*/
import * as fs from 'node:fs';
import * as fsPromises from 'node:fs/promises';
import * as path from 'node:path';
import type { IFileSystem } from './IFileSystem.js';
@@ -21,8 +22,12 @@ export class NodeFileSystem implements IFileSystem {
return fs.readFileSync(p, encoding);
}
writeFileSync(p: string, data: string, encoding: 'utf-8'): void {
fs.writeFileSync(p, data, encoding);
writeFileSync(p: string, data: string | Buffer, encoding?: 'utf-8'): void {
if (Buffer.isBuffer(data)) {
fs.writeFileSync(p, data);
} else {
fs.writeFileSync(p, data, encoding);
}
}
appendFileSync(p: string, data: string, encoding: 'utf-8'): void {
@@ -33,6 +38,14 @@ export class NodeFileSystem implements IFileSystem {
fs.mkdirSync(p, options);
}
async writeFile(p: string, data: string | Buffer): Promise<void> {
await fsPromises.writeFile(p, data);
}
async mkdir(p: string, options?: { recursive?: boolean }): Promise<void> {
await fsPromises.mkdir(p, options);
}
join(...paths: string[]): string {
return path.join(...paths);
}
@@ -11,6 +11,9 @@ import type { Content } from '@google/genai';
import { AgentChatHistory } from '../../core/agentChatHistory.js';
import { ContextManager } from '../contextManager.js';
import { InMemoryFileSystem } from '../system/InMemoryFileSystem.js';
import { DeterministicIdGenerator } from '../system/DeterministicIdGenerator.js';
export function createMockEnvironment(): ContextEnvironment {
return {
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
@@ -27,6 +30,8 @@ export function createMockEnvironment(): ContextEnvironment {
tracer: new ContextTracer({ targetDir: '/tmp', sessionId: 'mock-session' }),
charsPerToken: 1,
tokenCalculator: new ContextTokenCalculator(1),
fileSystem: new InMemoryFileSystem(),
idGenerator: new DeterministicIdGenerator('mock-uuid-'),
};
}