Compare commits

...

1 Commits

Author SHA1 Message Date
Sandy Tao 366036a197 feat(core): introduce MemoryProvider seam and MemoryService orchestrator
Refactors the existing background skill extractor into a layered memory
subsystem that GeminiClient can drive uniformly across interactive and
non-interactive sessions.

- Adds an internal `MemoryProvider` interface (not exported, not surfaced
  via extensions) as a typing seam between the orchestrator and its
  concrete implementation
- Refactors the existing skill-extraction logic into `DefaultMemoryProvider`,
  the sole implementation of the seam
- Introduces `MemoryService` as a thin orchestrator that owns a
  `DefaultMemoryProvider` and wraps every lifecycle call in try/catch so
  a buggy provider can't crash a turn
- Moves MemoryService ownership into `GeminiClient` (private field exposed
  via optional methods) so non-interactive entry points share the same
  lifecycle as the interactive UI
- Keeps builtin memory extraction asynchronous on startup to avoid
  blocking the first turn
- Adds unit tests for MemoryService, DefaultMemoryProvider, and the
  GeminiClient integration

The whole subsystem remains gated behind the existing
`isMemoryManagerEnabled()` experimental flag.
2026-04-17 11:08:33 -07:00
12 changed files with 2801 additions and 2068 deletions
+1
View File
@@ -529,6 +529,7 @@ export async function runNonInteractive(
// Cleanup stdin cancellation before other cleanup
cleanupStdinCancellation();
await config.getGeminiClient()?.shutdownSessionServices?.();
scheduler?.dispose();
consolePatcher.cleanup();
coreEvents.off(CoreEvent.UserFeedback, handleUserFeedback);
@@ -616,6 +616,7 @@ export async function runNonInteractive({
cleanupStdinCancellation();
abortController.signal.removeEventListener('abort', abortSession);
await config.getGeminiClient()?.shutdownSessionServices?.();
scheduler?.dispose();
consolePatcher.cleanup();
coreEvents.off(CoreEvent.UserFeedback, handleUserFeedback);
+1 -8
View File
@@ -92,7 +92,6 @@ import {
ApiKeyUpdatedEvent,
LegacyAgentProtocol,
type InjectionSource,
startMemoryService,
} from '@google/gemini-cli-core';
import { validateAuthMethod } from '../config/auth.js';
import process from 'node:process';
@@ -482,13 +481,6 @@ export const AppContainer = (props: AppContainerProps) => {
setConfigInitialized(true);
startupProfiler.flush(config);
// Fire-and-forget memory service (skill extraction from past sessions)
if (config.isMemoryManagerEnabled()) {
startMemoryService(config).catch((e) => {
debugLogger.error('Failed to start memory service:', e);
});
}
const sessionStartSource = resumedSessionData
? SessionStartSource.Resume
: SessionStartSource.Startup;
@@ -540,6 +532,7 @@ export const AppContainer = (props: AppContainerProps) => {
// Fire SessionEnd hook on cleanup (only if hooks are enabled)
await config?.getHookSystem()?.fireSessionEndEvent(SessionEndReason.Exit);
await config?.getGeminiClient()?.shutdownSessionServices?.();
};
registerCleanup(cleanupFn);
+1 -1
View File
@@ -20,7 +20,7 @@ import {
isProjectSkillPatchTarget,
validateParsedSkillPatchHeaders,
} from '../services/memoryPatchUtils.js';
import { readExtractionState } from '../services/memoryService.js';
import { readExtractionState } from '../services/defaultMemoryProvider.js';
import { refreshServerHierarchicalMemory } from '../utils/memoryDiscovery.js';
import type { MessageActionReturn, ToolActionReturn } from './types.js';
+144
View File
@@ -34,6 +34,7 @@ import {
import { getCoreSystemPrompt } from './prompts.js';
import { DEFAULT_GEMINI_MODEL_AUTO } from '../config/models.js';
import { FileDiscoveryService } from '../services/fileDiscoveryService.js';
import { MemoryService } from '../services/memoryService.js';
import { setSimulate429 } from '../utils/testUtils.js';
import { tokenLimit } from './tokenLimits.js';
import { ideContextStore } from '../ide/ideContext.js';
@@ -94,6 +95,29 @@ interface MockTurnContext {
}
const mockTurnRunFn = vi.fn();
const memoryServiceMocks = vi.hoisted(() => {
const onSessionStart = vi.fn();
const getSystemInstructions = vi.fn().mockResolvedValue('');
const getTurnContext = vi.fn().mockResolvedValue('');
const onTurnComplete = vi.fn();
const onSessionEnd = vi.fn();
const factory = vi.fn(() => ({
onSessionStart,
getSystemInstructions,
getTurnContext,
onTurnComplete,
onSessionEnd,
}));
return {
onSessionStart,
getSystemInstructions,
getTurnContext,
onTurnComplete,
onSessionEnd,
factory,
};
});
vi.mock('./turn', async (importOriginal) => {
const actual = await importOriginal<typeof import('./turn.js')>();
@@ -117,6 +141,9 @@ vi.mock('./turn', async (importOriginal) => {
});
vi.mock('../config/config.js');
vi.mock('../services/memoryService.js', () => ({
MemoryService: vi.fn().mockImplementation(() => memoryServiceMocks.factory()),
}));
vi.mock('./prompts');
vi.mock('../utils/getFolderStructure', () => ({
getFolderStructure: vi.fn().mockResolvedValue('Mock Folder Structure'),
@@ -172,6 +199,15 @@ describe('Gemini Client (client.ts)', () => {
vi.resetAllMocks();
ClearcutLogger.clearInstance();
vi.mocked(uiTelemetryService.setLastPromptTokenCount).mockClear();
vi.mocked(MemoryService).mockImplementation(
() => memoryServiceMocks.factory() as unknown as MemoryService,
);
memoryServiceMocks.factory.mockClear();
memoryServiceMocks.onSessionStart.mockReset();
memoryServiceMocks.getSystemInstructions.mockReset().mockResolvedValue('');
memoryServiceMocks.getTurnContext.mockReset().mockResolvedValue('');
memoryServiceMocks.onTurnComplete.mockReset();
memoryServiceMocks.onSessionEnd.mockReset();
mockGenerateContentFn = vi.fn().mockResolvedValue({
candidates: [{ content: { parts: [{ text: '{"key": "value"}' }] } }],
@@ -231,6 +267,8 @@ describe('Gemini Client (client.ts)', () => {
minPrunableThresholdTokens: 30000,
protectLatestTurn: true,
}),
getExtensions: vi.fn().mockReturnValue([]),
isMemoryManagerEnabled: vi.fn().mockReturnValue(false),
getSessionId: vi.fn().mockReturnValue('test-session-id'),
getProxy: vi.fn().mockReturnValue(undefined),
@@ -431,6 +469,112 @@ describe('Gemini Client (client.ts)', () => {
});
});
describe('memory service integration', () => {
it('does not construct the memory service when the experimental flag is off', async () => {
vi.mocked(mockConfig.isMemoryManagerEnabled).mockReturnValue(false);
client.dispose();
client = new GeminiClient(mockConfig as unknown as AgentLoopContext);
await client.initialize();
expect(memoryServiceMocks.factory).not.toHaveBeenCalled();
expect(memoryServiceMocks.onSessionStart).not.toHaveBeenCalled();
});
it('initializes the memory service and injects system instructions when the experimental flag is on', async () => {
vi.mocked(mockConfig.isMemoryManagerEnabled).mockReturnValue(true);
memoryServiceMocks.getSystemInstructions.mockResolvedValue(
'Memory instructions',
);
vi.mocked(getCoreSystemPrompt).mockReturnValue('Base prompt');
client.dispose();
client = new GeminiClient(mockConfig as unknown as AgentLoopContext);
await client.initialize();
const mockChat = {
setSystemInstruction: vi.fn(),
} as unknown as GeminiChat;
client['chat'] = mockChat;
client.updateSystemInstruction();
expect(memoryServiceMocks.factory).toHaveBeenCalledTimes(1);
expect(memoryServiceMocks.onSessionStart).toHaveBeenCalledWith(
'test-session-id',
);
expect(mockChat.setSystemInstruction).toHaveBeenCalledWith(
'Base prompt\n\n<memory_system_instructions>\nMemory instructions\n</memory_system_instructions>',
);
});
it('injects turn context and completes the prompt lifecycle after the final turn', async () => {
vi.mocked(mockConfig.isMemoryManagerEnabled).mockReturnValue(true);
memoryServiceMocks.getTurnContext.mockResolvedValue('recalled context');
mockTurnRunFn.mockReturnValue(
(async function* () {
yield {
type: GeminiEventType.Finished,
value: { reason: 'STOP', usageMetadata: undefined },
};
})(),
);
const stream = client.sendMessageStream(
[{ text: 'How do I deploy this?' }],
new AbortController().signal,
'prompt-id-memory',
);
await fromAsync(stream);
expect(memoryServiceMocks.getTurnContext).toHaveBeenCalledWith(
'How do I deploy this?',
);
expect(mockTurnRunFn).toHaveBeenCalledWith(
{ model: 'default-routed-model', isChatModel: true },
[
{ text: 'How do I deploy this?' },
{ text: '<memory_context>recalled context</memory_context>' },
],
expect.any(AbortSignal),
undefined,
);
expect(memoryServiceMocks.onTurnComplete).toHaveBeenCalledWith(
'How do I deploy this?',
'Mock Response',
);
});
it('invokes onSessionEnd when shutdownSessionServices is called', async () => {
vi.mocked(mockConfig.isMemoryManagerEnabled).mockReturnValue(true);
client.dispose();
client = new GeminiClient(mockConfig as unknown as AgentLoopContext);
await client.initialize();
await client.shutdownSessionServices();
expect(memoryServiceMocks.onSessionEnd).toHaveBeenCalledTimes(1);
});
it('skips the memory service for subagent-scoped clients', async () => {
vi.mocked(mockConfig.isMemoryManagerEnabled).mockReturnValue(true);
client.dispose();
const subagentContext = Object.assign(Object.create(mockConfig), {
config: mockConfig,
promptId: 'subagent-prompt-id',
parentSessionId: 'main-prompt-id',
}) as AgentLoopContext;
const subagentClient = new GeminiClient(subagentContext);
await subagentClient.initialize();
expect(memoryServiceMocks.factory).not.toHaveBeenCalled();
subagentClient.dispose();
});
});
describe('tryCompressChat', () => {
const mockGetHistory = vi.fn();
+211 -5
View File
@@ -43,6 +43,7 @@ import type {
} from '../services/chatRecordingService.js';
import type { ContentGenerator } from './contentGenerator.js';
import { LoopDetectionService } from '../services/loopDetectionService.js';
import { MemoryService } from '../services/memoryService.js';
import { ChatCompressionService } from '../context/chatCompressionService.js';
import { AgentHistoryProvider } from '../context/agentHistoryProvider.js';
import { ideContextStore } from '../ide/ideContext.js';
@@ -89,6 +90,14 @@ type BeforeAgentHookReturn =
| { additionalContext: string | undefined }
| undefined;
interface PromptLifecycleState {
originalUserMessage: string;
}
interface SendMessageStreamOptions {
internalContinuation?: boolean;
}
export class GeminiClient {
private chat?: GeminiChat;
private sessionTurnCount = 0;
@@ -107,6 +116,10 @@ export class GeminiClient {
* being forced and did it fail?
*/
private hasFailedCompressionAttempt = false;
private memoryService?: MemoryService;
private memorySystemInstructions = '';
private memorySessionStarted = false;
private promptLifecycleStateMap = new Map<string, PromptLifecycleState>();
constructor(private readonly context: AgentLoopContext) {
this.loopDetector = new LoopDetectionService(this.config);
@@ -248,6 +261,150 @@ export class GeminiClient {
}
}
private shouldUseMemoryService(): boolean {
if (this.context.parentSessionId) {
return false;
}
// The entire memory subsystem (built-in extractor and any
// extension-contributed providers) is gated behind a single
// experimental flag while the architecture stabilizes.
return this.config.isMemoryManagerEnabled();
}
private getOrCreateMemoryService(): MemoryService | undefined {
if (!this.shouldUseMemoryService()) {
return undefined;
}
if (!this.memoryService) {
this.memoryService = new MemoryService(this.config);
}
return this.memoryService;
}
private async refreshMemorySystemInstructions(): Promise<void> {
if (!this.memoryService) {
this.memorySystemInstructions = '';
return;
}
this.memorySystemInstructions =
await this.memoryService.getSystemInstructions();
}
private async ensureMemoryServiceInitialized(): Promise<void> {
const memoryService = this.getOrCreateMemoryService();
if (!memoryService) {
return;
}
if (!this.memorySessionStarted) {
this.memorySessionStarted = true;
await memoryService.onSessionStart(this.config.getSessionId());
}
await this.refreshMemorySystemInstructions();
}
private buildSystemInstruction(): string {
const systemMemory = this.config.getSystemInstructionMemory();
const systemInstruction = getCoreSystemPrompt(this.config, systemMemory);
const memoryInstructions = this.memorySystemInstructions.trim();
if (!memoryInstructions) {
return systemInstruction;
}
return `${systemInstruction}\n\n<memory_system_instructions>\n${memoryInstructions}\n</memory_system_instructions>`;
}
private getOrCreatePromptLifecycleState(
prompt_id: string,
request: PartListUnion,
): PromptLifecycleState {
let state = this.promptLifecycleStateMap.get(prompt_id);
if (!state) {
state = {
originalUserMessage: partListUnionToString(request),
};
this.promptLifecycleStateMap.set(prompt_id, state);
}
return state;
}
private clearPromptLifecycleState(prompt_id: string): void {
this.promptLifecycleStateMap.delete(prompt_id);
}
private requestHasFunctionResponse(request: PartListUnion): boolean {
const parts = Array.isArray(request) ? request : [request];
return parts.some(
(part) =>
typeof part === 'object' && part !== null && 'functionResponse' in part,
);
}
private appendMemoryContextToRequest(
request: PartListUnion,
memoryContext: string,
): PartListUnion {
const trimmedContext = memoryContext.trim();
if (!trimmedContext) {
return request;
}
const requestParts = Array.isArray(request) ? request : [request];
return [
...requestParts,
{ text: `<memory_context>${trimmedContext}</memory_context>` },
];
}
private async withMemoryTurnContext(
request: PartListUnion,
originalUserMessage: string,
isInternalContinuation: boolean,
): Promise<PartListUnion> {
if (isInternalContinuation || this.requestHasFunctionResponse(request)) {
return request;
}
await this.ensureMemoryServiceInitialized();
if (!this.memoryService || !originalUserMessage.trim()) {
return request;
}
const memoryContext =
await this.memoryService.getTurnContext(originalUserMessage);
return this.appendMemoryContextToRequest(request, memoryContext);
}
private async finalizePromptLifecycle(
prompt_id: string,
turn: Turn,
): Promise<void> {
try {
const assistantMessage = turn.getResponseText() || '';
const lifecycleState = this.promptLifecycleStateMap.get(prompt_id);
const userMessage = lifecycleState?.originalUserMessage ?? '';
if (
this.memoryService &&
(assistantMessage.trim().length > 0 || turn.finishReason !== undefined)
) {
// Fire-and-forget by contract — provider does its own async work.
this.memoryService.onTurnComplete(userMessage, assistantMessage);
await this.refreshMemorySystemInstructions();
this.updateSystemInstruction();
}
} finally {
this.clearPromptLifecycleState(prompt_id);
}
}
async initialize() {
this.chat = await this.startChat();
this.updateTelemetryTokenCount();
@@ -356,9 +513,7 @@ export class GeminiClient {
return;
}
const systemMemory = this.config.getSystemInstructionMemory();
const systemInstruction = getCoreSystemPrompt(this.config, systemMemory);
this.getChat().setSystemInstruction(systemInstruction);
this.getChat().setSystemInstruction(this.buildSystemInstruction());
}
async startChat(
@@ -376,8 +531,8 @@ export class GeminiClient {
const history = await getInitialChatHistory(this.config, extraHistory);
try {
const systemMemory = this.config.getSystemInstructionMemory();
const systemInstruction = getCoreSystemPrompt(this.config, systemMemory);
await this.ensureMemoryServiceInitialized();
const systemInstruction = this.buildSystemInstruction();
const chat = new GeminiChat(
this.config,
systemInstruction,
@@ -839,6 +994,8 @@ export class GeminiClient {
boundedTurns - 1,
true,
displayContent,
false,
{ internalContinuation: true },
);
return turn;
}
@@ -872,6 +1029,8 @@ export class GeminiClient {
boundedTurns - 1,
false, // isInvalidStreamRetry is false
displayContent,
false,
{ internalContinuation: true },
);
return turn;
}
@@ -888,7 +1047,14 @@ export class GeminiClient {
isInvalidStreamRetry: boolean = false,
displayContent?: PartListUnion,
stopHookActive: boolean = false,
options?: SendMessageStreamOptions,
): AsyncGenerator<ServerGeminiStreamEvent, Turn> {
const isInternalContinuation = options?.internalContinuation ?? false;
const promptLifecycleState = this.getOrCreatePromptLifecycleState(
prompt_id,
displayContent ?? request,
);
if (!isInvalidStreamRetry) {
this.config.resetTurn();
}
@@ -910,6 +1076,9 @@ export class GeminiClient {
'type' in hookResult &&
hookResult.type === GeminiEventType.AgentExecutionStopped
) {
if (!isInternalContinuation) {
this.clearPromptLifecycleState(prompt_id);
}
// Add user message to history before returning so it's kept in the transcript
this.getChat().addHistory(createUserContent(request));
yield hookResult;
@@ -918,6 +1087,9 @@ export class GeminiClient {
'type' in hookResult &&
hookResult.type === GeminiEventType.AgentExecutionBlocked
) {
if (!isInternalContinuation) {
this.clearPromptLifecycleState(prompt_id);
}
yield hookResult;
return new Turn(this.getChat(), prompt_id);
} else if ('additionalContext' in hookResult) {
@@ -933,6 +1105,12 @@ export class GeminiClient {
}
}
request = await this.withMemoryTurnContext(
request,
promptLifecycleState.originalUserMessage,
isInternalContinuation,
);
const boundedTurns = Math.min(turns, MAX_TURNS);
let turn = new Turn(this.getChat(), prompt_id);
let continuationHandled = false;
@@ -1008,11 +1186,15 @@ export class GeminiClient {
false,
displayContent,
true, // stopHookActive: signal retry to AfterAgent hooks
{ internalContinuation: true },
);
}
}
} catch (error) {
if (signal?.aborted || isAbortError(error)) {
if (!isInternalContinuation) {
this.clearPromptLifecycleState(prompt_id);
}
yield { type: GeminiEventType.UserCancelled };
return turn;
}
@@ -1035,9 +1217,31 @@ export class GeminiClient {
}
}
if (
!isInternalContinuation &&
!turn.pendingToolCalls.length &&
!signal.aborted
) {
await this.finalizePromptLifecycle(prompt_id, turn);
}
return turn;
}
async shutdownSessionServices(): Promise<void> {
if (!this.memoryService) {
this.promptLifecycleStateMap.clear();
return;
}
try {
await this.memoryService.onSessionEnd();
await this.refreshMemorySystemInstructions();
} finally {
this.promptLifecycleStateMap.clear();
}
}
async generateContent(
modelConfigKey: ModelConfigKey,
contents: Content[],
@@ -1277,6 +1481,8 @@ export class GeminiClient {
boundedTurns - 1,
isInvalidStreamRetry,
displayContent,
false,
{ internalContinuation: true },
);
}
}
+3 -2
View File
@@ -141,9 +141,10 @@ export * from './services/modelConfigService.js';
export * from './sandbox/windows/WindowsSandboxManager.js';
export * from './services/sessionSummaryUtils.js';
export {
startMemoryService,
DefaultMemoryProvider,
validatePatches,
} from './services/memoryService.js';
} from './services/defaultMemoryProvider.js';
export { MemoryService } from './services/memoryService.js';
export { isProjectSkillPatchTarget } from './services/memoryPatchUtils.js';
export * from './context/memoryContextManager.js';
export * from './services/trackerService.js';
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,756 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../config/config.js';
import type { MemoryProvider } from './memoryProvider.js';
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import { constants as fsConstants } from 'node:fs';
import { randomUUID } from 'node:crypto';
import * as Diff from 'diff';
import {
SESSION_FILE_PREFIX,
type ConversationRecord,
} from './chatRecordingService.js';
import { debugLogger } from '../utils/debugLogger.js';
import { coreEvents } from '../utils/events.js';
import { isNodeError } from '../utils/errors.js';
import { FRONTMATTER_REGEX, parseFrontmatter } from '../skills/skillLoader.js';
import { LocalAgentExecutor } from '../agents/local-executor.js';
import { SkillExtractionAgent } from '../agents/skill-extraction-agent.js';
import { getModelConfigAlias } from '../agents/registry.js';
import { ExecutionLifecycleService } from './executionLifecycleService.js';
import { PromptRegistry } from '../prompts/prompt-registry.js';
import { ResourceRegistry } from '../resources/resource-registry.js';
import { PolicyEngine } from '../policy/policy-engine.js';
import { PolicyDecision } from '../policy/types.js';
import { MessageBus } from '../confirmation-bus/message-bus.js';
import { Storage } from '../config/storage.js';
import type { AgentLoopContext } from '../config/agent-loop-context.js';
import {
applyParsedSkillPatches,
hasParsedPatchHunks,
} from './memoryPatchUtils.js';
const LOCK_FILENAME = '.extraction.lock';
const STATE_FILENAME = '.extraction-state.json';
const LOCK_STALE_MS = 35 * 60 * 1000; // 35 minutes (exceeds agent's 30-min time limit)
const MIN_USER_MESSAGES = 10;
const MIN_IDLE_MS = 3 * 60 * 60 * 1000; // 3 hours
const MAX_SESSION_INDEX_SIZE = 50;
// ... keeping all existing helper types and functions above startMemoryService
interface LockInfo {
pid: number;
startedAt: string;
}
export interface ExtractionRun {
runAt: string;
sessionIds: string[];
skillsCreated: string[];
}
export interface ExtractionState {
runs: ExtractionRun[];
}
export function getProcessedSessionIds(state: ExtractionState): Set<string> {
const ids = new Set<string>();
for (const run of state.runs) {
for (const id of run.sessionIds) {
ids.add(id);
}
}
return ids;
}
function isLockInfo(value: unknown): value is LockInfo {
return (
typeof value === 'object' &&
value !== null &&
'pid' in value &&
typeof value.pid === 'number' &&
'startedAt' in value &&
typeof value.startedAt === 'string'
);
}
function isConversationRecord(value: unknown): value is ConversationRecord {
return (
typeof value === 'object' &&
value !== null &&
'sessionId' in value &&
typeof value.sessionId === 'string' &&
'messages' in value &&
Array.isArray(value.messages) &&
'projectHash' in value &&
'startTime' in value &&
'lastUpdated' in value
);
}
function isExtractionRun(value: unknown): value is ExtractionRun {
return (
typeof value === 'object' &&
value !== null &&
'runAt' in value &&
typeof value.runAt === 'string' &&
'sessionIds' in value &&
Array.isArray(value.sessionIds) &&
'skillsCreated' in value &&
Array.isArray(value.skillsCreated)
);
}
function isExtractionState(value: unknown): value is { runs: unknown[] } {
return (
typeof value === 'object' &&
value !== null &&
'runs' in value &&
Array.isArray(value.runs)
);
}
export async function tryAcquireLock(
lockPath: string,
retries = 1,
): Promise<boolean> {
const lockInfo: LockInfo = {
pid: process.pid,
startedAt: new Date().toISOString(),
};
try {
const fd = await fs.open(
lockPath,
fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY,
);
try {
await fd.writeFile(JSON.stringify(lockInfo));
} finally {
await fd.close();
}
return true;
} catch (error: unknown) {
if (isNodeError(error) && error.code === 'EEXIST') {
if (retries > 0 && (await isLockStale(lockPath))) {
debugLogger.debug('[MemoryService] Cleaning up stale lock file');
await releaseLock(lockPath);
return tryAcquireLock(lockPath, retries - 1);
}
debugLogger.debug(
'[MemoryService] Lock held by another instance, skipping',
);
return false;
}
throw error;
}
}
export async function isLockStale(lockPath: string): Promise<boolean> {
try {
const content = await fs.readFile(lockPath, 'utf-8');
const parsed: unknown = JSON.parse(content);
if (!isLockInfo(parsed)) {
return true;
}
const lockInfo = parsed;
try {
process.kill(lockInfo.pid, 0);
} catch {
return true;
}
const lockAge = Date.now() - new Date(lockInfo.startedAt).getTime();
if (lockAge > LOCK_STALE_MS) {
return true;
}
return false;
} catch {
return true;
}
}
export async function releaseLock(lockPath: string): Promise<void> {
try {
await fs.unlink(lockPath);
} catch (error: unknown) {
if (isNodeError(error) && error.code === 'ENOENT') {
return;
}
debugLogger.warn(
`[MemoryService] Failed to release lock: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
export async function readExtractionState(
statePath: string,
): Promise<ExtractionState> {
try {
const content = await fs.readFile(statePath, 'utf-8');
const parsed: unknown = JSON.parse(content);
if (!isExtractionState(parsed)) {
return { runs: [] };
}
const runs: ExtractionRun[] = [];
for (const run of parsed.runs) {
if (!isExtractionRun(run)) continue;
runs.push({
runAt: run.runAt,
sessionIds: run.sessionIds.filter(
(sid): sid is string => typeof sid === 'string',
),
skillsCreated: run.skillsCreated.filter(
(sk): sk is string => typeof sk === 'string',
),
});
}
return { runs };
} catch (error) {
debugLogger.debug(
'[MemoryService] Failed to read extraction state:',
error,
);
return { runs: [] };
}
}
export async function writeExtractionState(
statePath: string,
state: ExtractionState,
): Promise<void> {
const tmpPath = `${statePath}.tmp`;
await fs.writeFile(tmpPath, JSON.stringify(state, null, 2));
await fs.rename(tmpPath, statePath);
}
function shouldProcessConversation(parsed: ConversationRecord): boolean {
if (parsed.kind === 'subagent') return false;
const lastUpdated = new Date(parsed.lastUpdated).getTime();
if (Date.now() - lastUpdated < MIN_IDLE_MS) return false;
const userMessageCount = parsed.messages.filter(
(m) => m.type === 'user',
).length;
if (userMessageCount < MIN_USER_MESSAGES) return false;
return true;
}
async function scanEligibleSessions(
chatsDir: string,
): Promise<Array<{ conversation: ConversationRecord; filePath: string }>> {
let allFiles: string[];
try {
allFiles = await fs.readdir(chatsDir);
} catch {
return [];
}
const sessionFiles = allFiles.filter(
(f) => f.startsWith(SESSION_FILE_PREFIX) && f.endsWith('.json'),
);
sessionFiles.sort((a, b) => b.localeCompare(a));
const results: Array<{ conversation: ConversationRecord; filePath: string }> =
[];
for (const file of sessionFiles) {
if (results.length >= MAX_SESSION_INDEX_SIZE) break;
const filePath = path.join(chatsDir, file);
try {
const content = await fs.readFile(filePath, 'utf-8');
const parsed: unknown = JSON.parse(content);
if (!isConversationRecord(parsed)) continue;
if (!shouldProcessConversation(parsed)) continue;
results.push({ conversation: parsed, filePath });
} catch {
// Skip unreadable files
}
}
return results;
}
export async function buildSessionIndex(
chatsDir: string,
state: ExtractionState,
): Promise<{ sessionIndex: string; newSessionIds: string[] }> {
const processedSet = getProcessedSessionIds(state);
const eligible = await scanEligibleSessions(chatsDir);
if (eligible.length === 0) {
return { sessionIndex: '', newSessionIds: [] };
}
const lines: string[] = [];
const newSessionIds: string[] = [];
for (const { conversation, filePath } of eligible) {
const userMessageCount = conversation.messages.filter(
(m) => m.type === 'user',
).length;
const isNew = !processedSet.has(conversation.sessionId);
if (isNew) {
newSessionIds.push(conversation.sessionId);
}
const status = isNew ? '[NEW]' : '[old]';
const summary = conversation.summary ?? '(no summary)';
lines.push(
`${status} ${summary} (${userMessageCount} user msgs) — ${filePath}`,
);
}
return { sessionIndex: lines.join('\n'), newSessionIds };
}
async function buildExistingSkillsSummary(
skillsDir: string,
config: Config,
): Promise<string> {
const sections: string[] = [];
const memorySkills: string[] = [];
try {
const entries = await fs.readdir(skillsDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const skillPath = path.join(skillsDir, entry.name, 'SKILL.md');
try {
const content = await fs.readFile(skillPath, 'utf-8');
const match = content.match(FRONTMATTER_REGEX);
if (match) {
const parsed = parseFrontmatter(match[1]);
const name = parsed?.name ?? entry.name;
const desc = parsed?.description ?? '';
memorySkills.push(`- **${name}**: ${desc}`);
} else {
memorySkills.push(`- **${entry.name}**`);
}
} catch {
// Skip unreadable files
}
}
} catch {
// Skip unreadable files
}
if (memorySkills.length > 0) {
sections.push(
`## Previously Extracted Skills (in ${skillsDir})\n${memorySkills.join('\n')}`,
);
}
try {
const discoveredSkills = config.getSkillManager().getSkills();
if (discoveredSkills.length > 0) {
const userSkillsDir = Storage.getUserSkillsDir();
const globalSkills: string[] = [];
const workspaceSkills: string[] = [];
const extensionSkills: string[] = [];
const builtinSkills: string[] = [];
for (const s of discoveredSkills) {
const loc = s.location;
if (loc.includes('/bundle/') || loc.includes('\\bundle\\')) {
builtinSkills.push(`- **${s.name}**: ${s.description}`);
} else if (loc.startsWith(userSkillsDir)) {
globalSkills.push(`- **${s.name}**: ${s.description} (${loc})`);
} else if (
loc.includes('/extensions/') ||
loc.includes('\\extensions\\')
) {
extensionSkills.push(`- **${s.name}**: ${s.description}`);
} else {
workspaceSkills.push(`- **${s.name}**: ${s.description} (${loc})`);
}
}
if (globalSkills.length > 0) {
sections.push(
`## Global Skills (~/.gemini/skills — do NOT duplicate)\n${globalSkills.join('\n')}`,
);
}
if (workspaceSkills.length > 0) {
sections.push(
`## Workspace Skills (.gemini/skills — do NOT duplicate)\n${workspaceSkills.join('\n')}`,
);
}
if (extensionSkills.length > 0) {
sections.push(
`## Extension Skills (from installed extensions — do NOT duplicate)\n${extensionSkills.join('\n')}`,
);
}
if (builtinSkills.length > 0) {
sections.push(
`## Builtin Skills (bundled with CLI — do NOT duplicate)\n${builtinSkills.join('\n')}`,
);
}
}
} catch {
// Skip unreadable files
}
return sections.join('\n\n');
}
function buildAgentLoopContext(config: Config): AgentLoopContext {
const autoApprovePolicy = new PolicyEngine({
rules: [
{
toolName: '*',
decision: PolicyDecision.ALLOW,
priority: 100,
},
],
});
const autoApproveBus = new MessageBus(autoApprovePolicy);
return {
config,
promptId: `skill-extraction-${randomUUID().slice(0, 8)}`,
toolRegistry: config.getToolRegistry(),
promptRegistry: new PromptRegistry(),
resourceRegistry: new ResourceRegistry(),
messageBus: autoApproveBus,
geminiClient: config.getGeminiClient(),
sandboxManager: config.sandboxManager,
};
}
export async function validatePatches(
skillsDir: string,
config: Config,
): Promise<string[]> {
let entries: string[];
try {
entries = await fs.readdir(skillsDir);
} catch {
return [];
}
const patchFiles = entries.filter((e) => e.endsWith('.patch'));
const validPatches: string[] = [];
for (const patchFile of patchFiles) {
const patchPath = path.join(skillsDir, patchFile);
let valid = true;
let reason = '';
try {
const patchContent = await fs.readFile(patchPath, 'utf-8');
const parsedPatches = Diff.parsePatch(patchContent);
if (!hasParsedPatchHunks(parsedPatches)) {
valid = false;
reason = 'no hunks found in patch';
} else {
const applied = await applyParsedSkillPatches(parsedPatches, config);
if (!applied.success) {
valid = false;
switch (applied.reason) {
case 'missingTargetPath':
reason = 'missing target file path in patch header';
break;
case 'invalidPatchHeaders':
reason = 'invalid diff headers';
break;
case 'outsideAllowedRoots':
reason = `target file is outside skill roots: ${applied.targetPath}`;
break;
case 'newFileAlreadyExists':
reason = `new file target already exists: ${applied.targetPath}`;
break;
case 'targetNotFound':
reason = `target file not found: ${applied.targetPath}`;
break;
case 'doesNotApply':
reason = `patch does not apply cleanly to ${applied.targetPath}`;
break;
default:
reason = 'unknown patch validation failure';
break;
}
}
}
} catch (err) {
valid = false;
reason = `failed to read or parse patch: ${err}`;
}
if (valid) {
validPatches.push(patchFile);
debugLogger.log(`[MemoryService] Patch validated: ${patchFile}`);
} else {
debugLogger.warn(
`[MemoryService] Removing invalid patch ${patchFile}: ${reason}`,
);
try {
await fs.unlink(patchPath);
} catch {
// Best-effort cleanup
}
}
}
return validPatches;
}
export class DefaultMemoryProvider implements MemoryProvider {
readonly id = 'gemini-cli-builtin-memory';
onSessionStart(config: Config, _sessionId: string): void {
void this.startSkillExtractionTask(config).catch((error) => {
debugLogger.warn(
'[MemoryService] Failed to start background skill extraction:',
error,
);
});
}
getSystemInstructions(): string {
return '';
}
getTurnContext(_query: string): string {
return '';
}
onTurnComplete(_userMessage: string, _assistantMessage: string): void {}
onSessionEnd(): void {}
private async startSkillExtractionTask(config: Config): Promise<void> {
const memoryDir = config.storage.getProjectMemoryTempDir();
const skillsDir = config.storage.getProjectSkillsMemoryDir();
const lockPath = path.join(memoryDir, LOCK_FILENAME);
const statePath = path.join(memoryDir, STATE_FILENAME);
const chatsDir = path.join(config.storage.getProjectTempDir(), 'chats');
await fs.mkdir(skillsDir, { recursive: true });
debugLogger.log(`[MemoryService] Starting. Skills dir: ${skillsDir}`);
if (!(await tryAcquireLock(lockPath))) {
debugLogger.log('[MemoryService] Skipped: lock held by another instance');
return;
}
debugLogger.log('[MemoryService] Lock acquired');
const abortController = new AbortController();
const handle = ExecutionLifecycleService.createExecution(
'',
() => abortController.abort(),
'none',
undefined,
'Skill extraction',
'silent',
);
const executionId = handle.pid;
const startTime = Date.now();
let completionResult: { error: Error } | undefined;
try {
const state = await readExtractionState(statePath);
const previousRuns = state.runs.length;
const previouslyProcessed = getProcessedSessionIds(state).size;
debugLogger.log(
`[MemoryService] State loaded: ${previousRuns} previous run(s), ${previouslyProcessed} session(s) already processed`,
);
const { sessionIndex, newSessionIds } = await buildSessionIndex(
chatsDir,
state,
);
const totalInIndex = sessionIndex ? sessionIndex.split('\n').length : 0;
debugLogger.log(
`[MemoryService] Session scan: ${totalInIndex} eligible session(s) found, ${newSessionIds.length} new`,
);
if (newSessionIds.length === 0) {
debugLogger.log('[MemoryService] Skipped: no new sessions to process');
return;
}
const skillsBefore = new Set<string>();
const patchContentsBefore = new Map<string, string>();
try {
const entries = await fs.readdir(skillsDir);
for (const e of entries) {
if (e.endsWith('.patch')) {
try {
patchContentsBefore.set(
e,
await fs.readFile(path.join(skillsDir, e), 'utf-8'),
);
} catch {
// Skip unreadable files
}
continue;
}
skillsBefore.add(e);
}
} catch {
// Skip unreadable files
}
debugLogger.log(
`[MemoryService] ${skillsBefore.size} existing skill(s) in memory`,
);
const existingSkillsSummary = await buildExistingSkillsSummary(
skillsDir,
config,
);
if (existingSkillsSummary) {
debugLogger.log(
`[MemoryService] Existing skills context:\n${existingSkillsSummary}`,
);
}
const agentDefinition = SkillExtractionAgent(
skillsDir,
sessionIndex,
existingSkillsSummary,
);
const context = buildAgentLoopContext(config);
const modelAlias = getModelConfigAlias(agentDefinition);
config.modelConfigService.registerRuntimeModelConfig(modelAlias, {
modelConfig: agentDefinition.modelConfig,
});
debugLogger.log(
`[MemoryService] Starting extraction agent (model: ${agentDefinition.modelConfig.model}, maxTurns: 30, maxTime: 30min)`,
);
const executor = await LocalAgentExecutor.create(
agentDefinition,
context,
);
await executor.run(
{ request: 'Extract skills from the provided sessions.' },
abortController.signal,
);
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
const skillsCreated: string[] = [];
try {
const entriesAfter = await fs.readdir(skillsDir);
for (const e of entriesAfter) {
if (!skillsBefore.has(e) && !e.endsWith('.patch')) {
skillsCreated.push(e);
}
}
} catch {
// Skip unreadable files
}
const validPatches = await validatePatches(skillsDir, config);
const patchesCreatedThisRun: string[] = [];
for (const patchFile of validPatches) {
const patchPath = path.join(skillsDir, patchFile);
let currentContent: string;
try {
currentContent = await fs.readFile(patchPath, 'utf-8');
} catch {
continue;
}
if (patchContentsBefore.get(patchFile) !== currentContent) {
patchesCreatedThisRun.push(patchFile);
}
}
if (validPatches.length > 0) {
debugLogger.log(
`[MemoryService] ${validPatches.length} valid patch(es) currently in inbox; ${patchesCreatedThisRun.length} created or updated this run`,
);
}
const run: ExtractionRun = {
runAt: new Date().toISOString(),
sessionIds: newSessionIds,
skillsCreated,
};
const updatedState: ExtractionState = {
runs: [...state.runs, run],
};
await writeExtractionState(statePath, updatedState);
if (skillsCreated.length > 0 || patchesCreatedThisRun.length > 0) {
const completionParts: string[] = [];
if (skillsCreated.length > 0) {
completionParts.push(
`created ${skillsCreated.length} skill(s): ${skillsCreated.join(', ')}`,
);
}
if (patchesCreatedThisRun.length > 0) {
completionParts.push(
`prepared ${patchesCreatedThisRun.length} patch(es): ${patchesCreatedThisRun.join(', ')}`,
);
}
debugLogger.log(
`[MemoryService] Completed in ${elapsed}s. ${completionParts.join('; ')} (processed ${newSessionIds.length} session(s))`,
);
const feedbackParts: string[] = [];
if (skillsCreated.length > 0) {
feedbackParts.push(
`${skillsCreated.length} new skill${skillsCreated.length > 1 ? 's' : ''} extracted from past sessions: ${skillsCreated.join(', ')}`,
);
}
if (patchesCreatedThisRun.length > 0) {
feedbackParts.push(
`${patchesCreatedThisRun.length} skill update${patchesCreatedThisRun.length > 1 ? 's' : ''} extracted from past sessions`,
);
}
coreEvents.emitFeedback(
'info',
`${feedbackParts.join('. ')}. Use /memory inbox to review.`,
);
} else {
debugLogger.log(
`[MemoryService] Completed in ${elapsed}s. No new skills or patches created (processed ${newSessionIds.length} session(s))`,
);
}
} catch (error) {
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
if (abortController.signal.aborted) {
debugLogger.log(`[MemoryService] Cancelled after ${elapsed}s`);
} else {
debugLogger.log(
`[MemoryService] Failed after ${elapsed}s: ${error instanceof Error ? error.message : String(error)}`,
);
}
completionResult = {
error: error instanceof Error ? error : new Error(String(error)),
};
return;
} finally {
await releaseLock(lockPath);
debugLogger.log('[MemoryService] Lock released');
if (executionId !== undefined) {
ExecutionLifecycleService.completeExecution(
executionId,
completionResult,
);
}
}
}
}
@@ -0,0 +1,50 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { Config } from '../config/config.js';
/**
* Internal contract implemented by the memory backend that {@link MemoryService}
* delegates to. This interface is intentionally *not* exported from the
* package's public surface (`src/index.ts`) and is *not* surfaced via
* `GeminiCLIExtension` it exists purely as a typing seam between
* `MemoryService` and its concrete implementation so that the two can evolve
* independently and be substituted in tests.
*/
export interface MemoryProvider {
/** Stable identifier used in diagnostic logs. */
readonly id: string;
/**
* Invoked once per session, when the owning `GeminiClient` initializes the
* memory subsystem. Implementations may kick off background work but must
* return synchronously.
*/
onSessionStart(config: Config, sessionId: string): void;
/**
* Returns static instructions to inject into the LLM's system prompt.
* Called once per session and again after `onTurnComplete`.
*/
getSystemInstructions(): string;
/**
* Returns dynamic, recalled context for the current turn based on the
* user's query. Called once per non-internal turn.
*/
getTurnContext(query: string): string;
/**
* Invoked after the LLM completes a turn. Must return synchronously; any
* persistence work should be fired fire-and-forget by the implementation.
*/
onTurnComplete(userMessage: string, assistantMessage: string): void;
/**
* Invoked when the owning session is shutting down gracefully.
*/
onSessionEnd(): void;
}
File diff suppressed because it is too large Load Diff
+52 -785
View File
@@ -4,814 +4,81 @@
* SPDX-License-Identifier: Apache-2.0
*/
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import { constants as fsConstants } from 'node:fs';
import { randomUUID } from 'node:crypto';
import * as Diff from 'diff';
import type { Config } from '../config/config.js';
import {
SESSION_FILE_PREFIX,
type ConversationRecord,
} from './chatRecordingService.js';
import { DefaultMemoryProvider } from './defaultMemoryProvider.js';
import type { MemoryProvider } from './memoryProvider.js';
import { debugLogger } from '../utils/debugLogger.js';
import { coreEvents } from '../utils/events.js';
import { isNodeError } from '../utils/errors.js';
import { FRONTMATTER_REGEX, parseFrontmatter } from '../skills/skillLoader.js';
import { LocalAgentExecutor } from '../agents/local-executor.js';
import { SkillExtractionAgent } from '../agents/skill-extraction-agent.js';
import { getModelConfigAlias } from '../agents/registry.js';
import { ExecutionLifecycleService } from './executionLifecycleService.js';
import { PromptRegistry } from '../prompts/prompt-registry.js';
import { ResourceRegistry } from '../resources/resource-registry.js';
import { PolicyEngine } from '../policy/policy-engine.js';
import { PolicyDecision } from '../policy/types.js';
import { MessageBus } from '../confirmation-bus/message-bus.js';
import { Storage } from '../config/storage.js';
import type { AgentLoopContext } from '../config/agent-loop-context.js';
import {
applyParsedSkillPatches,
hasParsedPatchHunks,
} from './memoryPatchUtils.js';
const LOCK_FILENAME = '.extraction.lock';
const STATE_FILENAME = '.extraction-state.json';
const LOCK_STALE_MS = 35 * 60 * 1000; // 35 minutes (exceeds agent's 30-min time limit)
const MIN_USER_MESSAGES = 10;
const MIN_IDLE_MS = 3 * 60 * 60 * 1000; // 3 hours
const MAX_SESSION_INDEX_SIZE = 50;
/**
* Lock file content for coordinating across CLI instances.
*/
interface LockInfo {
pid: number;
startedAt: string;
}
/**
* Metadata for a single extraction run.
*/
export interface ExtractionRun {
runAt: string;
sessionIds: string[];
skillsCreated: string[];
}
/**
* Tracks extraction history with per-run metadata.
*/
export interface ExtractionState {
runs: ExtractionRun[];
}
/**
* Returns all session IDs that have been processed across all runs.
*/
export function getProcessedSessionIds(state: ExtractionState): Set<string> {
const ids = new Set<string>();
for (const run of state.runs) {
for (const id of run.sessionIds) {
ids.add(id);
}
}
return ids;
}
function isLockInfo(value: unknown): value is LockInfo {
return (
typeof value === 'object' &&
value !== null &&
'pid' in value &&
typeof value.pid === 'number' &&
'startedAt' in value &&
typeof value.startedAt === 'string'
);
}
function isConversationRecord(value: unknown): value is ConversationRecord {
return (
typeof value === 'object' &&
value !== null &&
'sessionId' in value &&
typeof value.sessionId === 'string' &&
'messages' in value &&
Array.isArray(value.messages) &&
'projectHash' in value &&
'startTime' in value &&
'lastUpdated' in value
);
}
function isExtractionRun(value: unknown): value is ExtractionRun {
return (
typeof value === 'object' &&
value !== null &&
'runAt' in value &&
typeof value.runAt === 'string' &&
'sessionIds' in value &&
Array.isArray(value.sessionIds) &&
'skillsCreated' in value &&
Array.isArray(value.skillsCreated)
);
}
function isExtractionState(value: unknown): value is { runs: unknown[] } {
return (
typeof value === 'object' &&
value !== null &&
'runs' in value &&
Array.isArray(value.runs)
);
}
/**
* Attempts to acquire an exclusive lock file using O_CREAT | O_EXCL.
* Returns true if the lock was acquired, false if another instance owns it.
*/
export async function tryAcquireLock(
lockPath: string,
retries = 1,
): Promise<boolean> {
const lockInfo: LockInfo = {
pid: process.pid,
startedAt: new Date().toISOString(),
};
try {
// Atomic create-if-not-exists
const fd = await fs.open(
lockPath,
fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY,
);
try {
await fd.writeFile(JSON.stringify(lockInfo));
} finally {
await fd.close();
}
return true;
} catch (error: unknown) {
if (isNodeError(error) && error.code === 'EEXIST') {
// Lock exists — check if it's stale
if (retries > 0 && (await isLockStale(lockPath))) {
debugLogger.debug('[MemoryService] Cleaning up stale lock file');
await releaseLock(lockPath);
return tryAcquireLock(lockPath, retries - 1);
}
debugLogger.debug(
'[MemoryService] Lock held by another instance, skipping',
);
return false;
}
throw error;
}
}
/**
* Checks if a lock file is stale (owner PID is dead or lock is too old).
*/
export async function isLockStale(lockPath: string): Promise<boolean> {
try {
const content = await fs.readFile(lockPath, 'utf-8');
const parsed: unknown = JSON.parse(content);
if (!isLockInfo(parsed)) {
return true; // Invalid lock data — treat as stale
}
const lockInfo = parsed;
// Check if PID is still alive
try {
process.kill(lockInfo.pid, 0);
} catch {
// PID is dead — lock is stale
return true;
}
// Check if lock is too old
const lockAge = Date.now() - new Date(lockInfo.startedAt).getTime();
if (lockAge > LOCK_STALE_MS) {
return true;
}
return false;
} catch {
// Can't read lock — treat as stale
return true;
}
}
/**
* Releases the lock file.
*/
export async function releaseLock(lockPath: string): Promise<void> {
try {
await fs.unlink(lockPath);
} catch (error: unknown) {
if (isNodeError(error) && error.code === 'ENOENT') {
return; // Already removed
}
debugLogger.warn(
`[MemoryService] Failed to release lock: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
/**
* Reads the extraction state file, or returns a default state.
*/
export async function readExtractionState(
statePath: string,
): Promise<ExtractionState> {
try {
const content = await fs.readFile(statePath, 'utf-8');
const parsed: unknown = JSON.parse(content);
if (!isExtractionState(parsed)) {
return { runs: [] };
}
const runs: ExtractionRun[] = [];
for (const run of parsed.runs) {
if (!isExtractionRun(run)) continue;
runs.push({
runAt: run.runAt,
sessionIds: run.sessionIds.filter(
(sid): sid is string => typeof sid === 'string',
),
skillsCreated: run.skillsCreated.filter(
(sk): sk is string => typeof sk === 'string',
),
});
}
return { runs };
} catch (error) {
debugLogger.debug(
'[MemoryService] Failed to read extraction state:',
error,
);
return { runs: [] };
}
}
/**
* Writes the extraction state atomically (temp file + rename).
*/
export async function writeExtractionState(
statePath: string,
state: ExtractionState,
): Promise<void> {
const tmpPath = `${statePath}.tmp`;
await fs.writeFile(tmpPath, JSON.stringify(state, null, 2));
await fs.rename(tmpPath, statePath);
}
/**
* Determines if a conversation record should be considered for processing.
* Filters out subagent sessions, sessions that haven't been idle long enough,
* and sessions with too few user messages.
*/
function shouldProcessConversation(parsed: ConversationRecord): boolean {
// Skip subagent sessions
if (parsed.kind === 'subagent') return false;
// Skip sessions that are still active (not idle for 3+ hours)
const lastUpdated = new Date(parsed.lastUpdated).getTime();
if (Date.now() - lastUpdated < MIN_IDLE_MS) return false;
// Skip sessions with too few user messages
const userMessageCount = parsed.messages.filter(
(m) => m.type === 'user',
).length;
if (userMessageCount < MIN_USER_MESSAGES) return false;
return true;
}
/**
* Scans the chats directory for eligible session files (sorted most-recent-first,
* capped at MAX_SESSION_INDEX_SIZE). Shared by buildSessionIndex.
*/
async function scanEligibleSessions(
chatsDir: string,
): Promise<Array<{ conversation: ConversationRecord; filePath: string }>> {
let allFiles: string[];
try {
allFiles = await fs.readdir(chatsDir);
} catch {
return [];
}
const sessionFiles = allFiles.filter(
(f) => f.startsWith(SESSION_FILE_PREFIX) && f.endsWith('.json'),
);
// Sort by filename descending (most recent first)
sessionFiles.sort((a, b) => b.localeCompare(a));
const results: Array<{ conversation: ConversationRecord; filePath: string }> =
[];
for (const file of sessionFiles) {
if (results.length >= MAX_SESSION_INDEX_SIZE) break;
const filePath = path.join(chatsDir, file);
try {
const content = await fs.readFile(filePath, 'utf-8');
const parsed: unknown = JSON.parse(content);
if (!isConversationRecord(parsed)) continue;
if (!shouldProcessConversation(parsed)) continue;
results.push({ conversation: parsed, filePath });
} catch {
// Skip unreadable files
}
}
return results;
}
/**
* Builds a session index for the extraction agent: a compact listing of all
* eligible sessions with their summary, file path, and new/previously-processed status.
* The agent can use read_file on paths to inspect sessions that look promising.
* Orchestrates the memory subsystem for a single GeminiClient. The service
* owns a `DefaultMemoryProvider` and exposes a stable lifecycle surface
* (`onSessionStart`, `getSystemInstructions`, `getTurnContext`,
* `onTurnComplete`, `onSessionEnd`) so that GeminiClient can stay agnostic
* of the underlying implementation. Each lifecycle call is wrapped in a
* try/catch so a buggy provider can never crash a turn.
*
* Returns the index text and the list of new (unprocessed) session IDs.
* The methods return `Promise`s for callsite consistency even though the
* current provider is synchronous; this keeps the boundary stable should
* the provider ever need to do asynchronous work.
*/
export async function buildSessionIndex(
chatsDir: string,
state: ExtractionState,
): Promise<{ sessionIndex: string; newSessionIds: string[] }> {
const processedSet = getProcessedSessionIds(state);
const eligible = await scanEligibleSessions(chatsDir);
export class MemoryService {
private readonly provider: MemoryProvider = new DefaultMemoryProvider();
if (eligible.length === 0) {
return { sessionIndex: '', newSessionIds: [] };
}
const lines: string[] = [];
const newSessionIds: string[] = [];
for (const { conversation, filePath } of eligible) {
const userMessageCount = conversation.messages.filter(
(m) => m.type === 'user',
).length;
const isNew = !processedSet.has(conversation.sessionId);
if (isNew) {
newSessionIds.push(conversation.sessionId);
}
const status = isNew ? '[NEW]' : '[old]';
const summary = conversation.summary ?? '(no summary)';
lines.push(
`${status} ${summary} (${userMessageCount} user msgs) — ${filePath}`,
);
}
return { sessionIndex: lines.join('\n'), newSessionIds };
}
/**
* Builds a summary of all existing skills both memory-extracted skills
* in the skillsDir and globally/workspace-discovered skills from the SkillManager.
* This prevents the extraction agent from duplicating already-available skills.
*/
async function buildExistingSkillsSummary(
skillsDir: string,
config: Config,
): Promise<string> {
const sections: string[] = [];
// 1. Memory-extracted skills (from previous runs)
const memorySkills: string[] = [];
try {
const entries = await fs.readdir(skillsDir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const skillPath = path.join(skillsDir, entry.name, 'SKILL.md');
try {
const content = await fs.readFile(skillPath, 'utf-8');
const match = content.match(FRONTMATTER_REGEX);
if (match) {
const parsed = parseFrontmatter(match[1]);
const name = parsed?.name ?? entry.name;
const desc = parsed?.description ?? '';
memorySkills.push(`- **${name}**: ${desc}`);
} else {
memorySkills.push(`- **${entry.name}**`);
}
} catch {
// Skill directory without SKILL.md, skip
}
}
} catch {
// Skills directory doesn't exist yet
}
if (memorySkills.length > 0) {
sections.push(
`## Previously Extracted Skills (in ${skillsDir})\n${memorySkills.join('\n')}`,
);
}
// 2. Discovered skills — categorize by source location
try {
const discoveredSkills = config.getSkillManager().getSkills();
if (discoveredSkills.length > 0) {
const userSkillsDir = Storage.getUserSkillsDir();
const globalSkills: string[] = [];
const workspaceSkills: string[] = [];
const extensionSkills: string[] = [];
const builtinSkills: string[] = [];
for (const s of discoveredSkills) {
const loc = s.location;
if (loc.includes('/bundle/') || loc.includes('\\bundle\\')) {
builtinSkills.push(`- **${s.name}**: ${s.description}`);
} else if (loc.startsWith(userSkillsDir)) {
globalSkills.push(`- **${s.name}**: ${s.description} (${loc})`);
} else if (
loc.includes('/extensions/') ||
loc.includes('\\extensions\\')
) {
extensionSkills.push(`- **${s.name}**: ${s.description}`);
} else {
workspaceSkills.push(`- **${s.name}**: ${s.description} (${loc})`);
}
}
if (globalSkills.length > 0) {
sections.push(
`## Global Skills (~/.gemini/skills — do NOT duplicate)\n${globalSkills.join('\n')}`,
);
}
if (workspaceSkills.length > 0) {
sections.push(
`## Workspace Skills (.gemini/skills — do NOT duplicate)\n${workspaceSkills.join('\n')}`,
);
}
if (extensionSkills.length > 0) {
sections.push(
`## Extension Skills (from installed extensions — do NOT duplicate)\n${extensionSkills.join('\n')}`,
);
}
if (builtinSkills.length > 0) {
sections.push(
`## Builtin Skills (bundled with CLI — do NOT duplicate)\n${builtinSkills.join('\n')}`,
);
}
}
} catch {
// SkillManager not available
}
return sections.join('\n\n');
}
/**
* Builds an AgentLoopContext from a Config for background agent execution.
*/
function buildAgentLoopContext(config: Config): AgentLoopContext {
// Create a PolicyEngine that auto-approves all tool calls so the
// background sub-agent never prompts the user for confirmation.
const autoApprovePolicy = new PolicyEngine({
rules: [
{
toolName: '*',
decision: PolicyDecision.ALLOW,
priority: 100,
},
],
});
const autoApproveBus = new MessageBus(autoApprovePolicy);
return {
config,
promptId: `skill-extraction-${randomUUID().slice(0, 8)}`,
toolRegistry: config.getToolRegistry(),
promptRegistry: new PromptRegistry(),
resourceRegistry: new ResourceRegistry(),
messageBus: autoApproveBus,
geminiClient: config.getGeminiClient(),
sandboxManager: config.sandboxManager,
};
}
/**
* Validates all .patch files in the skills directory using the `diff` library.
* Parses each patch, reads the target file(s), and attempts a dry-run apply.
* Removes patches that fail validation. Returns the filenames of valid patches.
*/
export async function validatePatches(
skillsDir: string,
config: Config,
): Promise<string[]> {
let entries: string[];
try {
entries = await fs.readdir(skillsDir);
} catch {
return [];
}
const patchFiles = entries.filter((e) => e.endsWith('.patch'));
const validPatches: string[] = [];
for (const patchFile of patchFiles) {
const patchPath = path.join(skillsDir, patchFile);
let valid = true;
let reason = '';
constructor(private readonly config: Config) {}
async onSessionStart(sessionId: string): Promise<void> {
try {
const patchContent = await fs.readFile(patchPath, 'utf-8');
const parsedPatches = Diff.parsePatch(patchContent);
if (!hasParsedPatchHunks(parsedPatches)) {
valid = false;
reason = 'no hunks found in patch';
} else {
const applied = await applyParsedSkillPatches(parsedPatches, config);
if (!applied.success) {
valid = false;
switch (applied.reason) {
case 'missingTargetPath':
reason = 'missing target file path in patch header';
break;
case 'invalidPatchHeaders':
reason = 'invalid diff headers';
break;
case 'outsideAllowedRoots':
reason = `target file is outside skill roots: ${applied.targetPath}`;
break;
case 'newFileAlreadyExists':
reason = `new file target already exists: ${applied.targetPath}`;
break;
case 'targetNotFound':
reason = `target file not found: ${applied.targetPath}`;
break;
case 'doesNotApply':
reason = `patch does not apply cleanly to ${applied.targetPath}`;
break;
default:
reason = 'unknown patch validation failure';
break;
}
}
}
} catch (err) {
valid = false;
reason = `failed to read or parse patch: ${err}`;
}
if (valid) {
validPatches.push(patchFile);
debugLogger.log(`[MemoryService] Patch validated: ${patchFile}`);
} else {
this.provider.onSessionStart(this.config, sessionId);
} catch (error) {
debugLogger.warn(
`[MemoryService] Removing invalid patch ${patchFile}: ${reason}`,
`[MemoryService] Provider "${this.provider.id}" threw during onSessionStart:`,
error,
);
try {
await fs.unlink(patchPath);
} catch {
// Best-effort cleanup
}
}
}
return validPatches;
}
/**
* Main entry point for the skill extraction background task.
* Designed to be called fire-and-forget on session startup.
*
* Coordinates across multiple CLI instances via a lock file,
* scans past sessions for reusable patterns, and runs a sub-agent
* to extract and write SKILL.md files.
*/
export async function startMemoryService(config: Config): Promise<void> {
const memoryDir = config.storage.getProjectMemoryTempDir();
const skillsDir = config.storage.getProjectSkillsMemoryDir();
const lockPath = path.join(memoryDir, LOCK_FILENAME);
const statePath = path.join(memoryDir, STATE_FILENAME);
const chatsDir = path.join(config.storage.getProjectTempDir(), 'chats');
// Ensure directories exist
await fs.mkdir(skillsDir, { recursive: true });
debugLogger.log(`[MemoryService] Starting. Skills dir: ${skillsDir}`);
// Try to acquire exclusive lock
if (!(await tryAcquireLock(lockPath))) {
debugLogger.log('[MemoryService] Skipped: lock held by another instance');
return;
async getSystemInstructions(): Promise<string> {
try {
return this.provider.getSystemInstructions();
} catch (error) {
debugLogger.warn(
`[MemoryService] Provider "${this.provider.id}" threw during getSystemInstructions:`,
error,
);
return '';
}
}
debugLogger.log('[MemoryService] Lock acquired');
// Register with ExecutionLifecycleService for background tracking
const abortController = new AbortController();
const handle = ExecutionLifecycleService.createExecution(
'', // no initial output
() => abortController.abort(), // onKill
'none',
undefined, // no format injection
'Skill extraction',
'silent',
);
const executionId = handle.pid;
const startTime = Date.now();
let completionResult: { error: Error } | undefined;
try {
// Read extraction state
const state = await readExtractionState(statePath);
const previousRuns = state.runs.length;
const previouslyProcessed = getProcessedSessionIds(state).size;
debugLogger.log(
`[MemoryService] State loaded: ${previousRuns} previous run(s), ${previouslyProcessed} session(s) already processed`,
);
// Build session index: all eligible sessions with summaries + file paths.
// The agent decides which to read in full via read_file.
const { sessionIndex, newSessionIds } = await buildSessionIndex(
chatsDir,
state,
);
const totalInIndex = sessionIndex ? sessionIndex.split('\n').length : 0;
debugLogger.log(
`[MemoryService] Session scan: ${totalInIndex} eligible session(s) found, ${newSessionIds.length} new`,
);
if (newSessionIds.length === 0) {
debugLogger.log('[MemoryService] Skipped: no new sessions to process');
return;
}
// Snapshot existing skill directories before extraction
const skillsBefore = new Set<string>();
const patchContentsBefore = new Map<string, string>();
async getTurnContext(query: string): Promise<string> {
try {
const entries = await fs.readdir(skillsDir);
for (const e of entries) {
if (e.endsWith('.patch')) {
try {
patchContentsBefore.set(
e,
await fs.readFile(path.join(skillsDir, e), 'utf-8'),
);
} catch {
// Ignore unreadable existing patches.
}
continue;
}
skillsBefore.add(e);
}
} catch {
// Empty skills dir
}
debugLogger.log(
`[MemoryService] ${skillsBefore.size} existing skill(s) in memory`,
);
// Read existing skills for context (memory-extracted + global/workspace)
const existingSkillsSummary = await buildExistingSkillsSummary(
skillsDir,
config,
);
if (existingSkillsSummary) {
debugLogger.log(
`[MemoryService] Existing skills context:\n${existingSkillsSummary}`,
return this.provider.getTurnContext(query);
} catch (error) {
debugLogger.warn(
`[MemoryService] Provider "${this.provider.id}" threw during getTurnContext:`,
error,
);
return '';
}
}
// Build agent definition and context
const agentDefinition = SkillExtractionAgent(
skillsDir,
sessionIndex,
existingSkillsSummary,
);
const context = buildAgentLoopContext(config);
// Register the agent's model config since it's not going through AgentRegistry.
const modelAlias = getModelConfigAlias(agentDefinition);
config.modelConfigService.registerRuntimeModelConfig(modelAlias, {
modelConfig: agentDefinition.modelConfig,
});
debugLogger.log(
`[MemoryService] Starting extraction agent (model: ${agentDefinition.modelConfig.model}, maxTurns: 30, maxTime: 30min)`,
);
// Create and run the extraction agent
const executor = await LocalAgentExecutor.create(agentDefinition, context);
await executor.run(
{ request: 'Extract skills from the provided sessions.' },
abortController.signal,
);
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
// Diff skills directory to find newly created skills
const skillsCreated: string[] = [];
onTurnComplete(userMessage: string, assistantMessage: string): void {
try {
const entriesAfter = await fs.readdir(skillsDir);
for (const e of entriesAfter) {
if (!skillsBefore.has(e) && !e.endsWith('.patch')) {
skillsCreated.push(e);
}
}
} catch {
// Skills dir read failed
this.provider.onTurnComplete(userMessage, assistantMessage);
} catch (error) {
debugLogger.warn(
`[MemoryService] Provider "${this.provider.id}" threw during onTurnComplete:`,
error,
);
}
}
// Validate any .patch files the agent generated
const validPatches = await validatePatches(skillsDir, config);
const patchesCreatedThisRun: string[] = [];
for (const patchFile of validPatches) {
const patchPath = path.join(skillsDir, patchFile);
let currentContent: string;
try {
currentContent = await fs.readFile(patchPath, 'utf-8');
} catch {
continue;
}
if (patchContentsBefore.get(patchFile) !== currentContent) {
patchesCreatedThisRun.push(patchFile);
}
}
if (validPatches.length > 0) {
debugLogger.log(
`[MemoryService] ${validPatches.length} valid patch(es) currently in inbox; ${patchesCreatedThisRun.length} created or updated this run`,
);
}
// Record the run with full metadata
const run: ExtractionRun = {
runAt: new Date().toISOString(),
sessionIds: newSessionIds,
skillsCreated,
};
const updatedState: ExtractionState = {
runs: [...state.runs, run],
};
await writeExtractionState(statePath, updatedState);
if (skillsCreated.length > 0 || patchesCreatedThisRun.length > 0) {
const completionParts: string[] = [];
if (skillsCreated.length > 0) {
completionParts.push(
`created ${skillsCreated.length} skill(s): ${skillsCreated.join(', ')}`,
);
}
if (patchesCreatedThisRun.length > 0) {
completionParts.push(
`prepared ${patchesCreatedThisRun.length} patch(es): ${patchesCreatedThisRun.join(', ')}`,
);
}
debugLogger.log(
`[MemoryService] Completed in ${elapsed}s. ${completionParts.join('; ')} (processed ${newSessionIds.length} session(s))`,
);
const feedbackParts: string[] = [];
if (skillsCreated.length > 0) {
feedbackParts.push(
`${skillsCreated.length} new skill${skillsCreated.length > 1 ? 's' : ''} extracted from past sessions: ${skillsCreated.join(', ')}`,
);
}
if (patchesCreatedThisRun.length > 0) {
feedbackParts.push(
`${patchesCreatedThisRun.length} skill update${patchesCreatedThisRun.length > 1 ? 's' : ''} extracted from past sessions`,
);
}
coreEvents.emitFeedback(
'info',
`${feedbackParts.join('. ')}. Use /memory inbox to review.`,
);
} else {
debugLogger.log(
`[MemoryService] Completed in ${elapsed}s. No new skills or patches created (processed ${newSessionIds.length} session(s))`,
);
}
} catch (error) {
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
if (abortController.signal.aborted) {
debugLogger.log(`[MemoryService] Cancelled after ${elapsed}s`);
} else {
debugLogger.log(
`[MemoryService] Failed after ${elapsed}s: ${error instanceof Error ? error.message : String(error)}`,
);
}
completionResult = {
error: error instanceof Error ? error : new Error(String(error)),
};
return;
} finally {
await releaseLock(lockPath);
debugLogger.log('[MemoryService] Lock released');
if (executionId !== undefined) {
ExecutionLifecycleService.completeExecution(
executionId,
completionResult,
async onSessionEnd(): Promise<void> {
try {
this.provider.onSessionEnd();
} catch (error) {
debugLogger.warn(
`[MemoryService] Provider "${this.provider.id}" threw during onSessionEnd:`,
error,
);
}
}