From 5318610c1db7e090a8c6c35168ef529b510d0233 Mon Sep 17 00:00:00 2001 From: Sandy Tao Date: Wed, 22 Apr 2026 16:07:39 -0700 Subject: [PATCH] fix(core): support jsonl session logs in memory and summary services (#25816) --- .gemini/settings.json | 2 +- docs/cli/settings.md | 26 +- docs/reference/configuration.md | 7 +- packages/cli/src/config/settingsSchema.ts | 4 +- .../cli/src/nonInteractiveCliAgentSession.ts | 2 +- packages/core/src/config/config.test.ts | 16 +- packages/core/src/config/config.ts | 2 +- .../core/src/services/chatRecordingService.ts | 82 ++- .../core/src/services/memoryService.test.ts | 384 ++++++++++++- packages/core/src/services/memoryService.ts | 516 +++++++++++++++--- .../src/services/sessionSummaryUtils.test.ts | 407 +++++++++++--- .../core/src/services/sessionSummaryUtils.ts | 198 +++++-- schemas/settings.schema.json | 6 +- 13 files changed, 1396 insertions(+), 256 deletions(-) diff --git a/.gemini/settings.json b/.gemini/settings.json index 155abb7081..0fc36089f4 100644 --- a/.gemini/settings.json +++ b/.gemini/settings.json @@ -2,7 +2,7 @@ "experimental": { "extensionReloading": true, "modelSteering": true, - "memoryManager": true + "autoMemory": true }, "general": { "devtools": true diff --git a/docs/cli/settings.md b/docs/cli/settings.md index 7653afff08..94103dae32 100644 --- a/docs/cli/settings.md +++ b/docs/cli/settings.md @@ -161,19 +161,19 @@ they appear in the UI. ### Experimental -| UI Label | Setting | Description | Default | -| ---------------------------------------------------- | ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| Enable Git Worktrees | `experimental.worktrees` | Enable automated Git worktree management for parallel work. | `false` | -| Use OSC 52 Paste | `experimental.useOSC52Paste` | Use OSC 52 for pasting. This may be more robust than the default system when using remote terminal sessions (if your terminal is configured to allow it). | `false` | -| Use OSC 52 Copy | `experimental.useOSC52Copy` | Use OSC 52 for copying. This may be more robust than the default system when using remote terminal sessions (if your terminal is configured to allow it). | `false` | -| Model Steering | `experimental.modelSteering` | Enable model steering (user hints) to guide the model during tool execution. | `false` | -| Direct Web Fetch | `experimental.directWebFetch` | Enable web fetch behavior that bypasses LLM summarization. | `false` | -| Enable Gemma Model Router | `experimental.gemmaModelRouter.enabled` | Enable the Gemma Model Router (experimental). Requires a local endpoint serving Gemma via the Gemini API using LiteRT-LM shim. | `false` | -| Auto-start LiteRT Server | `experimental.gemmaModelRouter.autoStartServer` | Automatically start the LiteRT-LM server when Gemini CLI starts and the Gemma router is enabled. | `false` | -| Memory v2 | `experimental.memoryV2` | Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Routes facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits). | `false` | -| Auto Memory | `experimental.autoMemory` | Automatically extract reusable skills from past sessions in the background. Review results with /memory inbox. | `false` | -| Use the generalist profile to manage agent contexts. | `experimental.generalistProfile` | Suitable for general coding and software development tasks. | `false` | -| Enable Context Management | `experimental.contextManagement` | Enable logic for context management. | `false` | +| UI Label | Setting | Description | Default | +| ---------------------------------------------------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| Enable Git Worktrees | `experimental.worktrees` | Enable automated Git worktree management for parallel work. | `false` | +| Use OSC 52 Paste | `experimental.useOSC52Paste` | Use OSC 52 for pasting. This may be more robust than the default system when using remote terminal sessions (if your terminal is configured to allow it). | `false` | +| Use OSC 52 Copy | `experimental.useOSC52Copy` | Use OSC 52 for copying. This may be more robust than the default system when using remote terminal sessions (if your terminal is configured to allow it). | `false` | +| Model Steering | `experimental.modelSteering` | Enable model steering (user hints) to guide the model during tool execution. | `false` | +| Direct Web Fetch | `experimental.directWebFetch` | Enable web fetch behavior that bypasses LLM summarization. | `false` | +| Enable Gemma Model Router | `experimental.gemmaModelRouter.enabled` | Enable the Gemma Model Router (experimental). Requires a local endpoint serving Gemma via the Gemini API using LiteRT-LM shim. | `false` | +| Auto-start LiteRT Server | `experimental.gemmaModelRouter.autoStartServer` | Automatically start the LiteRT-LM server when Gemini CLI starts and the Gemma router is enabled. | `false` | +| Memory v2 | `experimental.memoryV2` | Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Route facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits). Set to false to fall back to the legacy save_memory tool. | `true` | +| Auto Memory | `experimental.autoMemory` | Automatically extract reusable skills from past sessions in the background. Review results with /memory inbox. | `false` | +| Use the generalist profile to manage agent contexts. | `experimental.generalistProfile` | Suitable for general coding and software development tasks. | `false` | +| Enable Context Management | `experimental.contextManagement` | Enable logic for context management. | `false` | ### Skills diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 97b880f84c..b582da4ea0 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1759,13 +1759,14 @@ their corresponding top-level category object in your `settings.json` file. - **`experimental.memoryV2`** (boolean): - **Description:** Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with - edit/write_file. Routes facts across four tiers: team-shared conventions go + edit/write_file. Route facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit - — settings, credentials, etc. remain off-limits). - - **Default:** `false` + — settings, credentials, etc. remain off-limits). Set to false to fall back + to the legacy save_memory tool. + - **Default:** `true` - **Requires restart:** Yes - **`experimental.autoMemory`** (boolean): diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 46d94a9692..f5da86b60a 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -2280,9 +2280,9 @@ const SETTINGS_SCHEMA = { label: 'Memory v2', category: 'Experimental', requiresRestart: true, - default: false, + default: true, description: - 'Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Routes facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits).', + 'Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Route facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits). Set to false to fall back to the legacy save_memory tool.', showInDialog: true, }, autoMemory: { diff --git a/packages/cli/src/nonInteractiveCliAgentSession.ts b/packages/cli/src/nonInteractiveCliAgentSession.ts index 4fee7eb610..0cf16da47d 100644 --- a/packages/cli/src/nonInteractiveCliAgentSession.ts +++ b/packages/cli/src/nonInteractiveCliAgentSession.ts @@ -80,7 +80,7 @@ export async function runNonInteractive({ const { setupInitialActivityLogger } = await import( './utils/devtoolsService.js' ); - await setupInitialActivityLogger(config); + setupInitialActivityLogger(config); } const { stdout: workingStdout } = createWorkingStdio(); diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index 6c3719eb49..05414b4945 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -3501,7 +3501,7 @@ describe('Config JIT Initialization', () => { }); describe('isMemoryV2Enabled', () => { - it('should default to false', () => { + it('should default to true', () => { const params: ConfigParameters = { sessionId: 'test-session', targetDir: '/tmp/test', @@ -3510,6 +3510,20 @@ describe('Config JIT Initialization', () => { cwd: '/tmp/test', }; + config = new Config(params); + expect(config.isMemoryV2Enabled()).toBe(true); + }); + + it('should return false when experimentalMemoryV2 is explicitly false', () => { + const params: ConfigParameters = { + sessionId: 'test-session', + targetDir: '/tmp/test', + debugMode: false, + model: 'test-model', + cwd: '/tmp/test', + experimentalMemoryV2: false, + }; + config = new Config(params); expect(config.isMemoryV2Enabled()).toBe(false); }); diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index d2bc6d9a4d..a6ca91d7b5 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -1172,7 +1172,7 @@ export class Config implements McpContext, AgentLoopContext { ); this.experimentalJitContext = params.experimentalJitContext ?? true; - this.experimentalMemoryV2 = params.experimentalMemoryV2 ?? false; + this.experimentalMemoryV2 = params.experimentalMemoryV2 ?? true; this.experimentalAutoMemory = params.experimentalAutoMemory ?? false; this.experimentalContextManagementConfig = params.experimentalContextManagementConfig; diff --git a/packages/core/src/services/chatRecordingService.ts b/packages/core/src/services/chatRecordingService.ts index cab67f80a1..b3cfb97527 100644 --- a/packages/core/src/services/chatRecordingService.ts +++ b/packages/core/src/services/chatRecordingService.ts @@ -109,6 +109,7 @@ export async function loadConversationRecord( ): Promise< | (ConversationRecord & { messageCount?: number; + userMessageCount?: number; firstUserMessage?: string; hasUserOrAssistantMessage?: boolean; }) @@ -128,8 +129,11 @@ export async function loadConversationRecord( let metadata: Partial = {}; const messagesMap = new Map(); const messageIds: string[] = []; + const messageKinds = new Map< + string, + { isUser: boolean; isUserOrAssistant: boolean } + >(); let firstUserMessageStr: string | undefined; - let hasUserOrAssistant = false; for await (const line of rl) { if (!line.trim()) continue; @@ -140,13 +144,14 @@ export async function loadConversationRecord( if (options?.metadataOnly) { const idx = messageIds.indexOf(rewindId); if (idx !== -1) { - messageIds.splice(idx); + const removedIds = messageIds.splice(idx); + for (const removedId of removedIds) { + messageKinds.delete(removedId); + } } else { messageIds.length = 0; + messageKinds.clear(); } - // For metadataOnly we can't perfectly un-track hasUserOrAssistant if it was rewinded, - // but we can assume false if messageIds is empty. - if (messageIds.length === 0) hasUserOrAssistant = false; } else { let found = false; const idsToDelete: string[] = []; @@ -164,20 +169,18 @@ export async function loadConversationRecord( } } else if (isMessageRecord(record)) { const id = record.id; - if ( + const isUser = hasProperty(record, 'type') && record.type === 'user'; + const isUserOrAssistant = hasProperty(record, 'type') && - (record.type === 'user' || record.type === 'gemini') - ) { - hasUserOrAssistant = true; - } + (record.type === 'user' || record.type === 'gemini'); // Track message count and first user message if (options?.metadataOnly) { messageIds.push(id); + messageKinds.set(id, { isUser, isUserOrAssistant }); } if ( !firstUserMessageStr && - hasProperty(record, 'type') && - record['type'] === 'user' && + isUser && hasProperty(record, 'content') && record['content'] ) { @@ -221,6 +224,33 @@ export async function loadConversationRecord( return await parseLegacyRecordFallback(filePath, options); } + const metadataMessages = Array.isArray(metadata.messages) + ? metadata.messages + : []; + const loadedMessages = + metadataMessages.length > 0 + ? metadataMessages + : Array.from(messagesMap.values()); + const metadataFirstUserMessage = + metadataMessages.find((message) => message.type === 'user') ?? null; + let fallbackFirstUserMessage = firstUserMessageStr; + if (!fallbackFirstUserMessage && metadataFirstUserMessage) { + const rawContent = metadataFirstUserMessage.content; + if (Array.isArray(rawContent)) { + fallbackFirstUserMessage = rawContent + .map((part: unknown) => (isTextPart(part) ? part['text'] : '')) + .join(''); + } else if (typeof rawContent === 'string') { + fallbackFirstUserMessage = rawContent; + } + } + const userMessageCount = options?.metadataOnly + ? Array.from(messageKinds.values()).filter((m) => m.isUser).length + : loadedMessages.filter((m) => m.type === 'user').length; + const hasUserOrAssistant = options?.metadataOnly + ? Array.from(messageKinds.values()).some((m) => m.isUserOrAssistant) + : loadedMessages.some((m) => m.type === 'user' || m.type === 'gemini'); + return { sessionId: metadata.sessionId, projectHash: metadata.projectHash, @@ -229,16 +259,21 @@ export async function loadConversationRecord( summary: metadata.summary, directories: metadata.directories, kind: metadata.kind, - messages: Array.from(messagesMap.values()), + messages: options?.metadataOnly ? [] : loadedMessages, messageCount: options?.metadataOnly - ? messageIds.length - : messagesMap.size, - firstUserMessage: firstUserMessageStr, - hasUserOrAssistantMessage: options?.metadataOnly - ? hasUserOrAssistant - : Array.from(messagesMap.values()).some( - (m) => m.type === 'user' || m.type === 'gemini', - ), + ? metadataMessages.length || messageIds.length + : loadedMessages.length, + userMessageCount: + options?.metadataOnly && metadataMessages.length > 0 + ? metadataMessages.filter((m) => m.type === 'user').length + : userMessageCount, + firstUserMessage: fallbackFirstUserMessage, + hasUserOrAssistantMessage: + options?.metadataOnly && metadataMessages.length > 0 + ? metadataMessages.some( + (m) => m.type === 'user' || m.type === 'gemini', + ) + : hasUserOrAssistant, }; } catch (error) { debugLogger.error('Error loading conversation record from JSONL:', error); @@ -816,6 +851,7 @@ async function parseLegacyRecordFallback( ): Promise< | (ConversationRecord & { messageCount?: number; + userMessageCount?: number; firstUserMessage?: string; hasUserOrAssistantMessage?: boolean; }) @@ -849,6 +885,8 @@ async function parseLegacyRecordFallback( ...legacyRecord, messages: [], messageCount: legacyRecord.messages?.length || 0, + userMessageCount: + legacyRecord.messages?.filter((m) => m.type === 'user').length || 0, firstUserMessage: fallbackFirstUserMessageStr, hasUserOrAssistantMessage: legacyRecord.messages?.some( @@ -858,6 +896,8 @@ async function parseLegacyRecordFallback( } return { ...legacyRecord, + userMessageCount: + legacyRecord.messages?.filter((m) => m.type === 'user').length || 0, hasUserOrAssistantMessage: legacyRecord.messages?.some( (m) => m.type === 'user' || m.type === 'gemini', diff --git a/packages/core/src/services/memoryService.test.ts b/packages/core/src/services/memoryService.test.ts index 69d7183ece..f0b191667b 100644 --- a/packages/core/src/services/memoryService.test.ts +++ b/packages/core/src/services/memoryService.test.ts @@ -117,6 +117,35 @@ function createConversation( }; } +async function writeConversationJsonl( + filePath: string, + conversation: ConversationRecord, +): Promise { + const metadata = { + sessionId: conversation.sessionId, + projectHash: conversation.projectHash, + startTime: conversation.startTime, + lastUpdated: conversation.lastUpdated, + summary: conversation.summary, + directories: conversation.directories, + kind: conversation.kind, + }; + + const records = [metadata, ...conversation.messages]; + await fs.writeFile( + filePath, + records.map((record) => JSON.stringify(record)).join('\n') + '\n', + ); +} + +async function setSessionMtime( + filePath: string, + timestamp: string, +): Promise { + const date = new Date(timestamp); + await fs.utimes(filePath, date, date); +} + describe('memoryService', () => { let tmpDir: string; @@ -535,6 +564,150 @@ describe('memoryService', () => { expect.stringContaining('/memory inbox'), ); }); + + it('records only sessions whose read_file calls succeed as processed', async () => { + const { startMemoryService, readExtractionState } = await import( + './memoryService.js' + ); + const { LocalAgentExecutor } = await import( + '../agents/local-executor.js' + ); + + vi.mocked(LocalAgentExecutor.create).mockReset(); + + const memoryDir = path.join(tmpDir, 'memory-read-tracking'); + const skillsDir = path.join(tmpDir, 'skills-read-tracking'); + const projectTempDir = path.join(tmpDir, 'temp-read-tracking'); + const chatsDir = path.join(projectTempDir, 'chats'); + await fs.mkdir(memoryDir, { recursive: true }); + await fs.mkdir(skillsDir, { recursive: true }); + await fs.mkdir(chatsDir, { recursive: true }); + + const openedConversation = createConversation({ + sessionId: 'opened-session', + summary: 'Read this one', + messageCount: 20, + lastUpdated: '2025-01-02T01:00:00Z', + }); + const skippedConversation = createConversation({ + sessionId: 'skipped-session', + summary: 'Do not read this one', + messageCount: 20, + lastUpdated: '2025-01-01T01:00:00Z', + }); + + const openedPath = path.join( + chatsDir, + `${SESSION_FILE_PREFIX}2025-01-02T00-00-opened.jsonl`, + ); + const skippedPath = path.join( + chatsDir, + `${SESSION_FILE_PREFIX}2025-01-01T00-00-skipped.jsonl`, + ); + await writeConversationJsonl(openedPath, openedConversation); + await writeConversationJsonl(skippedPath, skippedConversation); + + vi.mocked(LocalAgentExecutor.create).mockImplementationOnce( + async (_definition, _context, onActivity) => + ({ + run: vi.fn().mockImplementation(async () => { + onActivity?.({ + isSubagentActivityEvent: true, + agentName: 'Skill Extractor', + type: 'TOOL_CALL_START', + data: { + name: 'read_file', + args: { file_path: openedPath }, + callId: 'call-opened', + }, + }); + onActivity?.({ + isSubagentActivityEvent: true, + agentName: 'Skill Extractor', + type: 'TOOL_CALL_START', + data: { + name: 'read_file', + args: { file_path: skippedPath }, + callId: 'call-skipped', + }, + }); + onActivity?.({ + isSubagentActivityEvent: true, + agentName: 'Skill Extractor', + type: 'ERROR', + data: { + name: 'read_file', + callId: 'call-skipped', + error: 'access denied', + }, + }); + onActivity?.({ + isSubagentActivityEvent: true, + agentName: 'Skill Extractor', + type: 'TOOL_CALL_END', + data: { + name: 'read_file', + id: 'call-opened', + data: { content: 'Read this one' }, + }, + }); + onActivity?.({ + isSubagentActivityEvent: true, + agentName: 'Skill Extractor', + type: 'TOOL_CALL_START', + data: { + name: 'read_file', + args: { file_path: path.join(chatsDir, 'unrelated.jsonl') }, + callId: 'call-unrelated', + }, + }); + return undefined; + }), + }) as never, + ); + + const mockConfig = { + storage: { + getProjectMemoryDir: vi.fn().mockReturnValue(memoryDir), + getProjectMemoryTempDir: vi.fn().mockReturnValue(memoryDir), + getProjectSkillsMemoryDir: vi.fn().mockReturnValue(skillsDir), + getProjectTempDir: vi.fn().mockReturnValue(projectTempDir), + }, + getToolRegistry: vi.fn(), + getMessageBus: vi.fn(), + getGeminiClient: vi.fn(), + getSkillManager: vi.fn().mockReturnValue({ getSkills: () => [] }), + modelConfigService: { + registerRuntimeModelConfig: vi.fn(), + }, + getTargetDir: vi.fn().mockReturnValue(tmpDir), + sandboxManager: undefined, + } as unknown as Parameters[0]; + + await startMemoryService(mockConfig); + + const state = await readExtractionState( + path.join(memoryDir, '.extraction-state.json'), + ); + expect(state.runs).toHaveLength(1); + expect(state.runs[0].candidateSessions).toEqual([ + { + sessionId: 'opened-session', + lastUpdated: '2025-01-02T01:00:00Z', + }, + { + sessionId: 'skipped-session', + lastUpdated: '2025-01-01T01:00:00Z', + }, + ]); + expect(state.runs[0].processedSessions).toEqual([ + { + sessionId: 'opened-session', + lastUpdated: '2025-01-02T01:00:00Z', + }, + ]); + expect(state.runs[0].sessionIds).toEqual(['opened-session']); + }); }); describe('getProcessedSessionIds', () => { @@ -663,7 +836,7 @@ describe('memoryService', () => { const state: ExtractionState = { runs: [ { - runAt: '2025-01-01T00:00:00Z', + runAt: '2025-01-01T02:00:00Z', sessionIds: ['old-session'], skillsCreated: [], }, @@ -676,6 +849,39 @@ describe('memoryService', () => { expect(result.sessionIndex).not.toContain('[NEW]'); }); + it('treats resumed legacy sessions as [NEW] when lastUpdated moved past the old run', async () => { + const { buildSessionIndex } = await import('./memoryService.js'); + + const conversation = createConversation({ + sessionId: 'resumed-session', + summary: 'Resumed after extraction', + messageCount: 20, + lastUpdated: '2025-01-01T03:00:00Z', + }); + await fs.writeFile( + path.join( + chatsDir, + `${SESSION_FILE_PREFIX}2025-01-01T00-00-resumed01.json`, + ), + JSON.stringify(conversation), + ); + + const state: ExtractionState = { + runs: [ + { + runAt: '2025-01-01T02:00:00Z', + sessionIds: ['resumed-session'], + skillsCreated: [], + }, + ], + }; + + const result = await buildSessionIndex(chatsDir, state); + + expect(result.sessionIndex).toContain('[NEW]'); + expect(result.newSessionIds).toEqual(['resumed-session']); + }); + it('includes file path and summary in each line', async () => { const { buildSessionIndex } = await import('./memoryService.js'); @@ -800,7 +1006,7 @@ describe('memoryService', () => { const state: ExtractionState = { runs: [ { - runAt: '2025-01-01T00:00:00Z', + runAt: '2025-01-01T02:00:00Z', sessionIds: ['processed-one'], skillsCreated: [], }, @@ -815,6 +1021,136 @@ describe('memoryService', () => { expect(result.sessionIndex).toContain('[NEW]'); expect(result.sessionIndex).toContain('[old]'); }); + + it('reads JSONL sessions and sorts by actual lastUpdated instead of filename', async () => { + const { buildSessionIndex } = await import('./memoryService.js'); + + const olderByName = createConversation({ + sessionId: 'older-by-name', + summary: 'Filename looks newer', + messageCount: 20, + lastUpdated: '2025-01-01T01:00:00Z', + }); + const newerByActivity = createConversation({ + sessionId: 'newer-by-activity', + summary: 'Actually most recent', + messageCount: 20, + lastUpdated: '2025-02-01T01:00:00Z', + }); + + await writeConversationJsonl( + path.join( + chatsDir, + `${SESSION_FILE_PREFIX}2025-02-01T00-00-oldername.jsonl`, + ), + olderByName, + ); + await writeConversationJsonl( + path.join( + chatsDir, + `${SESSION_FILE_PREFIX}2025-01-01T00-00-neweractv.jsonl`, + ), + newerByActivity, + ); + + const result = await buildSessionIndex(chatsDir, { runs: [] }); + const firstLine = result.sessionIndex.split('\n')[0]; + + expect(firstLine).toContain('Actually most recent'); + expect(firstLine).not.toContain('Filename looks newer'); + }); + + it('rotates in older unprocessed sessions instead of starving them behind retried recent ones', async () => { + const { buildSessionIndex } = await import('./memoryService.js'); + + for (let i = 0; i < 11; i++) { + const day = String(11 - i).padStart(2, '0'); + const conversation = createConversation({ + sessionId: `backlog-${i}`, + summary: `Backlog ${i}`, + messageCount: 20, + lastUpdated: `2025-01-${day}T01:00:00Z`, + }); + await fs.writeFile( + path.join( + chatsDir, + `${SESSION_FILE_PREFIX}2025-01-${day}T00-00-backlog${i}.json`, + ), + JSON.stringify(conversation), + ); + } + + const state: ExtractionState = { + runs: [ + { + runAt: '2025-02-01T00:00:00Z', + sessionIds: [], + candidateSessions: Array.from({ length: 10 }, (_, i) => ({ + sessionId: `backlog-${i}`, + lastUpdated: `2025-01-${String(11 - i).padStart(2, '0')}T01:00:00Z`, + })), + skillsCreated: [], + }, + ], + }; + + const result = await buildSessionIndex(chatsDir, state); + + expect(result.newSessionIds).toContain('backlog-10'); + expect(result.newSessionIds).not.toContain('backlog-9'); + }); + + it('surfaces older unprocessed sessions even when the newest 100 files were already processed', async () => { + const { buildSessionIndex } = await import('./memoryService.js'); + + const processedSessions: ExtractionRun['processedSessions'] = []; + + for (let i = 0; i < 105; i++) { + const timestamp = new Date( + Date.UTC(2025, 0, 1, 0, 0, 105 - i), + ).toISOString(); + const conversation = createConversation({ + sessionId: `backlog-${i}`, + summary: `Backlog ${i}`, + messageCount: 20, + lastUpdated: timestamp, + }); + const filePath = path.join( + chatsDir, + `${SESSION_FILE_PREFIX}2025-01-01T00-00-backlog${String(i).padStart(3, '0')}.json`, + ); + await fs.writeFile(filePath, JSON.stringify(conversation)); + await setSessionMtime(filePath, timestamp); + + if (i < 100) { + processedSessions.push({ + sessionId: conversation.sessionId, + lastUpdated: conversation.lastUpdated, + }); + } + } + + const result = await buildSessionIndex(chatsDir, { + runs: [ + { + runAt: '2025-02-01T00:00:00Z', + sessionIds: processedSessions.map((session) => session.sessionId), + processedSessions, + skillsCreated: [], + }, + ], + }); + + expect(result.newSessionIds).toEqual([ + 'backlog-100', + 'backlog-101', + 'backlog-102', + 'backlog-103', + 'backlog-104', + ]); + expect(result.sessionIndex).toContain('Backlog 100'); + expect(result.sessionIndex).toContain('Backlog 104'); + }); }); describe('ExtractionState runs tracking', () => { @@ -827,6 +1163,18 @@ describe('memoryService', () => { { runAt: '2025-06-01T00:00:00Z', sessionIds: ['s1'], + candidateSessions: [ + { + sessionId: 's1', + lastUpdated: '2025-05-31T12:00:00Z', + }, + ], + processedSessions: [ + { + sessionId: 's1', + lastUpdated: '2025-05-31T12:00:00Z', + }, + ], skillsCreated: ['debug-helper', 'test-gen'], }, ], @@ -840,6 +1188,18 @@ describe('memoryService', () => { 'debug-helper', 'test-gen', ]); + expect(result.runs[0].candidateSessions).toEqual([ + { + sessionId: 's1', + lastUpdated: '2025-05-31T12:00:00Z', + }, + ]); + expect(result.runs[0].processedSessions).toEqual([ + { + sessionId: 's1', + lastUpdated: '2025-05-31T12:00:00Z', + }, + ]); expect(result.runs[0].sessionIds).toEqual(['s1']); expect(result.runs[0].runAt).toBe('2025-06-01T00:00:00Z'); }); @@ -854,6 +1214,26 @@ describe('memoryService', () => { { runAt: '2025-01-01T00:00:00Z', sessionIds: ['a', 'b'], + candidateSessions: [ + { + sessionId: 'a', + lastUpdated: '2024-12-31T23:00:00Z', + }, + { + sessionId: 'b', + lastUpdated: '2024-12-31T22:00:00Z', + }, + ], + processedSessions: [ + { + sessionId: 'a', + lastUpdated: '2024-12-31T23:00:00Z', + }, + { + sessionId: 'b', + lastUpdated: '2024-12-31T22:00:00Z', + }, + ], skillsCreated: ['skill-x'], }, { diff --git a/packages/core/src/services/memoryService.ts b/packages/core/src/services/memoryService.ts index 29b2b18701..4fdb51e50b 100644 --- a/packages/core/src/services/memoryService.ts +++ b/packages/core/src/services/memoryService.ts @@ -12,6 +12,7 @@ import * as Diff from 'diff'; import type { Config } from '../config/config.js'; import { SESSION_FILE_PREFIX, + loadConversationRecord, type ConversationRecord, } from './chatRecordingService.js'; import { debugLogger } from '../utils/debugLogger.js'; @@ -21,6 +22,7 @@ import { FRONTMATTER_REGEX, parseFrontmatter } from '../skills/skillLoader.js'; import { LocalAgentExecutor } from '../agents/local-executor.js'; import { SkillExtractionAgent } from '../agents/skill-extraction-agent.js'; import { getModelConfigAlias } from '../agents/registry.js'; +import type { SubagentActivityEvent } from '../agents/types.js'; import { ExecutionLifecycleService } from './executionLifecycleService.js'; import { PromptRegistry } from '../prompts/prompt-registry.js'; import { ResourceRegistry } from '../resources/resource-registry.js'; @@ -29,6 +31,7 @@ import { PolicyDecision } from '../policy/types.js'; import { MessageBus } from '../confirmation-bus/message-bus.js'; import { Storage } from '../config/storage.js'; import type { AgentLoopContext } from '../config/agent-loop-context.js'; +import { READ_FILE_TOOL_NAME } from '../tools/tool-names.js'; import { applyParsedSkillPatches, hasParsedPatchHunks, @@ -40,6 +43,7 @@ const LOCK_STALE_MS = 35 * 60 * 1000; // 35 minutes (exceeds agent's 30-min time const MIN_USER_MESSAGES = 10; const MIN_IDLE_MS = 3 * 60 * 60 * 1000; // 3 hours const MAX_SESSION_INDEX_SIZE = 50; +const MAX_NEW_SESSION_BATCH_SIZE = 10; /** * Lock file content for coordinating across CLI instances. @@ -49,12 +53,39 @@ interface LockInfo { startedAt: string; } +function hasProperty( + obj: unknown, + prop: T, +): obj is { [key in T]: unknown } { + return obj !== null && typeof obj === 'object' && prop in obj; +} + +function isStringProperty( + obj: unknown, + prop: T, +): obj is { [key in T]: string } { + return hasProperty(obj, prop) && typeof obj[prop] === 'string'; +} + +interface SessionVersion { + sessionId: string; + lastUpdated: string; +} + +interface IndexedSession extends SessionVersion { + filePath: string; + summary?: string; + userMessageCount: number; +} + /** * Metadata for a single extraction run. */ export interface ExtractionRun { runAt: string; sessionIds: string[]; + candidateSessions?: SessionVersion[]; + processedSessions?: SessionVersion[]; skillsCreated: string[]; } @@ -71,7 +102,10 @@ export interface ExtractionState { export function getProcessedSessionIds(state: ExtractionState): Set { const ids = new Set(); for (const run of state.runs) { - for (const id of run.sessionIds) { + const processedSessionIds = + run.processedSessions?.map((session) => session.sessionId) ?? + run.sessionIds; + for (const id of processedSessionIds) { ids.add(id); } } @@ -89,30 +123,49 @@ function isLockInfo(value: unknown): value is LockInfo { ); } -function isConversationRecord(value: unknown): value is ConversationRecord { +function isSessionVersion(value: unknown): value is SessionVersion { return ( typeof value === 'object' && value !== null && 'sessionId' in value && typeof value.sessionId === 'string' && - 'messages' in value && - Array.isArray(value.messages) && - 'projectHash' in value && - 'startTime' in value && - 'lastUpdated' in value + 'lastUpdated' in value && + typeof value.lastUpdated === 'string' ); } -function isExtractionRun(value: unknown): value is ExtractionRun { +function normalizeSessionVersions(value: unknown): SessionVersion[] { + if (!Array.isArray(value)) { + return []; + } + + return value.filter(isSessionVersion).map((session) => ({ + sessionId: session.sessionId, + lastUpdated: session.lastUpdated, + })); +} + +function normalizeStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + + return value.filter((item): item is string => typeof item === 'string'); +} + +function isExtractionRunLike(value: unknown): value is { + runAt: string; + sessionIds?: unknown; + candidateSessions?: unknown; + processedSessions?: unknown; + skillsCreated: unknown; +} { return ( typeof value === 'object' && value !== null && 'runAt' in value && typeof value.runAt === 'string' && - 'sessionIds' in value && - Array.isArray(value.sessionIds) && - 'skillsCreated' in value && - Array.isArray(value.skillsCreated) + 'skillsCreated' in value ); } @@ -125,6 +178,208 @@ function isExtractionState(value: unknown): value is { runs: unknown[] } { ); } +function buildExtractionRun(value: unknown): ExtractionRun | null { + if (!isExtractionRunLike(value)) { + return null; + } + + const candidateSessions = normalizeSessionVersions(value.candidateSessions); + const processedSessions = normalizeSessionVersions(value.processedSessions); + const sessionIds = normalizeStringArray(value.sessionIds); + + return { + runAt: value.runAt, + sessionIds: + sessionIds.length > 0 + ? sessionIds + : processedSessions.map((session) => session.sessionId), + candidateSessions: + candidateSessions.length > 0 ? candidateSessions : undefined, + processedSessions: + processedSessions.length > 0 ? processedSessions : undefined, + skillsCreated: normalizeStringArray(value.skillsCreated), + }; +} + +function getTimestampMs(timestamp: string): number { + const parsed = Date.parse(timestamp); + return Number.isNaN(parsed) ? 0 : parsed; +} + +function getSessionVersionKey(session: SessionVersion): string { + return `${session.sessionId}\u0000${session.lastUpdated}`; +} + +function hasLegacyRunProcessedSession( + run: ExtractionRun, + session: SessionVersion, +): boolean { + return ( + run.sessionIds.includes(session.sessionId) && + getTimestampMs(run.runAt) >= getTimestampMs(session.lastUpdated) + ); +} + +function isSessionVersionProcessed( + state: ExtractionState, + session: SessionVersion, +): boolean { + const sessionKey = getSessionVersionKey(session); + + for (const run of state.runs) { + if ( + run.processedSessions?.some( + (processed) => getSessionVersionKey(processed) === sessionKey, + ) + ) { + return true; + } + + if (!run.processedSessions && hasLegacyRunProcessedSession(run, session)) { + return true; + } + } + + return false; +} + +function getSessionAttemptCount( + state: ExtractionState, + session: SessionVersion, +): number { + const sessionKey = getSessionVersionKey(session); + let attempts = 0; + + for (const run of state.runs) { + if (run.candidateSessions) { + if ( + run.candidateSessions.some( + (candidate) => getSessionVersionKey(candidate) === sessionKey, + ) + ) { + attempts++; + } + continue; + } + + if (hasLegacyRunProcessedSession(run, session)) { + attempts++; + } + } + + return attempts; +} + +function compareIndexedSessions(a: IndexedSession, b: IndexedSession): number { + const timestampDelta = + getTimestampMs(b.lastUpdated) - getTimestampMs(a.lastUpdated); + if (timestampDelta !== 0) { + return timestampDelta; + } + + if (a.filePath.endsWith('.jsonl') !== b.filePath.endsWith('.jsonl')) { + return a.filePath.endsWith('.jsonl') ? -1 : 1; + } + + return b.filePath.localeCompare(a.filePath); +} + +function shouldReplaceIndexedSession( + existing: IndexedSession, + candidate: IndexedSession, +): boolean { + return compareIndexedSessions(candidate, existing) < 0; +} + +function isReadFileStartActivity( + activity: SubagentActivityEvent, +): activity is SubagentActivityEvent & { + data: { name: string; args?: { file_path?: unknown }; callId?: unknown }; +} { + return ( + activity.type === 'TOOL_CALL_START' && + activity.data['name'] === READ_FILE_TOOL_NAME + ); +} + +function getResolvedReadFilePath( + config: Config, + activity: SubagentActivityEvent, +): string | null { + if (!isReadFileStartActivity(activity)) { + return null; + } + + const args = activity.data.args; + if ( + typeof args !== 'object' || + args === null || + !('file_path' in args) || + typeof args.file_path !== 'string' + ) { + return null; + } + + return path.resolve(config.getTargetDir(), args.file_path); +} + +function getReadFileStartCallId( + activity: SubagentActivityEvent, +): string | null { + if ( + !isReadFileStartActivity(activity) || + !isStringProperty(activity.data, 'callId') + ) { + return null; + } + + return activity.data.callId; +} + +function getCompletedReadFileCallId( + activity: SubagentActivityEvent, +): string | null { + if ( + activity.type !== 'TOOL_CALL_END' || + activity.data['name'] !== READ_FILE_TOOL_NAME || + !isStringProperty(activity.data, 'id') + ) { + return null; + } + + return activity.data['id']; +} + +function getFailedReadFileCallId( + activity: SubagentActivityEvent, +): string | null { + if ( + activity.type !== 'ERROR' || + activity.data['name'] !== READ_FILE_TOOL_NAME || + !isStringProperty(activity.data, 'callId') + ) { + return null; + } + + return activity.data['callId']; +} + +function getUserMessageCount( + conversation: ConversationRecord & { userMessageCount?: number }, +): number { + return ( + conversation.userMessageCount ?? + conversation.messages.filter((message) => message.type === 'user').length + ); +} + +function isSupportedSessionFile(fileName: string): boolean { + return ( + fileName.startsWith(SESSION_FILE_PREFIX) && + (fileName.endsWith('.json') || fileName.endsWith('.jsonl')) + ); +} + /** * Attempts to acquire an exclusive lock file using O_CREAT | O_EXCL. * Returns true if the lock was acquired, false if another instance owns it. @@ -231,16 +486,9 @@ export async function readExtractionState( const runs: ExtractionRun[] = []; for (const run of parsed.runs) { - if (!isExtractionRun(run)) continue; - runs.push({ - runAt: run.runAt, - sessionIds: run.sessionIds.filter( - (sid): sid is string => typeof sid === 'string', - ), - skillsCreated: run.skillsCreated.filter( - (sk): sk is string => typeof sk === 'string', - ), - }); + const normalizedRun = buildExtractionRun(run); + if (!normalizedRun) continue; + runs.push(normalizedRun); } return { runs }; @@ -270,30 +518,32 @@ export async function writeExtractionState( * Filters out subagent sessions, sessions that haven't been idle long enough, * and sessions with too few user messages. */ -function shouldProcessConversation(parsed: ConversationRecord): boolean { +function shouldProcessConversation( + parsed: ConversationRecord & { userMessageCount?: number }, +): boolean { // Skip subagent sessions if (parsed.kind === 'subagent') return false; // Skip sessions that are still active (not idle for 3+ hours) - const lastUpdated = new Date(parsed.lastUpdated).getTime(); + const lastUpdated = getTimestampMs(parsed.lastUpdated); if (Date.now() - lastUpdated < MIN_IDLE_MS) return false; // Skip sessions with too few user messages - const userMessageCount = parsed.messages.filter( - (m) => m.type === 'user', - ).length; - if (userMessageCount < MIN_USER_MESSAGES) return false; + if (getUserMessageCount(parsed) < MIN_USER_MESSAGES) return false; return true; } /** - * Scans the chats directory for eligible session files (sorted most-recent-first, - * capped at MAX_SESSION_INDEX_SIZE). Shared by buildSessionIndex. + * Scans the chats directory for eligible session files, loading metadata from + * both JSONL and legacy JSON sessions, deduplicating migrated sessions by + * session ID, and sorting by actual lastUpdated. We scan the full directory + * here so already-processed recent sessions cannot permanently block older + * backlog sessions from surfacing as new candidates. */ async function scanEligibleSessions( chatsDir: string, -): Promise> { +): Promise { let allFiles: string[]; try { allFiles = await fs.readdir(chatsDir); @@ -301,33 +551,48 @@ async function scanEligibleSessions( return []; } - const sessionFiles = allFiles.filter( - (f) => f.startsWith(SESSION_FILE_PREFIX) && f.endsWith('.json'), - ); - - // Sort by filename descending (most recent first) - sessionFiles.sort((a, b) => b.localeCompare(a)); - - const results: Array<{ conversation: ConversationRecord; filePath: string }> = - []; - - for (const file of sessionFiles) { - if (results.length >= MAX_SESSION_INDEX_SIZE) break; - + const candidates: Array<{ filePath: string; mtimeMs: number }> = []; + for (const file of allFiles) { + if (!isSupportedSessionFile(file)) continue; const filePath = path.join(chatsDir, file); try { - const content = await fs.readFile(filePath, 'utf-8'); - const parsed: unknown = JSON.parse(content); - if (!isConversationRecord(parsed)) continue; - if (!shouldProcessConversation(parsed)) continue; + const stat = await fs.stat(filePath); + if (!stat.isFile()) continue; + candidates.push({ filePath, mtimeMs: stat.mtimeMs }); + } catch { + // Skip files that disappeared between readdir and stat. + } + } - results.push({ conversation: parsed, filePath }); + candidates.sort((a, b) => b.mtimeMs - a.mtimeMs); + + const latestBySessionId = new Map(); + + for (const { filePath } of candidates) { + try { + const conversation = await loadConversationRecord(filePath, { + metadataOnly: true, + }); + if (!conversation || !shouldProcessConversation(conversation)) continue; + + const indexedSession: IndexedSession = { + sessionId: conversation.sessionId, + lastUpdated: conversation.lastUpdated, + filePath, + summary: conversation.summary, + userMessageCount: getUserMessageCount(conversation), + }; + + const existing = latestBySessionId.get(indexedSession.sessionId); + if (!existing || shouldReplaceIndexedSession(existing, indexedSession)) { + latestBySessionId.set(indexedSession.sessionId, indexedSession); + } } catch { // Skip unreadable files } } - return results; + return Array.from(latestBySessionId.values()).sort(compareIndexedSessions); } /** @@ -335,39 +600,67 @@ async function scanEligibleSessions( * eligible sessions with their summary, file path, and new/previously-processed status. * The agent can use read_file on paths to inspect sessions that look promising. * - * Returns the index text and the list of new (unprocessed) session IDs. + * Returns the index text, the list of selected new (unprocessed) session IDs, + * and the surfaced candidate sessions for this run. */ export async function buildSessionIndex( chatsDir: string, state: ExtractionState, -): Promise<{ sessionIndex: string; newSessionIds: string[] }> { - const processedSet = getProcessedSessionIds(state); +): Promise<{ + sessionIndex: string; + newSessionIds: string[]; + candidateSessions: IndexedSession[]; +}> { const eligible = await scanEligibleSessions(chatsDir); if (eligible.length === 0) { - return { sessionIndex: '', newSessionIds: [] }; + return { sessionIndex: '', newSessionIds: [], candidateSessions: [] }; } - const lines: string[] = []; - const newSessionIds: string[] = []; - - for (const { conversation, filePath } of eligible) { - const userMessageCount = conversation.messages.filter( - (m) => m.type === 'user', - ).length; - const isNew = !processedSet.has(conversation.sessionId); - if (isNew) { - newSessionIds.push(conversation.sessionId); + const newSessions: IndexedSession[] = []; + const oldSessions: IndexedSession[] = []; + for (const session of eligible) { + if (isSessionVersionProcessed(state, session)) { + oldSessions.push(session); + } else { + newSessions.push(session); } - - const status = isNew ? '[NEW]' : '[old]'; - const summary = conversation.summary ?? '(no summary)'; - lines.push( - `${status} ${summary} (${userMessageCount} user msgs) — ${filePath}`, - ); } - return { sessionIndex: lines.join('\n'), newSessionIds }; + newSessions.sort((a, b) => { + const attemptDelta = + getSessionAttemptCount(state, a) - getSessionAttemptCount(state, b); + if (attemptDelta !== 0) { + return attemptDelta; + } + return compareIndexedSessions(a, b); + }); + + const candidateSessions = newSessions.slice(0, MAX_NEW_SESSION_BATCH_SIZE); + const remainingSlots = Math.max( + 0, + MAX_SESSION_INDEX_SIZE - candidateSessions.length, + ); + const displayedOldSessions = oldSessions.slice(0, remainingSlots); + const candidateSessionIds = new Set( + candidateSessions.map((session) => getSessionVersionKey(session)), + ); + + const lines = [...candidateSessions, ...displayedOldSessions].map( + (session) => { + const status = candidateSessionIds.has(getSessionVersionKey(session)) + ? '[NEW]' + : '[old]'; + const summary = session.summary ?? '(no summary)'; + return `${status} ${summary} (${session.userMessageCount} user msgs) — ${session.filePath}`; + }, + ); + + return { + sessionIndex: lines.join('\n'), + newSessionIds: candidateSessions.map((session) => session.sessionId), + candidateSessions, + }; } /** @@ -632,14 +925,12 @@ export async function startMemoryService(config: Config): Promise { // Build session index: all eligible sessions with summaries + file paths. // The agent decides which to read in full via read_file. - const { sessionIndex, newSessionIds } = await buildSessionIndex( - chatsDir, - state, - ); + const { sessionIndex, newSessionIds, candidateSessions } = + await buildSessionIndex(chatsDir, state); const totalInIndex = sessionIndex ? sessionIndex.split('\n').length : 0; debugLogger.log( - `[MemoryService] Session scan: ${totalInIndex} eligible session(s) found, ${newSessionIds.length} new`, + `[MemoryService] Session scan: ${totalInIndex} indexed session(s), ${candidateSessions.length} surfaced as new candidates`, ); if (newSessionIds.length === 0) { @@ -702,8 +993,59 @@ export async function startMemoryService(config: Config): Promise { `[MemoryService] Starting extraction agent (model: ${agentDefinition.modelConfig.model}, maxTurns: 30, maxTime: 30min)`, ); + const candidateSessionsByPath = new Map( + candidateSessions.map((session) => [ + path.resolve(session.filePath), + session, + ]), + ); + const processedSessionKeys = new Set(); + const pendingReadFileSessions = new Map(); + // Create and run the extraction agent - const executor = await LocalAgentExecutor.create(agentDefinition, context); + const executor = await LocalAgentExecutor.create( + agentDefinition, + context, + (activity) => { + const readFileCallId = getReadFileStartCallId(activity); + if (readFileCallId) { + const resolvedPath = getResolvedReadFilePath(config, activity); + if (!resolvedPath) { + return; + } + + const session = candidateSessionsByPath.get(resolvedPath); + if (!session) { + return; + } + + pendingReadFileSessions.set( + readFileCallId, + getSessionVersionKey(session), + ); + return; + } + + const completedReadFileCallId = getCompletedReadFileCallId(activity); + if (completedReadFileCallId) { + const sessionKey = pendingReadFileSessions.get( + completedReadFileCallId, + ); + if (!sessionKey) { + return; + } + + processedSessionKeys.add(sessionKey); + pendingReadFileSessions.delete(completedReadFileCallId); + return; + } + + const failedReadFileCallId = getFailedReadFileCallId(activity); + if (failedReadFileCallId) { + pendingReadFileSessions.delete(failedReadFileCallId); + } + }, + ); await executor.run( { request: 'Extract skills from the provided sessions.' }, @@ -746,10 +1088,24 @@ export async function startMemoryService(config: Config): Promise { ); } + const processedSessions = candidateSessions + .filter((session) => + processedSessionKeys.has(getSessionVersionKey(session)), + ) + .map((session) => ({ + sessionId: session.sessionId, + lastUpdated: session.lastUpdated, + })); + // Record the run with full metadata const run: ExtractionRun = { runAt: new Date().toISOString(), - sessionIds: newSessionIds, + sessionIds: processedSessions.map((session) => session.sessionId), + candidateSessions: candidateSessions.map((session) => ({ + sessionId: session.sessionId, + lastUpdated: session.lastUpdated, + })), + processedSessions, skillsCreated, }; const updatedState: ExtractionState = { @@ -770,7 +1126,7 @@ export async function startMemoryService(config: Config): Promise { ); } debugLogger.log( - `[MemoryService] Completed in ${elapsed}s. ${completionParts.join('; ')} (processed ${newSessionIds.length} session(s))`, + `[MemoryService] Completed in ${elapsed}s. ${completionParts.join('; ')} (read ${processedSessions.length}/${candidateSessions.length} surfaced session(s))`, ); const feedbackParts: string[] = []; if (skillsCreated.length > 0) { @@ -789,7 +1145,7 @@ export async function startMemoryService(config: Config): Promise { ); } else { debugLogger.log( - `[MemoryService] Completed in ${elapsed}s. No new skills or patches created (processed ${newSessionIds.length} session(s))`, + `[MemoryService] Completed in ${elapsed}s. No new skills or patches created (read ${processedSessions.length}/${candidateSessions.length} surfaced session(s))`, ); } } catch (error) { diff --git a/packages/core/src/services/sessionSummaryUtils.test.ts b/packages/core/src/services/sessionSummaryUtils.test.ts index 2314b7ca06..fa1a47a14f 100644 --- a/packages/core/src/services/sessionSummaryUtils.test.ts +++ b/packages/core/src/services/sessionSummaryUtils.test.ts @@ -8,12 +8,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { generateSummary, getPreviousSession } from './sessionSummaryUtils.js'; import type { Config } from '../config/config.js'; import type { ContentGenerator } from '../core/contentGenerator.js'; +import * as chatRecordingService from './chatRecordingService.js'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; - -// Mock fs/promises -vi.mock('node:fs/promises'); -const mockReaddir = fs.readdir as unknown as ReturnType; +import * as os from 'node:os'; // Mock the SessionSummaryService module vi.mock('./sessionSummaryService.js', () => ({ @@ -27,23 +25,84 @@ vi.mock('../core/baseLlmClient.js', () => ({ BaseLlmClient: vi.fn(), })); -// Helper to create a session with N user messages -function createSessionWithUserMessages( - count: number, - options: { summary?: string; sessionId?: string } = {}, -) { +vi.mock('./chatRecordingService.js', async () => { + const actual = await vi.importActual< + typeof import('./chatRecordingService.js') + >('./chatRecordingService.js'); + return { + ...actual, + loadConversationRecord: vi.fn(actual.loadConversationRecord), + }; +}); + +interface SessionFixture { + summary?: string; + sessionId?: string; + startTime?: string; + lastUpdated?: string; + userMessageCount: number; +} + +function buildLegacySessionJson(fixture: SessionFixture): string { return JSON.stringify({ - sessionId: options.sessionId ?? 'session-id', - summary: options.summary, - messages: Array.from({ length: count }, (_, i) => ({ + sessionId: fixture.sessionId ?? 'session-id', + projectHash: 'abc123', + startTime: fixture.startTime ?? '2024-01-01T00:00:00Z', + lastUpdated: fixture.lastUpdated ?? '2024-01-01T00:00:00Z', + summary: fixture.summary, + messages: Array.from({ length: fixture.userMessageCount }, (_, i) => ({ id: String(i + 1), + timestamp: '2024-01-01T00:00:00Z', type: 'user', content: [{ text: `Message ${i + 1}` }], })), }); } +function buildJsonlSession(fixture: SessionFixture): string { + const metadata = { + sessionId: fixture.sessionId ?? 'session-id', + projectHash: 'abc123', + startTime: fixture.startTime ?? '2024-01-01T00:00:00Z', + lastUpdated: fixture.lastUpdated ?? '2024-01-01T00:00:00Z', + ...(fixture.summary !== undefined ? { summary: fixture.summary } : {}), + }; + const lines: string[] = [JSON.stringify(metadata)]; + for (let i = 0; i < fixture.userMessageCount; i++) { + lines.push( + JSON.stringify({ + id: String(i + 1), + timestamp: '2024-01-01T00:00:00Z', + type: 'user', + content: [{ text: `Message ${i + 1}` }], + }), + ); + } + return lines.join('\n') + '\n'; +} + +async function writeSession( + chatsDir: string, + fileName: string, + contents: string, +): Promise { + const filePath = path.join(chatsDir, fileName); + await fs.writeFile(filePath, contents); + return filePath; +} + +async function setSessionMtime( + filePath: string, + timestamp: string, +): Promise { + const date = new Date(timestamp); + await fs.utimes(filePath, date, date); +} + describe('sessionSummaryUtils', () => { + let tmpDir: string; + let projectTempDir: string; + let chatsDir: string; let mockConfig: Config; let mockContentGenerator: ContentGenerator; let mockGenerateSummary: ReturnType; @@ -51,21 +110,23 @@ describe('sessionSummaryUtils', () => { beforeEach(async () => { vi.clearAllMocks(); - // Setup mock content generator + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'session-summary-utils-')); + projectTempDir = path.join(tmpDir, 'project'); + chatsDir = path.join(projectTempDir, 'chats'); + await fs.mkdir(chatsDir, { recursive: true }); + mockContentGenerator = {} as ContentGenerator; - // Setup mock config mockConfig = { getContentGenerator: vi.fn().mockReturnValue(mockContentGenerator), + getSessionId: vi.fn().mockReturnValue('current-session'), storage: { - getProjectTempDir: vi.fn().mockReturnValue('/tmp/project'), + getProjectTempDir: vi.fn().mockReturnValue(projectTempDir), }, } as unknown as Config; - // Setup mock generateSummary function mockGenerateSummary = vi.fn().mockResolvedValue('Add dark mode to the app'); - // Import the mocked module to access the constructor const { SessionSummaryService } = await import( './sessionSummaryService.js' ); @@ -76,13 +137,14 @@ describe('sessionSummaryUtils', () => { })); }); - afterEach(() => { + afterEach(async () => { vi.restoreAllMocks(); + await fs.rm(tmpDir, { recursive: true, force: true }); }); describe('getPreviousSession', () => { it('should return null if chats directory does not exist', async () => { - vi.mocked(fs.access).mockRejectedValue(new Error('ENOENT')); + await fs.rm(chatsDir, { recursive: true, force: true }); const result = await getPreviousSession(mockConfig); @@ -90,19 +152,19 @@ describe('sessionSummaryUtils', () => { }); it('should return null if no session files exist', async () => { - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue([]); - const result = await getPreviousSession(mockConfig); expect(result).toBeNull(); }); it('should return null if most recent session already has summary', async () => { - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue(['session-2024-01-01T10-00-abc12345.json']); - vi.mocked(fs.readFile).mockResolvedValue( - createSessionWithUserMessages(5, { summary: 'Existing summary' }), + await writeSession( + chatsDir, + 'session-2024-01-01T10-00-abc12345.json', + buildLegacySessionJson({ + userMessageCount: 5, + summary: 'Existing summary', + }), ); const result = await getPreviousSession(mockConfig); @@ -111,10 +173,10 @@ describe('sessionSummaryUtils', () => { }); it('should return null if most recent session has 1 or fewer user messages', async () => { - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue(['session-2024-01-01T10-00-abc12345.json']); - vi.mocked(fs.readFile).mockResolvedValue( - createSessionWithUserMessages(1), + await writeSession( + chatsDir, + 'session-2024-01-01T10-00-abc12345.json', + buildLegacySessionJson({ userMessageCount: 1 }), ); const result = await getPreviousSession(mockConfig); @@ -123,95 +185,282 @@ describe('sessionSummaryUtils', () => { }); it('should return path if most recent session has more than 1 user message and no summary', async () => { - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue(['session-2024-01-01T10-00-abc12345.json']); - vi.mocked(fs.readFile).mockResolvedValue( - createSessionWithUserMessages(2), + const filePath = await writeSession( + chatsDir, + 'session-2024-01-01T10-00-abc12345.json', + buildLegacySessionJson({ userMessageCount: 2 }), ); const result = await getPreviousSession(mockConfig); - expect(result).toBe( - path.join( - '/tmp/project', - 'chats', - 'session-2024-01-01T10-00-abc12345.json', - ), - ); + expect(result).toBe(filePath); }); - it('should select most recently created session by filename', async () => { - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue([ + it('should select most recently updated session', async () => { + await writeSession( + chatsDir, 'session-2024-01-01T10-00-older000.json', + buildLegacySessionJson({ + userMessageCount: 2, + lastUpdated: '2024-01-01T10:00:00Z', + }), + ); + const newerPath = await writeSession( + chatsDir, 'session-2024-01-02T10-00-newer000.json', - ]); - vi.mocked(fs.readFile).mockResolvedValue( - createSessionWithUserMessages(2), + buildLegacySessionJson({ + userMessageCount: 2, + lastUpdated: '2024-01-02T10:00:00Z', + }), ); const result = await getPreviousSession(mockConfig); - expect(result).toBe( - path.join( - '/tmp/project', - 'chats', - 'session-2024-01-02T10-00-newer000.json', - ), - ); + expect(result).toBe(newerPath); }); - it('should return null if most recent session file is corrupted', async () => { - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue(['session-2024-01-01T10-00-abc12345.json']); - vi.mocked(fs.readFile).mockResolvedValue('invalid json'); + it('should ignore corrupted session files', async () => { + await writeSession( + chatsDir, + 'session-2024-01-01T10-00-abc12345.json', + 'invalid json', + ); const result = await getPreviousSession(mockConfig); expect(result).toBeNull(); }); + + it('should support JSONL sessions and sort by lastUpdated instead of filename', async () => { + await writeSession( + chatsDir, + 'session-2024-01-02T10-00-older000.jsonl', + buildJsonlSession({ + userMessageCount: 2, + lastUpdated: '2024-01-01T10:00:00Z', + sessionId: 'older-session', + }), + ); + const newerPath = await writeSession( + chatsDir, + 'session-2024-01-01T10-00-newer000.jsonl', + buildJsonlSession({ + userMessageCount: 2, + lastUpdated: '2024-01-03T10:00:00Z', + sessionId: 'newer-session', + }), + ); + + const result = await getPreviousSession(mockConfig); + + expect(result).toBe(newerPath); + }); + + it('should stop scanning once older mtimes cannot beat the best lastUpdated', async () => { + const loadConversationRecord = vi.mocked( + chatRecordingService.loadConversationRecord, + ); + + const currentPath = await writeSession( + chatsDir, + 'session-2024-01-03T10-00-cur00001.jsonl', + buildJsonlSession({ + sessionId: 'current-session', + userMessageCount: 2, + lastUpdated: '2024-01-03T10:00:00Z', + }), + ); + await setSessionMtime(currentPath, '2024-01-03T10:00:00Z'); + + const bestPath = await writeSession( + chatsDir, + 'session-2024-01-02T10-00-best0001.jsonl', + buildJsonlSession({ + sessionId: 'best-session', + userMessageCount: 2, + lastUpdated: '2024-01-02T10:00:00Z', + }), + ); + await setSessionMtime(bestPath, '2024-01-02T10:00:00Z'); + + const olderPath = await writeSession( + chatsDir, + 'session-2024-01-01T10-00-older001.jsonl', + buildJsonlSession({ + sessionId: 'older-session', + userMessageCount: 2, + lastUpdated: '2024-01-01T10:00:00Z', + }), + ); + await setSessionMtime(olderPath, '2024-01-01T10:00:00Z'); + + const result = await getPreviousSession(mockConfig); + + expect(result).toBe(bestPath); + expect(loadConversationRecord).toHaveBeenCalledTimes(2); + expect(loadConversationRecord).not.toHaveBeenCalledWith(olderPath, { + metadataOnly: true, + }); + }); }); describe('generateSummary', () => { it('should not throw if getPreviousSession returns null', async () => { - vi.mocked(fs.access).mockRejectedValue(new Error('ENOENT')); + await fs.rm(chatsDir, { recursive: true, force: true }); await expect(generateSummary(mockConfig)).resolves.not.toThrow(); }); - it('should generate and save summary for session needing one', async () => { - const sessionPath = path.join( - '/tmp/project', - 'chats', + it('should generate and save summary for legacy JSON sessions', async () => { + const lastUpdated = '2024-01-01T10:00:00Z'; + const filePath = await writeSession( + chatsDir, 'session-2024-01-01T10-00-abc12345.json', + buildLegacySessionJson({ userMessageCount: 2, lastUpdated }), ); - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue(['session-2024-01-01T10-00-abc12345.json']); - vi.mocked(fs.readFile).mockResolvedValue( - createSessionWithUserMessages(2), - ); - vi.mocked(fs.writeFile).mockResolvedValue(undefined); - await generateSummary(mockConfig); expect(mockGenerateSummary).toHaveBeenCalledTimes(1); - expect(fs.writeFile).toHaveBeenCalledTimes(1); - expect(fs.writeFile).toHaveBeenCalledWith( - sessionPath, - expect.stringContaining('Add dark mode to the app'), - ); + const written = JSON.parse(await fs.readFile(filePath, 'utf-8')); + expect(written.summary).toBe('Add dark mode to the app'); + expect(written.lastUpdated).toBe(lastUpdated); }); it('should handle errors gracefully without throwing', async () => { - vi.mocked(fs.access).mockResolvedValue(undefined); - mockReaddir.mockResolvedValue(['session-2024-01-01T10-00-abc12345.json']); - vi.mocked(fs.readFile).mockResolvedValue( - createSessionWithUserMessages(2), + await writeSession( + chatsDir, + 'session-2024-01-01T10-00-abc12345.json', + buildLegacySessionJson({ userMessageCount: 2 }), ); mockGenerateSummary.mockRejectedValue(new Error('API Error')); await expect(generateSummary(mockConfig)).resolves.not.toThrow(); }); + + it('should append a metadata update when saving a summary to JSONL', async () => { + const lastUpdated = '2024-01-01T10:00:00Z'; + const filePath = await writeSession( + chatsDir, + 'session-2024-01-01T10-00-abc12345.jsonl', + buildJsonlSession({ userMessageCount: 2, lastUpdated }), + ); + + await generateSummary(mockConfig); + + expect(mockGenerateSummary).toHaveBeenCalledTimes(1); + const lines = (await fs.readFile(filePath, 'utf-8')) + .split('\n') + .filter(Boolean); + const lastRecord = JSON.parse(lines[lines.length - 1]); + expect(lastRecord).toEqual({ + $set: { + summary: 'Add dark mode to the app', + }, + }); + }); + + it('should preserve a newer JSONL lastUpdated written concurrently', async () => { + const initialLastUpdated = '2024-01-01T10:00:00Z'; + const newerLastUpdated = '2024-01-02T12:34:56Z'; + const filePath = await writeSession( + chatsDir, + 'session-2024-01-01T10-00-race.jsonl', + buildJsonlSession({ + userMessageCount: 2, + lastUpdated: initialLastUpdated, + }), + ); + + const actualChatRecordingService = await vi.importActual< + typeof import('./chatRecordingService.js') + >('./chatRecordingService.js'); + let injectedConcurrentUpdate = false; + let sessionReadCount = 0; + vi.mocked(chatRecordingService.loadConversationRecord).mockImplementation( + async (targetPath, options) => { + const conversation = + await actualChatRecordingService.loadConversationRecord( + targetPath, + options, + ); + + if (targetPath === filePath) { + sessionReadCount += 1; + } + + if ( + !injectedConcurrentUpdate && + targetPath === filePath && + sessionReadCount === 2 + ) { + injectedConcurrentUpdate = true; + await fs.appendFile( + filePath, + `${JSON.stringify({ $set: { lastUpdated: newerLastUpdated } })}\n`, + ); + } + + return conversation; + }, + ); + + await generateSummary(mockConfig); + + expect(injectedConcurrentUpdate).toBe(true); + const savedConversation = + await chatRecordingService.loadConversationRecord(filePath); + expect(savedConversation?.summary).toBe('Add dark mode to the app'); + expect(savedConversation?.lastUpdated).toBe(newerLastUpdated); + + const lines = (await fs.readFile(filePath, 'utf-8')) + .split('\n') + .filter(Boolean); + const lastRecord = JSON.parse(lines[lines.length - 1]); + expect(lastRecord).toEqual({ + $set: { + summary: 'Add dark mode to the app', + }, + }); + }); + + it('should skip the active startup session and summarize the previous session', async () => { + const previousPath = await writeSession( + chatsDir, + 'session-2024-01-01T10-00-prev0001.jsonl', + buildJsonlSession({ + sessionId: 'previous-session', + userMessageCount: 2, + lastUpdated: '2024-01-01T10:00:00Z', + }), + ); + const currentPath = await writeSession( + chatsDir, + 'session-2024-01-02T10-00-cur00001.jsonl', + buildJsonlSession({ + sessionId: 'current-session', + userMessageCount: 1, + lastUpdated: '2024-01-02T10:00:00Z', + }), + ); + + await generateSummary(mockConfig); + + expect(mockGenerateSummary).toHaveBeenCalledTimes(1); + + const previousLines = (await fs.readFile(previousPath, 'utf-8')) + .split('\n') + .filter(Boolean); + expect(JSON.parse(previousLines[previousLines.length - 1])).toEqual({ + $set: { + summary: 'Add dark mode to the app', + }, + }); + + const currentLines = (await fs.readFile(currentPath, 'utf-8')) + .split('\n') + .filter(Boolean); + expect(currentLines).toHaveLength(2); + }); }); }); diff --git a/packages/core/src/services/sessionSummaryUtils.ts b/packages/core/src/services/sessionSummaryUtils.ts index c64f19870d..592a0b42bf 100644 --- a/packages/core/src/services/sessionSummaryUtils.ts +++ b/packages/core/src/services/sessionSummaryUtils.ts @@ -10,6 +10,7 @@ import { BaseLlmClient } from '../core/baseLlmClient.js'; import { debugLogger } from '../utils/debugLogger.js'; import { SESSION_FILE_PREFIX, + loadConversationRecord, type ConversationRecord, } from './chatRecordingService.js'; import fs from 'node:fs/promises'; @@ -17,6 +18,60 @@ import path from 'node:path'; const MIN_MESSAGES_FOR_SUMMARY = 1; +type LoadedSession = ConversationRecord & { + messageCount?: number; + userMessageCount?: number; +}; + +interface SessionFileCandidate { + filePath: string; + mtimeMs: number; +} + +function isSupportedSessionFile(fileName: string): boolean { + return ( + fileName.startsWith(SESSION_FILE_PREFIX) && + (fileName.endsWith('.json') || fileName.endsWith('.jsonl')) + ); +} + +async function listSessionFileCandidates( + chatsDir: string, +): Promise { + const allFiles = await fs.readdir(chatsDir); + const candidates: SessionFileCandidate[] = []; + + for (const fileName of allFiles) { + if (!isSupportedSessionFile(fileName)) continue; + + const filePath = path.join(chatsDir, fileName); + try { + const stat = await fs.stat(filePath); + if (!stat.isFile()) continue; + candidates.push({ filePath, mtimeMs: stat.mtimeMs }); + } catch { + // Skip files that disappeared between readdir and stat. + } + } + + candidates.sort((a, b) => { + const mtimeDelta = b.mtimeMs - a.mtimeMs; + if (mtimeDelta !== 0) { + return mtimeDelta; + } + + return path.basename(b.filePath).localeCompare(path.basename(a.filePath)); + }); + + return candidates; +} + +function getSessionTimestampMs(session: LoadedSession): number { + if (!session.lastUpdated) return 0; + const parsed = Date.parse(session.lastUpdated); + return Number.isNaN(parsed) ? 0 : parsed; +} + /** * Generates and saves a summary for a session file. */ @@ -24,10 +79,11 @@ async function generateAndSaveSummary( config: Config, sessionPath: string, ): Promise { - // Read session file - const content = await fs.readFile(sessionPath, 'utf-8'); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const conversation: ConversationRecord = JSON.parse(content); + const conversation = await loadConversationRecord(sessionPath); + if (!conversation) { + debugLogger.debug(`[SessionSummary] Could not read session ${sessionPath}`); + return; + } // Skip if summary already exists if (conversation.summary) { @@ -68,10 +124,17 @@ async function generateAndSaveSummary( return; } - // Re-read the file before writing to handle race conditions - const freshContent = await fs.readFile(sessionPath, 'utf-8'); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const freshConversation: ConversationRecord = JSON.parse(freshContent); + // Re-read the file before writing to handle race conditions. For JSONL we + // only need the metadata; for legacy JSON we need the full record so we can + // round-trip the messages back to disk. + const isJsonl = sessionPath.endsWith('.jsonl'); + const freshConversation = await loadConversationRecord(sessionPath, { + metadataOnly: isJsonl, + }); + if (!freshConversation) { + debugLogger.debug(`[SessionSummary] Could not re-read ${sessionPath}`); + return; + } // Check if summary was added by another process if (freshConversation.summary) { @@ -81,17 +144,33 @@ async function generateAndSaveSummary( return; } - // Add summary and write back - freshConversation.summary = summary; - freshConversation.lastUpdated = new Date().toISOString(); - await fs.writeFile(sessionPath, JSON.stringify(freshConversation, null, 2)); + if (isJsonl) { + await fs.appendFile( + sessionPath, + `${JSON.stringify({ $set: { summary } })}\n`, + ); + } else { + const lastUpdated = freshConversation.lastUpdated; + await fs.writeFile( + sessionPath, + JSON.stringify( + { + ...freshConversation, + summary, + lastUpdated, + }, + null, + 2, + ), + ); + } debugLogger.debug( `[SessionSummary] Saved summary for ${sessionPath}: "${summary}"`, ); } /** - * Finds the most recently created session that needs a summary. + * Finds the most recently updated previous session that still needs a summary. * Returns the path if it needs a summary, null otherwise. */ export async function getPreviousSession( @@ -108,53 +187,74 @@ export async function getPreviousSession( return null; } - // List session files - const allFiles = await fs.readdir(chatsDir); - const sessionFiles = allFiles.filter( - (f) => f.startsWith(SESSION_FILE_PREFIX) && f.endsWith('.json'), - ); - + const sessionFiles = await listSessionFileCandidates(chatsDir); if (sessionFiles.length === 0) { debugLogger.debug('[SessionSummary] No session files found'); return null; } - // Sort by filename descending (most recently created first) - // Filename format: session-YYYY-MM-DDTHH-MM-XXXXXXXX.json - sessionFiles.sort((a, b) => b.localeCompare(a)); + let bestPreviousSession: { + filePath: string; + conversation: LoadedSession; + } | null = null; - // Check the most recently created session - const mostRecentFile = sessionFiles[0]; - const filePath = path.join(chatsDir, mostRecentFile); - - try { - const content = await fs.readFile(filePath, 'utf-8'); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const conversation: ConversationRecord = JSON.parse(content); - - if (conversation.summary) { - debugLogger.debug( - '[SessionSummary] Most recent session already has summary', - ); - return null; + for (const { filePath, mtimeMs } of sessionFiles) { + const bestTimestamp = bestPreviousSession + ? getSessionTimestampMs(bestPreviousSession.conversation) + : null; + if ( + bestPreviousSession && + bestTimestamp !== null && + bestTimestamp > 0 && + mtimeMs < bestTimestamp + ) { + break; } - // Only generate summaries for sessions with more than 1 user message - const userMessageCount = conversation.messages.filter( - (m) => m.type === 'user', - ).length; - if (userMessageCount <= MIN_MESSAGES_FOR_SUMMARY) { - debugLogger.debug( - `[SessionSummary] Most recent session has ${userMessageCount} user message(s), skipping (need more than ${MIN_MESSAGES_FOR_SUMMARY})`, - ); - return null; - } + try { + const conversation = await loadConversationRecord(filePath, { + metadataOnly: true, + }); + if (!conversation) continue; + if (conversation.sessionId === config.getSessionId()) continue; + if (conversation.summary) continue; - return filePath; - } catch { - debugLogger.debug('[SessionSummary] Could not read most recent session'); + // Only generate summaries for sessions with more than 1 user message. + // `loadConversationRecord` populates `userMessageCount` in metadataOnly + // mode; fall back to scanning messages for the legacy fallback path. + const userMessageCount = + conversation.userMessageCount ?? + conversation.messages.filter((message) => message.type === 'user') + .length; + if (userMessageCount <= MIN_MESSAGES_FOR_SUMMARY) { + continue; + } + + if ( + !bestPreviousSession || + getSessionTimestampMs(conversation) > + getSessionTimestampMs(bestPreviousSession.conversation) || + (getSessionTimestampMs(conversation) === + getSessionTimestampMs(bestPreviousSession.conversation) && + path + .basename(filePath) + .localeCompare(path.basename(bestPreviousSession.filePath)) > 0) + ) { + bestPreviousSession = { filePath, conversation }; + } + } catch { + // Ignore unreadable session files + } + } + + if (!bestPreviousSession) { + debugLogger.debug( + '[SessionSummary] No previous session needs summary generation', + ); return null; } + + return bestPreviousSession.filePath; } catch (error) { debugLogger.debug( `[SessionSummary] Error finding previous session: ${error instanceof Error ? error.message : String(error)}`, diff --git a/schemas/settings.schema.json b/schemas/settings.schema.json index d9fb8e3a11..e7b362fc4e 100644 --- a/schemas/settings.schema.json +++ b/schemas/settings.schema.json @@ -2993,9 +2993,9 @@ }, "memoryV2": { "title": "Memory v2", - "description": "Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Routes facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits).", - "markdownDescription": "Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Routes facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits).\n\n- Category: `Experimental`\n- Requires restart: `yes`\n- Default: `false`", - "default": false, + "description": "Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Route facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits). Set to false to fall back to the legacy save_memory tool.", + "markdownDescription": "Disable the built-in save_memory tool and let the main agent persist project context by editing markdown files directly with edit/write_file. Route facts across four tiers: team-shared conventions go to project GEMINI.md files, project-specific personal notes go to the per-project private memory folder (MEMORY.md as index + sibling .md files for detail), and cross-project personal preferences go to the global ~/.gemini/GEMINI.md (the only file under ~/.gemini/ that the agent can edit — settings, credentials, etc. remain off-limits). Set to false to fall back to the legacy save_memory tool.\n\n- Category: `Experimental`\n- Requires restart: `yes`\n- Default: `true`", + "default": true, "type": "boolean" }, "autoMemory": {