From 88985511c6bdaaa2157800ea424caf5d19e8fd42 Mon Sep 17 00:00:00 2001 From: Aishanee Shah Date: Mon, 23 Feb 2026 20:55:27 +0000 Subject: [PATCH] feat(prompts): redesign SI for modularity, skill activation, and essential workflows - Redesigned CORE_SI_SKELETON for maximum reasoning fidelity and minimum token usage. - Extracted Software Engineering and New Application workflows to dynamic skills. - Added 'Essential Workflows' section to maintain visibility for core user journeys. - Implemented precedence-based sorting for available skills (Workspace > User > Built-in). - Added behavioral tests in evals/skill_activation.eval.ts to verify proactive skill activation. - Fixed pre-existing build error in useGeminiStream.ts related to missing FinishReason values. --- evals/skill_activation.eval.ts | 110 ++++++++++++++++++ packages/cli/src/ui/hooks/useGeminiStream.ts | 3 + .../core/src/prompts/promptProvider.test.ts | 59 ++++++++++ packages/core/src/prompts/promptProvider.ts | 7 +- packages/core/src/prompts/skeleton.ts | 5 +- .../core/src/prompts/snippets.capability.ts | 53 ++++++++- packages/core/src/prompts/snippets.ts | 1 + 7 files changed, 228 insertions(+), 10 deletions(-) create mode 100644 evals/skill_activation.eval.ts diff --git a/evals/skill_activation.eval.ts b/evals/skill_activation.eval.ts new file mode 100644 index 0000000000..3f7dbdbfb4 --- /dev/null +++ b/evals/skill_activation.eval.ts @@ -0,0 +1,110 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe } from 'vitest'; +import { evalTest } from './test-helper.js'; + +describe('Skill Activation Behavioral Evals', () => { + /** + * Tests that the model proactively activates the software-engineering skill + * when faced with a typical engineering task like bug fixing. + */ + evalTest('ALWAYS_PASSES', { + name: 'should activate software-engineering skill for bug fixes', + prompt: + 'There is a bug in the greeting logic in src/index.ts. Please fix it.', + files: { + 'src/index.ts': + 'export const greet = (name: string) => `Hello, ${name}!`;', + 'src/index.test.ts': ` +import { greet } from './index'; +import { expect, test } from 'vitest'; +test('greet', () => { expect(greet('World')).toBe('Hello, World!'); }); + `, + }, + assert: async (rig) => { + await rig.expectToolCallSuccess(['activate_skill'], undefined, (args) => { + try { + const parsed = JSON.parse(args); + return parsed.name === 'software-engineering'; + } catch { + return false; + } + }); + }, + }); + + /** + * Tests that the model proactively activates the new-application skill + * when asked to scaffold a new project. + */ + evalTest('ALWAYS_PASSES', { + name: 'should activate new-application skill for prototyping', + prompt: 'Build me a new Todo app using React and Vanilla CSS.', + assert: async (rig) => { + await rig.expectToolCallSuccess(['activate_skill'], undefined, (args) => { + try { + const parsed = JSON.parse(args); + return parsed.name === 'new-application'; + } catch { + return false; + } + }); + }, + }); + + /** + * Tests that the model proactively activates the docs-writer skill + * when asked to update documentation files. + */ + evalTest('ALWAYS_PASSES', { + name: 'should activate docs-writer skill for documentation tasks', + prompt: + 'Update the documentation in docs/index.md to include the new features.', + files: { + 'docs/index.md': `# Documentation + +Existing content.`, + }, + assert: async (rig) => { + await rig.expectToolCallSuccess(['activate_skill'], undefined, (args) => { + try { + const parsed = JSON.parse(args); + return parsed.name === 'docs-writer'; + } catch { + return false; + } + }); + }, + }); + + /** + * Tests that the model can handle multi-step tasks that might require + * activating multiple skills sequentially (though usually it just activates one). + */ + evalTest('USUALLY_PASSES', { + name: 'should activate software-engineering even when the prompt is slightly indirect', + prompt: + 'The CI is failing on the main branch. Can you investigate and fix whatever is broken?', + files: { + 'package.json': '{ "scripts": { "test": "vitest run" } }', + 'src/logic.ts': + 'export const compute = () => { throw new Error("Broken"); };', + 'src/logic.test.ts': + 'import { compute } from "./logic"; import { test } from "vitest"; test("compute", () => { compute(); });', + }, + assert: async (rig) => { + await rig.expectToolCallSuccess(['activate_skill'], undefined, (args) => { + try { + const parsed = JSON.parse(args); + return parsed.name === 'software-engineering'; + } catch { + return false; + } + }); + }, + }); +}); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index 7a20557558..a1f175de2d 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -961,6 +961,9 @@ export const useGeminiStream = ( [FinishReason.SPII]: 'Response stopped due to sensitive personally identifiable information.', [FinishReason.OTHER]: 'Response stopped for other reasons.', + [FinishReason.IMAGE_RECITATION]: + 'Response stopped due to image recitation policy.', + [FinishReason.IMAGE_OTHER]: 'Response stopped for other image reasons.', [FinishReason.MALFORMED_FUNCTION_CALL]: 'Response stopped due to malformed function call.', [FinishReason.IMAGE_SAFETY]: diff --git a/packages/core/src/prompts/promptProvider.test.ts b/packages/core/src/prompts/promptProvider.test.ts index 6d79e90f28..278cda4870 100644 --- a/packages/core/src/prompts/promptProvider.test.ts +++ b/packages/core/src/prompts/promptProvider.test.ts @@ -118,6 +118,65 @@ describe('PromptProvider', () => { ); }); + it('should include skill activation guidance and placeholders in capability variant', () => { + vi.mocked(mockConfig.getActiveModel).mockReturnValue( + PREVIEW_GEMINI_FLASH_MODEL, + ); + vi.mocked(mockConfig.getSkillManager().getSkills).mockReturnValue([ + { + name: 'software-engineering', + description: 'Expert guidance.', + location: '/path/to/skill', + body: 'Skill body', + }, + ]); + + const provider = new PromptProvider(); + const prompt = provider.getCoreSystemPrompt(mockConfig); + + expect(prompt).toContain('## Essential Workflows'); + expect(prompt).toContain('software-engineering'); + expect(prompt).toContain( + 'Use `activate_skill` to enable specialized expert guidance', + ); + }); + + it('should sort available skills (workspace first) and elevate activated skills in capability variant', () => { + vi.mocked(mockConfig.getActiveModel).mockReturnValue( + PREVIEW_GEMINI_FLASH_MODEL, + ); + vi.mocked(mockConfig.getSkillManager().getSkills).mockReturnValue([ + { + name: 'builtin-skill', + description: 'Builtin description', + location: '/path/to/builtin', + isBuiltin: true, + }, + { + name: 'workspace-skill', + description: 'Workspace description', + location: '/path/to/workspace', + isBuiltin: false, + }, + ]); + vi.mocked(mockConfig.getSkillManager().isSkillActive).mockImplementation( + (name) => name === 'workspace-skill', + ); + + const provider = new PromptProvider(); + const prompt = provider.getCoreSystemPrompt(mockConfig); + + // Activated skills should be before available skills in the Capabilities section + const activatedIndex = prompt.indexOf('## Activated Skills'); + const availableIndex = prompt.indexOf('## Available Skills'); + expect(activatedIndex).toBeLessThan(availableIndex); + + // Workspace skill should be before built-in skill in Available Skills + const workspaceIndex = prompt.indexOf('workspace-skill', availableIndex); + const builtinIndex = prompt.indexOf('builtin-skill', availableIndex); + expect(workspaceIndex).toBeLessThan(builtinIndex); + }); + it('should handle multiple context filenames in user memory section', () => { vi.mocked(getAllGeminiMdFilenames).mockReturnValue([ DEFAULT_CONTEXT_FILENAME, diff --git a/packages/core/src/prompts/promptProvider.ts b/packages/core/src/prompts/promptProvider.ts index dd1901d64b..dc498327cd 100644 --- a/packages/core/src/prompts/promptProvider.ts +++ b/packages/core/src/prompts/promptProvider.ts @@ -128,6 +128,7 @@ export class PromptProvider { name: s.name, description: s.description, location: s.location, + isBuiltin: s.isBuiltin, })), ); basePrompt = applySubstitutions( @@ -171,6 +172,7 @@ export class PromptProvider { name: s.name, description: s.description, location: s.location, + isBuiltin: s.isBuiltin, })), skills.length > 0, ), @@ -241,7 +243,7 @@ export class PromptProvider { const getCoreSystemPrompt = activeSnippets.getCoreSystemPrompt as ( options: snippets.SystemPromptOptions, ) => string; - + basePrompt = getCoreSystemPrompt(options); } @@ -269,7 +271,6 @@ export class PromptProvider { /* eslint-enable @typescript-eslint/no-unsafe-assignment */ return sanitizedPrompt; - } getCompressionPrompt(config: Config): string { @@ -279,7 +280,7 @@ export class PromptProvider { ); const isModernModel = supportsModernFeatures(desiredModel); const activeSnippets = isModernModel ? snippets : legacySnippets; - + return activeSnippets.getCompressionPrompt(); } diff --git a/packages/core/src/prompts/skeleton.ts b/packages/core/src/prompts/skeleton.ts index 9e73baf2cf..2c49180f81 100644 --- a/packages/core/src/prompts/skeleton.ts +++ b/packages/core/src/prompts/skeleton.ts @@ -24,9 +24,10 @@ You are Gemini CLI, an expert agent. Help users safely and effectively. - **Self-Correction:** If progress stalls or deviates from the goal, pause and "take a step back." If you realize you are making fixes unrelated to the original objective, stop, revert to a stable state if necessary, and re-approach the problem. # Capabilities +{{ACTIVATED_SKILLS}} {{AVAILABLE_SUB_AGENTS}} {{AVAILABLE_SKILLS}} -{{ACTIVATED_SKILLS}} +- **Activation:** Use \`activate_skill\` to enable specialized expert guidance when a task matches a skill's description. # Operational Style - **Tone:** Professional, direct, senior engineer peer. @@ -37,4 +38,6 @@ You are Gemini CLI, an expert agent. Help users safely and effectively. {{HOOK_CONTEXT}} {{PLAN_MODE_OVERRIDE}} {{GIT_REPO_CONTEXT}} +{{SANDBOX_CONTEXT}} +{{YOLO_MODE_CONTEXT}} `.trim(); diff --git a/packages/core/src/prompts/snippets.capability.ts b/packages/core/src/prompts/snippets.capability.ts index de9efab854..fda53c9c82 100644 --- a/packages/core/src/prompts/snippets.capability.ts +++ b/packages/core/src/prompts/snippets.capability.ts @@ -56,10 +56,28 @@ export function getCoreSystemPrompt( '{{GIT_REPO_CONTEXT}}', renderGitRepo(options.gitRepo), ); + prompt = prompt.replace( + '{{SANDBOX_CONTEXT}}', + renderSandbox(options.sandbox), + ); + prompt = prompt.replace( + '{{YOLO_MODE_CONTEXT}}', + renderInteractiveYoloMode(options.interactiveYoloMode), + ); return prompt.trim(); } +function renderSandbox(mode?: snippets.SandboxMode): string { + if (!mode || mode === 'outside') return ''; + return `## Sandbox\nYou are in a ${mode} sandbox. Access to host resources and files outside the project is restricted. If a command fails with 'Operation not permitted', explain it might be due to sandboxing.`; +} + +function renderInteractiveYoloMode(enabled?: boolean): string { + if (!enabled) return ''; + return `## Autonomous Mode (YOLO)\nMinimal interruption requested. Use \`ask_user\` ONLY for critical architectural pivots or fundamental ambiguity. Otherwise, make expert decisions autonomously.`; +} + function renderSubAgents(subAgents?: snippets.SubAgentOptions[]): string { if (!subAgents || subAgents.length === 0) return ''; const agents = subAgents @@ -70,22 +88,45 @@ function renderSubAgents(subAgents?: snippets.SubAgentOptions[]): string { function renderAvailableSkills(skills?: snippets.AgentSkillOptions[]): string { if (!skills || skills.length === 0) return ''; - const available = skills - .map((s) => `- **${s.name}**: ${s.description}`) - .join('\n'); - return `## Available Skills\nActivate with \`activate_skill\`:\n${available}`; + + // Essential Workflows that we want to keep highly visible + const essentialNames = ['software-engineering', 'new-application']; + const essential = skills.filter((s) => essentialNames.includes(s.name)); + const others = skills.filter((s) => !essentialNames.includes(s.name)); + + // Sort others: Workspace/User skills first, Built-ins last + const sortedOthers = [...others].sort((a, b) => { + if (a.isBuiltin && !b.isBuiltin) return 1; + if (!a.isBuiltin && b.isBuiltin) return -1; + return a.name.localeCompare(b.name); + }); + + const renderList = (list: snippets.AgentSkillOptions[]) => + list.map((s) => `- **${s.name}**: ${s.description}`).join('\n'); + + let output = ''; + if (essential.length > 0) { + output += `## Essential Workflows\nActivate these for core agent behaviors:\n${renderList(essential)}\n\n`; + } + + if (sortedOthers.length > 0) { + output += `## Available Skills\nProactively activate a skill with \`activate_skill\` when a task matches its expertise. This provides specialized protocols and expert guidance.\n${renderList(sortedOthers)}`; + } + + return output.trim(); } function renderActivatedSkills( skills?: snippets.ActivatedSkillOptions[], ): string { if (!skills || skills.length === 0) return ''; - return skills + const skillsXml = skills .map( (s) => - `### \n${s.body}\n### `, + `\n${s.body}\n`, ) .join('\n\n'); + return `## Activated Skills\nFollow \`\` instructions as expert guidance. These rules supersede general workflows.\n${skillsXml}`; } function renderHookContext(enabled?: boolean): string { diff --git a/packages/core/src/prompts/snippets.ts b/packages/core/src/prompts/snippets.ts index fe0d025cfa..15c68acd8b 100644 --- a/packages/core/src/prompts/snippets.ts +++ b/packages/core/src/prompts/snippets.ts @@ -85,6 +85,7 @@ export interface AgentSkillOptions { name: string; description: string; location: string; + isBuiltin?: boolean; } export interface SubAgentOptions {