feat(optimization): finalized iterative-surgical optimization suite (checkpoint)

- Flatten directory structure by moving masking and evals to scripts root.
- Merge evaluation metrics into scripts/optimization/evals.
- Restore and verify extraction tests for the new structure.

.gitignore

@@ -62,3 +62,4 @@ gemini-debug.log
 .gemini-clipboard/
 .eslintcache
 evals/logs/
+data/optimization/

data/manifest.json

@@ -0,0 +1,80 @@
+{
+  "project": "Gemini CLI Tool Alignment Optimization",
+  "version": "1.0.0",
+  "optimization_constraints": {
+    "immutable_tokens": [
+      "glob",
+      "grep_search",
+      "list_directory",
+      "read_file",
+      "run_shell_command",
+      "write_file",
+      "replace",
+      "google_web_search",
+      "write_todos",
+      "web_fetch",
+      "read_many_files",
+      "save_memory",
+      "get_internal_docs",
+      "activate_skill",
+      "ask_user",
+      "exit_plan_mode",
+      "enter_plan_mode",
+      "codebase_investigator",
+      "cli_help",
+      "generalist"
+    ],
+    "protected_variables": [
+      "${FILE_PATH}",
+      "${DIR_PATH}",
+      "${PATTERN}",
+      "${OLD_STRING}",
+      "${NEW_STRING}",
+      "${GREP_PATTERN}",
+      "${SEARCH_PATTERN}"
+    ]
+  },
+  "data_inventory": {
+    "target_samples_per_tool": 5,
+    "overrides": {
+      "replace": 12,
+      "write_file": 10
+    },
+    "tools": {
+      "glob": { "description": "Find files by glob pattern" },
+      "grep_search": { "description": "Search text in files" },
+      "list_directory": { "description": "List files in a directory" },
+      "read_file": { "description": "Read a single file" },
+      "run_shell_command": { "description": "Execute shell commands" },
+      "write_file": { "description": "Write a complete file" },
+      "replace": { "description": "Surgical text replacement" },
+      "google_web_search": { "description": "Web search via Google" },
+      "write_todos": { "description": "Manage subtasks" },
+      "web_fetch": { "description": "Extract content from URLs" },
+      "read_many_files": { "description": "Read multiple files" },
+      "save_memory": { "description": "Global user preferences" },
+      "get_internal_docs": { "description": "Gemini CLI internal docs" },
+      "activate_skill": { "description": "Enable specialized skills" },
+      "ask_user": { "description": "Interactive user questions" },
+      "enter_plan_mode": { "description": "Start planning mode" },
+      "exit_plan_mode": { "description": "Exit planning mode" },
+      "codebase_investigator": {
+        "description": "High-level architecture mapping"
+      },
+      "cli_help": { "description": "Assistance with Gemini CLI" },
+      "generalist": { "description": "General purpose agent delegation" }
+    },
+    "file_descriptions": {
+      "data/tool_alignment.jsonl": "Ensures the model selects the correct built-in tool over generic shell commands and optimizes for brevity."
+    },
+    "optimization_targets": {
+      "snippets": [
+        "renderCoreMandates",
+        "renderPrimaryWorkflows",
+        "renderOperationalGuidelines",
+        "renderSubAgents",
+        "renderGitRepo"
+      ]
+    }
+  }
+}

data/tool_alignment.jsonl

@@ -0,0 +1,113 @@
+{"id":"read_file-01","metadata":{"tags":["tool:read_file","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z","platform":"darwin"},"input":{"user_query":"What are the contents of package.json?"},"expected":{"tool_calls":[{"name":"read_file","arguments":{"file_path":"package.json"}}],"rationale":"Directly use read_file for reading file contents instead of shell 'cat'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cat package.json"}}],"reason":"Generic shell command 'cat' is used instead of the specialized read_file tool.","severity":"high"},{"tool_calls":[{"name":"read_file","arguments":{"file_path":"package.json"}}],"output_text":"Certainly! I can help you read that file. Here are the contents of package.json:","reason":"Correct tool but excessive conversational filler.","severity":"low"}]}
+{"id":"read_file-02","metadata":{"tags":["tool:read_file","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z","platform":"win32"},"input":{"user_query":"Show me the content of the README.md file"},"expected":{"tool_calls":[{"name":"read_file","arguments":{"file_path":"README.md"}}],"rationale":"Directly use read_file for reading file contents instead of shell 'type'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"type README.md"}}],"reason":"Generic shell command 'type' is used on Windows instead of the specialized read_file tool.","severity":"high"}]}
+{"id":"read_file-03","metadata":{"tags":["tool:read_file","range-efficiency"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Read the first 50 lines of src/index.ts"},"expected":{"tool_calls":[{"name":"read_file","arguments":{"file_path":"src/index.ts","start_line":1,"end_line":50}}],"rationale":"Use line range parameters to read only the requested portion of a file."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"head -n 50 src/index.ts"}}],"reason":"Used shell 'head' instead of read_file range parameters.","severity":"high"}]}
+{"id":"read_file-04","metadata":{"tags":["tool:read_file","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Get the contents of the .env file"},"expected":{"tool_calls":[{"name":"read_file","arguments":{"file_path":".env"}}],"rationale":"Directly use read_file for reading file contents."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cat .env"}}],"reason":"Used shell 'cat' for a file read.","severity":"high"}]}
+{"id":"read_file-05","metadata":{"tags":["tool:read_file","precision"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Check the version in the VERSION file"},"expected":{"tool_calls":[{"name":"read_file","arguments":{"file_path":"VERSION"}}],"rationale":"Use read_file for quick file inspection."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cat VERSION"}}],"reason":"Used shell 'cat' instead of read_file.","severity":"high"}]}
+{"id":"list_directory-01","metadata":{"tags":["tool:list_directory","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z","platform":"darwin"},"input":{"user_query":"List the files in the src directory"},"expected":{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"src"}}],"rationale":"Use list_directory for viewing folder contents instead of shell 'ls'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"ls src"}}],"reason":"Used shell 'ls' instead of list_directory.","severity":"high"}]}
+{"id":"list_directory-02","metadata":{"tags":["tool:list_directory","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z","platform":"win32"},"input":{"user_query":"Show me what's in the docs folder"},"expected":{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"docs"}}],"rationale":"Use list_directory for viewing folder contents instead of shell 'dir'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"dir docs"}}],"reason":"Used shell 'dir' on Windows instead of list_directory.","severity":"high"}]}
+{"id":"list_directory-03","metadata":{"tags":["tool:list_directory","recursive-failure"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What files are in the project root?"},"expected":{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"."}}],"rationale":"Use list_directory for top-level directory listing."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"ls -F"}}],"reason":"Used shell 'ls' instead of list_directory.","severity":"high"}]}
+{"id":"list_directory-04","metadata":{"tags":["tool:list_directory","filter-efficiency"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"List files in 'tests' but ignore '.tmp' files"},"expected":{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"tests","ignore":["*.tmp"]}}],"rationale":"Use the 'ignore' parameter of list_directory to filter results efficiently."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"ls tests | grep -v '.tmp'"}}],"reason":"Used a shell pipeline (ls | grep) instead of built-in list_directory filtering.","severity":"high"}]}
+{"id":"list_directory-05","metadata":{"tags":["tool:list_directory","precision"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What is in the current folder?"},"expected":{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"."}}],"rationale":"Use list_directory for exploring the current working directory."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"ls"}}],"reason":"Used shell 'ls' instead of list_directory.","severity":"high"}]}
+{"id":"write_file-01","metadata":{"tags":["tool:write_file","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Create a file named hello.txt with the content 'Hello World'"},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"hello.txt","content":"Hello World"}}],"rationale":"Use write_file to create new files instead of shell 'echo'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"echo 'Hello World' > hello.txt"}}],"reason":"Used shell redirection 'echo >' to create a file.","severity":"high"}]}
+{"id":"write_file-02","metadata":{"tags":["tool:write_file","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Save this config to config.json: { \"active\": true }"},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"config.json","content":"{ \"active\": true }"}}],"rationale":"Use write_file for creating structured data files."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"echo '{ \"active\": true }' > config.json"}}],"reason":"Used shell 'echo' for file creation.","severity":"high"}]}
+{"id":"write_file-03","metadata":{"tags":["tool:write_file","precision"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Create a new empty file called .gitignore"},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":".gitignore","content":""}}],"rationale":"Use write_file with empty content to create empty files."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"touch .gitignore"}}],"reason":"Used shell 'touch' instead of write_file.","severity":"high"}]}
+{"id":"write_file-04","metadata":{"tags":["tool:write_file","boilerplate"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Initialize a new README.md with some basic info"},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"README.md","content":"# Project\n\nThis is a new project."}}],"rationale":"Use write_file for initial file creation."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cat <<EOF > README.md\n# Project\n\nThis is a new project.\nEOF"}}],"reason":"Used shell heredoc instead of write_file.","severity":"high"}]}
+{"id":"write_file-05","metadata":{"tags":["tool:write_file","overwrite"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Overwrite the current LICENSE file with the Apache-2.0 header"},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"LICENSE","content":"Apache-2.0 Header..."}}],"rationale":"Use write_file to overwrite existing file content entirely."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cp apache_header.txt LICENSE"}}],"reason":"Used shell 'cp' to overwrite a file instead of write_file.","severity":"high"}]}
+{"id":"write_file-06","metadata":{"tags":["tool:write_file","large-content"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Create a new utility file in src/utils/logger.ts with a full logger implementation"},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"src/utils/logger.ts","content":"export class Logger { ... }"}}],"rationale":"Use write_file for creating new source files with complete implementations."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"mkdir -p src/utils && echo 'export class Logger { ... }' > src/utils/logger.ts"}}],"reason":"Used shell 'mkdir' and 'echo' for file creation. write_file handles parent directory creation automatically.","severity":"high"}]}
+{"id":"write_file-07","metadata":{"tags":["tool:write_file","data-integrity"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Write the tool_alignment schema to data/schema.ts"},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"data/schema.ts","content":"export interface Scenario { ... }"}}],"rationale":"Use write_file for adding new code files to the project."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cat > data/schema.ts <<EOF\n...\nEOF"}}],"reason":"Used shell redirection instead of write_file tool.","severity":"high"}]}
+{"id":"write_file-08","metadata":{"tags":["tool:write_file","new-app"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Create a new index.html for the landing page."},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"index.html","content":"<!DOCTYPE html>..."}}],"rationale":"Use write_file for creating web assets."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"echo '<!DOCTYPE html>...' > index.html"}}],"reason":"Used shell echo for file creation.","severity":"high"}]}
+{"id":"write_file-09","metadata":{"tags":["tool:write_file","json-creation"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Create a manifest.json file for the app."},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"manifest.json","content":"{ \"name\": \"My App\" }"}}],"rationale":"Use write_file for structured data files."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"echo '{ \"name\": \"My App\" }' > manifest.json"}}],"reason":"Used shell echo instead of write_file.","severity":"high"}]}
+{"id":"write_file-10","metadata":{"tags":["tool:write_file","overwrite-safety"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Update the entire CONTRIBUTING.md with new instructions."},"expected":{"tool_calls":[{"name":"write_file","arguments":{"file_path":"CONTRIBUTING.md","content":"# Contributing..."}}],"rationale":"Use write_file for full-file updates when the entire content changes."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cat > CONTRIBUTING.md <<EOF\n...\nEOF"}}],"reason":"Used shell heredoc for full-file update.","severity":"high"}]}
+{"id":"replace-01","metadata":{"tags":["tool:replace","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Change the port from 3000 to 8080 in server.ts"},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"server.ts","old_string":"port: 3000","new_string":"port: 8080","instruction":"Change the server port to 8080."}}],"rationale":"Use replace for surgical text updates instead of shell 'sed'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"sed -i 's/port: 3000/port: 8080/g' server.ts"}}],"reason":"Used shell 'sed' for text replacement.","severity":"high"}]}
+{"id":"replace-02","metadata":{"tags":["tool:replace","inefficiency"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Fix a typo in index.ts: change 'recieve' to 'receive'"},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"index.ts","old_string":"recieve","new_string":"receive","instruction":"Fix typo: 'recieve' to 'receive'."}}],"rationale":"Use replace for small fixes instead of rewriting the entire file with write_file."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":"index.ts","content":"(full content with fix)"}}],"reason":"Rewrote entire file for a single word fix. replace is more efficient and safer for large files.","severity":"medium"}]}
+{"id":"replace-03","metadata":{"tags":["tool:replace","multi-occurrence"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Rename the variable 'userCount' to 'totalUsers' throughout app.ts"},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"app.ts","old_string":"userCount","new_string":"totalUsers","allow_multiple":true,"instruction":"Rename variable 'userCount' to 'totalUsers' globally in the file."}}],"rationale":"Use replace with allow_multiple=true for global renaming within a single file."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"sed -i 's/userCount/totalUsers/g' app.ts"}}],"reason":"Used shell 'sed' instead of replace.","severity":"high"}]}
+{"id":"replace-04","metadata":{"tags":["tool:replace","context-safety"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Update the deprecated API call in src/client.ts"},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"src/client.ts","old_string":"api.oldMethod()","new_string":"api.newMethod()","instruction":"Update deprecated API call from oldMethod to newMethod."}}],"rationale":"Use replace to target specific code blocks safely."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"grep -l 'api.oldMethod()' src/client.ts && sed -i 's/api.oldMethod()/api.newMethod()/g' src/client.ts"}}],"reason":"Used a combination of shell commands instead of replace.","severity":"high"}]}
+{"id":"replace-05","metadata":{"tags":["tool:replace","multiline"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Replace the existing try/catch block in main.ts with a more robust version"},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"main.ts","old_string":"try {\n  doWork();\n} catch (e) {\n  console.log(e);\n}","new_string":"try {\n  await doWork();\n} catch (e) {\n  logger.error('Failed to do work', e);\n  throw e;\n}","instruction":"Improve error handling in the main work loop."}}],"rationale":"Use replace for multiline code block updates. It's safer than complex sed commands."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":"main.ts","content":"(entire file)"}}],"reason":"Rewriting the whole file for a small block change is inefficient.","severity":"medium"}]}
+{"id":"replace-06","metadata":{"tags":["tool:replace","steering-example"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Add the following log line to server.ts: console.log('started');"},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"server.ts","old_string":"app.listen(3000, () => {","new_string":"app.listen(3000, () => {\n  console.log('started');","instruction":"Add a start log to the server listen callback."}}],"rationale":"Prefer using replace to add content to existing files instead of shell redirection (>>)."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"echo \"console.log('started');\" >> server.ts"}}],"reason":"Used shell redirection to append content. This is risky for source code as it doesn't guarantee correct placement or formatting.","severity":"high"}]}
+{"id":"replace-07","metadata":{"tags":["tool:replace","version-bump"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Bump the version in package.json from 1.0.0 to 1.1.0."},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"package.json","old_string":"\"version\": \"1.0.0\"","new_string":"\"version\": \"1.1.0\"","instruction":"Bump project version to 1.1.0."}}],"rationale":"Use replace for surgical updates to configuration files."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm version 1.1.0 --no-git-tag-version"}}],"reason":"While the command works, the optimizer wants to see the model use internal tools for precise control when asked.","severity":"low"}]}
+{"id":"replace-08","metadata":{"tags":["tool:replace","comment-update"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Update the TODO comment in database.ts to 'Fixed in v2'."},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"database.ts","old_string":"// TODO: optimize query","new_string":"// Fixed in v2","instruction":"Mark TODO as fixed."}}],"rationale":"Use replace for updating documentation and comments."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"sed -i 's|// TODO: optimize query|// Fixed in v2|' database.ts"}}],"reason":"Used shell sed for comment update.","severity":"high"}]}
+{"id":"replace-09","metadata":{"tags":["tool:replace","import-fix"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Fix the import in main.ts: change '../utils' to '@utils'."},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"main.ts","old_string":"import { log } from '../utils'","new_string":"import { log } from '@utils'","instruction":"Fix relative import path."}}],"rationale":"Use replace for fixing import paths."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"sed -i \"s|'../utils'|'@utils'|\" main.ts"}}],"reason":"Used shell sed for import fix.","severity":"high"}]}
+{"id":"replace-10","metadata":{"tags":["tool:replace","logic-patch"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Fix the off-by-one error in the loop in utils.ts."},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"utils.ts","old_string":"for (let i = 0; i <= arr.length; i++)","new_string":"for (let i = 0; i < arr.length; i++)","instruction":"Fix off-by-one error in loop condition."}}],"rationale":"Use replace for surgical logic fixes."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":"utils.ts","content":"..."}}],"reason":"Rewrote entire file for a one-character fix.","severity":"medium"}]}
+{"id":"replace-11","metadata":{"tags":["tool:replace","css-update"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Change the background color to #fff in styles.css."},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"styles.css","old_string":"background: #000;","new_string":"background: #fff;","instruction":"Update background color."}}],"rationale":"Use replace for styling updates."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"sed -i 's/#000/#fff/' styles.css"}}],"reason":"Used shell sed for CSS update.","severity":"high"}]}
+{"id":"replace-12","metadata":{"tags":["tool:replace","md-update"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Update the title in README.md."},"expected":{"tool_calls":[{"name":"replace","arguments":{"file_path":"README.md","old_string":"# Old Title","new_string":"# New Title","instruction":"Update project title."}}],"rationale":"Use replace for documentation updates."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"sed -i 's/# Old Title/# New Title/' README.md"}}],"reason":"Used shell sed for Markdown update.","severity":"high"}]}
+{"id":"grep_search-01","metadata":{"tags":["tool:grep_search","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Search for all occurrences of 'TODO' in the src directory"},"expected":{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"TODO","dir_path":"src"}}],"rationale":"Use grep_search for recursive text searching instead of shell 'grep'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"grep -r 'TODO' src"}}],"reason":"Used shell 'grep' instead of grep_search.","severity":"high"}]}
+{"id":"grep_search-02","metadata":{"tags":["tool:grep_search","precision"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Find where 'AuthService' is defined in the codebase"},"expected":{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"class AuthService","include_pattern":"**/*.ts"}}],"rationale":"Use grep_search with include_pattern for precise code discovery."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"grep -r 'class AuthService' ."}}],"reason":"Used shell 'grep' instead of grep_search.","severity":"high"}]}
+{"id":"grep_search-03","metadata":{"tags":["tool:grep_search","case-sensitive"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Search for 'ERROR' (case-sensitive) in logs/"},"expected":{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"ERROR","dir_path":"logs","case_sensitive":true}}],"rationale":"Use grep_search with case_sensitive parameter."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"grep 'ERROR' logs/*"}}],"reason":"Used shell 'grep' instead of grep_search.","severity":"high"}]}
+{"id":"grep_search-04","metadata":{"tags":["tool:grep_search","fixed-strings"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Find the exact string 'api/v1/users' in the config files"},"expected":{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"api/v1/users","fixed_strings":true,"include_pattern":"config/**"}}],"rationale":"Use fixed_strings:true for literal matches to avoid regex escaping issues."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"grep -F 'api/v1/users' config/*"}}],"reason":"Used shell 'grep -F' instead of grep_search.","severity":"high"}]}
+{"id":"grep_search-05","metadata":{"tags":["tool:grep_search","context"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Search for 'db.connect' and show 3 lines of context"},"expected":{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"db.connect","context":3}}],"rationale":"Use context parameter to get surrounding lines."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"grep -C 3 'db.connect' ."}}],"reason":"Used shell 'grep -C' instead of grep_search.","severity":"high"}]}
+{"id":"glob-01","metadata":{"tags":["tool:glob","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Find all .test.ts files in the integration-tests folder"},"expected":{"tool_calls":[{"name":"glob","arguments":{"pattern":"integration-tests/**/*.test.ts"}}],"rationale":"Use glob for recursive file discovery instead of shell 'find'."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"find integration-tests -name '*.test.ts'"}}],"reason":"Used shell 'find' instead of glob.","severity":"high"}]}
+{"id":"glob-02","metadata":{"tags":["tool:glob","precision"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Where are the CSS files located in the project?"},"expected":{"tool_calls":[{"name":"glob","arguments":{"pattern":"**/*.css"}}],"rationale":"Use glob with double asterisks for project-wide file discovery."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"find . -name '*.css'"}}],"reason":"Used shell 'find' instead of glob.","severity":"high"}]}
+{"id":"glob-03","metadata":{"tags":["tool:glob","shell-avoidance"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"List all markdown files in the docs directory"},"expected":{"tool_calls":[{"name":"glob","arguments":{"pattern":"docs/**/*.md"}}],"rationale":"Use glob for finding files matching a pattern."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"ls docs/**/*.md"}}],"reason":"Used shell 'ls' with globs instead of the specialized glob tool.","severity":"high"}]}
+{"id":"glob-04","metadata":{"tags":["tool:glob","ignore-patterns"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Find all .js files, ignoring node_modules"},"expected":{"tool_calls":[{"name":"glob","arguments":{"pattern":"**/*.js","respect_git_ignore":true}}],"rationale":"glob respects .gitignore by default, which usually includes node_modules."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"find . -name '*.js' -not -path '*/node_modules/*'"}}],"reason":"Used complex shell find instead of glob.","severity":"high"}]}
+{"id":"glob-05","metadata":{"tags":["tool:glob","sorting"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Find the most recently modified log files"},"expected":{"tool_calls":[{"name":"glob","arguments":{"pattern":"logs/*.log"}}],"rationale":"glob returns results sorted by modification time (newest first)."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"ls -t logs/*.log"}}],"reason":"Used shell 'ls -t' instead of glob.","severity":"high"}]}
+{"id":"codebase_investigator-01","metadata":{"tags":["tool:codebase_investigator","architecture"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"How does the authentication flow work in this project?"},"expected":{"tool_calls":[{"name":"codebase_investigator","arguments":{"objective":"Investigate and explain the authentication flow, including key files and logic."}}],"rationale":"Use codebase_investigator for high-level architectural understanding."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"grep -r 'auth' ."}}],"reason":"Used raw grep for a complex architectural question. codebase_investigator is better for mapping dependencies and structure.","severity":"high"},{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"auth"}}],"reason":"Used grep_search when a higher-level investigator is more appropriate for broad understanding.","severity":"medium"}]}
+{"id":"codebase_investigator-02","metadata":{"tags":["tool:codebase_investigator","refactoring"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I want to refactor the tool execution logic. Where should I start?"},"expected":{"tool_calls":[{"name":"codebase_investigator","arguments":{"objective":"Identify the core components and dependencies of the tool execution logic to prepare for a refactor."}}],"rationale":"Use codebase_investigator to map out dependencies before a major refactor."},"negatives":[{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"packages/core/src/tools"}}],"reason":"Used list_directory for a broad refactoring question. codebase_investigator provides more strategic insight.","severity":"medium"}]}
+{"id":"codebase_investigator-03","metadata":{"tags":["tool:codebase_investigator","onboarding"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What is the overall structure of this monorepo?"},"expected":{"tool_calls":[{"name":"codebase_investigator","arguments":{"objective":"Map out the monorepo structure, identifying the purpose of each package."}}],"rationale":"Use codebase_investigator for quick onboarding to a new codebase."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"ls -R"}}],"reason":"Used shell 'ls -R' which produces too much noise for structural understanding.","severity":"high"}]}
+{"id":"codebase_investigator-04","metadata":{"tags":["tool:codebase_investigator","debugging"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Investigate why the telemetry logs are not being sent to the server."},"expected":{"tool_calls":[{"name":"codebase_investigator","arguments":{"objective":"Find the telemetry logging logic and identify why it might be failing to send data."}}],"rationale":"Use codebase_investigator for root-cause analysis of system-wide issues."},"negatives":[{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"telemetry"}}],"reason":"Used simple grep when a systemic investigation is needed.","severity":"medium"}]}
+{"id":"codebase_investigator-05","metadata":{"tags":["tool:codebase_investigator","feature-planning"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I want to add a new tool for database access. Where do the existing tools live?"},"expected":{"tool_calls":[{"name":"codebase_investigator","arguments":{"objective":"Locate the tool definition directory and understand the pattern for adding new tools."}}],"rationale":"Use codebase_investigator to find patterns and locations for new features."},"negatives":[{"tool_calls":[{"name":"glob","arguments":{"pattern":"**/*tool*"}}],"reason":"Used glob for a discovery task that codebase_investigator can handle more intelligently.","severity":"medium"}]}
+{"id":"generalist-01","metadata":{"tags":["tool:generalist","delegation"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Fix all the linting errors in the 'packages/cli' directory."},"expected":{"tool_calls":[{"name":"generalist","arguments":{"request":"Fix all lint errors in the packages/cli directory using available tools like 'eslint --fix'."}}],"rationale":"Use the generalist agent for repetitive batch tasks across multiple files."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run lint -- --fix"}}],"reason":"While the shell command works, the generalist agent is preferred for 'autonomous' fixing that might require manual intervention for complex errors.","severity":"low"},{"tool_calls":[{"name":"read_file","arguments":{"file_path":"packages/cli/src/main.tsx"}}],"reason":"Attempted to manually read and fix files one by one instead of delegating the batch task.","severity":"medium"}]}
+{"id":"generalist-02","metadata":{"tags":["tool:generalist","high-volume"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Add a license header to every .ts file in the project."},"expected":{"tool_calls":[{"name":"generalist","arguments":{"request":"Add Apache-2.0 license headers to all .ts files in the project."}}],"rationale":"Use generalist for high-volume, repetitive file modifications."},"negatives":[{"tool_calls":[{"name":"replace","arguments":{"file_path":"src/index.ts","old_string":"","new_string":"// License..."}}],"reason":"Attempted to manually add headers file-by-file.","severity":"high"}]}
+{"id":"generalist-03","metadata":{"tags":["tool:generalist","speculative-research"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Try various ways to optimize the bundling process for the VS Code extension."},"expected":{"tool_calls":[{"name":"generalist","arguments":{"request":"Research and experiment with various bundling optimizations for the VS Code extension."}}],"rationale":"Use generalist for speculative research and trial-and-error investigations."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run build"}}],"reason":"Only ran the build once without systematic research.","severity":"medium"}]}
+{"id":"generalist-04","metadata":{"tags":["tool:generalist","batch-refactor"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Migrate all 'interface' definitions to 'type' definitions in 'packages/core'."},"expected":{"tool_calls":[{"name":"generalist","arguments":{"request":"Convert all interfaces to types in the packages/core directory."}}],"rationale":"Use generalist for systematic, codebase-wide refactoring tasks."},"negatives":[{"tool_calls":[{"name":"replace","arguments":{"file_path":"packages/core/src/types.ts","old_string":"interface","new_string":"type"}}],"reason":"Attempted to manually migrate file-by-file.","severity":"high"}]}
+{"id":"generalist-05","metadata":{"tags":["tool:generalist","delegation"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Clean up all temporary files and log files in the project."},"expected":{"tool_calls":[{"name":"generalist","arguments":{"request":"Find and delete all temporary files (.tmp, .log) across the workspace."}}],"rationale":"Use generalist for maintenance tasks that span multiple directories."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"rm -rf **/*.log"}}],"reason":"Used a potentially dangerous recursive shell command when a generalist can do it safely and verify.","severity":"medium"}]}
+{"id":"cli_help-01","metadata":{"tags":["tool:cli_help","meta"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"How do I enable the devtools in Gemini CLI?"},"expected":{"tool_calls":[{"name":"cli_help","arguments":{"question":"How to enable devtools in settings."}}],"rationale":"Use cli_help for questions about using the Gemini CLI itself."},"negatives":[{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"devtools","dir_path":"docs"}}],"reason":"Searched documentation manually when cli_help is the specialized assistant for this.","severity":"medium"}]}
+{"id":"cli_help-02","metadata":{"tags":["tool:cli_help","config"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What are the available settings in config.yaml?"},"expected":{"tool_calls":[{"name":"cli_help","arguments":{"question":"Available settings in config.yaml"}}],"rationale":"Use cli_help to understand CLI configuration options."},"negatives":[{"tool_calls":[{"name":"read_file","arguments":{"file_path":".gemini/config.yaml"}}],"reason":"Read the config file directly without understanding the possible options and their meanings.","severity":"medium"}]}
+{"id":"cli_help-03","metadata":{"tags":["tool:cli_help","features"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Does Gemini CLI support Model Context Protocol (MCP)?"},"expected":{"tool_calls":[{"name":"cli_help","arguments":{"question":"Support for Model Context Protocol (MCP)"}}],"rationale":"Use cli_help for feature-related inquiries."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"gemini cli mcp support"}}],"reason":"Used external search for a question about the internal features of the CLI.","severity":"medium"}]}
+{"id":"cli_help-04","metadata":{"tags":["tool:cli_help","shortcuts"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What is the keyboard shortcut to focus the shell?"},"expected":{"tool_calls":[{"name":"cli_help","arguments":{"question":"Keyboard shortcut for focusing the shell"}}],"rationale":"Use cli_help for questions about CLI interaction and shortcuts."},"negatives":[{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"keybinding"}}],"reason":"Used grep to search for keybindings.","severity":"medium"}]}
+{"id":"cli_help-05","metadata":{"tags":["tool:cli_help","troubleshooting"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I'm getting an 'invalid_request' error when using Google Search tool."},"expected":{"tool_calls":[{"name":"cli_help","arguments":{"question":"Troubleshooting 'invalid_request' error in Google Search tool"}}],"rationale":"Use cli_help for troubleshooting errors related to the CLI's tools and extensions."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run debug"}}],"reason":"Started debugging without checking the help system first for known issues.","severity":"low"}]}
+{"id":"activate_skill-01","metadata":{"tags":["tool:activate_skill","workflow"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I want to create a new pull request for my changes."},"expected":{"tool_calls":[{"name":"activate_skill","arguments":{"name":"pr-creator"}}],"rationale":"Use activate_skill to get specialized instructions for common workflows like PR creation."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"gh pr create"}}],"reason":"Using the CLI directly is fine, but activate_skill provides repository-specific guidance and templates.","severity":"low"},{"tool_calls":[{"name":"generalist","arguments":{"request":"Create a PR"}}],"reason":"Delegated to a generalist when a specialized skill exists.","severity":"medium"}]}
+{"id":"activate_skill-02","metadata":{"tags":["tool:activate_skill","docs"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Help me update the documentation in the docs/ folder."},"expected":{"tool_calls":[{"name":"activate_skill","arguments":{"name":"docs-writer"}}],"rationale":"Use docs-writer skill for any tasks involving Markdown or documentation files."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":"docs/index.md","content":"..."}}],"reason":"Attempted to write documentation directly without activating the specialized documentation skill.","severity":"medium"}]}
+{"id":"activate_skill-03","metadata":{"tags":["tool:activate_skill","review"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Can you review my local changes?"},"expected":{"tool_calls":[{"name":"activate_skill","arguments":{"name":"code-reviewer"}}],"rationale":"Use code-reviewer skill for high-quality code analysis."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"git diff"}}],"reason":"Just looking at the diff is not a proper review. activate_skill provides a specialized agent for reviews.","severity":"medium"}]}
+{"id":"activate_skill-04","metadata":{"tags":["tool:activate_skill","issues"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I found a bug. Create a GitHub issue for it."},"expected":{"tool_calls":[{"name":"activate_skill","arguments":{"name":"github-issue-creator"}}],"rationale":"Use github-issue-creator for standardized issue reporting."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"gh issue create"}}],"reason":"Direct CLI usage misses the benefit of project-specific templates provided by the skill.","severity":"low"}]}
+{"id":"activate_skill-05","metadata":{"tags":["tool:activate_skill","new-skill"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I want to create a new skill to automate my deployments."},"expected":{"tool_calls":[{"name":"activate_skill","arguments":{"name":"skill-creator"}}],"rationale":"Use skill-creator to extend the CLI's capabilities."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":".gemini/skills/deploy/SKILL.md","content":"..."}}],"reason":"Attempted to create a skill manually without the expert guidance of the skill-creator.","severity":"medium"}]}
+{"id":"ask_user-01","metadata":{"tags":["tool:ask_user","confirmation"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I'm ready to delete the old build artifacts. Should I proceed?"},"expected":{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"Are you sure you want to delete the old build artifacts?","header":"Confirm Delete","type":"yesno"}]}}],"rationale":"Use ask_user with type:'yesno' for critical confirmations."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"rm -rf dist/*"}}],"reason":"Proceeded with a destructive action without explicit user confirmation.","severity":"high"}]}
+{"id":"ask_user-02","metadata":{"tags":["tool:ask_user","selection"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I need to know which database you want to use for the new service."},"expected":{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"Which database should be used for the new service?","header":"Database Select","type":"choice","options":[{"label":"PostgreSQL","description":"Relational database"},{"label":"MongoDB","description":"NoSQL document store"}]}]}}],"rationale":"Use ask_user with type:'choice' for structured options."},"negatives":[{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"What database do you want?","header":"DB","type":"text"}]}}],"reason":"Used a free-form text input when specific choices were available, leading to potential ambiguity.","severity":"low"}]}
+{"id":"ask_user-03","metadata":{"tags":["tool:ask_user","multi-question"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I need your name and email to set up the git config."},"expected":{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"What is your name?","header":"Git Name","type":"text"},{"question":"What is your email?","header":"Git Email","type":"text"}]}}],"rationale":"Use ask_user to gather multiple pieces of information in one turn."},"negatives":[{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"What is your name?","header":"Name","type":"text"}]}}],"reason":"Only asked one question when two were needed, requiring an extra turn.","severity":"medium"}]}
+{"id":"ask_user-04","metadata":{"tags":["tool:ask_user","clarification"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Update the API."},"expected":{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"Which API do you want to update (Internal or External)?","header":"Clarification","type":"choice","options":[{"label":"Internal","description":"Packages/core/src/api"},{"label":"External","description":"Public facing API"}]}]}}],"rationale":"Use ask_user to clarify ambiguous user requests before acting."},"negatives":[{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"API"}}],"reason":"Attempted to search for 'API' without clarifying the user's vague intent.","severity":"medium"}]}
+{"id":"ask_user-05","metadata":{"tags":["tool:ask_user","multiselect"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Which packages should I run the tests for?"},"expected":{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"Select the packages to run tests for:","header":"Package Select","type":"choice","multiSelect":true,"options":[{"label":"cli","description":"Terminal UI"},{"label":"core","description":"Backend logic"},{"label":"devtools","description":"Inspector UI"}]}]}}],"rationale":"Use multiSelect:true for allowing multiple choices."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm test"}}],"reason":"Ran all tests without asking the user which specific packages they were interested in.","severity":"medium"}]}
+{"id":"enter_plan_mode-01","metadata":{"tags":["tool:enter_plan_mode","complex-task"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I want to implement a new Model Context Protocol (MCP) server."},"expected":{"tool_calls":[{"name":"enter_plan_mode","arguments":{"reason":"Design and plan the implementation of a new MCP server, including architecture and tool definitions."}}],"rationale":"Use enter_plan_mode for complex, multi-stage implementation tasks."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":"packages/mcp-server/src/index.ts","content":"..."}}],"reason":"Started coding a complex feature without a design phase.","severity":"high"}]}
+{"id":"enter_plan_mode-02","metadata":{"tags":["tool:enter_plan_mode","refactoring"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Refactor the entire telemetry system to use a new backend."},"expected":{"tool_calls":[{"name":"enter_plan_mode","arguments":{"reason":"Plan the refactoring of the telemetry system, ensuring all components are updated and tested."}}],"rationale":"Use enter_plan_mode for large-scale refactoring."},"negatives":[{"tool_calls":[{"name":"codebase_investigator","arguments":{"objective":"Refactor telemetry"}}],"reason":"Delegated to an investigator without entering plan mode first to manage the complex transition.","severity":"medium"}]}
+{"id":"enter_plan_mode-03","metadata":{"tags":["tool:enter_plan_mode","architecture"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"How should we handle secrets in our CI/CD pipeline?"},"expected":{"tool_calls":[{"name":"enter_plan_mode","arguments":{"reason":"Research and design a secure secret management strategy for the CI/CD pipeline."}}],"rationale":"Use enter_plan_mode for architectural design discussions."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"github actions secrets best practices"}}],"reason":"Started research without entering a formal planning state for the architectural decision.","severity":"low"}]}
+{"id":"enter_plan_mode-04","metadata":{"tags":["tool:enter_plan_mode","integration"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Integrate the new DevTools UI with the main CLI application."},"expected":{"tool_calls":[{"name":"enter_plan_mode","arguments":{"reason":"Plan the integration of DevTools UI, focusing on the connection between React/Ink and the DevTools server."}}],"rationale":"Use enter_plan_mode for complex integration tasks."},"negatives":[{"tool_calls":[{"name":"replace","arguments":{"file_path":"packages/cli/src/main.tsx","old_string":"...","new_string":"..."}}],"reason":"Attempted to integrate a major feature via a single file edit without planning.","severity":"high"}]}
+{"id":"enter_plan_mode-05","metadata":{"tags":["tool:enter_plan_mode","system-change"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Change the build system from esbuild to Rollup."},"expected":{"tool_calls":[{"name":"enter_plan_mode","arguments":{"reason":"Evaluate and plan the migration of the build system from esbuild to Rollup."}}],"rationale":"Use enter_plan_mode for fundamental system changes."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm install rollup"}}],"reason":"Started installing dependencies before planning the migration.","severity":"medium"}]}
+{"id":"exit_plan_mode-01","metadata":{"tags":["tool:exit_plan_mode","transition"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I've finished the design doc. Let's start implementing."},"expected":{"tool_calls":[{"name":"exit_plan_mode","arguments":{"plan_path":"plans/my-feature-plan.md"}}],"rationale":"Use exit_plan_mode to transition from planning to implementation."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":"src/feature.ts","content":"..."}}],"reason":"Started implementation without formally exiting plan mode.","severity":"medium"}]}
+{"id":"get_internal_docs-01","metadata":{"tags":["tool:get_internal_docs","discovery"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What are the coding standards for this project?"},"expected":{"tool_calls":[{"name":"get_internal_docs","arguments":{"path":"reference/coding-standards.md"}}],"rationale":"Use get_internal_docs for project-specific standards."},"negatives":[{"tool_calls":[{"name":"read_file","arguments":{"file_path":"docs/reference/coding-standards.md"}}],"reason":"Read the doc file directly instead of using the specialized internal docs tool.","severity":"low"}]}
+{"id":"get_internal_docs-02","metadata":{"tags":["tool:get_internal_docs","onboarding"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Show me all the internal documentation available."},"expected":{"tool_calls":[{"name":"get_internal_docs","arguments":{}}],"rationale":"Call get_internal_docs without arguments to list all available docs."},"negatives":[{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"docs"}}],"reason":"Listed the directory instead of using the docs tool to see organized documentation.","severity":"low"}]}
+{"id":"exit_plan_mode-02","metadata":{"tags":["tool:exit_plan_mode","rejection"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I don't like this plan. Can we try a different approach?"},"expected":{"tool_calls":[{"name":"exit_plan_mode","arguments":{"plan_path":"plans/rejected-plan.md"}}],"rationale":"Use exit_plan_mode when a plan is rejected and need to restart or pivot."},"negatives":[{"tool_calls":[{"name":"enter_plan_mode","arguments":{"reason":"Try again"}}],"reason":"Attempted to re-enter plan mode without exiting the current one first.","severity":"medium"}]}
+{"id":"exit_plan_mode-03","metadata":{"tags":["tool:exit_plan_mode","completion"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"The plan looks perfect. Go ahead."},"expected":{"tool_calls":[{"name":"exit_plan_mode","arguments":{"plan_path":"plans/approved-plan.md"}}],"rationale":"Use exit_plan_mode to signal that planning is over and implementation can begin."},"negatives":[{"tool_calls":[{"name":"generalist","arguments":{"request":"Start implementing"}}],"reason":"Delegated implementation while still in plan mode.","severity":"medium"}]}
+{"id":"exit_plan_mode-04","metadata":{"tags":["tool:exit_plan_mode","cancel"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Nevermind about this feature, let's stop planning."},"expected":{"tool_calls":[{"name":"exit_plan_mode","arguments":{"plan_path":"plans/cancelled-feature.md"}}],"rationale":"Use exit_plan_mode to gracefully stop a planning session."},"negatives":[{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"Should I stop?","header":"Stop","type":"yesno"}]}}],"reason":"Asked for confirmation to stop when the intent to stop was clear.","severity":"low"}]}
+{"id":"exit_plan_mode-05","metadata":{"tags":["tool:exit_plan_mode","handoff"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I'll take it from here, thanks for the plan."},"expected":{"tool_calls":[{"name":"exit_plan_mode","arguments":{"plan_path":"plans/handoff-plan.md"}}],"rationale":"Use exit_plan_mode to finalize the plan for the user."},"negatives":[{"tool_calls":[{"name":"save_memory","arguments":{"fact":"The user liked the plan"}}],"reason":"Saved a memory instead of closing the planning session.","severity":"low"}]}
+{"id":"get_internal_docs-03","metadata":{"tags":["tool:get_internal_docs","troubleshooting"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"How do I fix common authentication errors?"},"expected":{"tool_calls":[{"name":"get_internal_docs","arguments":{"path":"admin/auth-troubleshooting.md"}}],"rationale":"Use internal docs for specialized troubleshooting guides."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"gemini cli auth error"}}],"reason":"Searched the web for internal documentation.","severity":"medium"}]}
+{"id":"get_internal_docs-04","metadata":{"tags":["tool:get_internal_docs","tools"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What are the parameters for the 'replace' tool?"},"expected":{"tool_calls":[{"name":"get_internal_docs","arguments":{"path":"tools/replace.md"}}],"rationale":"Use internal docs to find detailed tool information."},"negatives":[{"tool_calls":[{"name":"cli_help","arguments":{"question":"replace tool params"}}],"reason":"While cli_help works, get_internal_docs provides direct access to the documentation source.","severity":"low"}]}
+{"id":"get_internal_docs-05","metadata":{"tags":["tool:get_internal_docs","contribution"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"How can I contribute to this project?"},"expected":{"tool_calls":[{"name":"get_internal_docs","arguments":{"path":"CONTRIBUTING.md"}}],"rationale":"Use get_internal_docs to access project meta-documentation."},"negatives":[{"tool_calls":[{"name":"read_file","arguments":{"file_path":"CONTRIBUTING.md"}}],"reason":"Read the file directly instead of using the docs tool.","severity":"low"}]}
+{"id":"google_web_search-01","metadata":{"tags":["tool:google_web_search","research"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What's the latest stable version of React?"},"expected":{"tool_calls":[{"name":"google_web_search","arguments":{"query":"latest stable version of React"}}],"rationale":"Use web search for up-to-date external information."},"negatives":[{"tool_calls":[{"name":"read_file","arguments":{"file_path":"package.json"}}],"reason":"Checked local file for global external information.","severity":"medium"}]}
+{"id":"google_web_search-02","metadata":{"tags":["tool:google_web_search","documentation"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"How do I use the 'Suspense' component in React 18?"},"expected":{"tool_calls":[{"name":"google_web_search","arguments":{"query":"React 18 Suspense documentation"}}],"rationale":"Use web search for library-specific documentation."},"negatives":[{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"Suspense"}}],"reason":"Searched local code for external library documentation.","severity":"medium"}]}
+{"id":"google_web_search-03","metadata":{"tags":["tool:google_web_search","troubleshooting"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Error: TS2322: Type 'string' is not assignable to type 'number'."},"expected":{"tool_calls":[{"name":"google_web_search","arguments":{"query":"TypeScript error TS2322 string not assignable to number"}}],"rationale":"Use web search to research common programming errors."},"negatives":[{"tool_calls":[{"name":"generalist","arguments":{"request":"Fix TS2322"}}],"reason":"Delegated to an agent before researching the error.","severity":"low"}]}
+{"id":"google_web_search-04","metadata":{"tags":["tool:google_web_search","comparison"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Compare Vitest vs Jest for a new TypeScript project."},"expected":{"tool_calls":[{"name":"google_web_search","arguments":{"query":"Vitest vs Jest comparison 2026"}}],"rationale":"Use web search for architectural comparisons."},"negatives":[{"tool_calls":[{"name":"cli_help","arguments":{"question":"Vitest vs Jest"}}],"reason":"Asked the internal help system for external library comparisons.","severity":"medium"}]}
+{"id":"google_web_search-05","metadata":{"tags":["tool:google_web_search","news"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What are the new features in the latest Node.js release?"},"expected":{"tool_calls":[{"name":"google_web_search","arguments":{"query":"latest Node.js release features"}}],"rationale":"Use web search for recent technical news."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"node -v"}}],"reason":"Checked local version instead of researching new features.","severity":"medium"}]}
+{"id":"read_many_files-01","metadata":{"tags":["tool:read_many_files","codebase-overview"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Show me all the CSS files in the project to check the styling."},"expected":{"tool_calls":[{"name":"read_many_files","arguments":{"include":["**/*.css"]}}],"rationale":"Use read_many_files to get the content of multiple files matching a pattern."},"negatives":[{"tool_calls":[{"name":"glob","arguments":{"pattern":"**/*.css"}}],"reason":"Only listed the files instead of reading their content as requested.","severity":"medium"},{"tool_calls":[{"name":"read_file","arguments":{"file_path":"src/app.css"}}],"reason":"Read only one file when all CSS files were requested.","severity":"medium"}]}
+{"id":"read_many_files-02","metadata":{"tags":["tool:read_many_files","batch-reading"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Read all the README files in the packages directory."},"expected":{"tool_calls":[{"name":"read_many_files","arguments":{"include":["packages/**/README.md"]}}],"rationale":"Use read_many_files for batch content retrieval."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"cat packages/**/README.md"}}],"reason":"Used shell 'cat' with globs instead of the specialized read_many_files tool.","severity":"high"}]}
+{"id":"read_many_files-03","metadata":{"tags":["tool:read_many_files","precision"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Show me the content of all .test.ts files in src/utils."},"expected":{"tool_calls":[{"name":"read_many_files","arguments":{"include":["src/utils/*.test.ts"]}}],"rationale":"Use read_many_files for targeted batch reading."},"negatives":[{"tool_calls":[{"name":"read_many_files","arguments":{"include":["**/*.test.ts"]}}],"reason":"Read too many files (all tests) instead of only those in src/utils.","severity":"low"}]}
+{"id":"read_many_files-04","metadata":{"tags":["tool:read_many_files","context-gathering"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I need to see all the configuration files (json and yaml)."},"expected":{"tool_calls":[{"name":"read_many_files","arguments":{"include":["**/*.json","**/*.yaml"]}}],"rationale":"Use read_many_files with multiple patterns to gather context."},"negatives":[{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":"."}}],"reason":"Only listed files instead of reading their content.","severity":"medium"}]}
+{"id":"read_many_files-05","metadata":{"tags":["tool:read_many_files","efficiency"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Read all source files in the current folder."},"expected":{"tool_calls":[{"name":"read_many_files","arguments":{"include":["*.ts","*.tsx","*.js"]}}],"rationale":"Use read_many_files to read specific source extensions."},"negatives":[{"tool_calls":[{"name":"read_file","arguments":{"file_path":"index.ts"}}],"reason":"Read only one file instead of all source files.","severity":"medium"}]}
+{"id":"save_memory-01","metadata":{"tags":["tool:save_memory","preference"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I prefer using tabs instead of spaces for indentation."},"expected":{"tool_calls":[{"name":"save_memory","arguments":{"fact":"The user prefers using tabs instead of spaces for indentation."}}],"rationale":"Use save_memory to persist global user preferences."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":".editorconfig","content":"..."}}],"reason":"Attempted to change project-specific config instead of saving a global user preference.","severity":"medium"}]}
+{"id":"save_memory-02","metadata":{"tags":["tool:save_memory","style"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Always use double quotes for strings in TypeScript."},"expected":{"tool_calls":[{"name":"save_memory","arguments":{"fact":"The user prefers double quotes for strings in TypeScript."}}],"rationale":"Use save_memory for coding style preferences."},"negatives":[{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"Should I use double quotes?","header":"Style","type":"yesno"}]}}],"reason":"Asked for confirmation instead of remembering the clear preference.","severity":"low"}]}
+{"id":"save_memory-03","metadata":{"tags":["tool:save_memory","project-context-failure"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Remember that the main server file is src/server.ts."},"expected":{"tool_calls":[{"name":"ask_user","arguments":{"questions":[{"question":"This seems specific to this project. Should I remember it globally or is it only for this workspace?","header":"Memory Scope","type":"choice","options":[{"label":"Global","description":"Save to global memory"},{"label":"Project","description":"Only for this project (do not save)"}]}]}}],"rationale":"Ask before saving project-specific facts to global memory to avoid cluttering cross-session context."},"negatives":[{"tool_calls":[{"name":"save_memory","arguments":{"fact":"The main server file is src/server.ts"}}],"reason":"Saved a project-specific fact to global memory without confirmation.","severity":"medium"}]}
+{"id":"save_memory-04","metadata":{"tags":["tool:save_memory","persona"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I am a senior backend engineer."},"expected":{"tool_calls":[{"name":"save_memory","arguments":{"fact":"The user is a senior backend engineer."}}],"rationale":"Use save_memory to remember user background/persona info."},"negatives":[{"tool_calls":[{"name":"generalist","arguments":{"request":"Remember user profile"}}],"reason":"Delegated to a simple memory task.","severity":"low"}]}
+{"id":"save_memory-05","metadata":{"tags":["tool:save_memory","workflow"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I like to run tests before every commit."},"expected":{"tool_calls":[{"name":"save_memory","arguments":{"fact":"The user prefers to run tests before every commit."}}],"rationale":"Use save_memory for workflow preferences."},"negatives":[{"tool_calls":[{"name":"write_todos","arguments":{"todos":[{"description":"Run tests","status":"pending"}]}}],"reason":"Added to current todos instead of saving a permanent preference.","severity":"low"}]}
+{"id":"run_shell_command-01","metadata":{"tags":["tool:run_shell_command","test-execution"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Run the unit tests for the core package."},"expected":{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run test -w @google/gemini-cli-core"}}],"rationale":"Use run_shell_command for executing ecosystem tools like test runners."},"negatives":[{"tool_calls":[{"name":"generalist","arguments":{"request":"Run tests"}}],"reason":"Delegated a simple command execution to an agent.","severity":"low"}]}
+{"id":"run_shell_command-02","metadata":{"tags":["tool:run_shell_command","git-ops"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Check the git status of the project."},"expected":{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"git status"}}],"rationale":"Use run_shell_command for git operations."},"negatives":[{"tool_calls":[{"name":"list_directory","arguments":{"dir_path":".git"}}],"reason":"Attempted to check git status by listing the .git directory.","severity":"medium"}]}
+{"id":"run_shell_command-03","metadata":{"tags":["tool:run_shell_command","background-process"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Start the DevTools server in the background."},"expected":{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run start --workspace @google/gemini-cli-devtools","is_background":true}}],"rationale":"Use is_background:true for long-running servers."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run start --workspace @google/gemini-cli-devtools"}}],"reason":"Started a server in the foreground, which would block the CLI.","severity":"medium"}]}
+{"id":"run_shell_command-04","metadata":{"tags":["tool:run_shell_command","env-check"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What version of Node.js am I running?"},"expected":{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"node -v"}}],"rationale":"Use run_shell_command for environment introspection."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"latest node version"}}],"reason":"Searched web instead of checking local environment.","severity":"medium"}]}
+{"id":"run_shell_command-05","metadata":{"tags":["tool:run_shell_command","build-system"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Build the whole project."},"expected":{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run build:all"}}],"rationale":"Use run_shell_command for build commands."},"negatives":[{"tool_calls":[{"name":"generalist","arguments":{"request":"Build project"}}],"reason":"Delegated a standard build command.","severity":"low"}]}
+{"id":"web_fetch-01","metadata":{"tags":["tool:web_fetch","documentation"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Summarize the breaking changes in this URL: https://example.com/changelog"},"expected":{"tool_calls":[{"name":"web_fetch","arguments":{"prompt":"Summarize the breaking changes from https://example.com/changelog"}}],"rationale":"Use web_fetch to extract and summarize content from a specific URL."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"breaking changes example.com"}}],"reason":"Used search instead of fetching the provided URL directly.","severity":"medium"}]}
+{"id":"web_fetch-02","metadata":{"tags":["tool:web_fetch","github-code"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Read the implementation of this function: https://github.com/user/repo/blob/main/src/lib.ts#L10"},"expected":{"tool_calls":[{"name":"web_fetch","arguments":{"prompt":"Read and explain the implementation at https://github.com/user/repo/blob/main/src/lib.ts#L10"}}],"rationale":"Use web_fetch for reading remote code snippets."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"implementation of src/lib.ts"}}],"reason":"Used search for a specific GitHub link.","severity":"medium"}]}
+{"id":"web_fetch-03","metadata":{"tags":["tool:web_fetch","multi-url"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Compare the feature sets of these two libraries: https://lib-a.com and https://lib-b.com"},"expected":{"tool_calls":[{"name":"web_fetch","arguments":{"prompt":"Compare the feature sets of https://lib-a.com and https://lib-b.com"}}],"rationale":"Use web_fetch to analyze and compare multiple URLs."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"compare lib-a vs lib-b"}}],"reason":"Used search instead of directly analyzing the provided URLs.","severity":"low"}]}
+{"id":"web_fetch-04","metadata":{"tags":["tool:web_fetch","research-depth"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"What is the main argument in this research paper? https://arxiv.org/pdf/1234.5678"},"expected":{"tool_calls":[{"name":"web_fetch","arguments":{"prompt":"Identify the main argument in the research paper at https://arxiv.org/pdf/1234.5678"}}],"rationale":"Use web_fetch for analyzing PDF content from a URL."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"main argument of arxiv 1234.5678"}}],"reason":"Used search for a direct PDF link analysis.","severity":"medium"}]}
+{"id":"web_fetch-05","metadata":{"tags":["tool:web_fetch","api-docs"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"How do I authenticate with this API? https://api-docs.com/auth"},"expected":{"tool_calls":[{"name":"web_fetch","arguments":{"prompt":"Find the authentication instructions at https://api-docs.com/auth"}}],"rationale":"Use web_fetch for extracting specific information from documentation URLs."},"negatives":[{"tool_calls":[{"name":"google_web_search","arguments":{"query":"how to authenticate with api-docs.com"}}],"reason":"Used search for a provided documentation link.","severity":"medium"}]}
+{"id":"write_todos-01","metadata":{"tags":["tool:write_todos","planning"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Implement a new feature to handle user profiles."},"expected":{"tool_calls":[{"name":"write_todos","arguments":{"todos":[{"description":"Design user profile schema","status":"pending"},{"description":"Implement database migrations","status":"pending"},{"description":"Create API endpoints for profiles","status":"pending"},{"description":"Add unit tests","status":"pending"}]}}],"rationale":"Use write_todos to break down a complex task into manageable subtasks."},"negatives":[{"tool_calls":[{"name":"write_file","arguments":{"file_path":"src/profiles.ts","content":"..."}}],"reason":"Started implementation without defining subtasks for a complex feature.","severity":"medium"}]}
+{"id":"write_todos-02","metadata":{"tags":["tool:write_todos","progress-tracking"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I've finished the schema design. What's next?"},"expected":{"tool_calls":[{"name":"write_todos","arguments":{"todos":[{"description":"Design user profile schema","status":"completed"},{"description":"Implement database migrations","status":"in_progress"},{"description":"Create API endpoints for profiles","status":"pending"},{"description":"Add unit tests","status":"pending"}]}}],"rationale":"Use write_todos to update progress as tasks are completed."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm run build"}}],"reason":"Just ran build instead of updating the task list.","severity":"low"}]}
+{"id":"write_todos-03","metadata":{"tags":["tool:write_todos","complex-debugging"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Debug the memory leak issue in the production server."},"expected":{"tool_calls":[{"name":"write_todos","arguments":{"todos":[{"description":"Analyze heap dumps","status":"in_progress"},{"description":"Identify leaking objects","status":"pending"},{"description":"Locate source of leak in code","status":"pending"},{"description":"Implement fix and verify","status":"pending"}]}}],"rationale":"Use write_todos for managing complex, non-linear debugging tasks."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"top"}}],"reason":"Only ran a monitoring command instead of planning the debugging process.","severity":"medium"}]}
+{"id":"write_todos-04","metadata":{"tags":["tool:write_todos","multi-stage-fix"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Fix the authentication bug and then update the docs."},"expected":{"tool_calls":[{"name":"write_todos","arguments":{"todos":[{"description":"Reproduce authentication bug","status":"in_progress"},{"description":"Implement fix","status":"pending"},{"description":"Verify fix with tests","status":"pending"},{"description":"Update admin/auth-troubleshooting.md","status":"pending"}]}}],"rationale":"Use write_todos to sequence multi-part fixes."},"negatives":[{"tool_calls":[{"name":"grep_search","arguments":{"pattern":"auth"}}],"reason":"Started searching without defining the multi-stage task.","severity":"low"}]}
+{"id":"write_todos-05","metadata":{"tags":["tool:write_todos","refactor-planning"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"Migrate the database to PostgreSQL."},"expected":{"tool_calls":[{"name":"write_todos","arguments":{"todos":[{"description":"Install PostgreSQL drivers","status":"pending"},{"description":"Update database configuration","status":"pending"},{"description":"Migrate existing data","status":"pending"},{"description":"Verify with integration tests","status":"pending"}]}}],"rationale":"Use write_todos for planning migrations."},"negatives":[{"tool_calls":[{"name":"run_shell_command","arguments":{"command":"npm install pg"}}],"reason":"Started installation before defining the full migration path.","severity":"medium"}]}
+{"id":"save_memory-06","metadata":{"tags":["tool:save_memory","persistence"],"created_at":"2026-03-02T13:00:00Z"},"input":{"user_query":"I want you to remember that my name is Abhijit."},"expected":{"tool_calls":[{"name":"save_memory","arguments":{"fact":"The user's name is Abhijit."}}],"rationale":"Use save_memory to persist personal user information across sessions."},"negatives":[{"tool_calls":[{"name":"generalist","arguments":{"request":"Remember user's name"}}],"reason":"Delegated a simple memory task to an agent.","severity":"low"}]}

eslint.config.js

@@ -68,7 +68,11 @@ export default tseslint.config(
   },
   {
     // Rules for packages/*/src (TS/TSX)
-    files: ['packages/*/src/**/*.{ts,tsx}'],
+    files: [
+      'packages/*/src/**/*.{ts,tsx}',
+      'data/**/*.ts',
+      'scripts/optimization/**/*.ts',
+    ],
     plugins: {
       import: importPlugin,
     },

package-lock.json

@@ -73,6 +73,7 @@
         "node": ">=20.0.0"
       },
       "optionalDependencies": {
+        "@ax-llm/ax": "^19.0.11",
         "@lydell/node-pty": "1.1.0",
         "@lydell/node-pty-darwin-arm64": "1.1.0",
         "@lydell/node-pty-darwin-x64": "1.1.0",
@@ -179,6 +180,21 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/@ax-llm/ax": {
+      "version": "19.0.11",
+      "resolved": "https://registry.npmjs.org/@ax-llm/ax/-/ax-19.0.11.tgz",
+      "integrity": "sha512-U3ZYzBrmMDTDst32jxgH873gC4c75aYjzdCZgwQWy+CwSDL2SskwQX2kZAWGDmmSzs8BxskleoASzQUXuqRLfQ==",
+      "hasInstallScript": true,
+      "license": "Apache-2.0",
+      "optional": true,
+      "dependencies": {
+        "@opentelemetry/api": "^1.9.0",
+        "dayjs": "^1.11.13"
+      },
+      "bin": {
+        "ax": "cli/index.mjs"
+      }
+    },
     "node_modules/@azu/format-text": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/@azu/format-text/-/format-text-1.0.2.tgz",
@@ -2292,6 +2308,7 @@
       "integrity": "sha512-t54CUOsFMappY1Jbzb7fetWeO0n6K0k/4+/ZpkS+3Joz8I4VcvY9OiEBFRYISqaI2fq5sCiPtAjRDOzVYG8m+Q==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@octokit/auth-token": "^6.0.0",
         "@octokit/graphql": "^9.0.2",
@@ -2472,6 +2489,7 @@
       "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
       "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
       "license": "Apache-2.0",
+      "peer": true,
       "engines": {
         "node": ">=8.0.0"
       }
@@ -2521,6 +2539,7 @@
       "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.0.tgz",
       "integrity": "sha512-ka4H8OM6+DlUhSAZpONu0cPBtPPTQKxbxVzC4CzVx5+K4JnroJVBtDzLAMx4/3CDTJXRvVFhpFjtl4SaiTNoyQ==",
       "license": "Apache-2.0",
+      "peer": true,
       "dependencies": {
         "@opentelemetry/semantic-conventions": "^1.29.0"
       },
@@ -2895,6 +2914,7 @@
       "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.5.0.tgz",
       "integrity": "sha512-F8W52ApePshpoSrfsSk1H2yJn9aKjCrbpQF1M9Qii0GHzbfVeFUB+rc3X4aggyZD8x9Gu3Slua+s6krmq6Dt8g==",
       "license": "Apache-2.0",
+      "peer": true,
       "dependencies": {
         "@opentelemetry/core": "2.5.0",
         "@opentelemetry/semantic-conventions": "^1.29.0"
@@ -2928,6 +2948,7 @@
       "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.5.0.tgz",
       "integrity": "sha512-BeJLtU+f5Gf905cJX9vXFQorAr6TAfK3SPvTFqP+scfIpDQEJfRaGJWta7sJgP+m4dNtBf9y3yvBKVAZZtJQVA==",
       "license": "Apache-2.0",
+      "peer": true,
       "dependencies": {
         "@opentelemetry/core": "2.5.0",
         "@opentelemetry/resources": "2.5.0"
@@ -2982,6 +3003,7 @@
       "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.0.tgz",
       "integrity": "sha512-VzRf8LzotASEyNDUxTdaJ9IRJ1/h692WyArDBInf5puLCjxbICD6XkHgpuudis56EndyS7LYFmtTMny6UABNdQ==",
       "license": "Apache-2.0",
+      "peer": true,
       "dependencies": {
         "@opentelemetry/core": "2.5.0",
         "@opentelemetry/resources": "2.5.0",
@@ -4178,6 +4200,7 @@
       "integrity": "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA==",
       "devOptional": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "csstype": "^3.0.2"
       }
@@ -4451,6 +4474,7 @@
       "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@typescript-eslint/scope-manager": "8.56.1",
         "@typescript-eslint/types": "8.56.1",
@@ -5298,6 +5322,7 @@
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
       "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
       "license": "MIT",
+      "peer": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -6879,6 +6904,13 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/dayjs": {
+      "version": "1.11.19",
+      "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.19.tgz",
+      "integrity": "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw==",
+      "license": "MIT",
+      "optional": true
+    },
     "node_modules/debug": {
       "version": "4.4.3",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
@@ -7901,6 +7933,7 @@
       "integrity": "sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.8.0",
         "@eslint-community/regexpp": "^4.12.1",
@@ -8533,6 +8566,7 @@
       "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz",
       "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "accepts": "^2.0.0",
         "body-parser": "^2.2.1",
@@ -9847,6 +9881,7 @@
       "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.2.tgz",
       "integrity": "sha512-gJnaDHXKDayjt8ue0n8Gs0A007yKXj4Xzb8+cNjZeYsSzzwKc0Lr+OZgYwVfB0pHfUs17EPoLvrOsEaJ9mj+Tg==",
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=16.9.0"
       }
@@ -10126,6 +10161,7 @@
       "resolved": "https://registry.npmjs.org/@jrichman/ink/-/ink-6.4.11.tgz",
       "integrity": "sha512-93LQlzT7vvZ1XJcmOMwN4s+6W334QegendeHOMnEJBlhnpIzr8bws6/aOEHG8ZCuVD/vNeeea5m1msHIdAY6ig==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@alcalzone/ansi-tokenize": "^0.2.1",
         "ansi-escapes": "^7.0.0",
@@ -13808,6 +13844,7 @@
       "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz",
       "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==",
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -13818,6 +13855,7 @@
       "integrity": "sha512-ePrwPfxAnB+7hgnEr8vpKxL9cmnp7F322t8oqcPshbIQQhDKgFDW4tjhF2wjVbdXF9O/nyuy3sQWd9JGpiLPvA==",
       "devOptional": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "shell-quote": "^1.6.1",
         "ws": "^7"
@@ -15906,6 +15944,7 @@
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
       "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=12"
       },
@@ -16129,7 +16168,8 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
       "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
       "dev": true,
-      "license": "0BSD"
+      "license": "0BSD",
+      "peer": true
     },
     "node_modules/tsx": {
       "version": "4.20.3",
@@ -16137,6 +16177,7 @@
       "integrity": "sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ==",
       "devOptional": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "esbuild": "~0.25.0",
         "get-tsconfig": "^4.7.5"
@@ -16296,6 +16337,7 @@
       "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==",
       "devOptional": true,
       "license": "Apache-2.0",
+      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -16519,6 +16561,7 @@
       "resolved": "https://registry.npmjs.org/vite/-/vite-7.2.2.tgz",
       "integrity": "sha512-BxAKBWmIbrDgrokdGZH1IgkIk/5mMHDreLDmCJ0qpyJaAteP8NvMhkwr/ZCQNqNH97bw/dANTE9PDzqwJghfMQ==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "esbuild": "^0.25.0",
         "fdir": "^6.5.0",
@@ -16632,6 +16675,7 @@
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
       "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=12"
       },
@@ -16644,6 +16688,7 @@
       "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz",
       "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@types/chai": "^5.2.2",
         "@vitest/expect": "3.2.4",
@@ -17288,6 +17333,7 @@
       "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
       "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
       "license": "MIT",
+      "peer": true,
       "funding": {
         "url": "https://github.com/sponsors/colinhacks"
       }
@@ -17687,6 +17733,7 @@
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
       "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=12"
       },

package.json

@@ -61,6 +61,11 @@
     "prepare:package": "node scripts/prepare-package.js",
     "release:version": "node scripts/version.js",
     "telemetry": "node scripts/telemetry.js",
+    "data:validate": "tsx scripts/validate-data.ts",
+    "data:format": "prettier --write 'data/*.json' 'scripts/validate-data.ts' 'scripts/optimization/**/*.ts'",
+    "data:lint": "eslint 'scripts/validate-data.ts' 'scripts/optimization/**/*.ts'",
+    "optimize": "tsx scripts/optimization/optimize.ts",
+    "optimize:extract": "tsx scripts/optimization/extract.ts",
     "check:lockfile": "node scripts/check-lockfile.js",
     "clean": "node scripts/clean.js",
     "pre-commit": "node scripts/pre-commit.js"
@@ -142,6 +147,7 @@
     "simple-git": "^3.28.0"
   },
   "optionalDependencies": {
+    "@ax-llm/ax": "^19.0.11",
     "@lydell/node-pty": "1.1.0",
     "@lydell/node-pty-darwin-arm64": "1.1.0",
     "@lydell/node-pty-darwin-x64": "1.1.0",

scripts/optimization/README.md

@@ -0,0 +1,61 @@
+# Prompt optimization pipeline
+
+This directory contains the infrastructure for optimizing Gemini CLI system
+instructions and tool descriptions. The pipeline uses a manifest-driven approach
+to isolate, mask, and refine instructions to achieve high tool-call accuracy
+with minimalist token usage.
+
+## Overview
+
+The pipeline automates the "Success Density" strategy, which aims for 100%
+functional correctness while minimizing prompt length. It achieves this through
+multi-objective optimization using the Genetic-Pareto Algorithm (GEPA).
+
+## Architecture
+
+The optimization infrastructure consists of three primary components that handle
+data extraction, variable protection, and evolutionary refinement.
+
+- `extract.ts`: The pre-processing engine. It uses character-aware parsing to
+  isolate prompt strings from TypeScript source files based on the targets
+  defined in `data/manifest.json`.
+- `masking.ts`: A security and integrity utility. It identifies and replaces
+  TypeScript template variables (for example, `${FILE_PATH}`) with indexed
+  tokens (for example, `[[GCLI_VAR_0]]`). This prevents the LLM from corrupting
+  program logic during the optimization phase.
+- `optimize.ts`: The core evolution engine. It runs the GEPA loop to find the
+  Pareto frontier between functional alignment and instruction brevity.
+
+## Metrics
+
+The pipeline evaluates every prompt candidate using two primary objectives. Both
+metrics return a score between 0.0 and 1.0, where 1.0 represents a perfect
+result.
+
+1.  **Functional alignment:** Measures how accurately the model selects the
+    correct tool for a given user query. It uses the 113 high-signal scenarios
+    in `data/tool_alignment.jsonl` to verify behavioral correctness.
+2.  **Brevity:** A 4-tier step function that rewards concise model responses. It
+    measures the word count of the model's output text (excluding tool calls) to
+    penalize unnecessary verbosity.
+    - **1.0:** 10 words or fewer.
+    - **0.7:** 25 words or fewer.
+    - **0.4:** 50 words or fewer.
+    - **0.1:** More than 50 words.
+
+## Workflow
+
+Follow these steps to run the optimization pipeline.
+
+1.  **Extraction:** Run `npm run optimize:extract` to pull instructions from the
+    source code and generate the `data/optimization/targets.json` artifact.
+2.  **Configuration:** Update `scripts/optimization/optimization.config.json` to
+    specify the student and teacher models and the number of trials.
+3.  **Optimization:** Run `npm run optimize` to start the GEPA evolution. This
+    process produces a refined prompt that maintains accuracy while reducing
+    tokens.
+
+## Next steps
+
+After a successful optimization run, use the generated Pareto-optimal
+instructions to update the core snippets and tool definitions.

scripts/optimization/evals/config.ts

@@ -0,0 +1,106 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Configuration for the Tool Alignment objective (The Accuracy Dimension).
+ */
+export interface AlignmentConfig {
+  /**
+   * The relative importance of accuracy vs other objectives in the Pareto frontier.
+   */
+  weight: number;
+
+  /**
+   * Strongest negative signal (0.0): used when model falls into a known shell trap.
+   */
+  hardFailureScore: number;
+
+  /**
+   * Neutral negative signal (0.1): used when model fails to produce a valid tool call.
+   */
+  invalidResponseScore: number;
+
+  /**
+   * Partial positive signal (0.4): model chose the right tool but hallucinated arguments.
+   */
+  toolNameMatchOnlyScore: number;
+
+  /**
+   * Maximum positive signal (1.0): model matched the golden signature perfectly.
+   */
+  functionalSuccessScore: number;
+}
+
+/**
+ * Configuration for the Brevity objective (The Density Dimension).
+ * Uses a word-count step-function to provide high-contrast signal for GEPA.
+ */
+export interface BrevityConfig {
+  /**
+   * Importance of brevity relative to accuracy.
+   */
+  weight: number;
+
+  /**
+   * TIER 1: Response is perfectly succinct (e.g., <= 10 words).
+   */
+  succinctThresholdWords: number;
+  succinctScore: number; // 1.0
+
+  /**
+   * TIER 2: Response is acceptable but slightly verbose (e.g., <= 25 words).
+   */
+  acceptableThresholdWords: number;
+  acceptableScore: number; // 0.7
+
+  /**
+   * TIER 3: Response is verbose (e.g., <= 50 words).
+   */
+  verboseThresholdWords: number;
+  verboseScore: number; // 0.4
+
+  /**
+   * TIER 4: Response is very heavy (e.g., > 50 words).
+   */
+  heavyScore: number; // 0.1
+}
+
+/**
+ * Global evaluation configuration for multi-objective optimization.
+ */
+export interface EvalConfig {
+  objectives: {
+    alignment: AlignmentConfig;
+    brevity: BrevityConfig;
+  };
+}
+
+/**
+ * Default weights and thresholds for the Genetic-Pareto (GEPA) engine.
+ * These constants drive the 'Selection Pressure' that evolves the prompt.
+ * GEPA always MAXIMIZES, so higher scores represent better performance.
+ */
+export const DEFAULT_EVAL_CONFIG: EvalConfig = {
+  objectives: {
+    alignment: {
+      weight: 1.0, // PRIMARY: Accuracy cannot be sacrificed.
+      hardFailureScore: 0.0,
+      invalidResponseScore: 0.1,
+      toolNameMatchOnlyScore: 0.4,
+      functionalSuccessScore: 1.0,
+    },
+    brevity: {
+      weight: 0.6, // SECONDARY: Reward brevity once accuracy is high.
+      succinctThresholdWords: 10,
+      succinctScore: 1.0,
+      acceptableThresholdWords: 25,
+      acceptableScore: 0.7,
+      verboseThresholdWords: 50,
+      verboseScore: 0.4,
+      heavyScore: 0.1, // Never hard-zero brevity to allow gradient improvement.
+    },
+  },
+};

scripts/optimization/evals/metrics/brevityMetric.test.ts

@@ -0,0 +1,54 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, it, expect } from 'vitest';
+import { evaluateBrevity } from './brevityMetric.js';
+
+describe('evaluateBrevity 4-tier step-function', () => {
+  it('should return 1.0 for a succinct response (<= 10 words)', () => {
+    const prediction = { output_text: 'I have updated the file for you now.' }; // 8 words
+    const result = evaluateBrevity(prediction);
+    expect(result.score).toBe(1.0);
+    expect(result.metadata?.tier).toBe('succinct');
+  });
+
+  it('should return 0.7 for an acceptable response (11-25 words)', () => {
+    const text =
+      'I have successfully updated the file. Everything looks good to proceed with the next step.';
+    // 16 words
+    const prediction = { output_text: text };
+    const result = evaluateBrevity(prediction);
+    expect(result.score).toBe(0.7);
+    expect(result.metadata?.tier).toBe('acceptable');
+  });
+
+  it('should return 0.4 for a verbose response (26-50 words)', () => {
+    const text =
+      'Certainly! I would be more than happy to assist you with that request. I am now proceeding to surgically update the file using the replace tool to ensure accuracy.';
+    // 29 words
+    const prediction = { output_text: text };
+    const result = evaluateBrevity(prediction);
+    expect(result.score).toBe(0.4);
+    expect(result.metadata?.tier).toBe('verbose');
+  });
+
+  it('should return 0.1 for a heavy response (> 50 words)', () => {
+    const text =
+      'Certainly! I would be more than happy to assist you with that request. I am now proceeding to surgically update the file using the replace tool to ensure accuracy. I will then verify the changes and let you know when I am finished with the task so we can move to the next stage of implementation.';
+    // 53 words
+    const prediction = { output_text: text };
+    const result = evaluateBrevity(prediction);
+    expect(result.score).toBe(0.1);
+    expect(result.metadata?.tier).toBe('heavy');
+  });
+
+  it('should handle missing output text as succinct (0 words)', () => {
+    const prediction = {};
+    const result = evaluateBrevity(prediction);
+    expect(result.score).toBe(1.0);
+    expect(result.metadata?.tier).toBe('succinct');
+  });
+});

scripts/optimization/evals/metrics/brevityMetric.ts

@@ -0,0 +1,62 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { debugLogger } from '../../../../packages/core/src/utils/debugLogger.js';
+import { DEFAULT_EVAL_CONFIG } from '../config.js';
+import { MetricObjective } from '../types.js';
+import type { MetricResult } from '../types.js';
+
+/**
+ * Evaluates the brevity of a model's response using a tiered 4-step word-count function.
+ * Focuses on rewarding succinctness and providing a non-zero gradient for verbose models.
+ */
+export function evaluateBrevity(
+  prediction: { output_text?: string },
+  config = DEFAULT_EVAL_CONFIG.objectives.brevity,
+): MetricResult {
+  const chatter = (prediction.output_text ?? '').trim();
+
+  // Simple word count: split by whitespace and filter out empty strings
+  const wordCount = chatter === '' ? 0 : chatter.split(/\s+/).length;
+
+  debugLogger.debug(
+    `[Eval:Brevity] Measuring output text word count: ${wordCount} words.`,
+  );
+
+  let score: number;
+  let reason: string;
+
+  if (wordCount <= config.succinctThresholdWords) {
+    score = config.succinctScore;
+    reason = `Succinct: Response is within ${config.succinctThresholdWords} words.`;
+  } else if (wordCount <= config.acceptableThresholdWords) {
+    score = config.acceptableScore;
+    reason = `Acceptable: Response is slightly verbose (${wordCount} words), exceeding ${config.succinctThresholdWords} words.`;
+  } else if (wordCount <= config.verboseThresholdWords) {
+    score = config.verboseScore;
+    reason = `Verbose: Response contains ${wordCount} words, exceeding acceptable limit of ${config.acceptableThresholdWords} words.`;
+  } else {
+    score = config.heavyScore;
+    reason = `Heavy: Response is excessively verbose (${wordCount} words).`;
+  }
+
+  return {
+    score,
+    objective: MetricObjective.BREVITY,
+    reason,
+    metadata: {
+      wordCount,
+      tier:
+        score === 1.0
+          ? 'succinct'
+          : score === 0.7
+            ? 'acceptable'
+            : score === 0.4
+              ? 'verbose'
+              : 'heavy',
+    },
+  };
+}

scripts/optimization/evals/metrics/toolAlignment.test.ts

@@ -0,0 +1,68 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, it, expect } from 'vitest';
+import { evaluateToolAlignment } from './toolAlignment.js';
+import { MetricObjective } from '../types.js';
+
+describe('evaluateToolAlignment', () => {
+  const mockScenario = {
+    id: 'test-scenario',
+    input: { user_query: 'test query' },
+    expected: {
+      tool_calls: [{ name: 'test_tool', arguments: { arg: 1 } }],
+    },
+    negatives: [
+      {
+        tool_calls: [{ name: 'shell', arguments: { cmd: 'rm -rf' } }],
+        reason: 'Matched negative shell pattern',
+        severity: 'high',
+      }
+    ],
+  } as any;
+
+  it('should return 1.0 for a perfect functional match', () => {
+    const prediction = {
+      tool_calls: [{ name: 'test_tool', arguments: { arg: 1 } }],
+    };
+    const result = evaluateToolAlignment(prediction, mockScenario);
+    expect(result.score).toBe(1.0);
+    expect(result.objective).toBe(MetricObjective.ALIGNMENT);
+  });
+
+  it('should return 0.0 for a hard failure (negative match)', () => {
+    const prediction = {
+      tool_calls: [{ name: 'shell', arguments: { cmd: 'rm -rf' } }],
+    };
+    const result = evaluateToolAlignment(prediction, mockScenario);
+    expect(result.score).toBe(0.0);
+    expect(result.reason).toContain('Matched negative shell pattern');
+  });
+
+  it('should return 0.1 for an incorrect tool selection', () => {
+    const prediction = {
+      tool_calls: [{ name: 'wrong_tool', arguments: { arg: 1 } }],
+    };
+    const result = evaluateToolAlignment(prediction, mockScenario);
+    expect(result.score).toBe(0.1);
+  });
+
+  it('should return 0.4 for correct tool but wrong arguments', () => {
+    const prediction = {
+      tool_calls: [{ name: 'test_tool', arguments: { arg: 999 } }],
+    };
+    const result = evaluateToolAlignment(prediction, mockScenario);
+    expect(result.score).toBe(0.4);
+  });
+
+  it('should return 0.1 for an empty tool call list', () => {
+    const prediction = {
+      tool_calls: [],
+    };
+    const result = evaluateToolAlignment(prediction, mockScenario);
+    expect(result.score).toBe(0.1);
+  });
+});

scripts/optimization/evals/metrics/toolAlignment.ts

@@ -0,0 +1,106 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { debugLogger } from '../../../../packages/core/src/utils/debugLogger.js';
+import type { Scenario, ToolCall } from '../schema.js';
+import { DEFAULT_EVAL_CONFIG } from '../config.js';
+import { MetricObjective } from '../types.js';
+import type { MetricResult } from '../types.js';
+
+/**
+ * Evaluates the alignment of a model's predicted tool calls against a golden scenario.
+ * Focuses strictly on functional correctness (tool selection and argument precision).
+ */
+export function evaluateToolAlignment(
+  prediction: { tool_calls: ToolCall[] },
+  example: Scenario,
+  config = DEFAULT_EVAL_CONFIG.objectives.alignment,
+): MetricResult {
+  const { tool_calls: predictedCalls } = prediction;
+  const { expected, negatives, id: scenarioId } = example;
+
+  debugLogger.debug(`[Eval:${scenarioId}] Evaluating tool alignment...`);
+
+  // 1. Check for Hard Failures (Explicit Negatives)
+  // These are for specific "Forbidden" tool uses (e.g., using shell instead of read_file)
+  for (const negative of negatives) {
+    const isNegativeMatch = negative.tool_calls.every((negCall: ToolCall) =>
+      predictedCalls.some(
+        (predCall: ToolCall) =>
+          predCall.name === negCall.name &&
+          areArgsMatching(negCall.arguments, predCall.arguments),
+      ),
+    );
+
+    if (isNegativeMatch && negative.tool_calls.length > 0) {
+      return {
+        score: config.hardFailureScore,
+        objective: MetricObjective.ALIGNMENT,
+        reason: `Hard Failure: ${negative.reason}`,
+        metadata: { matchedNegativeReason: negative.reason },
+      };
+    }
+  }
+
+  // 2. Structural Check
+  if (predictedCalls.length === 0) {
+    return {
+      score: config.invalidResponseScore,
+      objective: MetricObjective.ALIGNMENT,
+      reason: 'Model failed to produce any tool calls.',
+    };
+  }
+
+  // 3. Functional Alignment Check
+  const expectedCalls = expected.tool_calls;
+
+  // Check if all expected tool names are present
+  const namesMatch = expectedCalls.every((exp: ToolCall) =>
+    predictedCalls.some((pred: ToolCall) => pred.name === exp.name),
+  );
+
+  if (!namesMatch) {
+    return {
+      score: config.invalidResponseScore,
+      objective: MetricObjective.ALIGNMENT,
+      reason: 'Model selected the wrong tool(s).',
+    };
+  }
+
+  // Check for Argument Precision
+  const argsMatch = expectedCalls.every((exp: ToolCall) =>
+    predictedCalls.some(
+      (pred: ToolCall) =>
+        pred.name === exp.name &&
+        areArgsMatching(exp.arguments, pred.arguments),
+    ),
+  );
+
+  if (!argsMatch) {
+    return {
+      score: config.toolNameMatchOnlyScore,
+      objective: MetricObjective.ALIGNMENT,
+      reason: 'Correct tool selected, but arguments are incorrect or missing.',
+    };
+  }
+
+  // 4. Perfect Success
+  return {
+    score: config.functionalSuccessScore,
+    objective: MetricObjective.ALIGNMENT,
+    reason: 'Functional Success: Tool and arguments align perfectly.',
+  };
+}
+
+/**
+ * Deep equality check for tool arguments.
+ */
+function areArgsMatching(
+  expected: Record<string, unknown>,
+  predicted: Record<string, unknown>,
+): boolean {
+  return JSON.stringify(expected) === JSON.stringify(predicted);
+}

scripts/optimization/evals/schema.ts

@@ -0,0 +1,49 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * The core data interface for the Tool Alignment Dataset.
+ * Designed to be extensible for custom error reports and metrics.
+ */
+
+export interface ToolCall {
+  name: string;
+  arguments: Record<string, unknown>;
+}
+
+export interface NegativeExample {
+  id?: string;
+  tool_calls: ToolCall[];
+  output_text?: string; // For "too chatty" or "hallucination" failures
+  reason: string; // e.g., "Defaulted to shell 'cat'", "Included conversational filler"
+  severity: 'low' | 'medium' | 'high'; // Helps the optimizer prioritize fixes
+}
+
+export interface Scenario {
+  id: string; // Unique identifier (e.g., 'read_file-01')
+  metadata: {
+    tags: string[]; // e.g., ['tool-alignment', 'shell-avoidance']
+    created_at: string;
+    platform?: 'darwin' | 'linux' | 'win32'; // To handle platform-specific shell variations
+    model_info?: {
+      // Placeholder for future tracking
+      name?: string;
+      version?: string;
+    };
+  };
+  input: {
+    user_query: string;
+    context?: {
+      current_file?: string;
+      directory_structure?: string[];
+    };
+  };
+  expected: {
+    tool_calls: ToolCall[];
+    rationale: string; // Why this is the 'Golden' choice
+  };
+  negatives: NegativeExample[]; // Array of multiple failure modes
+}

scripts/optimization/evals/types.ts

@@ -0,0 +1,40 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * The specific dimensions being measured by the evaluation pipeline.
+ */
+export enum MetricObjective {
+  ALIGNMENT = 'alignment',
+  BREVITY = 'brevity',
+}
+
+/**
+ * Standardized result for any metric calculation.
+ * Designed for consumption by the Genetic-Pareto (GEPA) multi-objective function.
+ */
+export interface MetricResult {
+  /**
+   * The numeric score calculated by the metric.
+   * All metrics must provide a value where HIGHER is BETTER.
+   */
+  score: number;
+
+  /**
+   * The specific objective this result corresponds to.
+   */
+  objective: MetricObjective;
+
+  /**
+   * A human-readable (and optimizer-reflective) reason for the score.
+   */
+  reason: string;
+
+  /**
+   * Additional data points (e.g., char counts, matched negative IDs).
+   */
+  metadata?: Record<string, unknown>;
+}

scripts/optimization/extract.test.ts

@@ -0,0 +1,99 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import * as fs from 'node:fs';
+import { runExtraction } from './extract.js';
+
+vi.mock('node:fs');
+
+describe('extraction script', () => {
+  const mockManifest = {
+    data_inventory: {
+      optimization_targets: {
+        snippets: ['renderCoreMandates'],
+      },
+      tools: {
+        read_file: {},
+      },
+    },
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+    vi.mocked(fs.readFileSync).mockImplementation((path) => {
+      if (typeof path !== 'string') return '';
+      if (path.includes('manifest.json')) return JSON.stringify(mockManifest);
+
+      // Mock snippets.ts
+      if (path.includes('snippets.ts')) {
+        return `
+          export function renderCoreMandates(options: any): string {
+            const foo = "Ignore me";
+            return \`# Core Mandate Instruction \${USER_VAR}\`.trim();
+          }
+        `;
+      }
+
+      // Mock gemini-3.ts
+      if (path.includes('gemini-3.ts')) {
+        return `
+  read_file: {
+    description: 'Read file description.',
+  },
+`;
+      }
+
+      // Mock dynamic helpers
+      if (path.includes('dynamic-declaration-helpers.ts')) {
+        return `
+          return \`This tool executes a given shell command as \\\`bash -c <command>\\\`. \${backgroundInstructions}\`;
+          name: EXIT_PLAN_MODE_TOOL_NAME,
+          description: 'Exit Plan Mode.',
+          name: ACTIVATE_SKILL_TOOL_NAME,
+          description: \`Activate skill.\`,
+        `;
+      }
+      return '';
+    });
+  });
+
+  it('should extract snippets correctly (Step 1)', async () => {
+    const targets = await runExtraction();
+    const snippet = targets.find((t) => t.id === 'snippets:renderCoreMandates');
+    expect(snippet).toBeDefined();
+    expect(snippet?.originalText).toBe(
+      '# Core Mandate Instruction ${USER_VAR}',
+    );
+    expect(snippet?.maskedText).toContain('[[GCLI_VAR_0]]');
+  });
+
+  it('should extract tools correctly (Step 2)', async () => {
+    const targets = await runExtraction();
+    const tool = targets.find((t) => t.id === 'gemini3:read_file:description');
+    expect(tool).toBeDefined();
+    expect(tool?.originalText).toBe('Read file description.');
+  });
+
+  it('should extract dynamic helpers correctly (Step 3)', async () => {
+    const targets = await runExtraction();
+    const shell = targets.find((t) => t.id === 'shell:darwin:description');
+    expect(shell).toBeDefined();
+    expect(shell?.maskedText).toContain('[[GCLI_VAR_0]]');
+
+    const exitPlan = targets.find((t) => t.id === 'exit_plan_mode:description');
+    expect(exitPlan?.originalText).toBe('Exit Plan Mode.');
+  });
+
+  it('should write targets.json to the correct directory', async () => {
+    await runExtraction();
+    expect(fs.writeFileSync).toHaveBeenCalledWith(
+      expect.stringContaining('targets.json'),
+      expect.any(String),
+    );
+  });
+});

scripts/optimization/extract.ts

@@ -0,0 +1,175 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { maskVariables } from './masking.js';
+
+export interface OptimizationTarget {
+  id: string;
+  sourceFile: string;
+  originalText: string;
+  maskedText: string;
+  maskMap: Record<string, string>;
+}
+
+/**
+ * Robustly finds a block bounded by { } using character scanning.
+ */
+function findBlockBounds(
+  content: string,
+  startIdx: number,
+): { start: number; end: number } | null {
+  const blockStart = content.indexOf('{', startIdx);
+  if (blockStart === -1) return null;
+
+  let braceCount = 0;
+  for (let i = blockStart; i < content.length; i++) {
+    if (content[i] === '{') braceCount++;
+    if (content[i] === '}') braceCount--;
+    if (braceCount === 0) {
+      return { start: blockStart, end: i };
+    }
+  }
+  return null;
+}
+
+/**
+ * Main extraction function.
+ */
+export async function runExtraction() {
+  const manifest = JSON.parse(fs.readFileSync('data/manifest.json', 'utf8'));
+  const targets: OptimizationTarget[] = [];
+
+  // 1. Snippets
+  const snippetNames =
+    manifest.data_inventory?.optimization_targets?.snippets || [];
+  const snippetsPath = 'packages/core/src/prompts/snippets.ts';
+  if (fs.existsSync(snippetsPath)) {
+    const content = fs.readFileSync(snippetsPath, 'utf8');
+    for (const name of snippetNames) {
+      const startIdx = content.indexOf(`export function ${name}`);
+      if (startIdx === -1) continue;
+
+      const bounds = findBlockBounds(content, startIdx);
+      if (!bounds) continue;
+
+      const body = content.substring(bounds.start, bounds.end + 1);
+      // Capture the LAST template literal
+      const tickMatches = [...body.matchAll(/`((?:[^`\\]|\\.)*)`/g)];
+      if (tickMatches.length > 0) {
+        const text = tickMatches[tickMatches.length - 1][1].trim();
+        const { maskedText, maskMap } = maskVariables(text);
+        targets.push({
+          id: `snippets:${name}`,
+          sourceFile: snippetsPath,
+          originalText: text,
+          maskedText,
+          maskMap,
+        });
+      }
+    }
+  }
+
+  // 2. Tools
+  const toolNames = Object.keys(manifest.data_inventory?.tools || {});
+  const gemini3Path =
+    'packages/core/src/tools/definitions/model-family-sets/gemini-3.ts';
+  if (fs.existsSync(gemini3Path)) {
+    const content = fs.readFileSync(gemini3Path, 'utf8');
+    for (const name of toolNames) {
+      // Find tool key (2-space indent)
+      const toolRegex = new RegExp(`^\\s{2}${name}:\\s*\\{`, 'm');
+      const match = toolRegex.exec(content);
+      if (!match) continue;
+
+      const bounds = findBlockBounds(content, match.index);
+      if (!bounds) continue;
+
+      const toolBlock = content.substring(match.index, bounds.end + 1);
+      const descRegex =
+        /description:\s*(?:`((?:[^`\\]|\\.)*)`|'([^']*)'|"([^"]*)")/g;
+      const descMatch = descRegex.exec(toolBlock);
+
+      if (descMatch) {
+        const text = (descMatch[1] || descMatch[2] || descMatch[3]).trim();
+        const { maskedText, maskMap } = maskVariables(text);
+        targets.push({
+          id: `gemini3:${name}:description`,
+          sourceFile: gemini3Path,
+          originalText: text,
+          maskedText,
+          maskMap,
+        });
+      }
+    }
+  }
+
+  // 3. Dynamic Helpers
+  const helpersPath =
+    'packages/core/src/tools/definitions/dynamic-declaration-helpers.ts';
+  if (fs.existsSync(helpersPath)) {
+    const content = fs.readFileSync(helpersPath, 'utf8');
+    const specs = [
+      {
+        id: 'shell:darwin:description',
+        regex:
+          /return `This tool executes a given shell command as \\`bash -c <command>\\`. ([\s\S]*?)`;/,
+      },
+      {
+        id: 'shell:win32:description',
+        regex:
+          /return `This tool executes a given shell command as \\`powershell\.exe -NoProfile -Command <command>\\`. ([\s\S]*?)`;/,
+      },
+      {
+        id: 'exit_plan_mode:description',
+        regex:
+          /name: EXIT_PLAN_MODE_TOOL_NAME,[\s\S]*?description:\s*'([^']*)',/,
+      },
+      {
+        id: 'activate_skill:description',
+        regex:
+          /name: ACTIVATE_SKILL_TOOL_NAME,[\s\S]*?description:\s*`((?:[^`\\]|\\.)*)`,/,
+      },
+    ];
+    for (const s of specs) {
+      const m = s.regex.exec(content);
+      if (m && m[1]) {
+        const text = m[1].trim();
+        const { maskedText, maskMap } = maskVariables(text);
+        targets.push({
+          id: s.id,
+          sourceFile: helpersPath,
+          originalText: text,
+          maskedText,
+          maskMap,
+        });
+      }
+    }
+  }
+
+  const outputDir = 'data/optimization';
+  if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true });
+
+  fs.writeFileSync(
+    path.join(outputDir, 'targets.json'),
+    JSON.stringify(targets, null, 2),
+  );
+  return targets;
+}
+
+// CLI Entrypoint
+const isMain =
+  process.argv[1] &&
+  fileURLToPath(import.meta.url) === fs.realpathSync(process.argv[1]);
+if (isMain) {
+  runExtraction()
+    // eslint-disable-next-line no-console
+    .then((t) => console.log(`✅ Extracted ${t.length} targets.`))
+    // eslint-disable-next-line no-console
+    .catch(console.error);
+}

scripts/optimization/masking.test.ts

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, it, expect } from 'vitest';
+import { maskVariables, unmaskVariables } from './masking.js';
+
+describe('optimization masking utility', () => {
+  it('should mask unique template variables with indexed tokens', () => {
+    const input = 'Use ${TOOL_A} to read ${FILE_PATH}. ${TOOL_A} is efficient.';
+    const { maskedText, maskMap } = maskVariables(input);
+
+    expect(maskedText).toContain('[[GCLI_VAR_0]]');
+    expect(maskedText).toContain('[[GCLI_VAR_1]]');
+    // Ensure all occurrences of the same variable are replaced with the same token
+    const toolAToken = Object.keys(maskMap).find(
+      (key) => maskMap[key] === '${TOOL_A}',
+    )!;
+    const count = maskedText.split(toolAToken).length - 1;
+    expect(count).toBe(2);
+    expect(maskedText).not.toContain('${TOOL_A}');
+  });
+
+  it('should perfectly restore original text during unmasking', () => {
+    const original = 'Update ${OLD_STR} with ${NEW_STR} in ${FILE_PATH}.';
+    const { maskedText, maskMap } = maskVariables(original);
+    const restored = unmaskVariables(maskedText, maskMap);
+
+    expect(restored).toBe(original);
+  });
+
+  it('should handle text with no variables', () => {
+    const input = 'Static text with no placeholders.';
+    const { maskedText, maskMap } = maskVariables(input);
+
+    expect(maskedText).toBe(input);
+    expect(Object.keys(maskMap).length).toBe(0);
+  });
+});

scripts/optimization/masking.ts

@@ -0,0 +1,61 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Utility to protect TypeScript template variables from being "optimized" by the LLM.
+ * Replaces ${VAR} with unique stable tokens and allows for perfect restoration.
+ */
+
+export interface MaskResult {
+  maskedText: string;
+  maskMap: Record<string, string>;
+}
+
+const MASK_PREFIX = '[[GCLI_VAR_';
+const MASK_SUFFIX = ']]';
+
+/**
+ * Replaces all instances of ${VARIABLE_NAME} with indexed tokens.
+ * Supports both SCREAMING_SNAKE_CASE and camelCase variables.
+ */
+export function maskVariables(text: string): MaskResult {
+  const maskMap: Record<string, string> = {};
+  // Refined regex to capture any variable pattern like ${variableName} or ${VARIABLE_NAME}
+  const variableRegex = /\${[a-zA-Z0-9_.]+}/g;
+  let index = 0;
+  let maskedText = text;
+
+  // Find all unique variables
+  const uniqueVars = Array.from(new Set(text.match(variableRegex) || []));
+
+  uniqueVars.forEach((v) => {
+    const token = `${MASK_PREFIX}${index}${MASK_SUFFIX}`;
+    maskMap[token] = v;
+    // Use a global regex for the specific variable to replace all occurrences
+    maskedText = maskedText.split(v).join(token);
+    index++;
+  });
+
+  return { maskedText, maskMap };
+}
+
+/**
+ * Restores original ${VARIABLE_NAME} patterns using the provided mask map.
+ */
+export function unmaskVariables(
+  text: string,
+  maskMap: Record<string, string>,
+): string {
+  let unmaskedText = text;
+  // Sort tokens by length descending to prevent partial replacement (e.g. VAR_10 before VAR_1)
+  const sortedTokens = Object.keys(maskMap).sort((a, b) => b.length - a.length);
+
+  sortedTokens.forEach((token) => {
+    const originalVar = maskMap[token];
+    unmaskedText = unmaskedText.split(token).join(originalVar);
+  });
+  return unmaskedText;
+}

scripts/optimization/optimization.config.json

@@ -0,0 +1,22 @@
+{
+  "models": {
+    "student": {
+      "provider": "google-gemini",
+      "modelId": "gemini-3.1-pro-preview"
+    },
+    "teacher": {
+      "provider": "google-gemini",
+      "modelId": "gemini-3.1-pro-preview"
+    }
+  },
+  "gepa": {
+    "numTrials": 5,
+    "minibatch": true,
+    "maxMetricCalls": 10
+  },
+  "paths": {
+    "scenarios": "data/tool_alignment.jsonl",
+    "targets": "data/optimization/targets.json",
+    "outputDir": "data/optimization"
+  }
+}

scripts/optimization/optimize.test.ts

@@ -0,0 +1,37 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { describe, it, expect, vi } from 'vitest';
+import { maskVariables, unmaskVariables } from './masking.js';
+
+vi.mock('node:fs');
+
+describe('Optimization Pipeline Infrastructure', () => {
+  // --- Masking Tests ---
+  describe('Masking Utility', () => {
+    it('should mask unique template variables and restore them', () => {
+      const input =
+        'Use ${TOOL_A} to read ${FILE_PATH}. ${TOOL_A} is efficient.';
+      const { maskedText, maskMap } = maskVariables(input);
+
+      expect(maskedText).toContain('[[GCLI_VAR_0]]');
+      expect(maskedText).not.toContain('${TOOL_A}');
+
+      const restored = unmaskVariables(maskedText, maskMap);
+      expect(restored).toBe(input);
+    });
+
+    it('should handle text with no variables', () => {
+      const input = 'Static text.';
+      const { maskedText, maskMap } = maskVariables(input);
+      expect(maskedText).toBe(input);
+      expect(Object.keys(maskMap).length).toBe(0);
+    });
+  });
+
+  // Note: Extraction tests remain in extract.test.ts
+  // Optimization logic is verified via dry runs and Pareto frontier outputs.
+});

scripts/optimization/optimize.ts

@@ -0,0 +1,213 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ai, AxGEPA, ax } from '@ax-llm/ax';
+import { evaluateToolAlignment } from './evals/metrics/toolAlignment.js';
+import { evaluateBrevity } from './evals/metrics/brevityMetric.js';
+import type { Scenario, ToolCall } from './evals/schema.js';
+
+interface OptimizationConfig {
+  models: {
+    student: { provider: string; modelId: string };
+    teacher: { provider: string; modelId: string };
+  };
+  gepa: {
+    numTrials?: number;
+    minibatch?: boolean;
+    maxMetricCalls?: number;
+  };
+  paths?: {
+    scenarios?: string;
+    targets?: string;
+    outputDir?: string;
+  };
+}
+
+interface AxPrediction {
+  tool_calls?: ToolCall[];
+  output_text?: string;
+}
+
+interface MetricArgs {
+  prediction: AxPrediction;
+  example: Scenario;
+}
+
+let currentCallCount = 0;
+let maxCallsExpected = 0;
+
+/**
+ * multiObjectiveMetric: Evaluates model performance with structured logging.
+ */
+function multiObjectiveMetric({ prediction, example }: MetricArgs): Record<string, number> {
+  currentCallCount++;
+  
+  const modelOutput = {
+    tool_calls: prediction.tool_calls || [],
+    output_text: prediction.output_text || '',
+  };
+
+  const alignment = evaluateToolAlignment(modelOutput, example);
+  const brevity = evaluateBrevity(modelOutput);
+
+  // eslint-disable-next-line no-console
+  console.log(`\n[ EVAL: ${currentCallCount}/${maxCallsExpected} | ${example.id} ]`);
+  // eslint-disable-next-line no-console
+  console.log(`Scores: Acc=${alignment.score.toFixed(2)} | Brev=${brevity.score.toFixed(2)}`);
+
+  return {
+    accuracy: alignment.score,
+    brevity: brevity.score,
+  };
+}
+
+/**
+ * Evolve a specific target snippet using GEPA.
+ */
+async function evolveTarget(
+  id: string,
+  allTargets: any[],
+  scenarios: any[],
+  config: OptimizationConfig,
+  apiKey: string,
+  outputDir: string
+) {
+  // eslint-disable-next-line no-console
+  console.log(`\n🎯 TARGETED EVOLUTION: ${id}`);
+
+  const target = allTargets.find((t) => t.id === id);
+  const backgroundContext = allTargets
+    .filter((t) => t.id !== id)
+    .map((t) => `\n### ${t.id}\n${t.maskedText}`)
+    .join('\n');
+
+  const student = ai({
+    name: config.models.student.provider,
+    apiKey,
+    config: { model: config.models.student.modelId },
+  });
+
+  const teacher = ai({
+    name: config.models.teacher.provider,
+    apiKey,
+    config: { model: config.models.teacher.modelId },
+  });
+
+  // Standard field names to avoid signature validation errors.
+  const gcliProgram = ax(
+    'user_query:string, platform:string, tags:string[], background_context:string -> tool_calls:json, output_text:string',
+    { instructions: target.maskedText }
+  );
+
+  const dataset = scenarios.map(s => ({ ...s, background_context: backgroundContext }));
+
+  const optimizer = new AxGEPA({
+    studentAI: student,
+    teacherAI: teacher,
+    numTrials: config.gepa.numTrials || 16,
+    minibatch: config.gepa.minibatch !== false,
+    verbose: true,
+  });
+
+  currentCallCount = 0;
+  maxCallsExpected = config.gepa.maxMetricCalls || 100;
+
+  const result = (await optimizer.compile(
+    gcliProgram,
+    dataset,
+    multiObjectiveMetric,
+    {
+      maxMetricCalls: maxCallsExpected,
+    }
+  )) as any;
+
+  // Save to consolidated registry
+  const resultsPath = path.join(outputDir, 'results.json');
+  let registry: Record<string, any> = {};
+  if (fs.existsSync(resultsPath)) {
+    registry = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));
+  }
+
+  // The 'instruction' field in result.optimizedProgram contains the winner for the mutable part
+  const optimizedText = result.optimizedProgram?.instruction || "ERROR_EXTRACTING";
+
+  registry[id] = {
+    timestamp: new Date().toISOString(),
+    bestScore: result.optimizedProgram?.bestScore,
+    optimizedText,
+    stats: result.stats,
+    report: result.report,
+    paretoFront: result.paretoFront?.map((entry: any) => ({
+      scores: entry.scores,
+      isBest: entry.isBest,
+      text: entry.instruction || entry.program?.instruction
+    }))
+  };
+
+  fs.writeFileSync(resultsPath, JSON.stringify(registry, null, 2));
+
+  // eslint-disable-next-line no-console
+  console.log(`✅ Evolution complete for ${id}.`);
+}
+
+/**
+ * Main Optimization Runner.
+ */
+export async function runOptimization(configPath: string) {
+  const apiKey = process.env.GEMINI_API_KEY;
+  if (!apiKey) throw new Error('GEMINI_API_KEY is not set.');
+
+  if (!fs.existsSync(configPath)) throw new Error(`Config not found: ${configPath}`);
+
+  const config = JSON.parse(fs.readFileSync(configPath, 'utf8')) as OptimizationConfig;
+  const scenariosPath = config.paths?.scenarios || 'data/tool_alignment.jsonl';
+  const targetsPath = config.paths?.targets || 'data/optimization/targets.json';
+  const outputDir = config.paths?.outputDir || 'data/optimization';
+
+  if (!fs.existsSync(targetsPath)) throw new Error(`Targets file not found: ${targetsPath}`);
+
+  const allTargets = JSON.parse(fs.readFileSync(targetsPath, 'utf8'));
+  const scenarios = fs
+    .readFileSync(scenariosPath, 'utf8')
+    .split('\n')
+    .filter(Boolean)
+    .map((line) => {
+      const data = JSON.parse(line);
+      return {
+        id: data.id,
+        user_query: data.input.user_query,
+        platform: data.metadata?.platform || 'unknown',
+        tags: data.metadata?.tags || [],
+        expected: data.expected,
+        negatives: data.negatives || [],
+      };
+    });
+
+  // Iterative mode over ALL targets
+  const targetsToOptimize = allTargets;
+
+  if (config.gepa.numTrials === 1) {
+    // eslint-disable-next-line no-console
+    console.log('🧪 Micro-Trial detected: Optimizing first target.');
+    await evolveTarget(targetsToOptimize[0].id, allTargets, scenarios.slice(0, 2), config, apiKey, outputDir);
+  } else {
+    for (const t of targetsToOptimize) {
+      await evolveTarget(t.id, allTargets, scenarios, config, apiKey, outputDir);
+    }
+  }
+}
+
+// CLI Entrypoint
+const currentFilePath = fileURLToPath(import.meta.url);
+const isMain = process.argv[1] && fs.realpathSync(currentFilePath) === fs.realpathSync(process.argv[1]);
+
+if (isMain) {
+  const configPath = path.join(path.dirname(currentFilePath), 'optimization.config.json');
+  runOptimization(configPath).catch(console.error);
+}

scripts/tests/vitest.config.ts

@@ -10,7 +10,10 @@ export default defineConfig({
   test: {
     globals: true,
     environment: 'node',
-    include: ['scripts/tests/**/*.test.{js,ts}'],
+    include: [
+      'scripts/tests/**/*.test.{js,ts}',
+      'scripts/optimization/**/*.test.ts',
+    ],
     setupFiles: ['scripts/tests/test-setup.ts'],
     coverage: {
       provider: 'v8',

scripts/validate-data.ts

@@ -0,0 +1,166 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import type { Scenario } from './optimization/lib/evals/schema.ts';
+
+const MANIFEST_FILE = 'data/manifest.json';
+const DEFAULT_DATA_DIR = 'data';
+
+async function validateFile(
+  filePath: string,
+  manifest: {
+    data_inventory: {
+      file_descriptions: Record<string, string>;
+      tools: Record<string, unknown>;
+      target_samples_per_tool: number;
+      overrides: Record<string, number>;
+    };
+    optimization_constraints: { immutable_tokens: string[] };
+  },
+): Promise<{ success: boolean; counts: Record<string, number> }> {
+  const description =
+    manifest.data_inventory.file_descriptions?.[filePath] ||
+    'No description available.';
+  console.log(`\n🔍 Validating: ${filePath}`);
+  console.log(`   Purpose: ${description}`);
+
+  const immutableTools = new Set(
+    manifest.optimization_constraints.immutable_tokens,
+  );
+  const toolCounts: Record<string, number> = {};
+
+  // Initialize counts for all known tools
+  Object.keys(manifest.data_inventory.tools).forEach((tool) => {
+    toolCounts[tool] = 0;
+  });
+
+  const lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
+  let hasErrors = false;
+
+  lines.forEach((line, index) => {
+    const lineNum = index + 1;
+    try {
+      const scenario: Scenario = JSON.parse(line) as Scenario;
+
+      if (
+        !scenario.id ||
+        !scenario.input ||
+        !scenario.expected ||
+        !scenario.negatives
+      ) {
+        throw new Error(
+          `Missing required fields in scenario ${scenario.id || 'at line ' + lineNum}`,
+        );
+      }
+
+      scenario.expected.tool_calls.forEach((tc) => {
+        if (!immutableTools.has(tc.name)) {
+          console.error(
+            `  ❌ Line ${lineNum}: Unknown tool "${tc.name}" in expected output.`,
+          );
+          hasErrors = true;
+        } else {
+          toolCounts[tc.name]++;
+        }
+      });
+
+      scenario.negatives.forEach((neg) => {
+        neg.tool_calls.forEach((tc) => {
+          if (!immutableTools.has(tc.name)) {
+            console.error(
+              `  ❌ Line ${lineNum}: Unknown tool "${tc.name}" in negative example.`,
+            );
+            hasErrors = true;
+          }
+        });
+      });
+    } catch (e) {
+      console.error(
+        `  ❌ Line ${lineNum}: Invalid JSON or Schema.`,
+        e instanceof Error ? e.message : e,
+      );
+      hasErrors = true;
+    }
+  });
+
+  if (!hasErrors) {
+    console.log(`  ✅ ${lines.length} scenarios validated successfully.`);
+  }
+
+  return { success: !hasErrors, counts: toolCounts };
+}
+
+async function run() {
+  console.log('📊 Starting Data Layer Validation...');
+
+  if (!fs.existsSync(MANIFEST_FILE)) {
+    console.error(`❌ Manifest not found: ${MANIFEST_FILE}`);
+    process.exit(1);
+  }
+
+  const manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf8'));
+  const targetFiles = process.argv.slice(2);
+
+  const filesToValidate =
+    targetFiles.length > 0
+      ? targetFiles
+      : [path.join(DEFAULT_DATA_DIR, 'tool_alignment.jsonl')];
+
+  const globalToolCounts: Record<string, number> = {};
+  let allSuccess = true;
+
+  for (const file of filesToValidate) {
+    if (!fs.existsSync(file)) {
+      console.warn(`⚠️ File not found: ${file}`);
+      continue;
+    }
+    const result = await validateFile(file, manifest);
+    if (!result.success) {
+      allSuccess = false;
+    }
+
+    // Aggregate counts
+    Object.entries(result.counts).forEach(([tool, count]) => {
+      globalToolCounts[tool] = (globalToolCounts[tool] || 0) + count;
+    });
+  }
+
+  // Final Coverage Report
+  console.log('\n📈 Global Tool Coverage Report (Aggregated):');
+  console.log('-------------------------');
+
+  const targetInventory = manifest.data_inventory.tools;
+  const overrides = manifest.data_inventory.overrides || {};
+  let totalScenarios = 0;
+
+  Object.keys(targetInventory)
+    .sort()
+    .forEach((tool) => {
+      const count = globalToolCounts[tool] || 0;
+      const target =
+        overrides[tool] || manifest.data_inventory.target_samples_per_tool;
+      const status = count >= target ? '✅' : '⚠️';
+      console.log(`${status} ${tool.padEnd(25)}: ${count}/${target}`);
+      totalScenarios += count;
+    });
+
+  console.log('-------------------------');
+  console.log(`Total Valid Scenarios: ${totalScenarios}`);
+
+  if (!allSuccess) {
+    console.error('\n❌ Validation completed with errors.');
+    process.exit(1);
+  } else {
+    console.log('\n✅ Data integrity check passed.');
+  }
+}
+
+run().catch((err) => {
+  console.error('Fatal validation error:', err);
+  process.exit(1);
+});