diff --git a/tools/caretaker-agent/evals/triage/helpers/generate_golden_spec.md b/tools/caretaker-agent/evals/triage/helpers/generate_golden_spec.md new file mode 100644 index 0000000000..a42767c913 --- /dev/null +++ b/tools/caretaker-agent/evals/triage/helpers/generate_golden_spec.md @@ -0,0 +1,97 @@ +# Golden Workable Spec Generator System Instructions + +You are an expert software engineering spec synthesizer assistant. Your +objective is to analyze a completed GitHub Issue and its associated PR diff, +inspect the PR changes, and synthesize a 100% FAIR, high-precision Golden +Workable Spec JSON and its evaluation rationale. + +## REQUIRED REASONING WORKFLOW (CHAIN OF THOUGHT) + +Before producing the final JSON object, you MUST execute this 2-Phase reasoning +process: + +### Phase 1: PR File & Fix Analysis + +Examine the PR title, PR body, and code diff. Identify all files modified in the +PR diff and the changes made in each. + +### Phase 2: The Fairness Pruning Pass (CRITICAL FOR BENCHMARK FAIRNESS) + +For EACH file modified in the PR diff, cross-reference it against the original +Issue Description and ask: + +1. _"Was this file strictly required to resolve the user's reported symptom in + the issue text?"_ +2. _"Or is this file a secondary refactoring, un-reported feature extension, or + internal architecture cleanup added opportunistically by the PR author?"_ + +**STRICT PRUNING RULE:** You MUST PRUNE all secondary refactoring files from +`files_to_modify`. Keep ONLY the primary target source file(s) directly +responsible for resolving the reported bug. + +## Workable Spec Synthesis Rules + +1. **Golden Spec Rationale (`golden_spec_rationale`):** Focus STRICTLY on what + source files were NOT kept (PRUNED) from `files_to_modify` and WHY: + - If files modified in the PR diff were pruned (e.g., secondary refactorings, + un-reported feature extensions, or internal architecture cleanups), + explicitly name each pruned file and explain why it was excluded for + benchmark fairness. + - If NO source files were pruned, state: _"No source files were pruned; all + PR modifications directly address the reported issue."_ + - Do NOT state obvious rules (such as _"test files were excluded from + files_to_modify"_). Keep the rationale focused purely on non-obvious + pruning decisions. +2. **Source Files Only:** `files_to_modify` inside `workable_spec` MUST contain + ONLY primary source code files. Strictly EXCLUDE test files (`*.test.ts`), + lockfiles (`package-lock.json`, `yarn.lock`), documentation markdown files, + and version bump files. Test files belong ONLY in + `testing_strategy.test_file`. +3. **Test File Grounding:** + - If the PR diff modified or created an automated test file, set + `testing_strategy.test_file` to that exact path. + - If the PR diff did NOT touch any automated test file, set + `testing_strategy.test_file` strictly to `"N/A"`. +4. **Concrete Names (If Applicable):** `summary.root_cause` and + `implementation_plan.steps` MUST reference specific function names, regular + expressions, constants, or data structures modified to fix the reported + issue. +5. **No Hand-Waving:** Avoid vague, generic, or hand-wavy phrasing (such as + _"update the code as needed"_, _"fix the logic"_, or _"adjust accordingly"_). + Every step must give concrete, unambiguous technical guidance. + +## Output JSON Template Requirements + +Your final response MUST be a raw JSON object strictly matching this structure. +Do not wrap in markdown code blocks: + +```json +{ + "golden_spec_rationale": "Focus strictly on what source files were PRUNED and why (or state 'No source files were pruned; all PR modifications directly address the reported issue').", + "workable_spec": { + "issue_id": "{owner}/{repo}#{issue_number}", + "summary": { + "problem": "Concise statement of reported problem strictly matching the issue description.", + "root_cause": "Analysis of root cause referencing specific functions/regexes modified in the PR diff if applicable.", + "context": "Additional technical context from issue and PR." + }, + "implementation_plan": { + "files_to_modify": ["path/to/primary_source_file.ts"], + "steps": [ + "Ordered step-by-step instructions strictly required to implement the fix for reported issue." + ] + }, + "testing_strategy": { + "test_file": "path/to/test_file.test.ts", + "expected_behavior": "Description of expected behavior after fix.", + "verification_steps": [ + "Specific test assertions to add/modify or manual CLI verification steps." + ], + "framework": "Testing framework used (e.g. Vitest or 'N/A' if no automated test file is present)." + } + } +} +``` + +Do not include metadata like spam assessment or effort tags. Keep it focused +entirely on instructions for code generation and testing. diff --git a/tools/caretaker-agent/evals/triage/helpers/generate_golden_spec.py b/tools/caretaker-agent/evals/triage/helpers/generate_golden_spec.py new file mode 100644 index 0000000000..28f9d936fb --- /dev/null +++ b/tools/caretaker-agent/evals/triage/helpers/generate_golden_spec.py @@ -0,0 +1,125 @@ +""" +Golden Workable Spec Generator Module. + +Uses the Antigravity SDK (google.antigravity) to synthesize a clean, high-precision +Workable Spec JSON directly from Issue and PR Diff text using generate_golden_spec.md. +""" + +import re +import os +import json +import asyncio +from pathlib import Path +from dotenv import load_dotenv +import sys + +# Ensure cloudrun/triage-worker is in sys.path for worker utility imports +CARETAKER_DIR = Path(__file__).resolve().parents[3] +TRIAGE_WORKER_DIR = CARETAKER_DIR / "cloudrun" / "triage-worker" +if str(TRIAGE_WORKER_DIR) not in sys.path: + sys.path.insert(0, str(TRIAGE_WORKER_DIR)) + +load_dotenv() + +from utils.validator import validate_triage_result +from utils.agent_logger import extract_final_output +from google.antigravity import Agent, LocalAgentConfig +from google.antigravity.hooks.policy import deny + +PROMPT_FILE = Path(__file__).parent / "generate_golden_spec.md" + + +def _parse_llm_json(raw_text: str) -> dict: + """Strips markdown fences and parses LLM JSON with fallback unescaping.""" + clean = raw_text.strip() + if clean.startswith("```"): + clean = clean.split("\n", 1)[-1].rsplit("\n", 1)[0].strip() + try: + data = json.loads(clean, strict=False) + except Exception: + cleaned = re.sub(r'\\(?![/"bfnrtu]|u[0-9a-fA-F]{4})', r'\\\\', re.sub(r"(? str: + """Loads prompt instructions from generate_golden_spec.md.""" + if not PROMPT_FILE.exists(): + raise FileNotFoundError(f"Required prompt file missing at: {PROMPT_FILE}") + with open(PROMPT_FILE, "r", encoding="utf-8") as f: + return f.read() + + +def generate_golden_spec(owner: str, repo: str, issue_number: int, issue_data: dict, pr_data: dict) -> dict: + """ + Invokes the Antigravity SDK (google.antigravity) Agent using generate_golden_spec.md + instructions to synthesize a clean, high-precision Workable Spec JSON and its rationale. + Returns a dict with keys: 'workable_spec' and 'golden_spec_rationale'. + """ + system_instruction = _load_system_instruction() + + # Filter out lockfiles and non-code noise from diff preview + raw_diff = pr_data.get("diff", "") + filtered_diff_lines = [] + skip_file = False + for line in raw_diff.split("\n"): + if line.startswith("diff --git"): + if any(x in line for x in ["package-lock.json", "yarn.lock", "pnpm-lock.yaml"]): + skip_file = True + else: + skip_file = False + if not skip_file: + filtered_diff_lines.append(line) + + filtered_diff = "\n".join(filtered_diff_lines) + + prompt = f"""Target Issue & PR Data for {owner}/{repo}#{issue_number}: + +Issue #{issue_number} Title: {issue_data.get('title', '')} +Issue Description / Body: +{issue_data.get('body', '')} + +PR #{pr_data.get('number', '')} Title: {pr_data.get('title', '')} +PR Body: +{pr_data.get('body', '')} + +PR Filtered Code Diff: +{filtered_diff}""" + + policies = [deny("*")] + + async def run_spec_agent(): + config = LocalAgentConfig( + system_instructions=system_instruction, + api_key=os.environ.get("GEMINI_API_KEY"), + policies=policies, + ) + + print(f"[EVAL] Initializing Antigravity Spec Generator Agent for Issue #{issue_number}...") + async with Agent(config) as agent: + response = await agent.chat(prompt) + resolved_chunks = await response.resolve() + raw_text = extract_final_output(resolved_chunks).strip() + + data = _parse_llm_json(raw_text) + + golden_spec_rationale = data.get("golden_spec_rationale", "") + workable_spec = data.get("workable_spec", data) + + payload_to_validate = { + "triage_metadata": {"quality": "OK", "effort_estimate": "SMALL"}, + "workable_spec": workable_spec + } + validate_triage_result(payload_to_validate) + print("Schema validation successful!") + + return { + "workable_spec": workable_spec, + "golden_spec_rationale": golden_spec_rationale + } + + return asyncio.run(run_spec_agent()) diff --git a/tools/caretaker-agent/evals/triage/helpers/github_api.py b/tools/caretaker-agent/evals/triage/helpers/github_api.py new file mode 100644 index 0000000000..94d5e95807 --- /dev/null +++ b/tools/caretaker-agent/evals/triage/helpers/github_api.py @@ -0,0 +1,103 @@ +""" +GitHub Information & Target Commit SHA Resolution Utility. + +Provides helper functions for querying GitHub REST API, extracting issue/PR metadata, +resolving target repository commit SHAs, and assembling golden issue JSON templates. +""" + +import os +import requests +from typing import Optional, Dict, Any + + +def _get_github_headers() -> Dict[str, str]: + """ + Optionally retrieves GITHUB_TOKEN (or GH_TOKEN) to authenticate requests. + """ + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + headers = {"Accept": "application/vnd.github.v3+json"} + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def get_issue_details(owner: str, repo: str, issue_number: int) -> Dict[str, Any]: + """Queries GitHub REST API for issue details (title, body, createdAt, labels).""" + url = f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}" + resp = requests.get(url, headers=_get_github_headers(), timeout=15) + if resp.status_code != 200: + raise RuntimeError(f"Failed to fetch issue #{issue_number} from GitHub API ({resp.status_code}): {resp.text}") + + data = resp.json() + return { + "owner": owner, + "repo": repo, + "number": data.get("number"), + "title": data.get("title", ""), + "body": data.get("body", "") or "", + "createdAt": data.get("created_at", ""), + "labels": data.get("labels", []) + } + + +def get_pr_details(owner: str, repo: str, pr_number: int) -> Dict[str, Any]: + """Queries GitHub REST API for PR details (title, body, baseRefOid, patch/diff).""" + url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + headers = _get_github_headers() + resp = requests.get(url, headers=headers, timeout=15) + if resp.status_code != 200: + raise RuntimeError(f"Failed to fetch PR #{pr_number} from GitHub API ({resp.status_code}): {resp.text}") + + data = resp.json() + + # Fetch unified patch/diff + diff_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + diff_headers = headers.copy() + diff_headers["Accept"] = "application/vnd.github.v3.diff" + diff_resp = requests.get(diff_url, headers=diff_headers, timeout=15) + diff_content = diff_resp.text if diff_resp.status_code == 200 else "" + + return { + "number": data.get("number"), + "title": data.get("title", ""), + "body": data.get("body", "") or "", + "baseRefOid": data.get("base", {}).get("sha", ""), + "diff": diff_content + } + + +def _get_commit_sha_at_timestamp(owner: str, repo: str, created_at: str) -> str: + """Queries GitHub REST API to find the closest commit SHA at or before the given timestamp.""" + if not created_at: + return "" + url = f"https://api.github.com/repos/{owner}/{repo}/commits?until={created_at}&per_page=1" + resp = requests.get(url, headers=_get_github_headers(), timeout=15) + if resp.status_code == 200: + commits = resp.json() + if isinstance(commits, list) and len(commits) > 0: + return commits[0].get("sha", "") + return "" + + +def resolve_target_version(owner: str, repo: str, issue_data: Dict[str, Any], pr_data: Optional[Dict[str, Any]] = None) -> str: + """ + Resolves the target Git commit SHA for an issue: + 1. If PR data contains baseRefOid (base commit before PR fix was merged), use that. + 2. Otherwise, query GitHub REST API for the commit SHA at issue createdAt timestamp via get_commit_sha_at_timestamp(). + 3. Fallback to 'main'. + """ + if pr_data and pr_data.get("baseRefOid"): + return pr_data["baseRefOid"] + + created_at = issue_data.get("createdAt", "") + if created_at: + try: + sha = _get_commit_sha_at_timestamp(owner, repo, created_at) + if sha: + return sha + except Exception as e: + print(f"[FETCH_GITHUB] Warning: Could not resolve commit SHA at timestamp: {e}") + + return "main" + + diff --git a/tools/caretaker-agent/evals/triage/tools/__init__.py b/tools/caretaker-agent/evals/triage/tools/__init__.py new file mode 100644 index 0000000000..1dec597a76 --- /dev/null +++ b/tools/caretaker-agent/evals/triage/tools/__init__.py @@ -0,0 +1 @@ +"""Maintainer CLI tools for dataset management and metrics.""" diff --git a/tools/caretaker-agent/evals/triage/tools/dataset_metrics.py b/tools/caretaker-agent/evals/triage/tools/dataset_metrics.py new file mode 100644 index 0000000000..aa02c17a7e --- /dev/null +++ b/tools/caretaker-agent/evals/triage/tools/dataset_metrics.py @@ -0,0 +1,113 @@ +""" +Golden Dataset Quality & Effort Metrics Diagnostic CLI Tool. + +CLI Usage: + python3 -m evals.triage.tools.dataset_metrics +""" + +from collections import Counter +from evals.triage.helpers.dataset import load_issues + +VALID_QUALITIES = ["OK", "SPAM", "EMPTY", "NEEDS_INFO", "FEATURE"] +VALID_EFFORTS = ["SMALL", "MEDIUM", "LARGE"] + + +def _validate_spec_integrity(issues) -> bool: + """ + Validation helper that enforces spec & metadata integrity across the dataset: + - Quality MUST be one of: OK, SPAM, EMPTY, NEEDS_INFO, FEATURE. + - OK issues MUST have a valid workable spec and effort estimate (SMALL, MEDIUM, LARGE). + - Non-OK issues MUST NOT have a workable spec and MUST have an empty effort string (""). + Prints ONLY the specific issues causing errors (if any). + """ + errors = [] + for data in issues: + issue_num = data.get("issue_number", 0) + quality = data.get("expected_quality", "") + effort = data.get("expected_effort", "") + spec = data.get("expected_workable_spec", {}) + has_spec = bool(spec and isinstance(spec, dict) and len(spec) > 0) + + # 1. Quality validity check + if quality not in VALID_QUALITIES: + errors.append(f" ❌ Issue #{issue_num}: Quality '{quality}' is invalid! Must be one of: {VALID_QUALITIES}") + + # 2. Spec & Effort checks + if quality == "OK": + if not has_spec: + errors.append(f" ❌ Issue #{issue_num}: Quality is 'OK' but missing workable spec!") + elif not (isinstance(spec, dict) and spec.get("summary") and spec.get("implementation_plan")): + errors.append(f" ❌ Issue #{issue_num}: Quality is 'OK' but workable spec structure is incomplete!") + + if effort not in VALID_EFFORTS: + errors.append(f" ❌ Issue #{issue_num}: Quality is 'OK' but effort '{effort}' is invalid! Must be one of: {VALID_EFFORTS}") + else: + if has_spec: + errors.append(f" ❌ Issue #{issue_num}: Quality is '{quality}' but has unexpected workable spec content: {spec}") + if effort != "": + errors.append(f" ❌ Issue #{issue_num}: Quality is '{quality}' but has non-empty effort estimate ('{effort}')!") + + if errors: + print("\n--- ⚠️ Spec & Metadata Validation Errors ---") + for err in errors: + print(err) + return False + else: + print("\n ✅ Spec & Metadata Integrity Check: All issues correctly configured.") + return True + + +def compute_metrics() -> bool: + issues = load_issues() + total_issues = len(issues) + + if total_issues == 0: + print("[METRICS] No golden issues found in Firestore.") + return True + + qualities = Counter() + ok_efforts = Counter() + + for data in issues: + quality = data.get("expected_quality", "") + effort = data.get("expected_effort", "") + + qualities[quality] += 1 + if quality == "OK": + ok_efforts[effort] += 1 + + print("\n" + "=" * 70) + print(" 📊 GOLDEN DATASET DIAGNOSTIC REPORT (Firestore)") + print("=" * 70) + print(f"📦 Total Golden Issues: {total_issues}") + + print("\n--- 🏷️ Expected Quality Breakdown ---") + for q in VALID_QUALITIES: + count = qualities.get(q, 0) + pct = (count / total_issues * 100) if total_issues else 0 + bar = "█" * count + print(f" {q:<12}: {count:>2} ({pct:>5.1f}%) {bar}") + + ok_count = qualities.get("OK", 0) + print(f"\n--- ⚡ Expected Effort Breakdown (For {ok_count} OK Issues) ---") + for e in VALID_EFFORTS: + count = ok_efforts.get(e, 0) + pct = (count / ok_count * 100) if ok_count else 0 + bar = "█" * count + print(f" {e:<12}: {count:>2} ({pct:>5.1f}%) {bar}") + + # Run clean Spec & Metadata Integrity Check + success = _validate_spec_integrity(issues) + + print("=" * 70 + "\n") + return success + + +def main(): + import sys + if not compute_metrics(): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/caretaker-agent/evals/triage/tools/generate_golden_issue.py b/tools/caretaker-agent/evals/triage/tools/generate_golden_issue.py new file mode 100644 index 0000000000..aef9f1adf5 --- /dev/null +++ b/tools/caretaker-agent/evals/triage/tools/generate_golden_issue.py @@ -0,0 +1,88 @@ +""" +Golden Issue Generator CLI Tool (Main Entrypoint). + +CLI usage: + python3 -m evals.triage.tools.generate_golden_issue --issue [--pr ] +""" + +import json +import argparse +from pathlib import Path + +from evals.triage.helpers.github_api import ( + get_issue_details, + get_pr_details, + resolve_target_version +) +from evals.triage.helpers.generate_golden_spec import generate_golden_spec + +OUTPUT_DIR = Path(__file__).parent.parent / "dataset" / "golden-issues" + + +def generate_golden_issue(owner: str, repo: str, issue_number: int, pr_number: int = None): + """Main orchestrator for generating a brand-new Golden Issue JSON file.""" + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + file_path = OUTPUT_DIR / f"gemini_cli_{issue_number}.json" + + print(f"Fetching Issue #{issue_number} details from {owner}/{repo}...") + issue_data = get_issue_details(owner, repo, issue_number) + + pr_data = {} + if pr_number: + print(f"Fetching PR #{pr_number} details from {owner}/{repo}...") + pr_data = get_pr_details(owner, repo, pr_number) + + workable_spec = {} + golden_spec_rationale = "" + + if pr_number: + print(f"[EVAL] Generating Golden Workable Spec for Issue #{issue_number} using PR #{pr_number}...") + spec_res = generate_golden_spec(owner, repo, issue_number, issue_data, pr_data) + workable_spec = spec_res["workable_spec"] + golden_spec_rationale = spec_res["golden_spec_rationale"] + + # Extract effort from labels if present + labels = [l.get("name", "").lower() for l in issue_data.get("labels", []) if isinstance(l, dict)] + effort_from_labels = "" + for effort in ["small", "medium", "large"]: + if f"effort/{effort}" in labels: + effort_from_labels = effort.upper() + break + + # Default quality to 'OK' if a PR is attached, otherwise empty string '' + expected_quality_default = "OK" if pr_number else "" + + template = { + "owner": owner, + "repo": repo, + "issue_number": issue_number, + "issue_title": issue_data.get("title", ""), + "issue_body": issue_data.get("body", ""), + "pr_number": pr_number or 0, + "target_version": resolve_target_version(owner, repo, issue_data, pr_data), + "expected_quality": expected_quality_default, + "expected_effort": effort_from_labels, + "notes": f"Created at {issue_data.get('createdAt', '')} by automated generate_golden_issue.py", + "golden_spec_rationale": golden_spec_rationale, + "expected_workable_spec": workable_spec + } + + with open(file_path, "w", encoding="utf-8") as f: + json.dump(template, f, indent=2) + + print(f"Successfully saved golden issue file to: {file_path}") + + +def main(): + parser = argparse.ArgumentParser(description="Generate a Golden Issue JSON file.") + parser.add_argument("--issue", type=int, required=True, help="GitHub Issue number") + parser.add_argument("--pr", type=int, default=None, help="Associated PR number (optional)") + parser.add_argument("--owner", type=str, default="google-gemini", help="Repository owner") + parser.add_argument("--repo", type=str, default="gemini-cli", help="Repository name") + + args = parser.parse_args() + generate_golden_issue(args.owner, args.repo, args.issue, args.pr) + + +if __name__ == "__main__": + main() diff --git a/tools/caretaker-agent/evals/triage/tools/sync_firestore.py b/tools/caretaker-agent/evals/triage/tools/sync_firestore.py new file mode 100644 index 0000000000..be8e3ecbf8 --- /dev/null +++ b/tools/caretaker-agent/evals/triage/tools/sync_firestore.py @@ -0,0 +1,83 @@ +""" +Bidirectional Firestore Synchronization CLI Tool. + +CLI Usage: + python3 -m evals.triage.tools.sync_firestore --to-firestore + python3 -m evals.triage.tools.sync_firestore --from-firestore +""" + +import json +import argparse +from pathlib import Path +from dotenv import load_dotenv +from google.cloud import firestore +from evals.triage.helpers.dataset import get_env_var + +load_dotenv() + +TRIAGE_EVAL_DIR = Path(__file__).resolve().parent.parent +GOLDEN_ISSUES_DIR = TRIAGE_EVAL_DIR / "dataset" / "golden-issues" + + +def _get_db(): + project_id = get_env_var("PROJECT_ID") + db_id = get_env_var("FIRESTORE_DATABASE") + collection_name = get_env_var("FIRESTORE_EVAL_COLLECTION") + db = firestore.Client(project=project_id, database=db_id) + return db, collection_name + + +def sync_to_firestore(): + db, collection_name = _get_db() + json_files = sorted([f for f in GOLDEN_ISSUES_DIR.glob("**/gemini_cli_*.json") if not f.name.startswith(".")]) + if not json_files: + print(f"[SYNC] No JSON files found in {GOLDEN_ISSUES_DIR}.") + return + + print(f"[SYNC] Uploading {len(json_files)} JSON file(s) to Firestore collection '{collection_name}'...") + for file_path in json_files: + filename = file_path.name + try: + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + doc_id = f"github_{data['owner']}_{data['repo']}_{data['issue_number']}" + db.collection(collection_name).document(doc_id).set(data) + print(f" -> Uploaded '{filename}' as '{doc_id}'") + except Exception as e: + print(f" -> Failed to upload '{filename}': {e}") + + +def sync_from_firestore(): + db, collection_name = _get_db() + GOLDEN_ISSUES_DIR.mkdir(parents=True, exist_ok=True) + docs = db.collection(collection_name).stream() + + count = 0 + print(f"[SYNC] Downloading documents from Firestore collection '{collection_name}'...") + for doc in docs: + data = doc.to_dict() + issue_num = data.get("issue_number") + if not issue_num: + continue + file_path = GOLDEN_ISSUES_DIR / f"gemini_cli_{int(issue_num)}.json" + file_path.write_text(json.dumps(data, indent=2), encoding="utf-8") + print(f" -> Downloaded Issue #{issue_num} to '{file_path.name}'") + count += 1 + print(f"[SYNC] Downloaded {count} file(s) to {GOLDEN_ISSUES_DIR}.") + + +def main(): + parser = argparse.ArgumentParser(description="Bidirectional Firestore Synchronization CLI Tool.") + group = parser.add_mutually_exclusive_group() + group.add_argument("--to-firestore", action="store_true", help="Upload local JSONs to Firestore (Default)") + group.add_argument("--from-firestore", action="store_true", help="Download Firestore docs to local JSONs") + + args = parser.parse_args() + if args.from_firestore: + sync_from_firestore() + else: + sync_to_firestore() + + +if __name__ == "__main__": + main()